try (ArrowFileReader fileReader = new ArrowFileReader(dfs, jobOutputDir, resultMetadata.get(0), allocator)) { batchHolders.addAll(fileReader.read(0, 0)); try (ArrowFileReader fileReader = new ArrowFileReader(dfs, jobOutputDir, file, allocator)) { batchHolders.addAll(fileReader.read(fileOffset, fileLimit)); remaining -= fileLimit;
return ArrowFileReader.toBean(ArrowFileFormat.ArrowFileMetadata.parseFrom(metadataCaptor.getValue()));
start, limit, metadata.getRecordCount()); openFile(); batches.add(getEmptyBatch());
try(ArrowFileReader reader = new ArrowFileReader(FileSystem.get(FS_CONF), basePath, metadata, ALLOCATOR)) { List<RecordBatchHolder> batchHolders = reader.read(0, 5); assertEquals(1, batchHolders.size()); List<RecordBatchHolder> batchHolders = reader.read(0, 2); assertEquals(1, batchHolders.size()); List<RecordBatchHolder> batchHolders = reader.read(2, 2); assertEquals(1, batchHolders.size()); reader.read(-1, 1); fail("shouldn't be here"); } catch (IllegalArgumentException e) { reader.read(6, 1); fail("shouldn't be here"); } catch (IllegalArgumentException e) {
@Override public void execDataArrived(RpcOutcomeListener<Ack> outcomeListener, QueryWritableBatch result) { try (TimedBlock b = Timer.time("dataMetadataArrived"); QueryDataBatch dataBatch = LocalUserUtil.acquireData(allocator, outcomeListener, result); RecordBatchLoader loader = new RecordBatchLoader(allocator)) { b.addID("attempt=" + attemptId); loader.load(dataBatch.getHeader().getDef(), dataBatch.getData()); // Query output just contains the batch unique id and number of records in the batch. try (RecordBatchData batch = new RecordBatchData(loader, allocator)) { List<ValueVector> vectors = batch.getVectors(); if (vectors.size() < 4 || !(vectors.get(3) instanceof VarBinaryVector) ) { throw UserException.unsupportedError() .message("Job output contains invalid data") .build(logger); } VarBinaryVector metadataVector = (VarBinaryVector) vectors.get(3); for (int i = 0; i < batch.getRecordCount(); i++) { final ArrowFileFormat.ArrowFileMetadata metadata = ArrowFileFormat.ArrowFileMetadata.parseFrom(metadataVector.getObject(i)); job.getJobAttempt().getInfo().getResultMetadataList().add(ArrowFileReader.toBean(metadata)); } } } catch (Exception ex) { exception.addException(ex); } } }
@Test public void writeAndReadEmptyListVectors() throws Exception { try (final VectorContainer batchData = createBatch(1, testEmptyListVector())) { final Path basePath = new Path(dateGenFolder.getRoot().getPath()); final ArrowFileMetadata metadata = writeArrowFile(batchData); try (final ArrowFileReader reader = new ArrowFileReader(FileSystem.get(FS_CONF), basePath, metadata, ALLOCATOR)) { final List<RecordBatchHolder> batchHolders = reader.read(0, 1); assertEquals(1, batchHolders.size()); assertNotNull(batchHolders.get(0).getData()); assertEquals(0, batchHolders.get(0).getStart()); assertEquals(1, batchHolders.get(0).getEnd()); final BatchSchema schema = batchHolders.get(0).getData().getContainer().getSchema(); assertEquals(1, schema.getFieldCount()); assertEquals("emptyListVector", schema.getColumn(0).getName()); assertEquals(MinorType.LIST, Types.getMinorTypeForArrowType(schema.getColumn(0).getType())); final VectorContainer batchContainer = batchHolders.get(0).getData().getContainer(); assertTrue(Iterators.size(batchContainer.iterator()) == 1); for (final VectorWrapper<?> wrapper : batchContainer) { assertTrue(wrapper.getValueVector() instanceof ListVector); assertTrue(((ListVector) (wrapper.getValueVector())).getDataVector() instanceof ZeroVector); } releaseBatches(batchHolders); } } }
try(ArrowFileReader reader = new ArrowFileReader(FileSystem.get(FS_CONF), basePath, metadata, ALLOCATOR)) { List<RecordBatchHolder> batchHolders = reader.read(0, 15); assertEquals(3, batchHolders.size()); List<RecordBatchHolder> batchHolders = reader.read(0, 7); assertEquals(2, batchHolders.size()); List<RecordBatchHolder> batchHolders = reader.read(2, 11); assertEquals(3, batchHolders.size()); List<RecordBatchHolder> batchHolders = reader.read(7, 5); assertEquals(2, batchHolders.size());
testEmptyUnionVector()); ArrowFileMetadata metadata = writeArrowFile(batchData); try(ArrowFileReader reader = new ArrowFileReader(FileSystem.get(FS_CONF), basePath, metadata, ALLOCATOR)) { List<RecordBatchHolder> batchHolders = reader.read(0, 0); assertEquals(1, batchHolders.size()); reader.read(0, 1); fail("shouldn't be here"); } catch (IllegalArgumentException e) { reader.read(1, 1); fail("shouldn't be here"); } catch (IllegalArgumentException e) {