@Override protected Page computeNext() { try { if (Thread.currentThread().isInterrupted()) { throw new InterruptedIOException(); } int batchSize = reader.nextBatch(); if (batchSize <= 0) { return endOfData(); } Block[] blocks = new Block[types.size()]; for (int i = 0; i < types.size(); i++) { blocks[i] = reader.readBlock(types.get(i), i).getLoadedBlock(); } return new Page(batchSize, blocks); } catch (IOException e) { throw new PrestoException(HIVE_WRITER_DATA_ERROR, "Failed to read temporary data"); } } }
@Benchmark public Object readBooleanWithNull(BooleanWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BOOLEAN, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDecimalWithNull(DecimalWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DECIMAL_TYPE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readLongDirectNoNull(BigintNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BIGINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readSliceDictionaryNoNull(VarcharNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(VARCHAR, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readSliceDictionaryWithNull(VarcharWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(VARCHAR, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDecimal(BenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DECIMAL_TYPE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readBooleanNoNull(BooleanNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BOOLEAN, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readByteWithNull(TinyIntWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TINYINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDoubleNoNull(DoubleNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DOUBLE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readFloatWithNull(FloatWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(REAL, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readLongDirectWithNull(BigintWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(BIGINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readTimestampNoNull(TimestampNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TIMESTAMP, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readTimestampWithNull(TimestampWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TIMESTAMP, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readByteNoNull(TinyIntNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(TINYINT, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDecimalNoNull(DecimalNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DECIMAL_TYPE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readDoubleWithNull(DoubleWithNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(DOUBLE, 0); blocks.add(block); } return blocks; }
@Benchmark public Object readFloatNoNull(FloatNoNullBenchmarkData data) throws Throwable { OrcRecordReader recordReader = data.createRecordReader(); List<Block> blocks = new ArrayList<>(); while (recordReader.nextBatch() > 0) { Block block = recordReader.readBlock(REAL, 0); blocks.add(block); } return blocks; }
@Test public void testEntireFile() throws Exception { try (TempFile tempFile = new TempFile()) { createMultiStripeFile(tempFile.getFile()); try (OrcRecordReader reader = createCustomOrcRecordReader(tempFile, ORC, OrcPredicate.TRUE, BIGINT, MAX_BATCH_SIZE)) { assertEquals(reader.getReaderRowCount(), 100); assertEquals(reader.getReaderPosition(), 0); assertEquals(reader.getFileRowCount(), reader.getReaderRowCount()); assertEquals(reader.getFilePosition(), reader.getReaderPosition()); for (int i = 0; i < 5; i++) { assertEquals(reader.nextBatch(), 20); assertEquals(reader.getReaderPosition(), i * 20L); assertEquals(reader.getFilePosition(), reader.getReaderPosition()); assertCurrentBatch(reader, i); } assertEquals(reader.nextBatch(), -1); assertEquals(reader.getReaderPosition(), 100); assertEquals(reader.getFilePosition(), reader.getReaderPosition()); } } }
private RowBlock read(TempFile tempFile, Type readerType) throws IOException { DataSize dataSize = new DataSize(1, MEGABYTE); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), dataSize, dataSize, dataSize, true); OrcReader orcReader = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize, dataSize); Map<Integer, Type> includedColumns = new HashMap<>(); includedColumns.put(0, readerType); OrcRecordReader recordReader = orcReader.createRecordReader(includedColumns, OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), OrcReader.INITIAL_BATCH_SIZE); recordReader.nextBatch(); RowBlock block = (RowBlock) recordReader.readBlock(readerType, 0); recordReader.close(); return block; }