private RowBlock read(TempFile tempFile, Type readerType) throws IOException { DataSize dataSize = new DataSize(1, MEGABYTE); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), dataSize, dataSize, dataSize, true); OrcReader orcReader = new OrcReader(orcDataSource, ORC, dataSize, dataSize, dataSize, dataSize); Map<Integer, Type> includedColumns = new HashMap<>(); includedColumns.put(0, readerType); OrcRecordReader recordReader = orcReader.createRecordReader(includedColumns, OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), OrcReader.INITIAL_BATCH_SIZE); recordReader.nextBatch(); RowBlock block = (RowBlock) recordReader.readBlock(readerType, 0); recordReader.close(); return block; }
private static void writeOrcColumnPresto(File outputFile, Format format, CompressionKind compression, Type type, Iterator<?> values, OrcWriterStats stats) throws Exception { ImmutableMap.Builder<String, String> metadata = ImmutableMap.builder(); metadata.put("columns", "test"); metadata.put("columns.types", createSettableStructObjectInspector("test", type).getTypeName()); OrcWriter writer; writer = new OrcWriter( new OutputStreamOrcDataSink(new FileOutputStream(outputFile)), ImmutableList.of("test"), ImmutableList.of(type), format.getOrcEncoding(), compression, new OrcWriterOptions(), ImmutableMap.of(), HIVE_STORAGE_TIME_ZONE, true, BOTH, stats); BlockBuilder blockBuilder = type.createBlockBuilder(null, 1024); while (values.hasNext()) { Object value = values.next(); writeValue(type, blockBuilder, value); } writer.write(new Page(blockBuilder.build())); writer.close(); writer.validate(new FileOrcDataSource(outputFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true)); }
@Test public void testIntegration() throws IOException { // tiny file TestingOrcDataSource orcDataSource = new TestingOrcDataSource( new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), true)); doIntegration(orcDataSource, new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE)); assertEquals(orcDataSource.getReadCount(), 1); // read entire file at once // tiny stripes orcDataSource = new TestingOrcDataSource( new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), true)); doIntegration(orcDataSource, new DataSize(400, Unit.KILOBYTE), new DataSize(400, Unit.KILOBYTE), new DataSize(400, Unit.KILOBYTE)); assertEquals(orcDataSource.getReadCount(), 3); // footer, first few stripes, last few stripes }
@Test public void testReadUserMetadata() throws Exception { try (TempFile tempFile = new TempFile()) { Map<String, String> metadata = ImmutableMap.of( "a", "ala", "b", "ma", "c", "kota"); createFileWithOnlyUserMetadata(tempFile.getFile(), metadata); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(orcDataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); Footer footer = orcReader.getFooter(); Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii); assertEquals(readMetadata, metadata); } }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(tinyIntNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TINYINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(decimalNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DECIMAL_TYPE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(floatNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, REAL), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(doubleNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DOUBLE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(doubleWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DOUBLE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(bigintWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BIGINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(booleanWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BOOLEAN), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(tinyIntWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TINYINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(timestampNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TIMESTAMP), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(floatWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, REAL), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(bigintNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BIGINT), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(varcharWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, VARCHAR), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(timestampWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, TIMESTAMP), OrcPredicate.TRUE, UTC, newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(dataPath, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DECIMAL_TYPE), OrcPredicate.TRUE, DateTimeZone.UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(booleanNoNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, BOOLEAN), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }
private OrcRecordReader createRecordReader() throws IOException { OrcDataSource dataSource = new FileOrcDataSource(decimalWithNullFile, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(dataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); return orcReader.createRecordReader( ImmutableMap.of(0, DECIMAL_TYPE), OrcPredicate.TRUE, UTC, // arbitrary newSimpleAggregatedMemoryContext(), INITIAL_BATCH_SIZE); }