public List<String> getColumnNames() { return footer.getTypes().get(0).getFieldNames(); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { OrcProto.Footer footerProtobuf = OrcProto.Footer.newBuilder() .setWriter(PRESTO_WRITER_ID) .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(OrcMetadataWriter::toStripeInformation) .collect(toList())) .addAllTypes(footer.getTypes().stream() .map(OrcMetadataWriter::toType) .collect(toList())) .addAllStatistics(footer.getFileStats().stream() .map(OrcMetadataWriter::toColumnStatistics) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { DwrfProto.Footer footerProtobuf = DwrfProto.Footer.newBuilder() .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(DwrfMetadataWriter::toStripeInformation) .collect(toImmutableList())) .addAllTypes(footer.getTypes().stream() .map(DwrfMetadataWriter::toType) .collect(toImmutableList())) .addAllStatistics(footer.getFileStats().stream() .map(DwrfMetadataWriter::toColumnStatistics) .collect(toImmutableList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .addAllMetadata(STATIC_METADATA.entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .build(); return writeProtobufObject(output, footerProtobuf); }
offset, length, footer.getTypes(), decompressor, footer.getRowsInRowGroup(),
this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream); if (footer.getTypes().size() == 0) { throw new OrcCorruptionException(orcDataSource.getId(), "File has no columns");
orcDataSource.readFully(stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(), tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), Optional.empty(), newSimpleAggregatedMemoryContext(), tailBuffer.length)) { StripeFooter stripeFooter = ORC.createMetadataReader().readStripeFooter(footer.getTypes(), inputStream);
public List<String> getColumnNames() { return footer.getTypes().get(0).getFieldNames(); }
public List<String> getColumnNames() { return footer.getTypes().get(0).getFieldNames(); }
private List<ColumnInfo> getColumnInfo(OrcReader reader) { Optional<OrcFileMetadata> metadata = getOrcFileMetadata(reader); if (metadata.isPresent()) { return getColumnInfoFromOrcUserMetadata(metadata.get()); } // support for legacy files without metadata return getColumnInfoFromOrcColumnTypes(reader.getColumnNames(), reader.getFooter().getTypes()); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { OrcProto.Footer footerProtobuf = OrcProto.Footer.newBuilder() .setWriter(PRESTO_WRITER_ID) .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(OrcMetadataWriter::toStripeInformation) .collect(toList())) .addAllTypes(footer.getTypes().stream() .map(OrcMetadataWriter::toType) .collect(toList())) .addAllStatistics(footer.getFileStats().stream() .map(OrcMetadataWriter::toColumnStatistics) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { DwrfProto.Footer footerProtobuf = DwrfProto.Footer.newBuilder() .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(DwrfMetadataWriter::toStripeInformation) .collect(toImmutableList())) .addAllTypes(footer.getTypes().stream() .map(DwrfMetadataWriter::toType) .collect(toImmutableList())) .addAllStatistics(footer.getFileStats().stream() .map(DwrfMetadataWriter::toColumnStatistics) .collect(toImmutableList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .addAllMetadata(STATIC_METADATA.entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .build(); return writeProtobufObject(output, footerProtobuf); }
public OrcRecordReader createRecordReader( Map<Integer, Type> includedColumns, OrcPredicate predicate, long offset, long length, DateTimeZone hiveStorageTimeZone, AbstractAggregatedMemoryContext systemMemoryUsage) throws IOException { return new OrcRecordReader( requireNonNull(includedColumns, "includedColumns is null"), requireNonNull(predicate, "predicate is null"), footer.getNumberOfRows(), footer.getStripes(), footer.getFileStats(), metadata.getStripeStatsList(), orcDataSource, offset, length, footer.getTypes(), compressionKind, bufferSize, footer.getRowsInRowGroup(), requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"), metadataReader, maxMergeDistance, maxReadSize, systemMemoryUsage); }
offset, length, footer.getTypes(), decompressor, footer.getRowsInRowGroup(),
this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream); if (footer.getTypes().size() == 0) { throw new OrcCorruptionException(orcDataSource.getId(), "File has no columns");
orcDataSource.readFully(stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(), tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), Optional.empty(), newSimpleAggregatedMemoryContext(), tailBuffer.length)) { StripeFooter stripeFooter = ORC.createMetadataReader().readStripeFooter(footer.getTypes(), inputStream);