@Override public long getCompletedBytes() { return orcDataSource.getReadBytes(); }
@Override public long getReadTimeNanos() { return orcDataSource.getReadTimeNanos(); }
@Override public OrcDataSourceId getId() { return dataSource.getId(); }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, OrcDataSourceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, OrcDataSourceInput> entry : streamsData.entrySet()) { OrcDataSourceInput sourceInput = entry.getValue(); streamsBuilder.put(entry.getKey(), new OrcInputStream(orcDataSource.getId(), sourceInput.getInput(), decompressor, systemMemoryUsage, sourceInput.getRetainedSizeInBytes())); } return streamsBuilder.build(); }
this.orcDataSource = orcDataSource; requireNonNull(orcEncoding, "orcEncoding is null"); this.metadataReader = new ExceptionWrappingMetadataReader(orcDataSource.getId(), orcEncoding.createMetadataReader()); this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null"); this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null"); long size = orcDataSource.getSize(); if (size <= MAGIC.length()) { throw new OrcCorruptionException(orcDataSource.getId(), "Invalid file size %s", size); orcDataSource.readFully(size - buffer.length, buffer); throw new OrcCorruptionException(orcDataSource.getId(), "Invalid postscript length %s", postScriptSize); throw new OrcCorruptionException(orcDataSource.getId(), "Not an ORC file"); this.decompressor = createOrcDecompressor(orcDataSource.getId(), compressionKind, bufferSize); validateWrite(validation -> validation.getCompression() == compressionKind, "Unexpected compression"); orcDataSource.readFully(size - completeFooterSize, newBuffer, 0, completeFooterSize - buffer.length); try (InputStream metadataInputStream = new OrcInputStream(orcDataSource.getId(), metadataSlice.getInput(), decompressor, newSimpleAggregatedMemoryContext(), metadataSize)) { this.metadata = metadataReader.readMetadata(hiveWriterVersion, metadataInputStream); try (InputStream footerInputStream = new OrcInputStream(orcDataSource.getId(), footerSlice.getInput(), decompressor, newSimpleAggregatedMemoryContext(), footerSize)) { this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream); throw new OrcCorruptionException(orcDataSource.getId(), "File has no columns");
@Override public <K> Map<K, OrcDataSourceInput> readFully(Map<K, DiskRange> diskRanges) throws IOException { readCount += diskRanges.size(); lastReadRanges = ImmutableList.copyOf(diskRanges.values()); return delegate.readFully(diskRanges); } }
@Override public long getSize() { return dataSource.getSize(); }
@Override public void close() throws IOException { dataSource.close(); }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, OrcDataSourceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, OrcDataSourceInput> entry : streamsData.entrySet()) { OrcDataSourceInput sourceInput = entry.getValue(); streamsBuilder.put(entry.getKey(), new OrcInputStream(orcDataSource.getId(), sourceInput.getInput(), decompressor, systemMemoryUsage, sourceInput.getRetainedSizeInBytes())); } return streamsBuilder.build(); }
this.orcDataSource = orcDataSource; requireNonNull(orcEncoding, "orcEncoding is null"); this.metadataReader = new ExceptionWrappingMetadataReader(orcDataSource.getId(), orcEncoding.createMetadataReader()); this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null"); this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null"); long size = orcDataSource.getSize(); if (size <= MAGIC.length()) { throw new OrcCorruptionException(orcDataSource.getId(), "Invalid file size %s", size); orcDataSource.readFully(size - buffer.length, buffer); throw new OrcCorruptionException(orcDataSource.getId(), "Invalid postscript length %s", postScriptSize); throw new OrcCorruptionException(orcDataSource.getId(), "Not an ORC file"); this.decompressor = createOrcDecompressor(orcDataSource.getId(), compressionKind, bufferSize); validateWrite(validation -> validation.getCompression() == compressionKind, "Unexpected compression"); orcDataSource.readFully(size - completeFooterSize, newBuffer, 0, completeFooterSize - buffer.length); try (InputStream metadataInputStream = new OrcInputStream(orcDataSource.getId(), metadataSlice.getInput(), decompressor, newSimpleAggregatedMemoryContext(), metadataSize)) { this.metadata = metadataReader.readMetadata(hiveWriterVersion, metadataInputStream); try (InputStream footerInputStream = new OrcInputStream(orcDataSource.getId(), footerSlice.getInput(), decompressor, newSimpleAggregatedMemoryContext(), footerSize)) { this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream); throw new OrcCorruptionException(orcDataSource.getId(), "File has no columns");
@Override public <K> Map<K, OrcDataSourceInput> readFully(Map<K, DiskRange> diskRanges) throws IOException { readCount += diskRanges.size(); lastReadRanges = ImmutableList.copyOf(diskRanges.values()); return delegate.readFully(diskRanges); } }
@Override public long getSize() { return dataSource.getSize(); }
@Override public void close() throws IOException { dataSource.close(); }
@Override public OrcDataSourceId getId() { return dataSource.getId(); }
public StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) throws IOException { long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); int tailLength = toIntExact(stripe.getFooterLength()); // read the footer byte[] tailBuffer = new byte[tailLength]; orcDataSource.readFully(offset, tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), decompressor, systemMemoryUsage, tailLength)) { return metadataReader.readStripeFooter(types, inputStream); } }
@Override public long getReadTimeNanos() { return dataSource.getReadTimeNanos(); }
@Override public long getReadBytes() { return dataSource.getReadBytes(); }
@Override public void readFully(long position, byte[] buffer) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, buffer.length)); delegate.readFully(position, buffer); }
@Override public long getSize() { return delegate.getSize(); }
orcDataSource.close();