long length = stream.getLength(); int colIx = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind();
boolean hasIndexOnlyCols = false, hasAnyNonData = false; for (OrcProto.Stream stream : streamList) { long length = stream.getLength(); int colIx = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind();
boolean[] includedRgs = null; // Will always be the same for all cols at the moment. for (OrcProto.Stream stream : streamList) { long length = stream.getLength(); int colIx = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind();
ByteBuffer bb = range.getData().duplicate(); bb.position((int) (offset - range.getOffset())); bb.limit((int) (bb.position() + stream.getLength())); indexes[column] = OrcProto.RowIndex.parseFrom( InStream.createCodedInputStream("index", ReaderImpl.singleton(new BufferChunk(bb, 0)), stream.getLength(), codec, bufferSize)); ByteBuffer bb = range.getData().duplicate(); bb.position((int) (offset - range.getOffset())); bb.limit((int) (bb.position() + stream.getLength())); bloomFilterIndices[column] = OrcProto.BloomFilterIndex.parseFrom (InStream.createCodedInputStream("bloom_filter", ReaderImpl.singleton(new BufferChunk(bb, 0)), stream.getLength(), codec, bufferSize)); offset += stream.getLength();
CreateHelper list = new CreateHelper(); for (OrcProto.Stream stream : streamList) { long length = stream.getLength(); int column = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind();
/** * Plan the ranges of the file that we need to read given the list of * columns and row groups. * * @param streamList the list of streams available * @param includedColumns which columns are needed * @param doMergeBuffers * @return the list of disk ranges that will be loaded */ static DiskRangeList planReadPartialDataStreams (List<OrcProto.Stream> streamList, boolean[] includedColumns, boolean doMergeBuffers) { long offset = 0; // figure out which columns have a present stream DiskRangeList.CreateHelper list = new DiskRangeList.CreateHelper(); for (OrcProto.Stream stream : streamList) { long length = stream.getLength(); int column = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind(); // since stream kind is optional, first check if it exists if (stream.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamKind) == org.apache.orc.impl.StreamName.Area.DATA) && includedColumns[column]) { RecordReaderUtils.addEntireStreamToRanges(offset, length, list, doMergeBuffers); } offset += length; } return list.extract(); }
/** * Plan the ranges of the file that we need to read given the list of * columns and row groups. * * @param streamList the list of streams available * @param includedColumns which columns are needed * @param doMergeBuffers * @return the list of disk ranges that will be loaded */ static DiskRangeList planReadPartialDataStreams (List<OrcProto.Stream> streamList, boolean[] includedColumns, boolean doMergeBuffers) { long offset = 0; // figure out which columns have a present stream DiskRangeList.CreateHelper list = new DiskRangeList.CreateHelper(); for (OrcProto.Stream stream : streamList) { long length = stream.getLength(); int column = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind(); // since stream kind is optional, first check if it exists if (stream.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamKind) == org.apache.orc.impl.StreamName.Area.DATA) && includedColumns[column]) { RecordReaderUtils.addEntireStreamToRanges(offset, length, list, doMergeBuffers); } offset += length; } return list.extract(); }
void createStreams(List<OrcProto.Stream> streamDescriptions, DiskRangeList ranges, boolean[] includeColumn, CompressionCodec codec, int bufferSize, Map<org.apache.orc.impl.StreamName, InStream> streams) throws IOException { long streamOffset = 0; for (OrcProto.Stream streamDesc : streamDescriptions) { int column = streamDesc.getColumn(); if ((includeColumn != null && !includeColumn[column]) || streamDesc.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamDesc.getKind()) != org.apache.orc.impl.StreamName.Area.DATA)) { streamOffset += streamDesc.getLength(); continue; } List<DiskRange> buffers = RecordReaderUtils.getStreamBuffers( ranges, streamOffset, streamDesc.getLength()); org.apache.orc.impl.StreamName name = new StreamName(column, streamDesc.getKind()); streams.put(name, InStream.create(name.toString(), buffers, streamDesc.getLength(), codec, bufferSize)); streamOffset += streamDesc.getLength(); } }
void createStreams(List<OrcProto.Stream> streamDescriptions, DiskRangeList ranges, boolean[] includeColumn, CompressionCodec codec, int bufferSize, Map<org.apache.orc.impl.StreamName, InStream> streams) throws IOException { long streamOffset = 0; for (OrcProto.Stream streamDesc : streamDescriptions) { int column = streamDesc.getColumn(); if ((includeColumn != null && !includeColumn[column]) || streamDesc.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamDesc.getKind()) != org.apache.orc.impl.StreamName.Area.DATA)) { streamOffset += streamDesc.getLength(); continue; } List<DiskRange> buffers = RecordReaderUtils.getStreamBuffers( ranges, streamOffset, streamDesc.getLength()); org.apache.orc.impl.StreamName name = new StreamName(column, streamDesc.getKind()); streams.put(name, InStream.create(name.toString(), buffers, streamDesc.getLength(), codec, bufferSize)); streamOffset += streamDesc.getLength(); } }
void createStreams(List<OrcProto.Stream> streamDescriptions, DiskRangeList ranges, boolean[] includeColumn, CompressionCodec codec, int bufferSize, Map<StreamName, InStream> streams) throws IOException { long streamOffset = 0; for (OrcProto.Stream streamDesc : streamDescriptions) { int column = streamDesc.getColumn(); if ((includeColumn != null && (column < includeColumn.length && !includeColumn[column])) || streamDesc.hasKind() && (StreamName.getArea(streamDesc.getKind()) != StreamName.Area.DATA)) { streamOffset += streamDesc.getLength(); continue; } List<DiskRange> buffers = RecordReaderUtils.getStreamBuffers( ranges, streamOffset, streamDesc.getLength()); StreamName name = new StreamName(column, streamDesc.getKind()); streams.put(name, InStream.create(name.toString(), buffers, streamDesc.getLength(), codec, bufferSize)); streamOffset += streamDesc.getLength(); } }
public Builder mergeFrom(org.apache.orc.OrcProto.Stream other) { if (other == org.apache.orc.OrcProto.Stream.getDefaultInstance()) return this; if (other.hasKind()) { setKind(other.getKind()); } if (other.hasColumn()) { setColumn(other.getColumn()); } if (other.hasLength()) { setLength(other.getLength()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
public StreamContext(OrcProto.Stream stream, long streamOffset, int streamIndexOffset) { this.kind = stream.getKind(); this.length = stream.getLength(); this.offset = streamOffset; this.streamIndexOffset = streamIndexOffset; }
public StreamContext(OrcProto.Stream stream, long streamOffset, int streamIndexOffset) { this.kind = stream.getKind(); this.length = stream.getLength(); this.offset = streamOffset; this.streamIndexOffset = streamIndexOffset; }