protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException { BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader( ioManager, id, new LinkedBlockingQueue<>(), compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize); return new HeaderlessChannelReaderInputView(inReader, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), blockCount, lastSegmentLimit, false); }
@Override protected MemorySegment nextSegment(MemorySegment current) throws IOException { // check for end-of-stream if (this.numBlocksRemaining <= 0) { this.reader.close(); throw new EOFException(); } // send a request first. if we have only a single segment, this same segment will be the one obtained in // the next lines if (current != null) { sendReadRequest(current); } // get the next segment this.numBlocksRemaining--; return this.reader.getNextReturnedBlock(); }
this.externalInView.close(); this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false); return this.externalInView;
public HeaderlessChannelReaderInputView( BlockChannelReader<MemorySegment> reader, List<MemorySegment> memory, int numBlocks, int numBytesInLastBlock, boolean waitForFirstBlock, long offset) throws IOException { // postpone wait for first block after initializing offset // otherwise the offset is not set, and we can't seek input super(reader, memory, numBlocks, 0, false); checkArgument(offset >= 0, "`offset` can't be negative!"); this.numBlocksRemaining = numBlocks; this.lastBlockBytes = numBytesInLastBlock; this.offset = offset; if (waitForFirstBlock) { advance(); } }
this.currentSpilledBuildSide.closeAndDelete(); this.currentSpilledBuildSide = null;
this.externalInView.close(); this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false); return this.externalInView;
this.externalInView.close(); this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false); return this.externalInView;
public static AbstractChannelReaderInputView createInputView( IOManager ioManager, ChannelWithMeta channel, List<FileIOChannel> channels, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { CompressedHeaderlessChannelReaderInputView in = new CompressedHeaderlessChannelReaderInputView( channel.getChannel(), ioManager, compressionCodecFactory, compressionBlockSize, channel.getBlockCount()); channels.add(in.getReader()); return in; } else { BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel.getChannel()); channels.add(reader); return new HeaderlessChannelReaderInputView( reader, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), channel.getBlockCount(), channel.getNumBytesInLastBlock(), false); } }
@Override protected MemorySegment nextSegment(MemorySegment current) throws IOException { // check for end-of-stream if (this.numBlocksRemaining <= 0) { this.reader.close(); throw new EOFException(); } // send a request first. if we have only a single segment, this same segment will be the one obtained in // the next lines if (current != null) { sendReadRequest(current); } // get the next segment this.numBlocksRemaining--; return this.reader.getNextReturnedBlock(); }
this.externalInView.close(); this.externalInView = new HeaderlessChannelReaderInputView(reader, this.fullSegments, this.blockCount, this.numBytesInLastSegment, false); return this.externalInView;
private void newSpilledIterator(int channelID, long offset) throws IOException { ChannelWithMeta channel = channelIDs.get(channelID); currentChannelID = channelID; // close current reader first. closeCurrentFileReader(); // calculate segment number int segmentNum = (int) (offset / segmentSize); long seekPosition = segmentNum * segmentSize; // new reader. this.fileReader = ioManager.createBlockChannelReader(channel.getChannel()); if (offset > 0) { // seek to the beginning of that segment fileReader.seekToPosition(seekPosition); } ChannelReaderInputView inView = new HeaderlessChannelReaderInputView( fileReader, freeMemory, channel.getBlockCount() - segmentNum, channel.getNumBytesInLastBlock(), false, offset - seekPosition ); this.currentIterator = new PagedChannelReaderInputViewIterator<>( inView, null, binaryRowSerializer ); }
@Override protected MemorySegment nextSegment(MemorySegment current) throws IOException { // check for end-of-stream if (this.numBlocksRemaining <= 0) { this.reader.close(); throw new EOFException(); } // send a request first. if we have only a single segment, this same segment will be the one obtained in // the next lines if (current != null) { sendReadRequest(current); } // check if next segment is the first block isFirstBlock = (current == null); // get the next segment this.numBlocksRemaining--; return this.reader.getNextReturnedBlock(); }
final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(currentSpilledBuildSide, memory, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView,
@Override protected MemorySegment nextSegment(MemorySegment current) throws IOException { // check for end-of-stream if (this.numBlocksRemaining <= 0) { this.reader.close(); throw new EOFException(); } // send a request first. if we have only a single segment, this same segment will be the one obtained in // the next lines if (current != null) { sendReadRequest(current); } // get the next segment this.numBlocksRemaining--; return this.reader.getNextReturnedBlock(); }
final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(currentSpilledBuildSide, memory, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView,
final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(currentSpilledBuildSide, memory, p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false); final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView,