/** * Creates a block channel reader that reads blocks from the given channel. The reader pushed * full memory segments (with the read data) to its "return queue", to allow for asynchronous read * implementations. * * @param channelID The descriptor for the channel to write to. * @return A block channel reader that reads from the given channel. * @throws IOException Thrown, if the channel for the reader could not be opened. */ public BlockChannelReader<MemorySegment> createBlockChannelReader(FileIOChannel.ID channelID) throws IOException { return createBlockChannelReader(channelID, new LinkedBlockingQueue<MemorySegment>()); }
public ChannelReaderKVInputViewIterator( IOManager ioAccess, FileIOChannel.ID channel, LinkedBlockingQueue<MemorySegment> returnQueue, List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<K> keySerializer, TypeSerializer<V> valueSerializer, int numBlocks) throws IOException { this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue, segments, freeMemTarget, keySerializer, valueSerializer, numBlocks); }
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel, LinkedBlockingQueue<MemorySegment> returnQueue, List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks) throws IOException { this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue, segments, freeMemTarget, accessors, numBlocks); }
/** * Creates a block channel reader that reads blocks from the given channel. The reader pushed * full memory segments (with the read data) to its "return queue", to allow for asynchronous read * implementations. * * @param channelID The descriptor for the channel to write to. * @return A block channel reader that reads from the given channel. * @throws IOException Thrown, if the channel for the reader could not be opened. */ public BlockChannelReader<MemorySegment> createBlockChannelReader(FileIOChannel.ID channelID) throws IOException { return createBlockChannelReader(channelID, new LinkedBlockingQueue<MemorySegment>()); }
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel, LinkedBlockingQueue<MemorySegment> returnQueue, List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks) throws IOException { this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue, segments, freeMemTarget, accessors, numBlocks); }
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel, LinkedBlockingQueue<MemorySegment> returnQueue, List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks) throws IOException { this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue, segments, freeMemTarget, accessors, numBlocks); }
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel, LinkedBlockingQueue<MemorySegment> returnQueue, List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks) throws IOException { this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue, segments, freeMemTarget, accessors, numBlocks); }
/** * Creates a block channel reader that reads blocks from the given channel. The reader pushed * full memory segments (with the read data) to its "return queue", to allow for asynchronous read * implementations. * * @param channelID The descriptor for the channel to write to. * @return A block channel reader that reads from the given channel. * @throws IOException Thrown, if the channel for the reader could not be opened. */ public BlockChannelReader<MemorySegment> createBlockChannelReader(FileIOChannel.ID channelID) throws IOException { return createBlockChannelReader(channelID, new LinkedBlockingQueue<MemorySegment>()); }
/** * Creates a block channel reader that reads blocks from the given channel. The reader pushed * full memory segments (with the read data) to its "return queue", to allow for asynchronous read * implementations. * * @param channelID The descriptor for the channel to write to. * @return A block channel reader that reads from the given channel. * @throws IOException Thrown, if the channel for the reader could not be opened. */ public BlockChannelReader<MemorySegment> createBlockChannelReader(FileIOChannel.ID channelID) throws IOException { return createBlockChannelReader(channelID, new LinkedBlockingQueue<MemorySegment>()); }
public static BlockChannelReader<MemorySegment> createBlockChannelReader( IOManager ioManager, FileIOChannel.ID channel, LinkedBlockingQueue<MemorySegment> bufferReturnQueue, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { return new CompressedBlockChannelReader( ioManager, channel, bufferReturnQueue, compressionCodecFactory, compressionBlockSize, segmentSize); } else { return ioManager.createBlockChannelReader(channel, bufferReturnQueue); } } }
@Override public ChannelBackendMutableObjectIterator<T> createReader(List<MemorySegment> readMemory) throws IOException { checkState(isWritingFinished, ""); final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channelID); final ChannelReaderInputView inputView = new ChannelReaderInputView( reader, readMemory, channelWriterOutputView.getBlockCount(), false); return new ChannelReaderInputViewIterator<T>(inputView, null, this.serialize); } }
public void seek(long position) throws IOException { final int block = MathUtils.checkedDownCast(position / segmentSize); final int positionInBlock = (int) (position % segmentSize); if (position < 0 || block >= numBlocksTotal || (block == numBlocksTotal - 1 && positionInBlock > sizeOfLastBlock)) { throw new IllegalArgumentException("Position is out of range"); } clear(); if (reader != null) { reader.close(); } reader = ioManager.createBlockChannelReader(channelId); if (block > 0) { reader.seekToPosition(((long) block) * segmentSize); } this.numBlocksRemaining = this.numBlocksTotal - block; this.numRequestsRemaining = numBlocksRemaining; for (int i = 0; i < memory.size(); i++) { sendReadRequest(memory.get(i)); } numBlocksRemaining--; seekInput(reader.getNextReturnedBlock(), positionInBlock, numBlocksRemaining == 0 ? sizeOfLastBlock : segmentSize); }
public void seek(long position) throws IOException { final int block = MathUtils.checkedDownCast(position / segmentSize); final int positionInBlock = (int) (position % segmentSize); if (position < 0 || block >= numBlocksTotal || (block == numBlocksTotal - 1 && positionInBlock > sizeOfLastBlock)) { throw new IllegalArgumentException("Position is out of range"); } clear(); if (reader != null) { reader.close(); } reader = ioManager.createBlockChannelReader(channelId); if (block > 0) { reader.seekToPosition(((long) block) * segmentSize); } this.numBlocksRemaining = this.numBlocksTotal - block; this.numRequestsRemaining = numBlocksRemaining; for (int i = 0; i < memory.size(); i++) { sendReadRequest(memory.get(i)); } numBlocksRemaining--; seekInput(reader.getNextReturnedBlock(), positionInBlock, numBlocksRemaining == 0 ? sizeOfLastBlock : segmentSize); }
public void seek(long position) throws IOException { final int block = MathUtils.checkedDownCast(position / segmentSize); final int positionInBlock = (int) (position % segmentSize); if (position < 0 || block >= numBlocksTotal || (block == numBlocksTotal - 1 && positionInBlock > sizeOfLastBlock)) { throw new IllegalArgumentException("Position is out of range"); } clear(); if (reader != null) { reader.close(); } reader = ioManager.createBlockChannelReader(channelId); if (block > 0) { reader.seekToPosition(((long) block) * segmentSize); } this.numBlocksRemaining = this.numBlocksTotal - block; this.numRequestsRemaining = numBlocksRemaining; for (int i = 0; i < memory.size(); i++) { sendReadRequest(memory.get(i)); } numBlocksRemaining--; seekInput(reader.getNextReturnedBlock(), positionInBlock, numBlocksRemaining == 0 ? sizeOfLastBlock : segmentSize); }
public void seek(long position) throws IOException { final int block = MathUtils.checkedDownCast(position / segmentSize); final int positionInBlock = (int) (position % segmentSize); if (position < 0 || block >= numBlocksTotal || (block == numBlocksTotal - 1 && positionInBlock > sizeOfLastBlock)) { throw new IllegalArgumentException("Position is out of range"); } clear(); if (reader != null) { reader.close(); } reader = ioManager.createBlockChannelReader(channelId); if (block > 0) { reader.seekToPosition(((long) block) * segmentSize); } this.numBlocksRemaining = this.numBlocksTotal - block; this.numRequestsRemaining = numBlocksRemaining; for (int i = 0; i < memory.size(); i++) { sendReadRequest(memory.get(i)); } numBlocksRemaining--; seekInput(reader.getNextReturnedBlock(), positionInBlock, numBlocksRemaining == 0 ? sizeOfLastBlock : segmentSize); }
public static AbstractChannelReaderInputView createInputView( IOManager ioManager, ChannelWithMeta channel, List<FileIOChannel> channels, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { CompressedHeaderlessChannelReaderInputView in = new CompressedHeaderlessChannelReaderInputView( channel.getChannel(), ioManager, compressionCodecFactory, compressionBlockSize, channel.getBlockCount()); channels.add(in.getReader()); return in; } else { BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel.getChannel()); channels.add(reader); return new HeaderlessChannelReaderInputView( reader, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), channel.getBlockCount(), channel.getNumBytesInLastBlock(), false); } }
private void newSpilledIterator(int channelID, long offset) throws IOException { ChannelWithMeta channel = channelIDs.get(channelID); currentChannelID = channelID; // close current reader first. closeCurrentFileReader(); // calculate segment number int segmentNum = (int) (offset / segmentSize); long seekPosition = segmentNum * segmentSize; // new reader. this.fileReader = ioManager.createBlockChannelReader(channel.getChannel()); if (offset > 0) { // seek to the beginning of that segment fileReader.seekToPosition(seekPosition); } ChannelReaderInputView inView = new HeaderlessChannelReaderInputView( fileReader, freeMemory, channel.getBlockCount() - segmentNum, channel.getNumBytesInLastBlock(), false, offset - seekPosition ); this.currentIterator = new PagedChannelReaderInputViewIterator<>( inView, null, binaryRowSerializer ); }
this.segmentSize = memManager.getPageSize(); this.reader = ioManager.createBlockChannelReader(channelId);
this.segmentSize = memManager.getPageSize(); this.reader = ioManager.createBlockChannelReader(channelId);
this.segmentSize = memManager.getPageSize(); this.reader = ioManager.createBlockChannelReader(channelId);