/** * Creates a block channel writer that writes to the given channel. The writer adds the * written segment to its return-queue afterwards (to allow for asynchronous implementations). * * @param channelID The descriptor for the channel to write to. * @return A block channel writer that writes to the given channel. * @throws IOException Thrown, if the channel for the writer could not be opened. */ public BlockChannelWriter<MemorySegment> createBlockChannelWriter(FileIOChannel.ID channelID) throws IOException { return createBlockChannelWriter(channelID, new LinkedBlockingQueue<MemorySegment>()); }
/** * Creates a block channel writer that writes to the given channel. The writer adds the * written segment to its return-queue afterwards (to allow for asynchronous implementations). * * @param channelID The descriptor for the channel to write to. * @return A block channel writer that writes to the given channel. * @throws IOException Thrown, if the channel for the writer could not be opened. */ public BlockChannelWriter<MemorySegment> createBlockChannelWriter(FileIOChannel.ID channelID) throws IOException { return createBlockChannelWriter(channelID, new LinkedBlockingQueue<MemorySegment>()); }
/** * Creates a block channel writer that writes to the given channel. The writer adds the * written segment to its return-queue afterwards (to allow for asynchronous implementations). * * @param channelID The descriptor for the channel to write to. * @return A block channel writer that writes to the given channel. * @throws IOException Thrown, if the channel for the writer could not be opened. */ public BlockChannelWriter<MemorySegment> createBlockChannelWriter(FileIOChannel.ID channelID) throws IOException { return createBlockChannelWriter(channelID, new LinkedBlockingQueue<MemorySegment>()); }
/** * Creates a block channel writer that writes to the given channel. The writer adds the * written segment to its return-queue afterwards (to allow for asynchronous implementations). * * @param channelID The descriptor for the channel to write to. * @return A block channel writer that writes to the given channel. * @throws IOException Thrown, if the channel for the writer could not be opened. */ public BlockChannelWriter<MemorySegment> createBlockChannelWriter(FileIOChannel.ID channelID) throws IOException { return createBlockChannelWriter(channelID, new LinkedBlockingQueue<MemorySegment>()); }
public static BlockChannelWriter<MemorySegment> createBlockChannelWriter( IOManager ioManager, FileIOChannel.ID channel, LinkedBlockingQueue<MemorySegment> bufferReturnQueue, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { return new CompressedBlockChannelWriter( ioManager, channel, bufferReturnQueue, compressionCodecFactory, compressionBlockSize, segmentSize); } else { return ioManager.createBlockChannelWriter(channel, bufferReturnQueue); } }
public BlockSortedDataFile(FileIOChannel.ID channelID, TypeSerializer<T> serialize, IOManager ioManager, List<MemorySegment> writeMemory) throws IOException { checkArgument(writeMemory.size() > 0, "Write memory are required for the BlockSortedDataFile."); this.channelID = channelID; this.serialize = serialize; this.ioManager = ioManager; blockFileWriter = ioManager.createBlockChannelWriter(channelID); channelWriterOutputView = new ChannelWriterOutputView(blockFileWriter, writeMemory, writeMemory.get(0).size()); }
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
/** * Spills this partition to disk. This method is invoked once after the initial open() method * * @return Number of memorySegments in the writeBehindBuffers! */ int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException { this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap this.initialBuildSideChannel = targetChannel; initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers); final int numSegments = this.partitionBuffers.length; for (int i = 0; i < numSegments; i++) { initialBuildSideWriter.writeBlock(partitionBuffers[i]); } this.partitionBuffers = null; initialBuildSideWriter.close(); // num partitions are now in the writeBehindBuffers. We propagate this information back return numSegments; }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public void prepareProbePhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { if (isInMemory()) { return; } // ATTENTION: The following lines are duplicated code from finalizeBuildPhase this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); }
public static AbstractChannelWriterOutputView createOutputView( IOManager ioManager, FileIOChannel.ID channel, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel); return new CompressedHeaderlessChannelWriterOutputView( bufferWriter, compressionCodecFactory, compressionBlockSize); } else { BlockChannelWriter<MemorySegment> blockWriter = ioManager.createBlockChannelWriter(channel); return new HeaderlessChannelWriterOutputView( blockWriter, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), segmentSize); } }
private void spill() throws IOException { FileIOChannel.ID channel = ioManager.createChannel(); final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel); int numRecordBuffers = inMemoryBuffer.getNumRecordBuffers(); ArrayList<MemorySegment> segments = inMemoryBuffer.getRecordBufferSegments(); try { // spill in memory buffer in zero-copy. for (int i = 0; i < numRecordBuffers; i++) { writer.writeBlock(segments.get(i)); } LOG.info("here spill the reset buffer data with {} bytes", writer.getSize()); writer.close(); } catch (IOException e) { writer.closeAndDelete(); throw e; } spillSize += numRecordBuffers * segmentSize; channelIDs.add(new ChannelWithMeta( channel, inMemoryBuffer.getNumRecordBuffers(), inMemoryBuffer.getNumBytesInLastBuffer())); this.numRowsUntilThisChannel.add(numRows); inMemoryBuffer.reset(); }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator, LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException { this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment(); this.partitionBuffers = this.buildSideWriteBuffer.close(); if (!isInMemory()) { // close the channel. note that in the spilled case, the build-side-buffer will have sent off // the last segment and it will be returned to the write-behind-buffer queue. this.buildSideChannel.close(); // create the channel for the probe side and claim one buffer for it this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue); // creating the ChannelWriterOutputView without memory will cause it to draw one segment from the // write behind queue, which is the spare segment we had above. this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize); } }