public CompressedBlockChannelWriter(IOManager ioManager, ID channel, LinkedBlockingQueue<MemorySegment> blockQueue, BlockCompressionFactory codecFactory, int preferBlockSize, int segmentSize) throws IOException { this.writer = ioManager.createBufferFileWriter(channel); this.blockQueue = blockQueue; copyCompress = preferBlockSize > segmentSize * 2; int blockSize = copyCompress ? preferBlockSize : segmentSize; this.compressor = codecFactory.getCompressor(); if (copyCompress) { this.buf = new byte[blockSize]; this.bufWrapper = ByteBuffer.wrap(buf); } for (int i = 0; i < 2; i++) { compressedBuffers.add(MemorySegmentFactory.wrap( new byte[compressor.getMaxCompressedSize(blockSize)])); } }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("{}: Spilling {} bytes ({} buffers} for sub partition {} of {}.", parent.getOwningTaskName(), spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("Spilling {} bytes ({} buffers} for sub partition {} of {}.", spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("{}: Spilling {} bytes ({} buffers} for sub partition {} of {}.", parent.getOwningTaskName(), spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
public static AbstractChannelWriterOutputView createOutputView( IOManager ioManager, FileIOChannel.ID channel, boolean compressionEnable, BlockCompressionFactory compressionCodecFactory, int compressionBlockSize, int segmentSize) throws IOException { if (compressionEnable) { BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel); return new CompressedHeaderlessChannelWriterOutputView( bufferWriter, compressionCodecFactory, compressionBlockSize); } else { BlockChannelWriter<MemorySegment> blockWriter = ioManager.createBlockChannelWriter(channel); return new HeaderlessChannelWriterOutputView( blockWriter, Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)), segmentSize); } }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spilalble view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = 0; // Spill all buffers for (int i = 0; i < numberOfBuffers; i++) { Buffer buffer = buffers.remove(); spilledBytes += buffer.getSize(); spillWriter.writeBlock(buffer); } LOG.debug("Spilling {} bytes for sub partition {} of {}.", spilledBytes, index, parent.getPartitionId()); return numberOfBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());
BufferFileWriter spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel());