public void cleanup() throws IOException { // clear the buffers first. this part should not ever fail for (RecordDeserializer<?> deserializer : recordDeserializers) { Buffer buffer = deserializer.getCurrentBuffer(); if (buffer != null && !buffer.isRecycled()) { buffer.recycleBuffer(); } deserializer.clear(); } // cleanup the barrier handler resources barrierHandler.cleanup(); }
@Override public void close() { BufferOrEvent boe; while ((boe = currentBuffers.poll()) != null) { if (boe.isBuffer()) { boe.getBuffer().recycleBuffer(); } } }
private static BufferOrEvent createBuffer(int channel, int pageSize) { final int size = sizeCounter++; byte[] bytes = new byte[size]; RND.nextBytes(bytes); MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(pageSize); memory.put(0, bytes); Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE); buf.setSize(size); // retain an additional time so it does not get disposed after being read by the input gate buf.retainBuffer(); return new BufferOrEvent(buf, channel); }
ioMan = new IOManagerAsync(); networkBufferPool1 = new NetworkBufferPool(100, PAGE_SIZE); networkBufferPool2 = new NetworkBufferPool(100, PAGE_SIZE); BufferPool pool1 = networkBufferPool1.createBufferPool(100, 100); BufferPool pool2 = networkBufferPool2.createBufferPool(100, 100); BufferOrEvent boe = barrierBuffer.getNextNonBlocked(); if (boe.isBuffer()) { boe.getBuffer().recycleBuffer(); networkBufferPool1.destroyAllBufferPools(); networkBufferPool1.destroy(); networkBufferPool2.destroyAllBufferPools(); networkBufferPool2.destroy();
private static void check(BufferOrEvent expected, BufferOrEvent present, int pageSize) { assertNotNull(expected); assertNotNull(present); assertEquals(expected.isBuffer(), present.isBuffer()); if (expected.isBuffer()) { assertEquals(expected.getBuffer().getMaxCapacity(), present.getBuffer().getMaxCapacity()); assertEquals(expected.getBuffer().getSize(), present.getBuffer().getSize()); MemorySegment expectedMem = expected.getBuffer().getMemorySegment(); MemorySegment presentMem = present.getBuffer().getMemorySegment(); assertTrue("memory contents differs", expectedMem.compare(presentMem, 0, 0, pageSize) == 0); } else { assertEquals(expected.getEvent(), present.getEvent()); } }
public static BufferOrEvent generateRandomBuffer(int size, int channelIndex) { MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE); for (int i = 0; i < size; i++) { seg.put(i, (byte) i); } Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE); buf.setSize(size); return new BufferOrEvent(buf, channelIndex); }
private static void validateBuffer(BufferOrEvent boe, int expectedSize, int expectedChannelIndex) { assertEquals("wrong channel index", expectedChannelIndex, boe.getChannelIndex()); assertTrue("is not buffer", boe.isBuffer()); Buffer buf = boe.getBuffer(); assertEquals("wrong buffer size", expectedSize, buf.getSize()); MemorySegment seg = buf.getMemorySegment(); for (int i = 0; i < expectedSize; i++) { assertEquals("wrong buffer contents", (byte) i, seg.get(i)); } } }
if (boe.isBuffer()) { Buffer buf = boe.getBuffer(); contents = buf.getNioBufferReadable(); boe.getBuffer().recycleBuffer();
@Override public Optional<BufferOrEvent> getNextBufferOrEvent() throws IOException, InterruptedException { currentChannel = (currentChannel + 1) % numberOfChannels; if (barrierGens[currentChannel].isNextBarrier()) { return Optional.of( new BufferOrEvent( new CheckpointBarrier( ++currentBarriers[currentChannel], System.currentTimeMillis(), CheckpointOptions.forCheckpointWithDefaultLocation()), currentChannel)); } else { Buffer buffer = bufferPools[currentChannel].requestBuffer(); buffer.getMemorySegment().putLong(0, c++); return Optional.of(new BufferOrEvent(buffer, currentChannel)); } }
private static BufferOrEvent createBuffer(int channel) { return new BufferOrEvent( new NetworkBuffer(MemorySegmentFactory.wrap(new byte[]{1, 2}), FreeingBufferRecycler.INSTANCE), channel); }
@Override public void validateAlignmentBuffered(long actualBytesBuffered, BufferOrEvent... sequence) { long expectedBuffered = 0; for (BufferOrEvent boe : sequence) { if (boe.isBuffer()) { expectedBuffered += BufferSpiller.HEADER_SIZE + boe.getBuffer().getSize(); } } assertEquals("Wrong alignment buffered bytes", actualBytesBuffered, expectedBuffered); } }
BufferBuilder bufferBuilder = createBufferBuilder(bufferSize); recordSerializer.copyToBufferBuilder(bufferBuilder); bufferBuilder.finish(); return Optional.of(new BufferAndAvailability(buildSingleBuffer(bufferBuilder), moreAvailable, 0)); } else if (input != null && input.isEvent()) { AbstractEvent event = input.getEvent();
final NetworkBufferPool bufferPool = new NetworkBufferPool(bufferPoolSize, segmentSize);
private static BufferOrEvent createBuffer(int channel, int size) { byte[] bytes = new byte[size]; RND.nextBytes(bytes); MemorySegment memory = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE); memory.put(0, bytes); Buffer buf = new NetworkBuffer(memory, FreeingBufferRecycler.INSTANCE); buf.setSize(size); // retain an additional time so it does not get disposed after being read by the input gate buf.retainBuffer(); return new BufferOrEvent(buf, channel); }
private static void check(BufferOrEvent expected, BufferOrEvent present) { assertNotNull(expected); assertNotNull(present); assertEquals(expected.isBuffer(), present.isBuffer()); if (expected.isBuffer()) { assertEquals(expected.getBuffer().getMaxCapacity(), present.getBuffer().getMaxCapacity()); assertEquals(expected.getBuffer().getSize(), present.getBuffer().getSize()); MemorySegment expectedMem = expected.getBuffer().getMemorySegment(); MemorySegment presentMem = present.getBuffer().getMemorySegment(); assertTrue("memory contents differs", expectedMem.compare(presentMem, 0, 0, PAGE_SIZE) == 0); } else { assertEquals(expected.getEvent(), present.getEvent()); } }
Buffer buf = new NetworkBuffer(seg, FreeingBufferRecycler.INSTANCE); buf.setSize(length);
public void cleanup() throws IOException { // clear the buffers first. this part should not ever fail for (RecordDeserializer<?> deserializer : recordDeserializers) { Buffer buffer = deserializer.getCurrentBuffer(); if (buffer != null && !buffer.isRecycled()) { buffer.recycleBuffer(); } deserializer.clear(); } // cleanup the barrier handler resources barrierHandler.cleanup(); }
private static void validateBuffer(BufferOrEvent boe, int expectedSize, int expectedChannelIndex) { assertEquals("wrong channel index", expectedChannelIndex, boe.getChannelIndex()); assertTrue("is not buffer", boe.isBuffer()); Buffer buf = boe.getBuffer(); assertEquals("wrong buffer size", expectedSize, buf.getSize()); MemorySegment seg = buf.getMemorySegment(); for (int i = 0; i < expectedSize; i++) { byte expected = (byte) i; if (expected != seg.get(i)) { fail(String.format( "wrong buffer contents at position %s : expected=%d , found=%d", i, expected, seg.get(i))); } } }
@Override public void cleanup() { BufferOrEvent boe; while ((boe = queuedBuffers.poll()) != null) { if (boe.isBuffer()) { boe.getBuffer().recycleBuffer(); } } }
currentRecordDeserializer.getCurrentBuffer().recycleBuffer(); currentRecordDeserializer = null;