public void cleanup() throws IOException { // clear the buffers first. this part should not ever fail for (RecordDeserializer<?> deserializer : recordDeserializers) { Buffer buffer = deserializer.getCurrentBuffer(); if (buffer != null && !buffer.isRecycled()) { buffer.recycleBuffer(); } deserializer.clear(); } // cleanup the barrier handler resources barrierHandler.cleanup(); }
contents = EventSerializer.toSerializedEvent(boe.getEvent());
AbstractEvent evt = EventSerializer.fromSerializedEvent(buffer, getClass().getClassLoader()); buffer.limit(oldLimit);
DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate); if (result.isBufferConsumed()) { currentRecordDeserializer.getCurrentBuffer().recycleBuffer(); currentRecordDeserializer = null; if (result.isFullRecord()) { StreamElement recordOrMark = deserializationDelegate.getInstance(); currentChannel = bufferOrEvent.getChannelIndex(); currentRecordDeserializer = recordDeserializers[currentChannel]; currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
final RecordSerializer<SerializationDelegate<Object>> recordSerializer = new SpanningRecordSerializer<SerializationDelegate<Object>>(); final SerializationDelegate<Object> delegate = (SerializationDelegate<Object>) (SerializationDelegate<?>) new SerializationDelegate<StreamElement>(new StreamElementSerializer<T>(serializer)); return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE), moreAvailable, 0)); } else if (input != null && input.isStreamRecord()) { Object inputElement = input.getStreamRecord(); recordSerializer.serializeRecord(delegate); BufferBuilder bufferBuilder = createBufferBuilder(bufferSize); recordSerializer.copyToBufferBuilder(bufferBuilder); bufferBuilder.finish(); } else if (input != null && input.isEvent()) { AbstractEvent event = input.getEvent(); return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(event), moreAvailable, 0)); } else { return Optional.empty();
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) { this.nonSpanningWrapper = new NonSpanningWrapper(); this.spanningWrapper = new SpanningWrapper(tmpDirectories); }
@Override public void clear() { this.innerDeserializer.clear(); this.innerDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; this.subDeserializer.clear(); this.subDeserializationDelegate.clear(); this.subDeserializationResult = DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER; }
public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) { this.nonSpanningWrapper = new NonSpanningWrapper(); this.spanningWrapper = new SpanningWrapper(tmpDirectories); }
@Override public boolean hasUnfinishedData() { return this.nonSpanningWrapper.remaining() > 0 || this.spanningWrapper.getNumGatheredBytes() > 0; }
DeserializationResult result; if (currentChannel < numInputChannels1) { result = currentRecordDeserializer.getNextRecord(deserializationDelegate1); } else { result = currentRecordDeserializer.getNextRecord(deserializationDelegate2); if (result.isBufferConsumed()) { currentRecordDeserializer.getCurrentBuffer().recycleBuffer(); currentRecordDeserializer = null; if (result.isFullRecord()) { if (currentChannel < numInputChannels1) { StreamElement recordOrWatermark = deserializationDelegate1.getInstance(); currentChannel = bufferOrEvent.getChannelIndex(); currentRecordDeserializer = recordDeserializers[currentChannel]; currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
public void cleanup() throws IOException { // clear the buffers first. this part should not ever fail for (RecordDeserializer<?> deserializer : recordDeserializers) { Buffer buffer = deserializer.getCurrentBuffer(); if (buffer != null && !buffer.isRecycled()) { buffer.recycleBuffer(); } deserializer.clear(); } // cleanup the barrier handler resources barrierHandler.cleanup(); }
private static BufferOrEvent generateAndWriteEvent(FileChannel fileChannel, Random rnd, int numberOfChannels) throws IOException { long magicNumber = rnd.nextLong(); byte[] data = new byte[rnd.nextInt(1000)]; rnd.nextBytes(data); TestEvent evt = new TestEvent(magicNumber, data); int channelIndex = rnd.nextInt(numberOfChannels); ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt); ByteBuffer header = ByteBuffer.allocate(9); header.order(ByteOrder.LITTLE_ENDIAN); header.putInt(channelIndex); header.putInt(serializedEvent.remaining()); header.put((byte) 1); header.flip(); FileUtils.writeCompletely(fileChannel, header); FileUtils.writeCompletely(fileChannel, serializedEvent); return new BufferOrEvent(evt, channelIndex); }
recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>( ioManager.getSpillingDirectoriesPaths());
public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) { this.nonSpanningWrapper = new NonSpanningWrapper(); this.spanningWrapper = new SpanningWrapper(tmpDirectories); }