setIsLastBatch(other.getIsLastBatch());
public boolean batchArrived(final IncomingDataBatch incomingBatch) throws FragmentSetupException, IOException { // we want to make sure that we only generate local record batch reference in the case that we're not closed. // Otherwise we would leak memory. try (@SuppressWarnings("unused") AutoCloseables.Closeable lock = sharedIncomingBatchLock.open()) { if (closed) { return false; } if (incomingBatch.getHeader().getIsLastBatch()) { streamsRemaining.decrementAndGet(); } final int sendMajorFragmentId = incomingBatch.getHeader().getSendingMajorFragmentId(); DataCollector collector = collectorMap.get(sendMajorFragmentId); if (collector == null) { throw new FragmentSetupException(String.format( "We received a major fragment id that we were not expecting. The id was %d. %s", sendMajorFragmentId, Arrays.toString(collectorMap.values().toArray()))); } // Use the Data Collector's buffer allocator if set, otherwise the fragment's one BufferAllocator ownerAllocator = collector.getAllocator(); synchronized (collector) { final RawFragmentBatch newRawFragmentBatch = incomingBatch.newRawFragmentBatch(ownerAllocator); boolean decrementedToZero = collector .batchArrived(incomingBatch.getHeader().getSendingMinorFragmentId(), newRawFragmentBatch); newRawFragmentBatch.release(); // we should only return true if remaining required has been decremented and is currently equal to zero. return decrementedToZero; } } }
if (b.getHeader().getIsLastBatch()) { logger.debug("Got last batch from {}:{}", b.getHeader().getSendingMajorFragmentId(), b.getHeader() .getSendingMinorFragmentId());
setIsLastBatch(other.getIsLastBatch());
@Override protected void enqueueInner(RawFragmentBatch batch) throws IOException { assert batch.getHeader().getSendingMajorFragmentId() == oppositeId; logger.debug("Enqueue batch. Current buffer size: {}. Last batch: {}. Sending fragment: {}", bufferQueue.size(), batch.getHeader().getIsLastBatch(), batch.getHeader().getSendingMajorFragmentId()); RawFragmentBatchWrapper wrapper; boolean spoolCurrentBatch = isCurrentlySpooling(); wrapper = new RawFragmentBatchWrapper(batch, !spoolCurrentBatch); currentSizeInMemory += wrapper.getBodySize(); if (spoolCurrentBatch) { if (spooler == null) { initSpooler(); } spooler.addBatchForSpooling(wrapper); } bufferQueue.add(wrapper); if (!spoolCurrentBatch && currentSizeInMemory > threshold) { logger.debug("Buffer size {} greater than threshold {}. Start spooling to disk", currentSizeInMemory, threshold); startSpooling(); } }
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.BitData.FragmentRecordBatch message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasReceivingMajorFragmentId()) output.writeInt32(2, message.getReceivingMajorFragmentId(), false); for(int receivingMinorFragmentId : message.getReceivingMinorFragmentIdList()) output.writeInt32(3, receivingMinorFragmentId, true); if(message.hasSendingMajorFragmentId()) output.writeInt32(4, message.getSendingMajorFragmentId(), false); if(message.hasSendingMinorFragmentId()) output.writeInt32(5, message.getSendingMinorFragmentId(), false); if(message.hasDef()) output.writeObject(6, message.getDef(), org.apache.drill.exec.proto.SchemaUserBitShared.RecordBatchDef.WRITE, false); if(message.hasIsLastBatch()) output.writeBool(7, message.getIsLastBatch(), false); } public boolean isInitialized(org.apache.drill.exec.proto.BitData.FragmentRecordBatch message)
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.BitData.FragmentRecordBatch message) throws java.io.IOException { if(message.hasQueryId()) output.writeObject(1, message.getQueryId(), org.apache.drill.exec.proto.SchemaUserBitShared.QueryId.WRITE, false); if(message.hasReceivingMajorFragmentId()) output.writeInt32(2, message.getReceivingMajorFragmentId(), false); for(int receivingMinorFragmentId : message.getReceivingMinorFragmentIdList()) output.writeInt32(3, receivingMinorFragmentId, true); if(message.hasSendingMajorFragmentId()) output.writeInt32(4, message.getSendingMajorFragmentId(), false); if(message.hasSendingMinorFragmentId()) output.writeInt32(5, message.getSendingMinorFragmentId(), false); if(message.hasDef()) output.writeObject(6, message.getDef(), org.apache.drill.exec.proto.SchemaUserBitShared.RecordBatchDef.WRITE, false); if(message.hasIsLastBatch()) output.writeBool(7, message.getIsLastBatch(), false); } public boolean isInitialized(org.apache.drill.exec.proto.BitData.FragmentRecordBatch message)