@Override public PhysicalOperator visitStore(Store store, IndexedFragmentNode iNode) throws ExecutionSetupException { PhysicalOperator child = store.getChild().accept(this, iNode); iNode.addAllocation(store); try { PhysicalOperator o = store.getSpecificStore(child, iNode.getMinorFragmentId()); return o; } catch (PhysicalOperatorSetupException e) { throw new FragmentSetupException("Failure while generating a specific Store materialization.", e); } }
public boolean batchArrived(final IncomingDataBatch incomingBatch) throws FragmentSetupException, IOException { // we want to make sure that we only generate local record batch reference in the case that we're not closed. // Otherwise we would leak memory. try (@SuppressWarnings("unused") AutoCloseables.Closeable lock = sharedIncomingBatchLock.open()) { if (closed) { return false; } if (incomingBatch.getHeader().getIsLastBatch()) { streamsRemaining.decrementAndGet(); } final int sendMajorFragmentId = incomingBatch.getHeader().getSendingMajorFragmentId(); DataCollector collector = collectorMap.get(sendMajorFragmentId); if (collector == null) { throw new FragmentSetupException(String.format( "We received a major fragment id that we were not expecting. The id was %d. %s", sendMajorFragmentId, Arrays.toString(collectorMap.values().toArray()))); } // Use the Data Collector's buffer allocator if set, otherwise the fragment's one BufferAllocator ownerAllocator = collector.getAllocator(); synchronized (collector) { final RawFragmentBatch newRawFragmentBatch = incomingBatch.newRawFragmentBatch(ownerAllocator); boolean decrementedToZero = collector .batchArrived(incomingBatch.getHeader().getSendingMinorFragmentId(), newRawFragmentBatch); newRawFragmentBatch.release(); // we should only return true if remaining required has been decremented and is currently equal to zero. return decrementedToZero; } } }
@Override public PhysicalOperator visitStore(Store store, IndexedFragmentNode iNode) throws ExecutionSetupException { PhysicalOperator child = store.getChild().accept(this, iNode); iNode.addAllocation(store); try { PhysicalOperator o = store.getSpecificStore(child, iNode.getMinorFragmentId()); o.setOperatorId(Short.MAX_VALUE & store.getOperatorId()); // logger.debug("New materialized store node {} with child {}", o, child); return o; } catch (PhysicalOperatorSetupException e) { throw new FragmentSetupException("Failure while generating a specific Store materialization."); } }