/** * Set all of the {@code inputs} as predecessors of the {@code operatorContext} each of the {@code outputs}. * * @param inputs input {@link ChannelInstance}s * @param executionLineageNode in-between {@link ExecutionLineageNode} * @param outputs output {@link ChannelInstance}s * @see #addPredecessor(LazyExecutionLineageNode) */ public static void connectAll(ChannelInstance[] inputs, ExecutionLineageNode executionLineageNode, ChannelInstance[] outputs) { for (ChannelInstance input : inputs) { if (input != null) executionLineageNode.addPredecessor(input.getLineage()); } for (ChannelInstance output : outputs) { if (output != null) output.getLineage().addPredecessor(executionLineageNode); } }
/** * Registers the given {@link ChannelInstance} with this instance. * * @param channelInstance that should be registered */ public void register(ChannelInstance channelInstance) { channelInstance.noteObtainedReference(); final ChannelInstance oldInstance = this.channelInstances.put(channelInstance.getChannel(), channelInstance); if (oldInstance != null) oldInstance.noteDiscardedReference(true); }
/** * If the given {@link ChannelInstance} has a measured cardinality, then register this cardinality in the * {@link #crossPlatformExecutor} with the corresponding {@link Channel} and all its siblings. * * @param channelInstance the said {@link ChannelInstance} */ protected void registerMeasuredCardinality(ChannelInstance channelInstance) { // Check if a cardinality was measured in the first place. final OptionalLong optionalCardinality = channelInstance.getMeasuredCardinality(); if (!optionalCardinality.isPresent()) { if (channelInstance.getChannel().isMarkedForInstrumentation()) { this.logger.warn( "No cardinality available for {}, although it was requested.", channelInstance.getChannel() ); } return; } this.crossPlatformExecutor.addCardinalityMeasurement(channelInstance); }
public ChannelLineageNode(final ChannelInstance channelInstance) { assert !channelInstance.wasProduced(); this.channelInstance = channelInstance; this.channelInstance.noteObtainedReference(); }
@Override protected void markAsExecuted() { super.markAsExecuted(); assert !this.channelInstance.wasProduced(); this.channelInstance.markProduced(); this.channelInstance.noteDiscardedReference(false); }
/** * Injects the measured cardinality of a {@link ChannelInstance}. */ private void injectMeasuredCardinality(ChannelInstance channelInstance) { assert channelInstance.wasProduced(); assert channelInstance.isMarkedForInstrumentation(); // Obtain cardinality measurement. final long cardinality = channelInstance.getMeasuredCardinality().getAsLong(); // Try to inject into the RheemPlan Operator output. final OutputSlot<?> rheemPlanOutput = OptimizationUtils.findRheemPlanOutputSlotFor(channelInstance.getChannel()); int outputIndex = rheemPlanOutput == null ? 0 : rheemPlanOutput.getIndex(); OptimizationContext optimizationContext = channelInstance.getProducerOperatorContext().getOptimizationContext(); final OptimizationContext.OperatorContext rheemPlanOperatorCtx = optimizationContext.getOperatorContext(rheemPlanOutput.getOwner()); if (rheemPlanOperatorCtx != null) { this.injectMeasuredCardinality(cardinality, rheemPlanOperatorCtx, outputIndex); } else { this.logger.warn("Could not inject cardinality measurement {} for {}.", cardinality, rheemPlanOutput); } }
/** * Execute one single {@link ExecutionStage} */ private void executeSingleStage(boolean isBreakpointsDisabled, StageActivator stageActivator) { // Check if #breakpoint permits the execution. if (!isBreakpointsDisabled && this.suspendIfBreakpointRequest(stageActivator)) { return; } // Otherwise, execute the stage. this.execute(stageActivator); // Try to activate the successor stages. this.tryToActivateSuccessors(stageActivator); // We can now dispose the stageActivator that collected the input ChannelInstances. stageActivator.dispose(); // Dispose obsolete ChannelInstances. final Iterator<Map.Entry<Channel, ChannelInstance>> iterator = this.channelInstances.entrySet().iterator(); while (iterator.hasNext()) { final Map.Entry<Channel, ChannelInstance> channelInstanceEntry = iterator.next(); final ChannelInstance channelInstance = channelInstanceEntry.getValue(); // If this is instance is the only one to still use this ChannelInstance, discard it. if (channelInstance.getNumReferences() == 1) { channelInstance.noteDiscardedReference(true); iterator.remove(); } } }
/** * Select the produced {@link ChannelInstance}s that are marked for instrumentation and register them if they * contain a measured cardinality. * * @param producedChannelInstances the {@link ChannelInstance}s */ protected void registerMeasuredCardinalities(Collection<ChannelInstance> producedChannelInstances) { for (ChannelInstance producedChannelInstance : producedChannelInstances) { if (!producedChannelInstance.wasProduced()) { this.logger.error("Expected {} to be produced, but is not flagged as such.", producedChannelInstance); continue; } if (producedChannelInstance.isMarkedForInstrumentation()) { this.registerMeasuredCardinality(producedChannelInstance); } } }
/** * Put new {@link ChannelInstance}s to the {@link #executionState} and release input {@link ChannelInstance}s. */ private void updateExecutionState() { for (final ChannelInstance channelInstance : this.allChannelInstances) { // Capture outbound ChannelInstances. if (channelInstance.getChannel().isBetweenStages() || channelInstance.getChannel().getConsumers().stream() .anyMatch(consumer -> consumer.getOperator().isLoopHead())) { this.executionState.register(channelInstance); } // Release the ChannelInstance. channelInstance.noteDiscardedReference(true); } this.allChannelInstances.clear(); } }
/** * Registers the {@code inputChannelInstance} as input of the wrapped {@link #task}. * * @param inputChannelInstance the input {@link ChannelInstance} */ public void accept(ChannelInstance inputChannelInstance) { // Identify the input index of the inputChannelInstance wrt. the ExecutionTask. final Channel channel = inputChannelInstance.getChannel(); final int inputIndex; if (this.task.getOperator().getNumInputs() == 0) { // If we have a ChannelInstance but no InputSlots, we fall-back to index 0 as per convention. assert this.inputChannelInstances.isEmpty(); this.inputChannelInstances.add(null); inputIndex = 0; } else { final InputSlot<?> inputSlot = this.task.getInputSlotFor(channel); assert inputSlot != null : String.format("Could not identify an InputSlot in %s for %s.", this.task, inputChannelInstance); inputIndex = inputSlot.getIndex(); } // Register the inputChannelInstance (and check for conflicts). assert this.inputChannelInstances.get(inputIndex) == null : String.format("Tried to replace %s with %s as %dth input of %s.", this.inputChannelInstances.get(inputIndex), inputChannelInstance, inputIndex, this.task); this.inputChannelInstances.set(inputIndex, inputChannelInstance); inputChannelInstance.noteObtainedReference(); }
/** * Disposes this instance. */ protected void dispose() { // Drop references towards the #inputChannelInstances. for (ChannelInstance inputChannelInstance : this.inputChannelInstances) { if (inputChannelInstance != null) { inputChannelInstance.noteDiscardedReference(true); } } }
/** * Tells whether this instance should be instrumented */ default boolean isMarkedForInstrumentation() { return this.getChannel().isMarkedForInstrumentation(); }
/** * Stores the {@link ChannelInstance}. * * @param channelInstance should be stored */ private void store(ChannelInstance channelInstance) { this.allChannelInstances.add(channelInstance); channelInstance.noteObtainedReference(); }
if (channelInstance.getNumReferences() == 1) { channelInstance.noteDiscardedReference(true); iterator.remove();
@Override protected void disposeUnreferenced() { for (ChannelInstance channelInstance : channelInstances.values()) { channelInstance.noteDiscardedReference(true); } this.channelInstances = null; }
private void activateSuccessorTasks(ExecutionTask task, Collection<ChannelInstance> outputChannelInstances) { for (ChannelInstance outputChannelInstance : outputChannelInstances) { if (outputChannelInstance == null) continue; // Partial results possible (cf. LoopHeadOperator). final Channel channel = outputChannelInstance.getChannel(); for (ExecutionTask consumer : channel.getConsumers()) { // Stay within ExecutionStage. if (consumer.getStage() != task.getStage() || consumer.getOperator().isLoopHead()) continue; // Get or create the TaskActivator. final TaskActivator consumerActivator = this.stagedActivators.computeIfAbsent( consumer, (key) -> new TaskActivator(key, this.fetchOperatorContext(key), this.executionState) ); // Register the outputChannelInstance. consumerActivator.accept(outputChannelInstance); // Schedule the consumerActivator if it isReady. if (consumerActivator.isReady()) { this.stagedActivators.remove(consumer); this.readyActivators.add(consumerActivator); } } } }
/** * Adds a predecessor. * * @param predecessor the predecessor */ public void addPredecessor(LazyExecutionLineageNode predecessor) { assert !this.predecessors.contains(predecessor) : String.format("Lineage predecessor %s is already present.", predecessor); this.predecessors.add(predecessor); // TODO: Pinning the input ChannelInstances down like this is not very elegant. // A better solution would be to incorporate all LazyExecutionLineageNodes into the // reference counting scheme. However, this would imply considerable effort to get it right. if (!this.isExecuted && predecessor instanceof ChannelLineageNode) { ChannelInstance channelInstance = ((ChannelLineageNode) predecessor).getChannelInstance(); this.pinnedDownChannelInstances.add(channelInstance); channelInstance.noteObtainedReference(); } }
/** * Utility method to forward a {@link JavaChannelInstance} to another. * * @param input that should be forwarded * @param output to that should be forwarded */ static void forward(ChannelInstance input, ChannelInstance output) { // Do the forward. if (output instanceof CollectionChannel.Instance) { ((CollectionChannel.Instance) output).accept(((CollectionChannel.Instance) input).provideCollection()); } else if (output instanceof StreamChannel.Instance) { ((StreamChannel.Instance) output).accept(((JavaChannelInstance) input).provideStream()); } else { throw new RheemException(String.format("Cannot forward %s to %s.", input, output)); } // Manipulate the lineage. output.getLineage().addPredecessor(input.getLineage()); }
@Override public void register(ChannelInstance channelInstance) { final ExecutionStageLoop loop = getExecutionStageLoop(channelInstance.getChannel()); if (loop == null) { final ChannelInstance oldChannelInstance = this.channelInstances.put(channelInstance.getChannel(), channelInstance); channelInstance.noteObtainedReference(); if (oldChannelInstance != null) { oldChannelInstance.noteDiscardedReference(true); } } else { final ExecutionStageLoopContext loopContext = this.getOrCreateLoopContext(loop); loopContext.register(channelInstance); } }
/** * Mark that this instance should not be traversed any more. */ protected void markAsExecuted() { LoggerFactory.getLogger(this.getClass()).debug("Marking {} as executed.", this); this.isExecuted = true; // Free pinned down ChannelInstances. for (ChannelInstance channelInstance : this.pinnedDownChannelInstances) { channelInstance.noteDiscardedReference(true); } this.pinnedDownChannelInstances.clear(); }