@Override public void execute(final ExecutionStage stage, OptimizationContext optimizationContext, ExecutionState executionState) { Queue<ExecutionTask> scheduledTasks = new LinkedList<>(stage.getStartTasks()); Set<ExecutionTask> executedTasks = new HashSet<>(); while (!scheduledTasks.isEmpty()) { final ExecutionTask task = scheduledTasks.poll(); if (executedTasks.contains(task)) continue; this.execute(task, optimizationContext, executionState); executedTasks.add(task); Arrays.stream(task.getOutputChannels()) .flatMap(channel -> channel.getConsumers().stream()) .filter(consumer -> consumer.getStage() == stage) .forEach(scheduledTasks::add); } }
/** * Checks if <i>all</i> output {@link Channel}s of the given {@code task} are outbound w.r.t. to the * {@link InterimStage}. */ private boolean checkIfTerminalTask(ExecutionTask task) { for (Channel channel : task.getOutputChannels()) { if (this.checkIfFeedforwardChannel(task, channel)) continue; for (ExecutionTask consumer : channel.getConsumers()) { if (this.equals(StageAssignmentTraversal.this.assignedInterimStages.get(consumer))) { return false; } } } return true; }
/** * Copies the consumers of the given {@code channel} into this instance. */ private void copyConsumersFrom(Channel channel) { assert channel.getOriginal() == this; for (ExecutionTask consumer : new ArrayList<>(channel.getConsumers())) { // We must take care not to copy back channels, that we already have in the original. assert this.consumers.stream() .noneMatch(existingConsumer -> existingConsumer.getOperator().equals(consumer.getOperator())) : String.format("Conflict when copying consumers from %s (%s) to %s (%s).", this, this.consumers, channel, channel.getConsumers() ); consumer.exchangeInputChannel(channel, this); } }
@Override public String toString() { return String.format("%s[%s->%s]", this.getClass().getSimpleName(), this.getProducer() == null ? this.getProducerSlot() : this.getProducer(), this.getConsumers()); }
/** * Collects all {@link ExecutionTask}s of this instance. */ public Set<ExecutionTask> getAllTasks() { final Queue<ExecutionTask> nextTasks = new LinkedList<>(this.startTasks); final Set<ExecutionTask> allTasks = new HashSet<>(); while (!nextTasks.isEmpty()) { final ExecutionTask task = nextTasks.poll(); assert task.getStage() == this; if (allTasks.add(task) && !this.terminalTasks.contains(task)) { Arrays.stream(task.getOutputChannels()) .flatMap(channel -> channel.getConsumers().stream()) .filter(consumer -> consumer.getStage() == this) .forEach(nextTasks::add); } } return allTasks; }
/** * Removes the given {@link Channel} as input of this instance. * * @return the former input index the {@link Channel} */ public int removeInputChannel(Channel inputChannel) { int inputIndex; for (inputIndex = 0; inputIndex < this.getNumInputChannels(); inputIndex++) { if (this.getInputChannel(inputIndex) == inputChannel) { this.getInputChannels()[inputIndex] = null; inputChannel.getConsumers().remove(this); return inputIndex; } } throw new IllegalArgumentException(String.format("%s is not an input of %s.", inputChannel, this)); }
/** * Exchanges input {@link Channel}. Will also update the {@link Channel}'s consumers appropriately. */ public void exchangeInputChannel(Channel currentChannel, Channel newChannel) { for (int inputIndex = 0; inputIndex < this.getNumInputChannels(); inputIndex++) { if (this.getInputChannel(inputIndex) == currentChannel) { currentChannel.getConsumers().remove(this); this.setInputChannel(inputIndex, null); newChannel.addConsumer(this, inputIndex); return; } } throw new IllegalArgumentException(String.format("%s is not an input of %s.", currentChannel, this)); }
/** * Find {@link ChannelDescriptor}s of {@link Channel}s that can be reached from the current * {@link ChannelDescriptor}. This is relevant only when there are {@link #existingChannels}. * * @param descriptor from which successor {@link ChannelDescriptor}s are requested * @return the successor {@link ChannelDescriptor}s or {@code null} if no restrictions apply */ private Set<ChannelDescriptor> getSuccessorChannelDescriptors(ChannelDescriptor descriptor) { final Channel channel = this.existingChannels.get(descriptor); if (channel == null || this.openChannelDescriptors.contains(descriptor)) return null; Set<ChannelDescriptor> result = new HashSet<>(); for (ExecutionTask consumer : channel.getConsumers()) { if (!consumer.getOperator().isAuxiliary()) continue; for (Channel successorChannel : consumer.getOutputChannels()) { result.add(successorChannel.getDescriptor()); } } return result; }
private void toExtensiveStringAux(ExecutionTask task, Set<ExecutionTask> seenTasks, StringBuilder sb, String indent) { if (!seenTasks.add(task)) { return; } for (Channel channel : task.getOutputChannels()) { for (ExecutionTask consumer : channel.getConsumers()) { if (consumer.getStage() == this) { sb.append(indent) .append(" ") .append(this.prettyPrint(task)) .append(" => ") .append(this.prettyPrint(channel)) .append(" => ") .append(this.prettyPrint(consumer)).append('\n'); this.toExtensiveStringAux(consumer, seenTasks, sb, indent); } else { sb.append(indent) .append("Out ") .append(this.prettyPrint(task)) .append(" => ") .append(this.prettyPrint(channel)).append('\n'); } } } }
for (int outputIndex = 0; outputIndex < task.getNumOuputChannels(); outputIndex++) { Channel outputChannel = task.getOutputChannels()[outputIndex]; boolean isInterStageRequired = outputChannel.getConsumers().stream() .anyMatch(consumer -> !this.allTasks.contains(consumer)); if (!isInterStageRequired) continue;
/** * Checks whether the given {@link Channel} is inside of a {@link ExecutionStageLoop}. * * @param channel the said {@link Channel} * @return whether the {@link Channel} is in a {@link ExecutionStageLoop} */ private static boolean checkIfIsInLoopChannel(Channel channel) { final ExecutionStageLoop producerLoop = channel.getProducer().getStage().getLoop(); return producerLoop != null && channel.getConsumers().stream().anyMatch( consumer -> consumer.getStage().getLoop() == producerLoop ); }
/** * Handle the upstream neighbors of the given {@code task} by expanding the {@code expandableStage}. */ private void expandDownstream(ExecutionTask task, InterimStage expandableStage) { for (Channel channel : task.getOutputChannels()) { assert channel != null : String.format("%s has null output channels.", task); if (channel.isExecutionBreaker()) { expandableStage.setOutbound(task); } for (ExecutionTask consumer : channel.getConsumers()) { final InterimStage assignedStage = this.assignedInterimStages.get(consumer); if (assignedStage == null) { this.handleTaskWithoutPlatformExecution(consumer, /*channel.isExecutionBreaker() ? null : */ expandableStage); } } } }
/** * Determine the consuming {@link InputSlot}s of the given {@link Channel} that lie within a {@link RheemPlan} and * have not been executed yet. * We follow non-RheemPlan {@link ExecutionOperator}s because they should merely forward data. */ private Collection<InputSlot<?>> findRheemPlanInputSlotFor(Channel channel, Set<ExecutionStage> executedStages) { Collection<InputSlot<?>> result = new LinkedList<>(); for (ExecutionTask consumerTask : channel.getConsumers()) { if (executedStages.contains(consumerTask.getStage())) continue; if (!consumerTask.getOperator().isAuxiliary()) { result.add(consumerTask.getInputSlotFor(channel)); } else { for (Channel consumerOutputChannel : consumerTask.getOutputChannels()) { result.addAll(this.findRheemPlanInputSlotFor(consumerOutputChannel, executedStages)); } } } return result; }
jsonConnectsTo.put(thisOutIndex.toString(), perOutputThatList); for (ExecutionTask consumer : channel.getConsumers()) { HashMap<String, Object> jsonThatOp = new HashMap<>(); jsonThatOp.put(consumer.getOperator().getName(),
/** * Put new {@link ChannelInstance}s to the {@link #executionState} and release input {@link ChannelInstance}s. */ private void updateExecutionState() { for (final ChannelInstance channelInstance : this.allChannelInstances) { // Capture outbound ChannelInstances. if (channelInstance.getChannel().isBetweenStages() || channelInstance.getChannel().getConsumers().stream() .anyMatch(consumer -> consumer.getOperator().isLoopHead())) { this.executionState.register(channelInstance); } // Release the ChannelInstance. channelInstance.noteDiscardedReference(true); } this.allChannelInstances.clear(); } }
/** * Tells whether the given constellation of producing and consuming {@link ExecutionTask}, linked by the * {@link Channel} can be handled within a single {@link PlatformExecution} of this {@link Platform} * * @param producerTask an {@link ExecutionTask} running on this {@link Platform} * @param channel links the {@code producerTask} and {@code consumerTask} * @param consumerTask an {@link ExecutionTask} running on this {@link Platform} * @return whether the {@link ExecutionTask}s can be executed in a single {@link PlatformExecution} */ public boolean isSinglePlatformExecutionPossible(ExecutionTask producerTask, Channel channel, ExecutionTask consumerTask) { assert producerTask.getOperator().getPlatform() == this; assert consumerTask.getOperator().getPlatform() == this; assert channel.getProducer() == producerTask; assert channel.getConsumers().contains(consumerTask); // Overwrite as necessary. return true; }
/** * Traverse the given {@link Channel} downstream and collect any {@link Channel} that feeds some * {@link InputSlot} of a non-conversion {@link ExecutionOperator}. * * @param channel the {@link Channel} to traverse from * @see #existingChannels * @see #existingDestinationChannels */ private void collectExistingChannels(Channel channel) { this.existingChannels.put(channel.getDescriptor(), channel); for (ExecutionTask consumer : channel.getConsumers()) { final ExecutionOperator operator = consumer.getOperator(); if (!operator.isAuxiliary()) { final InputSlot<?> input = consumer.getInputSlotFor(channel); this.existingDestinationChannels.put(input, channel); int destIndex = 0; while (this.destInputs.get(destIndex) != input) destIndex++; this.existingDestinationChannelIndices.set(destIndex); } else { for (Channel outputChannel : consumer.getOutputChannels()) { if (outputChannel != null) this.collectExistingChannels(outputChannel); } } } }
private void activateSuccessorTasks(ExecutionTask task, Collection<ChannelInstance> outputChannelInstances) { for (ChannelInstance outputChannelInstance : outputChannelInstances) { if (outputChannelInstance == null) continue; // Partial results possible (cf. LoopHeadOperator). final Channel channel = outputChannelInstance.getChannel(); for (ExecutionTask consumer : channel.getConsumers()) { // Stay within ExecutionStage. if (consumer.getStage() != task.getStage() || consumer.getOperator().isLoopHead()) continue; // Get or create the TaskActivator. final TaskActivator consumerActivator = this.stagedActivators.computeIfAbsent( consumer, (key) -> new TaskActivator(key, this.fetchOperatorContext(key), this.executionState) ); // Register the outputChannelInstance. consumerActivator.accept(outputChannelInstance); // Schedule the consumerActivator if it isReady. if (consumerActivator.isReady()) { this.stagedActivators.remove(consumer); this.readyActivators.add(consumerActivator); } } } }
/** * The given instance should build upon the open {@link Channel}s of this instance. Then, this instance will be * expanded with the content of the given instance. * * @param expansion extends this instance, but they are not overlapping */ public void expand(ExecutionPlan expansion) { for (Channel openChannel : expansion.getOpenInputChannels()) { openChannel.mergeIntoOriginal(); final Channel original = openChannel.getOriginal(); final ExecutionStage producerStage = original.getProducer().getStage(); assert producerStage != null : String.format("No stage found for %s.", original.getProducer()); for (ExecutionTask consumer : original.getConsumers()) { final ExecutionStage consumerStage = consumer.getStage(); assert consumerStage != null : String.format("No stage found for %s.", consumer); // Equal stages possible on "partially open" Channels. if (producerStage != consumerStage) { producerStage.addSuccessor(consumerStage); } } } }
/** * Registers the given {@link ChannelInstance} with appropriate {@link ExecutionStageLoopIterationContext}s. * * @param channelInstance the {@link ChannelInstance} */ public void register(ChannelInstance channelInstance) { final Channel channel = channelInstance.getChannel(); boolean isFeedback = false, isIterationLocal = false; for (ExecutionTask consumer : channel.getConsumers()) { if (consumer.isFeedbackInput(channel)) { isFeedback = true; } else { isIterationLocal = true; } } if (isIterationLocal) this.currentIteration.register(channelInstance); if (isFeedback) this.getOrCreateNextTransition().register(channelInstance); }