private StageExecution(ExecutionStage stage, OptimizationContext optimizationContext, ExecutionState executionState) { this.executionState = executionState; this.optimizationContext = optimizationContext; // Initialize the readyActivators. assert !stage.getStartTasks().isEmpty() : String.format("No start tasks for %s.", stage); stage.getStartTasks().forEach(this::scheduleStartTask); // Initialize the terminalTasks. this.terminalTasks = RheemCollections.asSet(stage.getTerminalTasks()); }
public ExecutionStage createStage(ExecutionStageLoop executionStageLoop, int sequenceNumber) { return new ExecutionStage(this, executionStageLoop, sequenceNumber); }
private boolean checkForLoopHead(ExecutionStage executionStage) { return executionStage.getAllTasks().stream().anyMatch(this::isLoopHead); }
@Override public boolean permitsExecutionOf(ExecutionStage stage, ExecutionState state, OptimizationContext context) { // TODO: We could break, if we enter a loop, however, multi-stage loop heads have feedback predecessors. return stage.getLoop() != null && stage.getPredecessors().stream().anyMatch( predecessor -> predecessor.getLoop() != null ); }
/** * Retrieves the {@link LoopHeadOperator} {@link ExecutionTask} in this instance. This instance must be a * loop head. * * @return the {@link LoopHeadOperator} {@link ExecutionTask} * @see #isLoopHead() */ public ExecutionTask getLoopHeadTask() { assert this.isLoopHead(); return RheemCollections.getSingle(this.getAllTasks()); }
this.optimizationContext = optimizationContext; if (this.stage.getLoop() != null) { this.loopContext = CrossPlatformExecutor.this.getOrCreateLoopContext(this.stage.getLoop()); this.loopContext.noteObtainedReference(); } else { final Collection<Channel> inboundChannels = this.stage.getInboundChannels(); if (this.stage.isLoopHead()) { assert this.stage.getAllTasks().size() == 1 : String.format("Loop head stage looks like this:\n%s", this.stage.getPlanAsString("! "));
/** * Creates a {@link String} representation of this instance. * * @param isStriclyOrdering whether {@link ExecutionStage}s should be listed only after <i>all</i> their predecessors * @return the {@link String} representation */ public String toExtensiveString(boolean isStriclyOrdering) { StringBuilder sb = new StringBuilder(); Counter<ExecutionStage> stageActivationCounter = new Counter<>(); Queue<ExecutionStage> activatedStages = new LinkedList<>(this.startingStages); Set<ExecutionStage> seenStages = new HashSet<>(); while (!activatedStages.isEmpty()) { while (!activatedStages.isEmpty()) { final ExecutionStage stage = activatedStages.poll(); if (!seenStages.add(stage)) continue; sb.append(">>> ").append(stage).append(":\n"); stage.getPlanAsString(sb, "> "); sb.append("\n"); for (ExecutionStage successor : stage.getSuccessors()) { final int count = stageActivationCounter.add(successor, 1); if (!isStriclyOrdering || count == successor.getPredecessors().size() || successor.isLoopHead()) { activatedStages.add(successor); } } } } return sb.toString(); }
@Override public ExecutionStage toExecutionStage() { final Iterator<ExecutionTask> iterator = this.allTasks.iterator(); final LoopSubplan loop = iterator.next().getOperator().getInnermostLoop(); assert Iterators.allMatch(iterator, task -> task.getOperator().getInnermostLoop() == loop, true ) : String.format("There are different loops in the stage with the tasks %s.", this.allTasks.stream() .map(task -> new Tuple<>(task, task.getOperator().getInnermostLoop())) .collect(Collectors.toList()) ); ExecutionStageLoop executionStageLoop = null; if (loop != null) { executionStageLoop = StageAssignmentTraversal.this.stageLoops.computeIfAbsent(loop, ExecutionStageLoop::new); } final ExecutionStage executionStage = this.platformExecution.createStage(executionStageLoop, this.sequenceNumber); for (ExecutionTask task : this.allTasks) { executionStage.addTask(task); if (this.checkIfStartTask(task)) { executionStage.markAsStartTask(task); } if (this.checkIfTerminalTask(task)) { executionStage.markAsTerminalTask(task); } } assert !executionStage.getTerminalTasks().isEmpty() : String.format("No terminal tasks among %s.", this.allTasks); return executionStage; }
/** * Tries to execute the given {@link ExecutionStage}. * * @param stageActivator that should be executed * @return whether the {@link ExecutionStage} was really executed */ private void execute(StageActivator stageActivator) { final ExecutionStage stage = stageActivator.getStage(); final OptimizationContext optimizationContext = stageActivator.getOptimizationContext(); // Find parts of the stage to instrument. this.instrumentationStrategy.applyTo(stage); // Obtain an Executor for the stage. Executor executor = this.getOrCreateExecutorFor(stage); // Have the execution done. CrossPlatformExecutor.this.logger.info("Having {} execute {}:\n{}", executor, stage, stage.getPlanAsString("> ")); long startTime = System.currentTimeMillis(); executor.execute(stage, optimizationContext, this); long finishTime = System.currentTimeMillis(); CrossPlatformExecutor.this.logger.info("Executed {} in {}.", stage, Formats.formatDuration(finishTime - startTime, true)); // Remember that we have executed the stage. this.completedStages.add(stage); if (stage.isLoopHead()) { this.getOrCreateLoopContext(stage.getLoop()).scrapPreviousTransitionContext(); } }
/** * Scrap {@link Channel}s and {@link ExecutionTask}s that are not within the given {@link ExecutionStage}s. * * @return {@link Channel}s from that consumer {@link ExecutionTask}s have been removed */ public Set<Channel> retain(Set<ExecutionStage> retainableStages) { Set<Channel> openChannels = new HashSet<>(); for (ExecutionStage stage : retainableStages) { for (Channel channel : stage.getOutboundChannels()) { if (channel.retain(retainableStages)) { openChannels.add(channel); } } stage.retainSuccessors(retainableStages); stage.getPlatformExecution().retain(retainableStages); } return openChannels; }
@Override public String toString() { return String.format("%s%s", this.getClass().getSimpleName(), this.getStartTasks()); }
private void logStages(ExecutionPlan executionPlan) { if (this.logger.isInfoEnabled()) { StringBuilder sb = new StringBuilder(); Set<ExecutionStage> seenStages = new HashSet<>(); Queue<ExecutionStage> stagedStages = new LinkedList<>(executionPlan.getStartingStages()); ExecutionStage nextStage; while ((nextStage = stagedStages.poll()) != null) { sb.append(nextStage).append(":\n"); nextStage.getPlanAsString(sb, "* "); nextStage.getSuccessors().stream() .filter(seenStages::add) .forEach(stagedStages::add); } this.logger.info("Current execution plan:\n{}", executionPlan.toExtensiveString()); } }
&& !executionStage.getSuccessors().contains(successorExecutionStage)) { executionStage.addSuccessor(successorExecutionStage);
/** * Try to satisfy the input {@link ChannelInstance} requirements by updating {@link #inputChannelInstances}. * * @return whether the activation is possible */ boolean updateInputChannelInstances() { boolean isMiscChannelsReady = this.updateChannelInstances(this.miscInboundChannels, false); boolean isLoopChannelsReady = true; if (this.stage.isLoopHead()) { LoopHeadOperator loopOperator = (LoopHeadOperator) this.stage.getLoopHeadTask().getOperator(); switch (loopOperator.getState()) { case NOT_STARTED: isLoopChannelsReady = this.updateChannelInstances(this.initializationInboundChannels, false); break; case RUNNING: isLoopChannelsReady = this.updateChannelInstances(this.iterationInboundChannels, true); break; default: logger.warn("Tried to update input channel instances for finished {}.", this.stage); isLoopChannelsReady = false; } } return isMiscChannelsReady && isLoopChannelsReady; }
if (successorStage.getLoop() == null) { return prevOptimizationContext.getRootParent(); if (processedStage.getLoop() == successorStage.getLoop()) { if (successorStage.isLoopHead()) { final LoopSubplan loopSubplan = successorStage.getLoop().getLoopSubplan(); return prevOptimizationContext .getRootParent()
final Collection<Channel> outboundChannels = processedStage.getOutboundChannels(); Set<ExecutionStage> successorStages = new HashSet<>(outboundChannels.size()); for (Channel outboundChannel : outboundChannels) { final ExecutionStage consumerStage = consumer.getStage(); if (consumerStage != processedStage && !consumerStage.isInFinishedLoop()) { successorStages.add(consumerStage);
/** * Checks if the given {@link Channel} (inbound to {@link #stage}), is entering an {@link ExecutionStageLoop} * when serving the {@link #stage}. * * @param inboundChannel {@link Channel} that is inbound to {@link #stage} and that might be a {@link ExecutionStageLoop} input * @return whether the {@link Channel} is a {@link ExecutionStageLoop} input, i.e., it is not produced in an {@link ExecutionStageLoop} * while this {@link #stage} is part of a {@link ExecutionStageLoop} */ private boolean checkIfIsLoopInput(Channel inboundChannel) { // NB: This code assumes no nested loops. return this.stage.getLoop() != null && this.stage.getLoop() != inboundChannel.getProducer().getStage().getLoop(); }
/** * Prints the instance's {@link ExecutionTask}s and {@link Channel}s. * * @return a {@link String} containing the textual representation */ public String getPlanAsString() { return this.getPlanAsString(""); }
@Override public void applyTo(ExecutionStage stage) { stage.getOutboundChannels().forEach(Channel::markForInstrumentation); } }
@Override public boolean permitsExecutionOf(ExecutionStage stage, ExecutionState state, OptimizationContext optimizationContext) { for (Channel channel : stage.getInboundChannels()) { final CardinalityEstimate cardinalityEstimate = this.getCardinalityEstimate(channel, optimizationContext); if (cardinalityEstimate == null) { // TODO: We might need to look inside of LoopContexts. this.logger.warn("Could not find a cardinality estimate for {}.", channel); // Be conservative here. return false; } else { if (!this.approves(cardinalityEstimate)) { return false; } } } return true; }