/** * Create a new instance and register it with the given {@link PlatformExecution}. */ ExecutionStage(PlatformExecution platformExecution, ExecutionStageLoop executionStageLoop, int sequenceNumber) { this.platformExecution = platformExecution; this.sequenceNumber = sequenceNumber; this.executionStageLoop = executionStageLoop; if (this.executionStageLoop != null) { this.executionStageLoop.add(this); } this.platformExecution.addStage(this); }
@SuppressWarnings("unused") public String toNameString() { return String.format("%s[%s-%d:%d-%6x]", this.getClass().getSimpleName(), this.platformExecution.getPlatform().getName(), this.platformExecution.getSequenceNumber(), this.sequenceNumber, this.hashCode()); }
@Override public Platform getPlatform() { return this.platformExecution.getPlatform(); }
/** * Scrap {@link Channel}s and {@link ExecutionTask}s that are not within the given {@link ExecutionStage}s. * * @return {@link Channel}s from that consumer {@link ExecutionTask}s have been removed */ public Set<Channel> retain(Set<ExecutionStage> retainableStages) { Set<Channel> openChannels = new HashSet<>(); for (ExecutionStage stage : retainableStages) { for (Channel channel : stage.getOutboundChannels()) { if (channel.retain(retainableStages)) { openChannels.add(channel); } } stage.retainSuccessors(retainableStages); stage.getPlatformExecution().retain(retainableStages); } return openChannels; }
@Override public ExecutionStage toExecutionStage() { final Iterator<ExecutionTask> iterator = this.allTasks.iterator(); final LoopSubplan loop = iterator.next().getOperator().getInnermostLoop(); assert Iterators.allMatch(iterator, task -> task.getOperator().getInnermostLoop() == loop, true ) : String.format("There are different loops in the stage with the tasks %s.", this.allTasks.stream() .map(task -> new Tuple<>(task, task.getOperator().getInnermostLoop())) .collect(Collectors.toList()) ); ExecutionStageLoop executionStageLoop = null; if (loop != null) { executionStageLoop = StageAssignmentTraversal.this.stageLoops.computeIfAbsent(loop, ExecutionStageLoop::new); } final ExecutionStage executionStage = this.platformExecution.createStage(executionStageLoop, this.sequenceNumber); for (ExecutionTask task : this.allTasks) { executionStage.addTask(task); if (this.checkIfStartTask(task)) { executionStage.markAsStartTask(task); } if (this.checkIfTerminalTask(task)) { executionStage.markAsTerminalTask(task); } } assert !executionStage.getTerminalTasks().isEmpty() : String.format("No terminal tasks among %s.", this.allTasks); return executionStage; }
/** * Starts building an {@link InterimStage} starting from the given {@link ExecutionTask} unless there is one already. * If a {@link PlatformExecution} is provided, the {@link InterimStage} will be associated with it. */ private void createStageFor(ExecutionTask task, PlatformExecution platformExecution) { assert task.getStage() == null : String.format("%s has already stage %s.", task, task.getStage()); // See if there is already an InterimStage. if (this.assignedInterimStages.containsKey(task)) { return; } // Create a new PlatformExecution if none. if (platformExecution == null) { Platform platform = task.getOperator().getPlatform(); platformExecution = new PlatformExecution(platform); } // Create the InterimStage and expand it. InterimStage initialStage = new InterimStageImpl(platformExecution); this.addStage(initialStage); this.assignTaskAndExpand(task, initialStage); }
public Map toJsonMap() { HashMap<String, Object> jsonMap = new HashMap<>(); ArrayList<Map> operators = new ArrayList<>(); jsonMap.put("platform", this.getPlatformExecution().getPlatform().getName()); jsonMap.put("operators", operators); Set<ExecutionTask> seenTasks = new HashSet<>(); for (ExecutionTask startTask : this.startTasks) { this.toJsonMapAux(startTask, seenTasks, operators); } return jsonMap; }
private Executor getOrCreateExecutorFor(ExecutionStage stage) { return this.executors.computeIfAbsent( stage.getPlatformExecution().getPlatform(), platform -> { // It is important to register the Executor. This way, we ensure that it will also not be disposed // among disconnected PlatformExecutions. The downside is, that we only remove it, once the // execution is done. final Executor executor = platform.getExecutorFactory().create(this.job); this.registerGlobal(executor); return executor; } ); }