public Channel getOutputChannel(int index) { return this.getOutputChannels()[index]; }
public int getNumOuputChannels() { return this.getOutputChannels().length; }
@Override public void applyTo(ExecutionStage stage) { stage.getAllTasks().stream() .flatMap(task -> Arrays.stream(task.getOutputChannels())) .forEach(Channel::markForInstrumentation); } }
/** * @return all {@link Channel}s of this instance that connect to other {@link ExecutionStage}s */ public Collection<Channel> getOutboundChannels() { return this.getAllTasks().stream() .flatMap( task -> Arrays.stream(task.getOutputChannels()).filter(Channel::isBetweenStages) ).collect(Collectors.toList()); }
public boolean isComplete() { final Logger logger = LoggerFactory.getLogger(this.getClass()); final Set<ExecutionTask> allTasks = this.collectAllTasks(); if (allTasks.isEmpty()) { logger.warn("Instance has not tasks."); return false; } for (ExecutionTask task : allTasks) { if (Arrays.stream(task.getOutputChannels()).anyMatch(Objects::isNull)) { logger.warn("{} has missing output channels among {}.", task, Arrays.toString(task.getOutputChannels())); return false; } if (Arrays.stream(task.getInputChannels()).anyMatch(Objects::isNull)) { logger.warn("{} has missing input channels among {}.", task, Arrays.toString(task.getInputChannels())); return false; } } return true; }
/** * Checks if <i>all</i> output {@link Channel}s of the given {@code task} are outbound w.r.t. to the * {@link InterimStage}. */ private boolean checkIfTerminalTask(ExecutionTask task) { for (Channel channel : task.getOutputChannels()) { if (this.checkIfFeedforwardChannel(task, channel)) continue; for (ExecutionTask consumer : channel.getConsumers()) { if (this.equals(StageAssignmentTraversal.this.assignedInterimStages.get(consumer))) { return false; } } } return true; }
@Override public void execute(final ExecutionStage stage, OptimizationContext optimizationContext, ExecutionState executionState) { Queue<ExecutionTask> scheduledTasks = new LinkedList<>(stage.getStartTasks()); Set<ExecutionTask> executedTasks = new HashSet<>(); while (!scheduledTasks.isEmpty()) { final ExecutionTask task = scheduledTasks.poll(); if (executedTasks.contains(task)) continue; this.execute(task, optimizationContext, executionState); executedTasks.add(task); Arrays.stream(task.getOutputChannels()) .flatMap(channel -> channel.getConsumers().stream()) .filter(consumer -> consumer.getStage() == stage) .forEach(scheduledTasks::add); } }
/** * Collects all {@link ExecutionTask}s of this instance. */ public Set<ExecutionTask> getAllTasks() { final Queue<ExecutionTask> nextTasks = new LinkedList<>(this.startTasks); final Set<ExecutionTask> allTasks = new HashSet<>(); while (!nextTasks.isEmpty()) { final ExecutionTask task = nextTasks.poll(); assert task.getStage() == this; if (allTasks.add(task) && !this.terminalTasks.contains(task)) { Arrays.stream(task.getOutputChannels()) .flatMap(channel -> channel.getConsumers().stream()) .filter(consumer -> consumer.getStage() == this) .forEach(nextTasks::add); } } return allTasks; }
/** * Removes the given {@link Channel} as output of this instance. * * @return the former output index the {@link Channel} */ public int removeOutputChannel(Channel outputChannel) { int outputIndex; for (outputIndex = 0; outputIndex < this.getNumOuputChannels(); outputIndex++) { if (this.getOutputChannel(outputIndex) == outputChannel) { this.getOutputChannels()[outputIndex] = null; outputChannel.setProducer(null); return outputIndex; } } throw new IllegalArgumentException(String.format("%s is not an output of %s.", outputChannel, this)); }
/** * Sets an output {@link Channel} for this instance. */ public void setOutputChannel(int index, Channel channel) { assert this.getOutputChannel(index) == null : String.format("Output channel %d of %s is already set to %s.", index, this, this.getOutputChannel(index)); this.getOutputChannels()[index] = channel; channel.setProducer(this); }
for (int i = 0; i < executionTask.getOutputChannels().length; i++) { Channel channel = executionTask.getOutputChannels()[i]; if (channel == null) { logger.warn("{} does not have an output channel @{}.", executionTask, i);
/** * Implements various sanity checks. Problems are logged. */ public boolean isSane() { // 1. Check if every ExecutionTask is assigned an ExecutionStage. final Set<ExecutionTask> allTasks = this.collectAllTasks(); boolean isAllTasksAssigned = allTasks.stream().allMatch(task -> task.getStage() != null); if (!isAllTasksAssigned) { this.logger.error("There are tasks without stages."); } final Set<Channel> allChannels = allTasks.stream() .flatMap(task -> Stream.concat(Arrays.stream(task.getInputChannels()), Arrays.stream(task.getOutputChannels()))) .collect(Collectors.toSet()); boolean isAllChannelsOriginal = allChannels.stream() .allMatch(channel -> !channel.isCopy()); if (!isAllChannelsOriginal) { this.logger.error("There are channels that are copies."); } boolean isAllSiblingsConsistent = true; for (Channel channel : allChannels) { for (Channel sibling : channel.getSiblings()) { if (!allChannels.contains(sibling)) { this.logger.error("A sibling of {}, namely {}, seems to be invalid.", channel, sibling); isAllSiblingsConsistent = false; } } } return isAllTasksAssigned && isAllChannelsOriginal && isAllSiblingsConsistent; }
/** * Find {@link ChannelDescriptor}s of {@link Channel}s that can be reached from the current * {@link ChannelDescriptor}. This is relevant only when there are {@link #existingChannels}. * * @param descriptor from which successor {@link ChannelDescriptor}s are requested * @return the successor {@link ChannelDescriptor}s or {@code null} if no restrictions apply */ private Set<ChannelDescriptor> getSuccessorChannelDescriptors(ChannelDescriptor descriptor) { final Channel channel = this.existingChannels.get(descriptor); if (channel == null || this.openChannelDescriptors.contains(descriptor)) return null; Set<ChannelDescriptor> result = new HashSet<>(); for (ExecutionTask consumer : channel.getConsumers()) { if (!consumer.getOperator().isAuxiliary()) continue; for (Channel successorChannel : consumer.getOutputChannels()) { result.add(successorChannel.getDescriptor()); } } return result; }
private void toExtensiveStringAux(ExecutionTask task, Set<ExecutionTask> seenTasks, StringBuilder sb, String indent) { if (!seenTasks.add(task)) { return; } for (Channel channel : task.getOutputChannels()) { for (ExecutionTask consumer : channel.getConsumers()) { if (consumer.getStage() == this) { sb.append(indent) .append(" ") .append(this.prettyPrint(task)) .append(" => ") .append(this.prettyPrint(channel)) .append(" => ") .append(this.prettyPrint(consumer)).append('\n'); this.toExtensiveStringAux(consumer, seenTasks, sb, indent); } else { sb.append(indent) .append("Out ") .append(this.prettyPrint(task)) .append(" => ") .append(this.prettyPrint(channel)).append('\n'); } } } }
Channel outputChannel = task.getOutputChannels()[outputIndex]; boolean isInterStageRequired = outputChannel.getConsumers().stream() .anyMatch(consumer -> !this.allTasks.contains(consumer));
for (Channel channel : outboundTask.getOutputChannels()) { for (ExecutionTask consumer : channel.getConsumers()) { if (!consumer.isFeedbackInput(channel)) {
/** * Handle the upstream neighbors of the given {@code task} by expanding the {@code expandableStage}. */ private void expandDownstream(ExecutionTask task, InterimStage expandableStage) { for (Channel channel : task.getOutputChannels()) { assert channel != null : String.format("%s has null output channels.", task); if (channel.isExecutionBreaker()) { expandableStage.setOutbound(task); } for (ExecutionTask consumer : channel.getConsumers()) { final InterimStage assignedStage = this.assignedInterimStages.get(consumer); if (assignedStage == null) { this.handleTaskWithoutPlatformExecution(consumer, /*channel.isExecutionBreaker() ? null : */ expandableStage); } } } }
/** * Determine the consuming {@link InputSlot}s of the given {@link Channel} that lie within a {@link RheemPlan} and * have not been executed yet. * We follow non-RheemPlan {@link ExecutionOperator}s because they should merely forward data. */ private Collection<InputSlot<?>> findRheemPlanInputSlotFor(Channel channel, Set<ExecutionStage> executedStages) { Collection<InputSlot<?>> result = new LinkedList<>(); for (ExecutionTask consumerTask : channel.getConsumers()) { if (executedStages.contains(consumerTask.getStage())) continue; if (!consumerTask.getOperator().isAuxiliary()) { result.add(consumerTask.getInputSlotFor(channel)); } else { for (Channel consumerOutputChannel : consumerTask.getOutputChannels()) { result.addAll(this.findRheemPlanInputSlotFor(consumerOutputChannel, executedStages)); } } } return result; }
operators.add(operator); for (Channel channel : task.getOutputChannels()) { ArrayList<HashMap<String, Object>> perOutputThatList = new ArrayList<>(); Integer thisOutIndex = channel.getProducerSlot()==null ? 0 : channel.getProducerSlot().getIndex();
/** * Traverse the given {@link Channel} downstream and collect any {@link Channel} that feeds some * {@link InputSlot} of a non-conversion {@link ExecutionOperator}. * * @param channel the {@link Channel} to traverse from * @see #existingChannels * @see #existingDestinationChannels */ private void collectExistingChannels(Channel channel) { this.existingChannels.put(channel.getDescriptor(), channel); for (ExecutionTask consumer : channel.getConsumers()) { final ExecutionOperator operator = consumer.getOperator(); if (!operator.isAuxiliary()) { final InputSlot<?> input = consumer.getInputSlotFor(channel); this.existingDestinationChannels.put(input, channel); int destIndex = 0; while (this.destInputs.get(destIndex) != input) destIndex++; this.existingDestinationChannelIndices.set(destIndex); } else { for (Channel outputChannel : consumer.getOutputChannels()) { if (outputChannel != null) this.collectExistingChannels(outputChannel); } } } }