@Override public Iterable<OutputType> call(Iterator<InputType> it) throws Exception { List<OutputType> out = new ArrayList<>(); while (it.hasNext()) { final Iterable<OutputType> mappedPartition = this.function.apply(Iterators.wrapWithIterable(it)); for (OutputType dataQuantum : mappedPartition) { out.add(dataQuantum); } } return out; } }
@Override public ExecutionStage toExecutionStage() { final Iterator<ExecutionTask> iterator = this.allTasks.iterator(); final LoopSubplan loop = iterator.next().getOperator().getInnermostLoop(); assert Iterators.allMatch(iterator, task -> task.getOperator().getInnermostLoop() == loop, true ) : String.format("There are different loops in the stage with the tasks %s.", this.allTasks.stream() .map(task -> new Tuple<>(task, task.getOperator().getInnermostLoop())) .collect(Collectors.toList()) ); ExecutionStageLoop executionStageLoop = null; if (loop != null) { executionStageLoop = StageAssignmentTraversal.this.stageLoops.computeIfAbsent(loop, ExecutionStageLoop::new); } final ExecutionStage executionStage = this.platformExecution.createStage(executionStageLoop, this.sequenceNumber); for (ExecutionTask task : this.allTasks) { executionStage.addTask(task); if (this.checkIfStartTask(task)) { executionStage.markAsStartTask(task); } if (this.checkIfTerminalTask(task)) { executionStage.markAsTerminalTask(task); } } assert !executionStage.getTerminalTasks().isEmpty() : String.format("No terminal tasks among %s.", this.allTasks); return executionStage; }
@Override public Iterable<OutputType> call(Iterator<InputType> it) throws Exception { this.impl.open(executionContext); List<OutputType> out = new ArrayList<>(); while (it.hasNext()) { final Iterable<OutputType> mappedPartition = this.impl.apply(Iterators.wrapWithIterable(it)); for (OutputType dataQuantum : mappedPartition) { out.add(dataQuantum); } } return out; } }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Function<Iterable<InputType>, Iterable<OutputType>> function = javaExecutor.getCompiler().compile(this.functionDescriptor); JavaExecutor.openFunction(this, function, inputs, operatorContext); final Iterable<OutputType> outputDataQuanta = function.apply(Iterators.wrapWithIterable(((JavaChannelInstance) inputs[0]).<InputType>provideStream().iterator())); ((StreamChannel.Instance) outputs[0]).accept(StreamSupport.stream( Spliterators.spliteratorUnknownSize( outputDataQuanta.iterator(), Spliterator.ORDERED), false )); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }