@Override public Instance createInstance(Executor executor, OptimizationContext.OperatorContext producerOperatorContext, int producerOutputIndex) { return new Instance(executor, producerOperatorContext, producerOutputIndex); }
@Override public Instance createInstance(Executor executor, OptimizationContext.OperatorContext producerOperatorContext, int producerOutputIndex) { return new Instance(executor, producerOperatorContext, producerOutputIndex); }
@Override public StreamChannel copy() { return new StreamChannel(this); }
/** * Utility method to forward a {@link JavaChannelInstance} to another. * * @param input that should be forwarded * @param output to that should be forwarded */ static void forward(ChannelInstance input, ChannelInstance output) { // Do the forward. if (output instanceof CollectionChannel.Instance) { ((CollectionChannel.Instance) output).accept(((CollectionChannel.Instance) input).provideCollection()); } else if (output instanceof StreamChannel.Instance) { ((StreamChannel.Instance) output).accept(((JavaChannelInstance) input).provideStream()); } else { throw new RheemException(String.format("Cannot forward %s to %s.", input, output)); } // Manipulate the lineage. output.getLineage().addPredecessor(input.getLineage()); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); ((StreamChannel.Instance) outputs[0]).accept(Stream.concat( ((JavaChannelInstance) inputs[0]).provideStream(), ((JavaChannelInstance) inputs[1]).provideStream()) ); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final JavaChannelInstance input = (JavaChannelInstance) inputs[0]; final long count; if (input instanceof CollectionChannel.Instance) { count = ((CollectionChannel.Instance) input).provideCollection().size(); } else { count = input.provideStream().count(); } ((CollectionChannel.Instance) outputs[0]).accept(Collections.singleton(count)); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { CollectionChannel.Instance input = (CollectionChannel.Instance) inputs[0]; StreamChannel.Instance output = (StreamChannel.Instance) outputs[0]; final Collection<Tuple2<Long, Long>> edges = input.provideCollection(); final TLongFloatMap pageRanks = this.pageRank(edges); final Stream<Tuple2<Long, Float>> pageRankStream = this.stream(pageRanks); output.accept(pageRankStream); return ExecutionOperator.modelQuasiEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { final StreamChannel.Instance streamChannelInstance = (StreamChannel.Instance) inputs[0]; final CollectionChannel.Instance collectionChannelInstance = (CollectionChannel.Instance) outputs[0]; final List<?> collection = streamChannelInstance.provideStream().collect(Collectors.toList()); collectionChannelInstance.accept(collection); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override @SuppressWarnings("unchecked") public <T> Collection<T> getBroadcast(String name) { for (int i = 0; i < this.operator.getNumInputs(); i++) { final InputSlot<?> input = this.operator.getInput(i); if (input.isBroadcast() && input.getName().equals(name)) { final CollectionChannel.Instance broadcastChannelInstance = (CollectionChannel.Instance) this.inputs[i]; return (Collection<T>) broadcastChannelInstance.provideCollection(); } } throw new RheemException("No such broadcast found: " + name); }
@Override @SuppressWarnings("unchecked") public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == 0; assert outputs.length == 1; ((CollectionChannel.Instance) outputs[0]).accept(this.getCollection()); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor executor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); ((JavaChannelInstance) inputs[0]).<T>provideStream().forEach(this.callback); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public CollectionChannel copy() { return new CollectionChannel(this); }
public <T> void accept(Stream<T> stream) { assert this.stream == null; this.stream = stream; if (this.isMarkedForInstrumentation()) { this.stream = this.stream.filter(dataQuantum -> { this.cardinality += 1; return true; }); } }
public void accept(Collection<?> collection) { this.collection = collection; this.setMeasuredCardinality(this.collection.size()); }
public void accept(Collection<?> collection) { assert this.stream == null; this.stream = collection.stream(); this.setMeasuredCardinality(collection.size()); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); ((StreamChannel.Instance) outputs[0]).accept(((JavaChannelInstance) inputs[0]).provideStream().distinct()); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, SparkExecutor sparkExecutor, OptimizationContext.OperatorContext operatorContext) { RddChannel.Instance input = (RddChannel.Instance) inputs[0]; CollectionChannel.Instance output = (CollectionChannel.Instance) outputs[0]; @SuppressWarnings("unchecked") final List<Type> collectedRdd = (List<Type>) input.provideRdd().collect(); output.accept(collectedRdd); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Function<Type, Key> keyExtractor = javaExecutor.getCompiler().compile(this.keyDescriptor); ((StreamChannel.Instance) outputs[0]).accept(((JavaChannelInstance) inputs[0]).<Type>provideStream() .sorted((e1, e2) -> ((Comparable)keyExtractor.apply(e1)).compareTo(keyExtractor.apply(e2)))); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override @SuppressWarnings("unchecked") public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Predicate<Type> filterFunction = javaExecutor.getCompiler().compile(this.predicateDescriptor); JavaExecutor.openFunction(this, filterFunction, inputs, operatorContext); ((StreamChannel.Instance) outputs[0]).accept(((JavaChannelInstance) inputs[0]).<Type>provideStream().filter(filterFunction)); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final JavaChannelInstance input = (JavaChannelInstance) inputs[0]; final StreamChannel.Instance output = (StreamChannel.Instance) outputs[0]; final Function<InputType, OutputType> function = javaExecutor.getCompiler().compile(this.functionDescriptor); JavaExecutor.openFunction(this, function, inputs, operatorContext); output.accept(input.<InputType>provideStream().map(function)); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }