@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Function<Type, Key> keyExtractor = javaExecutor.getCompiler().compile(this.keyDescriptor); ((StreamChannel.Instance) outputs[0]).accept(((JavaChannelInstance) inputs[0]).<Type>provideStream() .sorted((e1, e2) -> ((Comparable)keyExtractor.apply(e1)).compareTo(keyExtractor.apply(e2)))); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Function<InputType, Iterable<OutputType>> flatmapFunction = javaExecutor.getCompiler().compile(this.functionDescriptor); JavaExecutor.openFunction(this, flatmapFunction, inputs, operatorContext); ((StreamChannel.Instance) outputs[0]).accept( ((JavaChannelInstance) inputs[0]).<InputType>provideStream().flatMap(dataQuantum -> StreamSupport.stream( Spliterators.spliteratorUnknownSize( flatmapFunction.apply(dataQuantum).iterator(), Spliterator.ORDERED), false ) ) ); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Function<Iterable<InputType>, Iterable<OutputType>> function = javaExecutor.getCompiler().compile(this.functionDescriptor); JavaExecutor.openFunction(this, function, inputs, operatorContext); final Iterable<OutputType> outputDataQuanta = function.apply(Iterators.wrapWithIterable(((JavaChannelInstance) inputs[0]).<InputType>provideStream().iterator())); ((StreamChannel.Instance) outputs[0]).accept(StreamSupport.stream( Spliterators.spliteratorUnknownSize( outputDataQuanta.iterator(), Spliterator.ORDERED), false )); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
final Collection<InputType0> collection = ((CollectionChannel.Instance) inputs[0]).provideCollection(); final Stream<InputType1> stream = ((JavaChannelInstance) inputs[1]).provideStream(); output.<Tuple2<InputType0, InputType1>>accept( stream.flatMap(e1 -> collection.stream().map( e0 -> new Tuple2<>(e0, e1) final Stream<InputType0> stream = ((JavaChannelInstance) inputs[0]).provideStream(); final Collection<InputType1> collection = ((CollectionChannel.Instance) inputs[1]).provideCollection(); output.<Tuple2<InputType0, InputType1>>accept( stream.flatMap(e0 -> collection.stream().map( e1 -> new Tuple2<>(e0, e1) output.<Tuple2<InputType0, InputType1>>accept( stream.flatMap(e1 -> collection.stream().map( e0 -> new Tuple2<>(e0, e1) final Collection<InputType1> collection = (Collection<InputType1>) ((JavaChannelInstance) inputs[1]).provideStream().collect(Collectors.toList()); final Stream<InputType0> stream = ((JavaChannelInstance) inputs[0]).provideStream(); output.<Tuple2<InputType0, InputType1>>accept( stream.flatMap(e0 -> collection.stream().map( e1 -> new Tuple2<>(e0, e1)
outputChannelInstance.accept(outputStream);
((StreamChannel.Instance) outputs[0]).accept(joinStream); outputs[0].getLineage().addPredecessor(probingExecutionLineageNode);
outputChannel.accept(result2);
((StreamChannel.Instance) outputs[0]).accept(intersectStream); outputs[0].getLineage().addPredecessor(probingExecutionLineageNode);
outputChannel.<org.qcri.rheem.basic.data.Tuple2<Input, Input>>accept(result2.stream());
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); String url = this.getInputUrl().trim(); FileSystem fs = FileSystems.getFileSystem(url).orElseThrow( () -> new RheemException(String.format("Cannot access file system of %s.", url)) ); try { final InputStream inputStream = fs.open(url); Stream<String> lines = new BufferedReader(new InputStreamReader(inputStream)).lines(); ((StreamChannel.Instance) outputs[0]).accept(lines); } catch (IOException e) { throw new RheemException(String.format("Reading %s failed.", url), e); } ExecutionLineageNode prepareLineageNode = new ExecutionLineageNode(operatorContext); prepareLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.textfilesource.load.prepare", javaExecutor.getConfiguration() )); ExecutionLineageNode mainLineageNode = new ExecutionLineageNode(operatorContext); mainLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.textfilesource.load.main", javaExecutor.getConfiguration() )); outputs[0].getLineage().addPredecessor(mainLineageNode); return prepareLineageNode.collectAndMark(); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert outputs.length == this.getNumOutputs(); SequenceFileIterator sequenceFileIterator; final String path; if (this.sourcePath == null) { final FileChannel.Instance input = (FileChannel.Instance) inputs[0]; path = input.getSinglePath(); } else { assert inputs.length == 0; path = this.sourcePath; } try { final String actualInputPath = FileSystems.findActualSingleInputPath(path); sequenceFileIterator = new SequenceFileIterator<>(actualInputPath); Stream<?> sequenceFileStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(sequenceFileIterator, 0), false); ((StreamChannel.Instance) outputs[0]).accept(sequenceFileStream); } catch (IOException e) { throw new RheemException(String.format("%s failed to read from %s.", this, path), e); } return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert outputs.length == this.getNumOutputs(); final String path; if (this.sourcePath == null) { final FileChannel.Instance input = (FileChannel.Instance) inputs[0]; path = input.getSinglePath(); } else { assert inputs.length == 0; path = this.sourcePath; } final String actualInputPath = FileSystems.findActualSingleInputPath(path); Stream<T> stream = this.createStream(actualInputPath); ((StreamChannel.Instance) outputs[0]).accept(stream); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
/** * Utility method to forward a {@link JavaChannelInstance} to another. * * @param input that should be forwarded * @param output to that should be forwarded */ static void forward(ChannelInstance input, ChannelInstance output) { // Do the forward. if (output instanceof CollectionChannel.Instance) { ((CollectionChannel.Instance) output).accept(((CollectionChannel.Instance) input).provideCollection()); } else if (output instanceof StreamChannel.Instance) { ((StreamChannel.Instance) output).accept(((JavaChannelInstance) input).provideStream()); } else { throw new RheemException(String.format("Cannot forward %s to %s.", input, output)); } // Manipulate the lineage. output.getLineage().addPredecessor(input.getLineage()); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final JavaChannelInstance input = (JavaChannelInstance) inputs[0]; final StreamChannel.Instance output = (StreamChannel.Instance) outputs[0]; final Function<InputType, OutputType> function = javaExecutor.getCompiler().compile(this.functionDescriptor); JavaExecutor.openFunction(this, function, inputs, operatorContext); output.accept(input.<InputType>provideStream().map(function)); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); ((StreamChannel.Instance) outputs[0]).accept(Stream.concat( ((JavaChannelInstance) inputs[0]).provideStream(), ((JavaChannelInstance) inputs[1]).provideStream()) ); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override @SuppressWarnings("unchecked") public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Predicate<Type> filterFunction = javaExecutor.getCompiler().compile(this.predicateDescriptor); JavaExecutor.openFunction(this, filterFunction, inputs, operatorContext); ((StreamChannel.Instance) outputs[0]).accept(((JavaChannelInstance) inputs[0]).<Type>provideStream().filter(filterFunction)); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { CollectionChannel.Instance input = (CollectionChannel.Instance) inputs[0]; StreamChannel.Instance output = (StreamChannel.Instance) outputs[0]; final Collection<Tuple2<Long, Long>> edges = input.provideCollection(); final TLongFloatMap pageRanks = this.pageRank(edges); final Stream<Tuple2<Long, Float>> pageRankStream = this.stream(pageRanks); output.accept(pageRankStream); return ExecutionOperator.modelQuasiEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); ((StreamChannel.Instance) outputs[0]).accept(((JavaChannelInstance) inputs[0]).provideStream().distinct()); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }