/** * Models lazy execution by not marking any {@link LazyExecutionLineageNode}s. * * @param inputs the input {@link ChannelInstance}s * @param outputs the output {@link ChannelInstance}s * @param operatorContext the executed {@link OptimizationContext.OperatorContext} * @return the executed {@link OptimizationContext.OperatorContext} and produced {@link ChannelInstance}s */ static Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> modelLazyExecution(ChannelInstance[] inputs, ChannelInstance[] outputs, OptimizationContext.OperatorContext operatorContext) { final ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext(); LazyExecutionLineageNode.connectAll(inputs, executionLineageNode, outputs); return new Tuple<>(Collections.emptyList(), Collections.emptyList()); }
/** * Models eager execution by marking all {@link LazyExecutionLineageNode}s as executed and collecting all marked ones. * However, the output {@link ChannelInstance}s are not yet produced. * * @param inputs the input {@link ChannelInstance}s * @param outputs the output {@link ChannelInstance}s * @param operatorContext the executed {@link OptimizationContext.OperatorContext} * @return the executed {@link OptimizationContext.OperatorContext} and produced {@link ChannelInstance}s */ static Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> modelQuasiEagerExecution( ChannelInstance[] inputs, ChannelInstance[] outputs, OptimizationContext.OperatorContext operatorContext) { final ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext(); LazyExecutionLineageNode.connectAll(inputs, executionLineageNode, outputs); return executionLineageNode.collectAndMark(); }
/** * Models eager execution by marking all {@link LazyExecutionLineageNode}s as executed and collecting all marked ones. * * @param inputs the input {@link ChannelInstance}s * @param outputs the output {@link ChannelInstance}s * @param operatorContext the executed {@link OptimizationContext.OperatorContext} * @return the executed {@link OptimizationContext.OperatorContext} and produced {@link ChannelInstance}s */ static Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> modelEagerExecution( ChannelInstance[] inputs, ChannelInstance[] outputs, OptimizationContext.OperatorContext operatorContext) { final ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext(); LazyExecutionLineageNode.connectAll(inputs, executionLineageNode, outputs); final Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> collectors; if (outputs.length == 0) { collectors = executionLineageNode.collectAndMark(); } else { collectors = new Tuple<>(new LinkedList<>(), new LinkedList<>()); for (ChannelInstance output : outputs) { output.getLineage().collectAndMark(collectors.getField0(), collectors.getField1()); } } return collectors; }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); String url = this.getInputUrl().trim(); FileSystem fs = FileSystems.getFileSystem(url).orElseThrow( () -> new RheemException(String.format("Cannot access file system of %s.", url)) ); try { final InputStream inputStream = fs.open(url); Stream<String> lines = new BufferedReader(new InputStreamReader(inputStream)).lines(); ((StreamChannel.Instance) outputs[0]).accept(lines); } catch (IOException e) { throw new RheemException(String.format("Reading %s failed.", url), e); } ExecutionLineageNode prepareLineageNode = new ExecutionLineageNode(operatorContext); prepareLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.textfilesource.load.prepare", javaExecutor.getConfiguration() )); ExecutionLineageNode mainLineageNode = new ExecutionLineageNode(operatorContext); mainLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.textfilesource.load.main", javaExecutor.getConfiguration() )); outputs[0].getLineage().addPredecessor(mainLineageNode); return prepareLineageNode.collectAndMark(); }
final CardinalityEstimate cardinalityEstimate1 = operatorContext.getInputCardinality(1); ExecutionLineageNode indexingExecutionLineageNode = new ExecutionLineageNode(operatorContext); indexingExecutionLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.join.load.indexing", javaExecutor.getConfiguration() )); ExecutionLineageNode probingExecutionLineageNode = new ExecutionLineageNode(operatorContext); probingExecutionLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.join.load.probing", javaExecutor.getConfiguration()
ExecutionLineageNode probingExecutionLineageNode = new ExecutionLineageNode(operatorContext); probingExecutionLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.cartesian.load.probing", javaExecutor.getConfiguration() )); ExecutionLineageNode indexingExecutionLineageNode = new ExecutionLineageNode(operatorContext); indexingExecutionLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.cartesian.load.indexing", javaExecutor.getConfiguration()
assert outputs.length == this.getNumOutputs(); ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext();
assert outputs.length == this.getNumOutputs(); final ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext();
final ExecutionLineageNode mainExecutionLineage = new ExecutionLineageNode(operatorContext); mainExecutionLineage.add(LoadProfileEstimators.createFromSpecification( "rheem.graphchi.pagerank.load.main", configuration mainExecutionLineage.addPredecessor(inputFileChannelInstance.getLineage()); final ExecutionLineageNode outputExecutionLineage = new ExecutionLineageNode(operatorContext); outputExecutionLineage.add(LoadProfileEstimators.createFromSpecification( "rheem.graphchi.pagerank.load.output", configuration
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, SparkExecutor sparkExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); RddChannel.Instance output = (RddChannel.Instance) outputs[0]; final JavaRDD<String> rdd = sparkExecutor.sc.textFile(this.getInputUrl()); this.name(rdd); output.accept(rdd, sparkExecutor); ExecutionLineageNode prepareLineageNode = new ExecutionLineageNode(operatorContext); prepareLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.spark.textfilesource.load.prepare", sparkExecutor.getConfiguration() )); ExecutionLineageNode mainLineageNode = new ExecutionLineageNode(operatorContext); mainLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.spark.textfilesource.load.main", sparkExecutor.getConfiguration() )); output.getLineage().addPredecessor(mainLineageNode); return prepareLineageNode.collectAndMark(); }
assert outputs.length == this.getNumOutputs(); ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext();
assert outputs.length == this.getNumOutputs(); ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext();
assert outputs.length == this.getNumOutputs(); final ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext();
assert outputs.length == this.getNumOutputs(); ExecutionLineageNode executionLineageNode = new ExecutionLineageNode(operatorContext); executionLineageNode.addAtomicExecutionFromOperatorContext();
final CardinalityEstimate cardinalityEstimate1 = operatorContext.getOutputCardinality(0); ExecutionLineageNode indexingExecutionLineageNode = new ExecutionLineageNode(operatorContext); indexingExecutionLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.intersect.load.indexing", javaExecutor.getConfiguration() )); ExecutionLineageNode probingExecutionLineageNode = new ExecutionLineageNode(operatorContext); probingExecutionLineageNode.add(LoadProfileEstimators.createFromSpecification( "rheem.java.intersect.load.probing", javaExecutor.getConfiguration()