public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, PrintWriter writer) { Collection<SinkPlanNode> sinks = plan.getDataSinks(); if (sinks instanceof List) { dumpOptimizerPlanAsJSON((List<SinkPlanNode>) sinks, writer); } else { List<SinkPlanNode> n = new ArrayList<SinkPlanNode>(); n.addAll(sinks); dumpOptimizerPlanAsJSON(n, writer); } }
program.accept(this); JobGraph graph = new JobGraph(jobId, program.getJobName()); try { graph.setExecutionConfig(program.getOriginalPlan().getExecutionConfig()); graph.setSessionTimeout(program.getOriginalPlan().getSessionTimeout()); vertex.setInputDependencyConstraint(program.getOriginalPlan().getExecutionConfig().getDefaultInputDependencyConstraint()); graph.addVertex(vertex); program.getOriginalPlan().getCachedFiles().stream() .map(entry -> Tuple2.of(entry.getKey(), entry.getValue())) .collect(Collectors.toList());
final int targetParallelism = targetNode.getParallelism(); final Costs defaultZeroCosts = new Costs(0, 0, 0); final TypeComparatorFactory<?> comparator = Utils.getShipComparator(channel, this.plan.getOriginalPlan().getExecutionConfig()); sipPlanNode.setCosts(defaultZeroCosts); sipChannel.setTarget(sipPlanNode); this.plan.getAllNodes().add(sipPlanNode); sourceNewOutputChannels.add(sipChannel); sicChannel.setTarget(sicPlanNode); sipPlanNode.addOutgoingChannel(sicChannel); this.plan.getAllNodes().add(sicPlanNode); rbChannel.setTarget(rbPlanNode); sicPlanNode.addOutgoingChannel(rbChannel); this.plan.getAllNodes().add(rbPlanNode); ariPlanNode.setCosts(defaultZeroCosts); ariChannel.setTarget(ariPlanNode); this.plan.getAllNodes().add(ariPlanNode); sourceNewOutputChannels.add(ariChannel); prPlanNode.initProperties(globalProperties, new LocalProperties()); prPlanNode.setCosts(defaultZeroCosts); this.plan.getAllNodes().add(prPlanNode);
@Override public void postPass(OptimizedPlan plan) { executionConfig = plan.getOriginalPlan().getExecutionConfig(); for (SinkPlanNode sink : plan.getDataSinks()) { traverse(sink); } }
return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
public OptimizerPlanNodeResolver(OptimizedPlan p) { HashMap<String, ArrayList<PlanNode>> map = new HashMap<String, ArrayList<PlanNode>>(); for (PlanNode n : p.getAllNodes()) { Operator<?> c = n.getOriginalOptimizerNode().getOperator(); String name = c.getName();
@Override public void postPass(OptimizedPlan plan) { executionConfig = plan.getOriginalPlan().getExecutionConfig(); for (SinkPlanNode sink : plan.getDataSinks()) { traverse(sink); } }
oPlan.accept(new Visitor<PlanNode>() {
return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
@Test public void testWorksetIterationNotDependingOnSolutionSet() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 100).map(new Duplicator<Long>()); DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 100, 1); DataSet<Tuple2<Long, Long>> iterEnd = iteration.getWorkset().map(new TestMapper<Tuple2<Long,Long>>()); iteration.closeWith(iterEnd, iterEnd) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource(); assertTrue(wipn.getSolutionSetPlanNode().getOutgoingChannels().isEmpty()); JobGraphGenerator jgg = new JobGraphGenerator(); jgg.compileJobGraph(op); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
program.accept(this); JobGraph graph = new JobGraph(jobId, program.getJobName()); try { graph.setExecutionConfig(program.getOriginalPlan().getExecutionConfig()); graph.setSessionTimeout(program.getOriginalPlan().getSessionTimeout()); program.getOriginalPlan().getCachedFiles().stream() .map(entry -> Tuple2.of(entry.getKey(), entry.getValue())) .collect(Collectors.toList());
@Override public void postPass(OptimizedPlan plan) { executionConfig = plan.getOriginalPlan().getExecutionConfig(); for (SinkPlanNode sink : plan.getDataSinks()) { traverse(sink); } }
final int targetParallelism = targetNode.getParallelism(); final Costs defaultZeroCosts = new Costs(0, 0, 0); final TypeComparatorFactory<?> comparator = Utils.getShipComparator(channel, this.plan.getOriginalPlan().getExecutionConfig()); sipPlanNode.setCosts(defaultZeroCosts); sipChannel.setTarget(sipPlanNode); this.plan.getAllNodes().add(sipPlanNode); sourceNewOutputChannels.add(sipChannel); sicChannel.setTarget(sicPlanNode); sipPlanNode.addOutgoingChannel(sicChannel); this.plan.getAllNodes().add(sicPlanNode); rbChannel.setTarget(rbPlanNode); sicPlanNode.addOutgoingChannel(rbChannel); this.plan.getAllNodes().add(rbPlanNode); ariPlanNode.setCosts(defaultZeroCosts); ariChannel.setTarget(ariPlanNode); this.plan.getAllNodes().add(ariPlanNode); sourceNewOutputChannels.add(ariChannel); prPlanNode.initProperties(globalProperties, new LocalProperties()); prPlanNode.setCosts(defaultZeroCosts); this.plan.getAllNodes().add(prPlanNode);
oPlan.accept(new Visitor<PlanNode>() {
return new OptimizedPlan(this.sources, this.sinks, this.allNodes, jobName, originalPlan);
@Test public void testNoBreakerForIndependentVariable() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> source1 = env.fromElements("test"); DataSet<String> source2 = env.fromElements("test"); source1.map(new IdentityMapper<String>()).withBroadcastSet(source2, "some name") .output(new DiscardingOutputFormat<String>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); SinkPlanNode sink = op.getDataSinks().iterator().next(); SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource(); assertEquals(TempMode.NONE, mapper.getInput().getTempMode()); assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode()); assertEquals(DataExchangeMode.PIPELINED, mapper.getInput().getDataExchangeMode()); assertEquals(DataExchangeMode.PIPELINED, mapper.getBroadcastInputs().get(0).getDataExchangeMode()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
program.accept(this); JobGraph graph = new JobGraph(jobId, program.getJobName()); try { graph.setExecutionConfig(program.getOriginalPlan().getExecutionConfig()); graph.setSessionTimeout(program.getOriginalPlan().getSessionTimeout()); program.getOriginalPlan().getCachedFiles().stream() .map(entry -> Tuple2.of(entry.getKey(), entry.getValue())) .collect(Collectors.toList());
@Override public void postPass(OptimizedPlan plan) { executionConfig = plan.getOriginalPlan().getExecutionConfig(); for (SinkPlanNode sink : plan.getDataSinks()) { traverse(sink); } }
final int targetParallelism = targetNode.getParallelism(); final Costs defaultZeroCosts = new Costs(0, 0, 0); final TypeComparatorFactory<?> comparator = Utils.getShipComparator(channel, this.plan.getOriginalPlan().getExecutionConfig()); sipPlanNode.setCosts(defaultZeroCosts); sipChannel.setTarget(sipPlanNode); this.plan.getAllNodes().add(sipPlanNode); sourceNewOutputChannels.add(sipChannel); sicChannel.setTarget(sicPlanNode); sipPlanNode.addOutgoingChannel(sicChannel); this.plan.getAllNodes().add(sicPlanNode); rbChannel.setTarget(rbPlanNode); sicPlanNode.addOutgoingChannel(rbChannel); this.plan.getAllNodes().add(rbPlanNode); ariPlanNode.setCosts(defaultZeroCosts); ariChannel.setTarget(ariPlanNode); this.plan.getAllNodes().add(ariPlanNode); sourceNewOutputChannels.add(ariChannel); prPlanNode.initProperties(globalProperties, new LocalProperties()); prPlanNode.setCosts(defaultZeroCosts); this.plan.getAllNodes().add(prPlanNode);