private JobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) { final BulkIterationPlanNode iteration = pspn.getContainingIterationNode(); if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) { final Channel c = pspn.getOutgoingChannels().get(0); final PlanNode successor = c.getTarget(); merge = c.getShipStrategy() == ShipStrategyType.FORWARD && c.getLocalStrategy() == LocalStrategy.NONE && c.getTempMode() == TempMode.NONE && successor.getParallelism() == pspn.getParallelism() && !(successor instanceof NAryUnionPlanNode) && successor != iteration.getRootOfStepFunction() && final TaskConfig headConfig; if (merge) { final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget(); headVertex = this.vertices.get(successor);
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) { if (this.cachedPlans != null) { throw new IllegalStateException(); } else { this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this, "PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput)); } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
@Test public void testIterationNotPushingWorkOut() throws Exception { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(8); DataSet<Tuple2<Long, Long>> input1 = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue()); DataSet<Tuple2<Long, Long>> input2 = env.readCsvFile("/some/file/path").types(Long.class, Long.class); // Use input1 as partial solution. Partial solution is used in a single join operation --> it is cheaper // to do the hash partitioning between the partial solution node and the join node // instead of pushing the partitioning out doSimpleBulkIteration(input1, input2).output(new DiscardingOutputFormat<Tuple2<Long,Long>>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); assertEquals(1, op.getDataSinks().size()); assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof BulkIterationPlanNode); BulkIterationPlanNode bipn = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource(); // check that work has not been pushed out for (Channel c : bipn.getPartialSolutionPlanNode().getOutgoingChannels()) { assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy()); } assertEquals(ShipStrategyType.FORWARD, bipn.getInput().getShipStrategy()); new JobGraphGenerator().compileJobGraph(op); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
"Cannot associate the node for a partial solutions with its containing iteration."); pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs().iterator();
for (Channel c : bipn.getPartialSolutionPlanNode().getOutgoingChannels()) { assertEquals(ShipStrategyType.FORWARD, c.getShipStrategy());
"Cannot associate the node for a partial solutions with its containing iteration."); pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
private JobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) { final BulkIterationPlanNode iteration = pspn.getContainingIterationNode(); if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) { final Channel c = pspn.getOutgoingChannels().get(0); final PlanNode successor = c.getTarget(); merge = c.getShipStrategy() == ShipStrategyType.FORWARD && c.getLocalStrategy() == LocalStrategy.NONE && c.getTempMode() == TempMode.NONE && successor.getParallelism() == pspn.getParallelism() && !(successor instanceof NAryUnionPlanNode) && successor != iteration.getRootOfStepFunction() && final TaskConfig headConfig; if (merge) { final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget(); headVertex = this.vertices.get(successor);
final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
Assert.assertEquals(ShipStrategyType.FORWARD, partSolPlanNode.getOutgoingChannels().get(0).getShipStrategy());
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) { if (this.cachedPlans != null) { throw new IllegalStateException(); } else { this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this, "PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput)); } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
"Cannot associate the node for a partial solutions with its containing iteration."); pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);
private JobVertex createBulkIterationHead(BulkPartialSolutionPlanNode pspn) { final BulkIterationPlanNode iteration = pspn.getContainingIterationNode(); if (mergeIterationAuxTasks && pspn.getOutgoingChannels().size() == 1) { final Channel c = pspn.getOutgoingChannels().get(0); final PlanNode successor = c.getTarget(); merge = c.getShipStrategy() == ShipStrategyType.FORWARD && c.getLocalStrategy() == LocalStrategy.NONE && c.getTempMode() == TempMode.NONE && successor.getParallelism() == pspn.getParallelism() && !(successor instanceof NAryUnionPlanNode) && successor != iteration.getRootOfStepFunction() && final TaskConfig headConfig; if (merge) { final PlanNode successor = pspn.getOutgoingChannels().get(0).getTarget(); headVertex = this.vertices.get(successor);
final BulkIterationPlanNode iterationNode = pspn.getContainingIterationNode();
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) { if (this.cachedPlans != null) { throw new IllegalStateException(); } else { this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this, "PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput)); } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
"Cannot associate the node for a partial solutions with its containing iteration."); pspn.setContainingIterationNode((BulkIterationPlanNode) iteration);