successor.getParallelism() == pspn.getParallelism() && !(successor instanceof NAryUnionPlanNode) && successor != iteration.getRootOfStepFunction() && iteration.getInput().getLocalStrategy() == LocalStrategy.NONE; } else { merge = false; headVertex = new JobVertex("PartialSolution ("+iteration.getNodeName()+")"); headVertex.setResources(iteration.getMinResources(), iteration.getPreferredResources()); headVertex.setInvokableClass(IterationHeadTask.class); headConfig = new TaskConfig(headVertex.getConfiguration());
((BulkIterationPlanNode) node).getRootOfStepFunction(); ((BulkIterationPlanNode) node).getPartialSolutionPlanNode();
final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask(); if (relativeMemForBackChannel <= 0) { throw new CompilerException("Bug: No memory has been assigned to the iteration back channel."); final JobVertex sync = new JobVertex("Sync (" + bulkNode.getNodeName() + ")"); sync.setResources(bulkNode.getMinResources(), bulkNode.getPreferredResources()); sync.setInvokableClass(IterationSynchronizationSinkTask.class); sync.setParallelism(1); final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations(); if (maxNumIterations < 1) { throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations."); final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion(); final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction(); final TaskConfig tailConfig; tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel()); tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel()); AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators(); Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
BulkIterationPlanNode node = new BulkIterationPlanNode(this, this.getOperator().getName(), in, pspn, candidate); GlobalProperties gProps = candidate.getGlobalProperties().clone(); LocalProperties lProps = candidate.getLocalProperties().clone(); node.initProperties(gProps, lProps); target.add(node); for (PlanNode terminationCandidate : terminationCriterionCandidates) { if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) { BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate, terminationCandidate); GlobalProperties gProps = candidate.getGlobalProperties().clone(); LocalProperties lProps = candidate.getLocalProperties().clone(); node.initProperties(gProps, lProps); target.add(node);
BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node; if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) { throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node."); if (iterationNode.getRootOfTerminationCriterion() != null) { SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion(); traverseChannel(addMapper.getInput()); BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator(); iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType())); traverseChannel(iterationNode.getInput()); traverse(iterationNode.getRootOfStepFunction());
addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName()); if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) { return; if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) { throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node."); if (iterationNode.getRootOfTerminationCriterion() != null) { SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion(); traverse(addMapper.getInput().getSource(), createEmptySchema(), false); try { traverse(iterationNode.getRootOfStepFunction(), schema, false); T pss = (T) iterationNode.getPartialSolutionPlanNode().postPassHelper; if (pss == null) { throw new CompilerException("Error in Optimizer Post Pass: Partial solution schema is null after first traversal of the step function."); traverse(iterationNode.getRootOfStepFunction(), pss, createUtilities); if (iterationNode.getRootOfTerminationCriterion() != null) { SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion(); traverse(addMapper.getInput().getSource(), createEmptySchema(), createUtilities); try { addSchemaToSchema(pss, schema, iterationNode.getProgramOperator().getName()); iterationNode.setSerializerForIterationChannel(createSerializer(pss, iterationNode.getPartialSolutionPlanNode()));
for (Channel c : bipn.getPartialSolutionPlanNode().getOutgoingChannels()) { assertEquals(ShipStrategyType.FORWARD, c.getShipStrategy()); for (Channel c : bipn.getRootOfStepFunction().getInputs()) { assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy()); assertEquals(ShipStrategyType.PARTITION_HASH, bipn.getInput().getShipStrategy());
if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){ chaining = false; }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred || wspn.getRootOfTerminationCriterion() == pred)) { chaining = false;
@Test public void testIterationNotPushingWorkOut() throws Exception { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(8); DataSet<Tuple2<Long, Long>> input1 = env.readCsvFile("/some/file/path").types(Long.class).map(new DuplicateValue()); DataSet<Tuple2<Long, Long>> input2 = env.readCsvFile("/some/file/path").types(Long.class, Long.class); // Use input1 as partial solution. Partial solution is used in a single join operation --> it is cheaper // to do the hash partitioning between the partial solution node and the join node // instead of pushing the partitioning out doSimpleBulkIteration(input1, input2).output(new DiscardingOutputFormat<Tuple2<Long,Long>>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); assertEquals(1, op.getDataSinks().size()); assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof BulkIterationPlanNode); BulkIterationPlanNode bipn = (BulkIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource(); // check that work has not been pushed out for (Channel c : bipn.getPartialSolutionPlanNode().getOutgoingChannels()) { assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy()); } assertEquals(ShipStrategyType.FORWARD, bipn.getInput().getShipStrategy()); new JobGraphGenerator().compileJobGraph(op); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
PlanNode root = iterationNode.getRootOfStepFunction(); if (root.getParallelism() != node.getParallelism())
SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction(); BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource(); NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource(); assertTrue(mixedUnion.getInput1().getTempMode().isCached()); assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0); assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0); assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);
inConns = ((BulkPartialSolutionPlanNode) node).getContainingIterationNode().getInputs().iterator();
BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node; if (iterationNode.getRootOfStepFunction() instanceof NAryUnionPlanNode) { throw new CompilerException("Optimizer cannot compile an iteration step function where next partial solution is created by a Union node."); if (iterationNode.getRootOfTerminationCriterion() != null) { SingleInputPlanNode addMapper = (SingleInputPlanNode) iterationNode.getRootOfTerminationCriterion(); traverseChannel(addMapper.getInput()); BulkIterationBase<?> operator = (BulkIterationBase<?>) iterationNode.getProgramOperator(); iterationNode.setSerializerForIterationChannel(createSerializer(operator.getOperatorInfo().getOutputType())); traverseChannel(iterationNode.getInput()); traverse(iterationNode.getRootOfStepFunction());
if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){ chaining = false; }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred || wspn.getRootOfTerminationCriterion() == pred)) { chaining = false;
BulkIterationPlanNode node = new BulkIterationPlanNode(this, this.getOperator().getName(), in, pspn, candidate); GlobalProperties gProps = candidate.getGlobalProperties().clone(); LocalProperties lProps = candidate.getLocalProperties().clone(); node.initProperties(gProps, lProps); target.add(node); for (PlanNode terminationCandidate : terminationCriterionCandidates) { if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) { BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate, terminationCandidate); GlobalProperties gProps = candidate.getGlobalProperties().clone(); LocalProperties lProps = candidate.getLocalProperties().clone(); node.initProperties(gProps, lProps); target.add(node);
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iterPlanNode.getInput().getShipStrategy()); Assert.assertEquals(LocalStrategy.NONE, iterPlanNode.getInput().getLocalStrategy()); BulkPartialSolutionPlanNode partSolPlanNode = iterPlanNode.getPartialSolutionPlanNode(); Assert.assertEquals(ShipStrategyType.FORWARD, partSolPlanNode.getOutgoingChannels().get(0).getShipStrategy());
for (Channel c : bipn.getRootOfStepFunction().getInputs()) { assertEquals(ShipStrategyType.PARTITION_HASH, c.getShipStrategy());
SingleInputPlanNode noopNode = (SingleInputPlanNode) iterPlan.getRootOfStepFunction(); BinaryUnionPlanNode mixedUnion = (BinaryUnionPlanNode) noopNode.getInput().getSource(); NAryUnionPlanNode staticUnion = (NAryUnionPlanNode) mixedUnion.getInput1().getSource(); assertEquals(0.5, iterPlan.getRelativeMemoryPerSubTask(), 0.0); assertEquals(0.5, mixedUnion.getInput1().getRelativeTempMemory(), 0.0); assertEquals(0.0, mixedUnion.getInput2().getRelativeTempMemory(), 0.0);