int inputIndex = headConfig.getDriverStrategy().getNumInputs(); headConfig.setIterationHeadSolutionSetInputIndex(inputIndex); translateChannel(wsNode.getInitialSolutionSetInput(), inputIndex, headVertex, headConfig, false); WorksetPlanNode wspn = (WorksetPlanNode) node; inConns = Collections.singleton(wspn.getContainingIterationNode().getInput2()).iterator();
successor.getParallelism() == wspn.getParallelism() && !(successor instanceof NAryUnionPlanNode) && successor != iteration.getNextWorkSetPlanNode() && iteration.getInitialWorksetInput().getLocalStrategy() == LocalStrategy.NONE; } else { merge = false; headVertex = new JobVertex("IterationHead("+iteration.getNodeName()+")"); headVertex.setResources(iteration.getMinResources(), iteration.getPreferredResources()); headVertex.setInvokableClass(IterationHeadTask.class); headConfig = new TaskConfig(headVertex.getConfiguration()); headConfig.setSolutionSetUnmanaged(iteration.getIterationNode().getIterationContract().isSolutionSetUnManaged());
WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node; if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) { throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node."); if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) { throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node."); DeltaIterationBase<?, ?> operator = (DeltaIterationBase<?, ?>) iterationNode.getProgramOperator(); iterationNode.setSolutionSetSerializer(createSerializer(operator.getOperatorInfo().getFirstInputType())); iterationNode.setWorksetSerializer(createSerializer(operator.getOperatorInfo().getSecondInputType())); iterationNode.setSolutionSetComparator(createComparator(operator.getOperatorInfo().getFirstInputType(), iterationNode.getSolutionSetKeyFields(), getSortOrders(iterationNode.getSolutionSetKeyFields(), null))); traverseChannel(iterationNode.getInput1()); traverseChannel(iterationNode.getInput2()); traverse(iterationNode.getSolutionSetDeltaPlanNode()); traverse(iterationNode.getNextWorkSetPlanNode());
if(selectedCandidate == null && getSolutionSetDeltaPlanNode() != null && getSolutionSetDeltaPlanNode() .branchPlan != null){ selectedCandidate = getSolutionSetDeltaPlanNode().branchPlan.get(brancher); if(selectedCandidate == null && getNextWorkSetPlanNode() != null && getNextWorkSetPlanNode() .branchPlan != null){ selectedCandidate = getNextWorkSetPlanNode().branchPlan.get(brancher);
final double relativeMemory = iterNode.getRelativeMemoryPerSubTask(); if (relativeMemory <= 0) { throw new CompilerException("Bug: No memory has been assigned to the workset iteration."); headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer()); headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator()); final JobVertex sync = new JobVertex("Sync (" + iterNode.getNodeName() + ")"); sync.setResources(iterNode.getMinResources(), iterNode.getPreferredResources()); sync.setInvokableClass(IterationSynchronizationSinkTask.class); sync.setParallelism(1); final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations(); if (maxNumIterations < 1) { throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations."); final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode(); final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode(); final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail); worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer()); solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer()); if (!iterNode.isImmediateSolutionSetUpdate()) { throw new CompilerException("A solution set update without dedicated tail is not set to perform immediate updates."); AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
addSchemaToSchema(parentSchema, schema, iterationNode.getProgramOperator().getName()); if (schema.getNumConnectionsThatContributed() < iterationNode.getOutgoingChannels().size()) { return; if (iterationNode.getNextWorkSetPlanNode() instanceof NAryUnionPlanNode) { throw new CompilerException("Optimizer cannot compile a workset iteration step function where the next workset is produced by a Union node."); if (iterationNode.getSolutionSetDeltaPlanNode() instanceof NAryUnionPlanNode) { throw new CompilerException("Optimizer cannot compile a workset iteration step function where the solution set delta is produced by a Union node."); traverse(iterationNode.getNextWorkSetPlanNode(), createEmptySchema(), false); traverse(iterationNode.getSolutionSetDeltaPlanNode(), schema, false); T wss = (T) iterationNode.getWorksetPlanNode().postPassHelper; T sss = (T) iterationNode.getSolutionSetPlanNode().postPassHelper; traverse(iterationNode.getNextWorkSetPlanNode(), wss, createUtilities); traverse(iterationNode.getSolutionSetDeltaPlanNode(), sss, createUtilities); + " in node '" + iterationNode.getProgramOperator().getName() + "'. Contradicting types between the " + "result of the iteration and the solution set schema: " + e.getPreviousType() + " and " + e.getNewType() + ". Most probable cause: Invalid constant field annotations."); WorksetIterationNode optNode = iterationNode.getIterationNode(); iterationNode.setWorksetSerializer(createSerializer(wss, iterationNode.getWorksetPlanNode())); iterationNode.setSolutionSetSerializer(createSerializer(sss, iterationNode.getSolutionSetPlanNode())); try { iterationNode.setSolutionSetComparator(createComparator(optNode.getSolutionSetKeyFields(), null, sss));
((WorksetIterationPlanNode) node).getNextWorkSetPlanNode(); DumpableNode<?> solutionDelta = node instanceof WorksetIterationNode ? ((WorksetIterationNode) node).getSolutionSetDelta() : ((WorksetIterationPlanNode) node).getSolutionSetDeltaPlanNode(); ((WorksetIterationPlanNode) node).getWorksetPlanNode(); DumpableNode<?> solutionSet = node instanceof WorksetIterationNode ? ((WorksetIterationNode) node).getSolutionSetNode() : ((WorksetIterationPlanNode) node).getSolutionSetPlanNode();
WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(this, this.getOperator().getName(), solutionSetIn, worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate); wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate); wsNode.initProperties(gp, lp); target.add(wsNode);
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialWorksetInput().getLocalStrategy()); Assert.assertEquals(TempMode.NONE, iter.getInitialWorksetInput().getTempMode()); Assert.assertEquals(TempMode.NONE, iter.getInitialSolutionSetInput().getTempMode()); Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialWorksetInput().getDataExchangeMode()); Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialSolutionSetInput().getDataExchangeMode());
assertEquals(ShipStrategyType.FORWARD, wipn.getInput1().getShipStrategy()); assertEquals(DataExchangeMode.BATCH, wipn.getInput1().getDataExchangeMode()); assertEquals(DataExchangeMode.BATCH, wipn.getInput2().getDataExchangeMode()); assertEquals(TempMode.NONE, wipn.getInput1().getTempMode()); assertEquals(TempMode.NONE, wipn.getInput2().getTempMode());
public Channel getInitialWorksetInput() { return getInput2(); }
public Channel getInitialSolutionSetInput() { return getInput1(); }
public FieldList getSolutionSetKeyFields() { return getIterationNode().getSolutionSetKeyFields(); }
@Test public void testWorksetIterationNotDependingOnSolutionSet() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 100).map(new Duplicator<Long>()); DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 100, 1); DataSet<Tuple2<Long, Long>> iterEnd = iteration.getWorkset().map(new TestMapper<Tuple2<Long,Long>>()); iteration.closeWith(iterEnd, iterEnd) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource(); assertTrue(wipn.getSolutionSetPlanNode().getOutgoingChannels().isEmpty()); JobGraphGenerator jgg = new JobGraphGenerator(); jgg.compileJobGraph(op); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Override public boolean preVisit(PlanNode visitable) { if (visitable instanceof WorksetIterationPlanNode) { PlanNode deltaNode = ((WorksetIterationPlanNode) visitable).getSolutionSetDeltaPlanNode(); //get the CoGroup DualInputPlanNode dpn = (DualInputPlanNode) deltaNode.getInputs().iterator().next().getSource(); Channel in1 = dpn.getInput1(); Channel in2 = dpn.getInput2(); Assert.assertTrue(in1.getLocalProperties().getOrdering() == null); Assert.assertTrue(in2.getLocalProperties().getOrdering() != null); Assert.assertTrue(in2.getLocalProperties().getOrdering().getInvolvedIndexes().contains(0)); Assert.assertTrue(in1.getShipStrategy() == ShipStrategyType.FORWARD); Assert.assertTrue(in2.getShipStrategy() == ShipStrategyType.PARTITION_HASH); return false; } return true; }
final double relativeMemory = iterNode.getRelativeMemoryPerSubTask(); if (relativeMemory <= 0) { throw new CompilerException("Bug: No memory has been assigned to the workset iteration."); headConfig.setSolutionSetSerializer(iterNode.getSolutionSetSerializer()); headConfig.setSolutionSetComparator(iterNode.getSolutionSetComparator()); final JobVertex sync = new JobVertex("Sync (" + iterNode.getNodeName() + ")"); sync.setResources(iterNode.getMinResources(), iterNode.getPreferredResources()); sync.setInvokableClass(IterationSynchronizationSinkTask.class); sync.setParallelism(1); final int maxNumIterations = iterNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations(); if (maxNumIterations < 1) { throw new CompilerException("Cannot create workset iteration with unspecified maximum number of iterations."); final PlanNode nextWorksetNode = iterNode.getNextWorkSetPlanNode(); final PlanNode solutionDeltaNode = iterNode.getSolutionSetDeltaPlanNode(); final boolean hasSolutionSetTail = (!iterNode.isImmediateSolutionSetUpdate()) || (!hasWorksetTail); worksetTailConfig.setOutputSerializer(iterNode.getWorksetSerializer()); solutionDeltaConfig.setOutputSerializer(iterNode.getSolutionSetSerializer()); if (!iterNode.isImmediateSolutionSetUpdate()) { throw new CompilerException("A solution set update without dedicated tail is not set to perform immediate updates."); AggregatorRegistry aggs = iterNode.getIterationNode().getIterationContract().getAggregators();
((WorksetIterationPlanNode) node).getNextWorkSetPlanNode(); DumpableNode<?> solutionDelta = node instanceof WorksetIterationNode ? ((WorksetIterationNode) node).getSolutionSetDelta() : ((WorksetIterationPlanNode) node).getSolutionSetDeltaPlanNode(); ((WorksetIterationPlanNode) node).getWorksetPlanNode(); DumpableNode<?> solutionSet = node instanceof WorksetIterationNode ? ((WorksetIterationNode) node).getSolutionSetNode() : ((WorksetIterationPlanNode) node).getSolutionSetPlanNode();
PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode(); PlanNode solutionSetDelta = iterationNode.getSolutionSetDeltaPlanNode();
WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(this, this.getOperator().getName(), solutionSetIn, worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate); wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate); wsNode.initProperties(gp, lp); target.add(wsNode);
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialWorksetInput().getLocalStrategy()); Assert.assertEquals(TempMode.NONE, iter.getInitialWorksetInput().getTempMode()); Assert.assertEquals(TempMode.NONE, iter.getInitialSolutionSetInput().getTempMode()); Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialWorksetInput().getDataExchangeMode()); Assert.assertEquals(DataExchangeMode.BATCH, iter.getInitialSolutionSetInput().getDataExchangeMode());