public void collect(Channel in, List<Channel> inputs) { if (in.getSource() instanceof NAryUnionPlanNode) { // sanity check if (in.getShipStrategy() != ShipStrategyType.FORWARD) { throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators."); } if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) { throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators."); } inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs()); } else { // is not a collapsed union node, so we take the channel directly inputs.add(in); } } }
if (in.getLocalStrategy() == LocalStrategy.SORT) { if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) { throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
config.setBroadcastInputSerializer(channel.getSerializer(), inputNum); if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) { throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel."); } else { if (channel.getLocalStrategy() != LocalStrategy.NONE) { config.setInputLocalStrategy(inputNum, channel.getLocalStrategy()); if (channel.getLocalStrategyComparator() != null) { config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) { if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys()); Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys()); Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder())); Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys()); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
if (c.getLocalStrategy().dams()) { final double relativeMem = 1.0 / this.memoryConsumerWeights; c.setRelativeMemoryLocalStrategy(relativeMem);
private boolean checkRepartitionMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) { if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set01, join.getInput1().getLocalStrategyKeys()); Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys()); Assert.assertTrue(join.getInput1().getLocalStrategySortOrder()[0] == join.getInput2().getLocalStrategySortOrder()[0]); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
private boolean checkHashJoinStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, boolean buildFirst) { if ((buildFirst && DriverStrategy.HYBRIDHASH_BUILD_FIRST == join.getDriverStrategy()) || (!buildFirst && DriverStrategy.HYBRIDHASH_BUILD_SECOND == join.getDriverStrategy())) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.NONE, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys()); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }
if (channel.getLocalStrategy().requiresComparator()) { channel.setLocalStrategyComparator(createComparator(type, channel.getLocalStrategyKeys(), getSortOrders(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder())));
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.input.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.input.getLocalStrategy().dams() || this.input.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { // NOT_FOUND // check the broadcast inputs for (NamedChannel nc : getBroadcastInputs()) { SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source); if (bcRes != NOT_FOUND) { // broadcast inputs are always dams return FOUND_SOURCE_AND_DAM; } } return NOT_FOUND; } }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD) { // locally connected, directly instantiate return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE); } else { // non forward case.plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); Channel toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", toReducer, DriverStrategy.ALL_GROUP_REDUCE); } }
if (this.input1.getLocalStrategy().dams() || this.input1.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM; if (this.input2.getLocalStrategy().dams() || this.input2.getTempMode().breaksPipeline() || getDriverStrategy().secondDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM;
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD) { // locally connected, directly instantiate return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_REDUCE); } else { // non forward case.plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); Channel toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", toReducer, DriverStrategy.ALL_REDUCE); } }
assertEquals(LocalStrategy.NONE, reducer2.getInput().getLocalStrategy()); assertEquals(LocalStrategy.COMBININGSORT, reducer1.getInput().getLocalStrategy());
assertEquals(ShipStrategyType.BROADCAST, mapper.getBroadcastInputs().get(0).getShipStrategy()); assertEquals(LocalStrategy.NONE, mapper.getInput().getLocalStrategy()); assertEquals(LocalStrategy.NONE, mapper.getBroadcastInputs().get(0).getLocalStrategy()); assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy()); assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy()); assertNull(combiner.getInput().getLocalStrategyKeys()); assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy()); assertEquals(set0, reducer.getKeys(0)); assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
assertEquals(LocalStrategy.COMBININGSORT, reducer2.getInput().getLocalStrategy()); assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy()); assertEquals(LocalStrategy.COMBININGSORT, reducer1.getInput().getLocalStrategy());
Assert.assertEquals(LocalStrategy.SORT, c.getLocalStrategy());
private void checkStandardStrategies(SingleInputPlanNode map, DualInputPlanNode join, SingleInputPlanNode combiner, SingleInputPlanNode reducer, SinkPlanNode sink) { // check ship strategies that are always fix Assert.assertEquals(ShipStrategyType.FORWARD, map.getInput().getShipStrategy()); Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy()); // check the driver strategies that are always fix Assert.assertEquals(DriverStrategy.FLAT_MAP, map.getDriverStrategy()); Assert.assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy()); if (combiner != null) { Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy()); Assert.assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy()); } }
LocalStrategy.SORT == red2Node.getInput().getLocalStrategy() || LocalStrategy.SORT == map2Node.getInput().getLocalStrategy());