DualInputPlanNode dn = (DualInputPlanNode) node; if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) { throw new RuntimeException("Wrong operator type found in post pass."); DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator(); if (dn.getDriverStrategy().getNumRequiredComparators() > 0) { dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(), getSortOrders(dn.getKeysForInput1(), dn.getSortOrders()))); dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(), getSortOrders(dn.getKeysForInput2(), dn.getSortOrders()))); dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dualInputOperator.getOperatorInfo().getSecondInputType())); traverseChannel(dn.getInput1()); traverseChannel(dn.getInput2()); for (Channel c: dn.getBroadcastInputs()) { traverseChannel(c);
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { String nodeName = "RightOuterJoin ("+node.getOperator().getName()+")"; return new DualInputPlanNode(node, nodeName, in1, in2, getStrategy(), this.keys1, this.keys2); }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
@Test public void testOptimizerChoosesTest() { try { DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.OPTIMIZER_CHOOSES); assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput1().getShipStrategy()); assertEquals(ShipStrategyType.PARTITION_HASH, node.getInput2().getShipStrategy()); assertTrue(DriverStrategy.HYBRIDHASH_BUILD_FIRST == node.getDriverStrategy() || DriverStrategy.HYBRIDHASH_BUILD_SECOND == node.getDriverStrategy()); } catch (Exception e) { e.printStackTrace(); fail(e.getClass().getSimpleName() + ": " + e.getMessage()); } }
@Override public boolean preVisit(PlanNode visitable) { if (visitable instanceof DualInputPlanNode) { DualInputPlanNode node = (DualInputPlanNode) visitable; Channel c1 = node.getInput1(); Channel c2 = node.getInput2(); Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.FORWARD, c1.getShipStrategy()); Assert.assertEquals("Incompatible shipping strategy chosen for match", ShipStrategyType.PARTITION_HASH, c2.getShipStrategy()); return false; } return true; }
private boolean checkRepartitionMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) { if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set01, join.getInput1().getLocalStrategyKeys()); Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys()); Assert.assertTrue(join.getInput1().getLocalStrategySortOrder()[0] == join.getInput2().getLocalStrategySortOrder()[0]); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
TwoInputNode optNode = dn.getTwoInputNode(); if (schema1.getNumConnectionsThatContributed() < dn.getOutgoingChannels().size()) { return; if (dn.getDriverStrategy().getNumRequiredComparators() > 0) { dn.setComparator1(createComparator(dn.getKeysForInput1(), dn.getSortOrders(), schema1)); dn.setComparator2(createComparator(dn.getKeysForInput2(), dn.getSortOrders(), schema2)); } catch (MissingFieldTypeInfoException e) { throw new CompilerPostPassException("Could not set up runtime strategy for node '" + dn.setPairComparator(createPairComparator(dn.getKeysForInput1(), dn.getKeysForInput2(), dn.getSortOrders(), schema1, schema2)); } catch (MissingFieldTypeInfoException e) { throw new CompilerPostPassException("Could not set up runtime strategy for node '" + propagateToChannel(schema1, dn.getInput1(), createUtilities); } catch (MissingFieldTypeInfoException e) { throw new CompilerPostPassException("Could not set up runtime strategy for the first input channel to node '" propagateToChannel(schema2, dn.getInput2(), createUtilities); } catch (MissingFieldTypeInfoException e) { throw new CompilerPostPassException("Could not set up runtime strategy for the second input channel to node '" for (Channel c: dn.getBroadcastInputs()) { try { propagateToChannel(createEmptySchema(), c, createUtilities);
Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput1().getShipStrategy()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, coGroupNode.getInput2().getShipStrategy()); Channel c1 = coGroupNode.getInput1(); Channel c2 = coGroupNode.getInput2(); Assert.assertEquals(local2, c2.getLocalStrategyKeys()); Assert.assertTrue(c1.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]); Assert.assertTrue(c1.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]); Assert.assertTrue(c2.getLocalStrategySortOrder()[0] == coGroupNode.getSortOrders()[0]); Assert.assertTrue(c2.getLocalStrategySortOrder()[1] == coGroupNode.getSortOrders()[1]);
getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM; } else { getDriverStrategy().secondDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM; } else { for (NamedChannel nc : getBroadcastInputs()) { SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source); if (bcRes != NOT_FOUND) {
@Override public Iterable<PlanNode> getPredecessors() { if (getBroadcastInputs() == null || getBroadcastInputs().isEmpty()) { return Arrays.asList(this.input1.getSource(), this.input2.getSource()); } else { List<PlanNode> preds = new ArrayList<PlanNode>(); preds.add(input1.getSource()); preds.add(input2.getSource()); for (Channel c : getBroadcastInputs()) { preds.add(c.getSource()); } return preds; } }
toJoin2.setLocalStrategy(LocalStrategy.NONE); DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST); FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties()); assertEquals(NO_PARTIAL_SOLUTION, report);
private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) { if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys()); Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys()); Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder())); Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys()); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
@Test public void testBroadcastHashFirstTest() { try { DualInputPlanNode node = createPlanAndGetJoinNode(JoinHint.BROADCAST_HASH_FIRST); assertEquals(ShipStrategyType.BROADCAST, node.getInput1().getShipStrategy()); assertEquals(ShipStrategyType.FORWARD, node.getInput2().getShipStrategy()); assertEquals(DriverStrategy.HYBRIDHASH_BUILD_FIRST, node.getDriverStrategy()); } catch (Exception e) { e.printStackTrace(); fail(e.getClass().getSimpleName() + ": " + e.getMessage()); } }
private boolean checkRepartitionShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, SingleInputPlanNode combiner) { if (ShipStrategyType.PARTITION_HASH == join.getInput1().getShipStrategy() && ShipStrategyType.PARTITION_HASH == join.getInput2().getShipStrategy() && ShipStrategyType.FORWARD == reducer.getInput().getShipStrategy()) { // check combiner Assert.assertNull("Plan should not have a combiner", combiner); return true; } else { return false; } }
getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM; } else { getDriverStrategy().secondDam() == DamBehavior.FULL_DAM) { return FOUND_SOURCE_AND_DAM; } else { for (NamedChannel nc : getBroadcastInputs()) { SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source); if (bcRes != NOT_FOUND) {
@Override public void accept(Visitor<PlanNode> visitor) { if (visitor.preVisit(this)) { this.input1.getSource().accept(visitor); this.input2.getSource().accept(visitor); for (Channel broadcastInput : getBroadcastInputs()) { broadcastInput.getSource().accept(visitor); } visitor.postVisit(this); } }
Channel toJoin2 = new Channel(map2); DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST); LocalProperties lp = LocalProperties.EMPTY; FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp); assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET); LocalProperties lp = LocalProperties.forGrouping(new FieldList(2, 1)); FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp); assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET); toJoin1.setLocalStrategy(LocalStrategy.NONE); FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp); assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET); FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp); assertEquals(NOT_MET, report); FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp); assertEquals(MET, report); FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, gp, lp); assertTrue(report != null && report != NO_PARTIAL_SOLUTION && report != NOT_MET);
DualInputPlanNode dn = (DualInputPlanNode) node; if (!(dn.getOptimizerNode().getOperator() instanceof DualInputOperator)) { throw new RuntimeException("Wrong operator type found in post pass."); DualInputOperator<?, ?, ?, ?> dualInputOperator = (DualInputOperator<?, ?, ?, ?>) dn.getOptimizerNode().getOperator(); if (dn.getDriverStrategy().getNumRequiredComparators() > 0) { dn.setComparator1(createComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dn.getKeysForInput1(), getSortOrders(dn.getKeysForInput1(), dn.getSortOrders()))); dn.setComparator2(createComparator(dualInputOperator.getOperatorInfo().getSecondInputType(), dn.getKeysForInput2(), getSortOrders(dn.getKeysForInput2(), dn.getSortOrders()))); dn.setPairComparator(createPairComparator(dualInputOperator.getOperatorInfo().getFirstInputType(), dualInputOperator.getOperatorInfo().getSecondInputType())); traverseChannel(dn.getInput1()); traverseChannel(dn.getInput2()); for (Channel c: dn.getBroadcastInputs()) { traverseChannel(c);
private boolean checkHashJoinStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, boolean buildFirst) { if ((buildFirst && DriverStrategy.HYBRIDHASH_BUILD_FIRST == join.getDriverStrategy()) || (!buildFirst && DriverStrategy.HYBRIDHASH_BUILD_SECOND == join.getDriverStrategy())) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.NONE, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys()); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }