final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator); final Set<RequestedGlobalProperties> intGlobal = this.inConn.getInterestingProperties().getGlobalProperties(); List<DagConnection> broadcastConnections = getBroadcastConnections(); List<String> broadcastConnectionNames = getBroadcastConnectionNames(); for (OperatorDescriptorSingle ods : getPossibleProperties()) { pairs.addAll(ods.getPossibleGlobalProperties()); final int parallelism = getParallelism(); final int inParallelism = getPredecessorNode().getParallelism(); if (rgps.isMetBy(c.getGlobalProperties())) { c.setRequiredGlobalProps(rgps); addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator); break; addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator); break; estimator.costOperator(node); prunePlanAlternatives(outputPlans); outputPlans.trimToSize();
@Override public void computeUnclosedBranchStack() { if (this.openBranches != null) { return; } addClosedBranches(getPredecessorNode().closedBranchingNodes); List<UnclosedBranchDescriptor> fromInput = getPredecessorNode().getBranchesForParent(this.inConn); // handle the data flow branching for the broadcast inputs List<UnclosedBranchDescriptor> result = computeUnclosedBranchStackForBroadcastInputs(fromInput); this.openBranches = (result == null || result.isEmpty()) ? Collections.<UnclosedBranchDescriptor>emptyList() : result; }
@Override public void clearInterestingProperties() { super.clearInterestingProperties(); this.singleRoot.accept(InterestingPropertiesClearer.INSTANCE); this.rootConnection.clearInterestingProperties(); }
@Override public void accept(Visitor<OptimizerNode> visitor) { if (visitor.preVisit(this)) { if (getPredecessorNode() != null) { getPredecessorNode().accept(visitor); } else { throw new CompilerException(); } for (DagConnection connection : getBroadcastConnections()) { connection.getSource().accept(visitor); } visitor.postVisit(this); } } }
final Configuration conf = getOperator().getParameters(); final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null); final ShipStrategyType preSet; Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput(); throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input."); } else { pred = contractToNode.get(children); setIncomingConnection(conn); pred.addOutgoingConnection(conn);
if (!areBranchCompatible(bcSource, inputSource)) { validCombination = false; break; PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource(); if (!areBranchCompatible(bcSource, otherBcSource)) { validCombination = false; break; gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0); lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0); node.updatePropertiesWithUniqueSets(getUniqueFields()); target.add(node);
/** * Gets the operator represented by this optimizer node. * * @return The operator represented by this optimizer node. */ @Override public GroupReduceOperatorBase<?, ?, ?> getOperator() { return (GroupReduceOperatorBase<?, ?, ?>) super.getOperator(); }
@Override public void computeInterestingPropertiesForInputs(CostEstimator estimator) { final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0); for (OperatorDescriptorSingle dps : getPossibleProperties()) { for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) { for (DagConnection conn : getBroadcastConnections()) { conn.setInterestingProperties(new InterestingProperties());
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { Channel toReducer = in; if (in.getShipStrategy() == ShipStrategyType.FORWARD || (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) { if (in.getSource().getOptimizerNode() instanceof PartitionNode) { LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator."); } } else if (combinerStrategy != DriverStrategy.NONE) { // non forward case. all local properties are killed anyways, so we can safely plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, this.combinerStrategy, this.keyList); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); } return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer, DriverStrategy.SORTED_REDUCE, this.keyList); }
OptimizerNode pred = ((SingleInputNode) this).getPredecessorNode(); if (pred != null && pred.getEstimatedNumRecords() >= 0) { this.estimatedNumRecords = (long) (pred.getEstimatedNumRecords() * hints.getFilterFactor());
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { node.setParallelism(in.getSource().getParallelism()); // sorting key info SingleInputPlanNode singleInputPlanNode = new SingleInputPlanNode( node, "GroupCombine (" + node.getOperator().getName() + ")", in, // reuse the combine strategy also used in the group reduce DriverStrategy.SORTED_GROUP_COMBINE, this.keyList); // set sorting comparator key info singleInputPlanNode.setDriverKeyInfo(this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections(), 0); // set grouping comparator key info singleInputPlanNode.setDriverKeyInfo(this.keyList, 1); return singleInputPlanNode; }
SingleInputNode keyExtractorNode = (SingleInputNode) reduceNode.getPredecessorNode(); SingleInputNode filterNode = (SingleInputNode) keyExtractorNode.getPredecessorNode(); SingleInputNode mapNode = (SingleInputNode) filterNode.getPredecessorNode(); assertFalse(reduceNode.getIncomingConnection().isBreakingPipeline()); assertFalse(keyExtractorNode.getIncomingConnection().isBreakingPipeline()); assertFalse(filterNode.getIncomingConnection().isBreakingPipeline()); assertFalse(mapNode.getIncomingConnection().isBreakingPipeline());
if (!areBranchCompatible(bcSource, inputSource)) { validCombination = false; break; PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource(); if (!areBranchCompatible(bcSource, otherBcSource)) { validCombination = false; break; gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0); lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0); node.updatePropertiesWithUniqueSets(getUniqueFields()); target.add(node);
/** * Gets the operator represented by this optimizer node. * * @return The operator represented by this optimizer node. */ @Override public GroupCombineOperatorBase<?, ?, ?> getOperator() { return (GroupCombineOperatorBase<?, ?, ?>) super.getOperator(); }
@Override public void computeInterestingPropertiesForInputs(CostEstimator estimator) { final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0); for (OperatorDescriptorSingle dps : getPossibleProperties()) { for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) { for (DagConnection conn : getBroadcastConnections()) { conn.setInterestingProperties(new InterestingProperties());
@Override public void accept(Visitor<OptimizerNode> visitor) { if (visitor.preVisit(this)) { if (getPredecessorNode() != null) { getPredecessorNode().accept(visitor); } else { throw new CompilerException(); } for (DagConnection connection : getBroadcastConnections()) { connection.getSource().accept(visitor); } visitor.postVisit(this); } } }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD || (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_REDUCE, this.keyList); "Combine ("+node.getOperator().getName()+")", toCombiner, this.combinerStrategy, this.keyList); toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); return new SingleInputPlanNode(node, "Reduce("+node.getOperator().getName()+")", toReducer, DriverStrategy.SORTED_REDUCE, this.keyList);
OptimizerNode pred = ((SingleInputNode) this).getPredecessorNode(); if (pred != null && pred.getEstimatedNumRecords() >= 0) { this.estimatedNumRecords = (long) (pred.getEstimatedNumRecords() * hints.getFilterFactor());
SingleInputNode mapNode = (SingleInputNode) filterNode.getPredecessorNode(); assertFalse(sinkDirect.getInputConnection().isBreakingPipeline()); assertFalse(filterNode.getIncomingConnection().isBreakingPipeline()); assertFalse(mapNode.getIncomingConnection().isBreakingPipeline()); assertFalse(joinInput.getIncomingConnection().isBreakingPipeline()); assertEquals(mapNode, ((SingleInputNode) joinNode.getFirstPredecessorNode()).getPredecessorNode()); assertEquals(mapNode, sinkDirect.getPredecessorNode());
final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator); final Set<RequestedGlobalProperties> intGlobal = this.inConn.getInterestingProperties().getGlobalProperties(); List<DagConnection> broadcastConnections = getBroadcastConnections(); List<String> broadcastConnectionNames = getBroadcastConnectionNames(); for (OperatorDescriptorSingle ods : getPossibleProperties()) { pairs.addAll(ods.getPossibleGlobalProperties()); final int parallelism = getParallelism(); final int inParallelism = getPredecessorNode().getParallelism(); if (rgps.isMetBy(c.getGlobalProperties())) { c.setRequiredGlobalProps(rgps); addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator); break; addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator); break; estimator.costOperator(node); prunePlanAlternatives(outputPlans); outputPlans.trimToSize();