PlanNode candidate = planDeleter.next(); GlobalProperties atEndGlobal = candidate.getGlobalProperties(); LocalProperties atEndLocal = candidate.getLocalProperties(); FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobal, atEndLocal); if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) { noOpUnaryUdfOp.setInput(candidate.getProgramOperator()); UnaryOperatorNode rebuildPropertiesNode = new UnaryOperatorNode("Rebuild Partial Solution Properties", noOpUnaryUdfOp, true); rebuildPropertiesNode.setParallelism(candidate.getParallelism()); FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(pspn, atEndGlobalModified, atEndLocalModified); for (PlanNode candidate : candidates) { BulkIterationPlanNode node = new BulkIterationPlanNode(this, this.getOperator().getName(), in, pspn, candidate); GlobalProperties gProps = candidate.getGlobalProperties().clone(); LocalProperties lProps = candidate.getLocalProperties().clone(); node.initProperties(gProps, lProps); target.add(node); if (singleRoot.areBranchCompatible(candidate, terminationCandidate)) { BulkIterationPlanNode node = new BulkIterationPlanNode(this, "BulkIteration ("+this.getOperator().getName()+")", in, pspn, candidate, terminationCandidate); GlobalProperties gProps = candidate.getGlobalProperties().clone(); LocalProperties lProps = candidate.getLocalProperties().clone(); node.initProperties(gProps, lProps); target.add(node);
final long availableMemory = n.getGuaranteedAvailableMemory(); for (Channel channel : n.getInputs()) { final Costs costs = new Costs(); if (n.isOnDynamicPath()) { costWeight = n.getCostWeight(); Iterator<Channel> channels = n.getInputs().iterator(); if (channels.hasNext()) { firstInput = channels.next(); switch (n.getDriverStrategy()) { case NONE: case UNARY_NO_OP: break; default: throw new CompilerException("Unknown local strategy: " + n.getDriverStrategy().name()); n.setCosts(totalCosts);
@Override public Iterable<DumpableConnection<PlanNode>> getDumpableInputs() { List<DumpableConnection<PlanNode>> allInputs = new ArrayList<DumpableConnection<PlanNode>>(); for (Channel c : getInputs()) { allInputs.add(c); } for (NamedChannel c : getBroadcastInputs()) { allInputs.add(c); } return allInputs; }
@Override public void postVisit(PlanNode node) { if(node instanceof IterationPlanNode) { IterationPlanNode iNode = (IterationPlanNode)node; if(!visitedIterationNodes.contains(iNode)) { visitedIterationNodes.add(iNode); iNode.acceptForStepFunction(this); } } final Iterable<Channel> inputChannels = node.getInputs(); for (Channel channel : inputChannels) { ShipStrategyType shipStrategy = channel.getShipStrategy(); // Make sure we only optimize the DAG for range partition, and do not optimize multi times. if (shipStrategy == ShipStrategyType.PARTITION_RANGE) { if(channel.getDataDistribution() == null) { if (node.isOnDynamicPath()) { throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution."); } PlanNode channelSource = channel.getSource(); List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel); channelSource.getOutgoingChannels().remove(channel); channelSource.getOutgoingChannels().addAll(newSourceOutputChannels); } } } }
if (cheapest == null || (cheapest.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) { cheapest = candidate; if (gps[i].isMetBy(candidate.getGlobalProperties())) { if (cheapestForGlobal[i] == null || (cheapestForGlobal[i].getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) { cheapestForGlobal[i] = candidate; if (lps[k].isMetBy(candidate.getLocalProperties())) { final PlanNode previous = localMatches[k]; if (previous == null || previous.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0) { cheapest.setPruningMarker(); // remember that that plan is in the set final PlanNode[] localMatches = toKeep[i]; for (final PlanNode n : localMatches) { if (n != null && !n.isPruneMarkerSet()) { n.setPruningMarker(); plans.add(n); if (!n.isPruneMarkerSet()) { n.setPruningMarker(); plans.add(n);
for (Channel conn : visitable.getInputs()) { conn.setTarget(visitable); conn.getSource().addOutgoingChannel(conn); for (Channel c : visitable.getBroadcastInputs()) { c.setTarget(visitable); c.getSource().addOutgoingChannel(c); this.memoryConsumerWeights += visitable.getMemoryConsumerWeight(); for (Channel c : visitable.getInputs()) { if (c.getLocalStrategy().dams()) { this.memoryConsumerWeights++; for (Channel c : visitable.getBroadcastInputs()) { if (c.getLocalStrategy().dams()) { this.memoryConsumerWeights++;
if (node.isOnDynamicPath()) { throw new CompilerException("Nested Iterations are not possible at the moment!"); if ((chainedTask = this.chainedTasks.get(node)) != null) { final Iterator<Channel> inConns = node.getInputs().iterator(); if (!inConns.hasNext()) { throw new CompilerException("Bug: Found chained task with no input."); container.setResources(container.getMinResources().merge(node.getMinResources()), container.getPreferredResources().merge(node.getPreferredResources())); if (node.isOnDynamicPath()) { targetVertex.setStrictlyCoLocatedWith(head); targetVertexConfig.setIterationHeadSolutionSetInputIndex(1); } else { inConns = node.getInputs().iterator(); for (NamedChannel broadcastInput: node.getBroadcastInputs()) { int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true); targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex);
final JsonGenerator gen = jsonFactory.createGenerator(writer); final OptimizerNode optNode = node.getOptimizerNode(); if (node.getGlobalProperties() != null) { GlobalProperties gp = node.getGlobalProperties(); gen.writeArrayFieldStart("global_properties"); if (node.getLocalProperties() != null) { LocalProperties lp = node.getLocalProperties(); gen.writeArrayFieldStart("local_properties"); if (node.getNodeCosts() != null) { gen.writeArrayFieldStart("costs"); addProperty(gen, "Network", node.getNodeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(node.getNodeCosts().getNetworkCost(), "B")); addProperty(gen, "Disk I/O", node.getNodeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(node.getNodeCosts().getDiskCost(), "B")); addProperty(gen, "CPU", node.getNodeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(node.getNodeCosts().getCpuCost(), "")); addProperty(gen, "Cumulative Network", node.getCumulativeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(node.getCumulativeCosts().getNetworkCost(), "B")); addProperty(gen, "Cumulative Disk I/O", node.getCumulativeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(node.getCumulativeCosts().getDiskCost(), "B")); addProperty(gen, "Cumulative CPU", node.getCumulativeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(node.getCumulativeCosts().getCpuCost(), ""));
if (p.getDriverStrategy() != null) { switch (p.getDriverStrategy()) { case NONE: case BINARY_NO_OP: locString = p.getDriverStrategy().name(); break; final GlobalProperties gp = p.getGlobalProperties(); LocalProperties lp = p.getLocalProperties(); if (p.getNodeCosts() != null) { writer.print(",\n\t\t\"costs\": [\n"); addProperty(writer, "Network", p.getNodeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(p.getNodeCosts().getNetworkCost(), "B"), true); addProperty(writer, "Disk I/O", p.getNodeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(p.getNodeCosts().getDiskCost(), "B"), false); addProperty(writer, "CPU", p.getNodeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(p.getNodeCosts().getCpuCost(), ""), false); p.getCumulativeCosts().getNetworkCost() == -1 ? "(unknown)" : formatNumber(p .getCumulativeCosts().getNetworkCost(), "B"), false); addProperty(writer, "Cumulative Disk I/O", p.getCumulativeCosts().getDiskCost() == -1 ? "(unknown)" : formatNumber(p .getCumulativeCosts().getDiskCost(), "B"), false); addProperty(writer, "Cumulative CPU", p.getCumulativeCosts().getCpuCost() == -1 ? "(unknown)" : formatNumber(p
for (PlanNode node : this.allNodes) { final int consumerWeight = node.getMemoryConsumerWeight(); if (consumerWeight > 0) { final double relativeMem = (double)consumerWeight / this.memoryConsumerWeights; node.setRelativeMemoryPerSubtask(relativeMem); if (Optimizer.LOG.isDebugEnabled()) { Optimizer.LOG.debug("Assigned " + relativeMem + " of total memory to each subtask of " + node.getProgramOperator().getName() + "."); for (Channel c : node.getInputs()) { if (c.getLocalStrategy().dams()) { final double relativeMem = 1.0 / this.memoryConsumerWeights;
@Override public boolean preVisit(PlanNode visitable) { if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof ReduceOperatorBase) { for (Channel input: visitable.getInputs()) { GlobalProperties gprops = visitable.getGlobalProperties(); LocalProperties lprops = visitable.getLocalProperties(); if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof MapOperatorBase) { for (Channel input: visitable.getInputs()) { GlobalProperties gprops = visitable.getGlobalProperties(); LocalProperties lprops = visitable.getLocalProperties();
boolean allLocallyMet = true; for (Channel input : getInputs()) { FeedbackPropertiesMeetRequirementsReport inputState = input.getSource().checkPartialSolutionPropertiesMet(partialSolution, feedbackGlobal, feedbackLocal);
String opName = planNode.getOptimizerNode().getOperatorName(); if (vertex.getOperatorName() == null) { vertex.setOperatorName(opName); String opDescription = JsonMapper.getOperatorStrategyString(planNode.getDriverStrategy()); if (vertex.getOperatorDescription() == null) { vertex.setOperatorDescription(opDescription); String prettyName = StringUtils.showControlCharacters(planNode.getNodeName()); if (vertex.getOperatorPrettyName() == null) { vertex.setOperatorPrettyName(prettyName); String input2name = null; int num = 0; for (Channel c : node.getInputs()) { if (num == 0) { input1name = c.getSource().getNodeName(); input2name = c.getSource().getNodeName(); String opName = node.getOptimizerNode().getOperatorName(); if (vertex.getOperatorName() == null) { vertex.setOperatorName(opName); node.getDriverStrategy(), input1name != null ? input1name : "(unnamed)", input2name != null ? input2name : "(unnamed)"); String prettyName = StringUtils.showControlCharacters(node.getNodeName());
@Override public boolean preVisit(PlanNode visitable) { if (visitable instanceof SingleInputPlanNode && visitable.getProgramOperator() instanceof GroupReduceOperatorBase) { for (Channel inConn : visitable.getInputs()) { Assert.assertTrue("Reduce should just forward the input if it is already partitioned", inConn.getShipStrategy() == ShipStrategyType.FORWARD); } //just check latest ReduceNode return false; } return true; }
if (child.getGlobalProperties().isFullyReplicated()) {
@Override public int compare(PlanNode o1, PlanNode o2) { for (OptimizerNode branchDeterminer : branchDeterminers) { PlanNode n1 = o1.getCandidateAtBranchPoint(branchDeterminer); PlanNode n2 = o2.getCandidateAtBranchPoint(branchDeterminer); int hash1 = System.identityHashCode(n1); int hash2 = System.identityHashCode(n2); if (hash1 != hash2) { return hash1 - hash2; } } return 0; } };
@Override public String toString() { return this.template.getOperatorName() + " \"" + getProgramOperator().getName() + "\" : " + this.driverStrategy + " [[ " + this.globalProps + " ]] [[ " + this.localProps + " ]]"; }
} else { for (OptimizerNode brancher : this.hereJoinedBranches) { PlanNode candAtBrancher = in1.getSource().getCandidateAtBranchPoint(brancher); SourceAndDamReport res = in1.getSource().hasDamOnPathDownTo(candAtBrancher); if (res == NOT_FOUND) { throw new CompilerException("Bug: Tracing dams for deadlock detection is broken."); } else { for (OptimizerNode brancher : this.hereJoinedBranches) { PlanNode candAtBrancher = in2.getSource().getCandidateAtBranchPoint(brancher); SourceAndDamReport res = in2.getSource().hasDamOnPathDownTo(candAtBrancher); if (res == NOT_FOUND) { throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
private void computeLocalPropertiesAfterShippingOnly() { switch (this.shipStrategy) { case BROADCAST: case PARTITION_HASH: case PARTITION_CUSTOM: case PARTITION_RANGE: case PARTITION_RANDOM: case PARTITION_FORCED_REBALANCE: this.localProps = new LocalProperties(); break; case FORWARD: this.localProps = this.source.getLocalProperties(); break; case NONE: throw new CompilerException("ShipStrategy has not yet been set."); default: throw new CompilerException("Unknown ShipStrategy."); } }