public Statistics getStatistics() { return (conf == null) ? null : conf.getStatistics(); }
public Statistics getStatistics() { if (conf != null) { return conf.getStatistics(); } return null; }
private static void annotateRuntimeStats(Operator<? extends OperatorDesc> op, ParseContext pctx) { Long runTimeNumRows = pctx.getContext().getExplainConfig().getOpIdToRuntimeNumRows() .get(op.getOperatorId()); if (op.getConf() != null && op.getConf().getStatistics() != null && runTimeNumRows != null) { LOG.info("annotateRuntimeStats for " + op.getOperatorId()); op.getConf().getStatistics().setRunTimeNumRows(runTimeNumRows); } else { LOG.debug("skip annotateRuntimeStats for " + op.getOperatorId()); } }
private static void annotateRuntimeStats(Operator<? extends OperatorDesc> op, ParseContext pctx) { Long runTimeNumRows = pctx.getContext().getExplainConfig().getOpIdToRuntimeNumRows() .get(op.getOperatorId()); if (op.getConf() != null && op.getConf().getStatistics() != null && runTimeNumRows != null) { LOG.info("annotateRuntimeStats for " + op.getOperatorId()); op.getConf().getStatistics().setRunTimeNumRows(runTimeNumRows); } else { LOG.debug("skip annotateRuntimeStats for " + op.getOperatorId()); } }
private void setStatistics(Operator<? extends OperatorDesc> origin, Operator<? extends OperatorDesc> clone) { clone.getConf().setStatistics(origin.getConf().getStatistics()); clone.getConf().setTraits(origin.getConf().getTraits()); if (origin.getChildOperators().size() == clone.getChildOperators().size()) { for (int i = 0; i < clone.getChildOperators().size(); i++) { setStatistics(origin.getChildOperators().get(i), clone.getChildOperators().get(i)); } } } }
private void setStatistics(Operator<? extends OperatorDesc> origin, Operator<? extends OperatorDesc> clone) { clone.getConf().setStatistics(origin.getConf().getStatistics()); clone.getConf().setTraits(origin.getConf().getTraits()); if (origin.getChildOperators().size() == clone.getChildOperators().size()) { for (int i = 0; i < clone.getChildOperators().size(); i++) { setStatistics(origin.getChildOperators().get(i), clone.getChildOperators().get(i)); } } } }
private void removeDPPOperator(Set<Operator<?>> component, OptimizeSparkProcContext context) { SparkPartitionPruningSinkOperator toRemove = null; for (Operator<?> o : component) { if (o instanceof SparkPartitionPruningSinkOperator) { // we want to remove the DPP with bigger data size if (toRemove == null || o.getConf().getStatistics().getDataSize() > toRemove.getConf().getStatistics() .getDataSize()) { toRemove = (SparkPartitionPruningSinkOperator) o; } } } if (toRemove == null) { return; } OperatorUtils.removeBranch(toRemove); // at this point we've found the fork in the op pipeline that has the pruning as a child plan. LOG.info("Disabling dynamic pruning for: " + toRemove.getConf().getTableScanNames() + ". Needed to break cyclic dependency"); }
private void removeDPPOperator(Set<Operator<?>> component, OptimizeSparkProcContext context) { SparkPartitionPruningSinkOperator toRemove = null; for (Operator<?> o : component) { if (o instanceof SparkPartitionPruningSinkOperator) { // we want to remove the DPP with bigger data size if (toRemove == null || o.getConf().getStatistics().getDataSize() > toRemove.getConf().getStatistics() .getDataSize()) { toRemove = (SparkPartitionPruningSinkOperator) o; } } } if (toRemove == null) { return; } OperatorUtils.removeBranch(toRemove); // at this point we've found the fork in the op pipeline that has the pruning as a child plan. LOG.info("Disabling dynamic pruning for: " + toRemove.getConf().getTableScan().toString() + ". Needed to break cyclic dependency"); }
Statistics stats = conf.getStatistics(); if (stats == null && op.getParentOperators() != null) {
Statistics stats = conf.getStatistics(); if (stats == null) { if (op.getParentOperators() != null) {
public Statistics getStatistics() { if (conf != null) { return conf.getStatistics(); } return null; }
private void setStatistics(Operator<? extends OperatorDesc> origin, Operator<? extends OperatorDesc> clone) { clone.getConf().setStatistics(origin.getConf().getStatistics()); clone.getConf().setTraits(origin.getConf().getTraits()); if (origin.getChildOperators().size() == clone.getChildOperators().size()) { for (int i = 0; i < clone.getChildOperators().size(); i++) { setStatistics(origin.getChildOperators().get(i), clone.getChildOperators().get(i)); } } } }
private void removeEventOperator(Set<Operator<?>> component, OptimizeTezProcContext context) { AppMasterEventOperator victim = null; for (Operator<?> o : component) { if (o instanceof AppMasterEventOperator) { if (victim == null || o.getConf().getStatistics().getDataSize() < victim.getConf().getStatistics() .getDataSize()) { victim = (AppMasterEventOperator) o; } } } if (victim == null || (!context.pruningOpsRemovedByPriorOpt.isEmpty() && context.pruningOpsRemovedByPriorOpt.contains(victim))) { return; } GenTezUtils.getUtils().removeBranch(victim); // at this point we've found the fork in the op pipeline that has the pruning as a child plan. LOG.info("Disabling dynamic pruning for: " + ((DynamicPruningEventDesc) victim.getConf()).getTableScan().toString() + ". Needed to break cyclic dependency"); }
Statistics stats = conf.getStatistics(); if (stats == null) { if (op.getParentOperators() != null) {