private static ExprWalkerInfo getChildWalkerInfo(Operator<?> current, OpWalkerInfo owi) throws SemanticException { if (current.getNumChild() == 0) { return null; } if (current.getNumChild() > 1) { // ppd for multi-insert query is not yet implemented // we assume that nothing can is pushed beyond this operator List<Operator<? extends OperatorDesc>> children = Lists.newArrayList(current.getChildOperators()); for (Operator<?> child : children) { ExprWalkerInfo childInfo = owi.getPrunedPreds(child); createFilter(child, childInfo, owi); } return null; } return owi.getPrunedPreds(current.getChildOperators().get(0)); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; // The lateral view forward operator has 2 children, a SELECT(*) and // a SELECT(cols) (for the UDTF operator) The child at index 0 is the // SELECT(*) because that's the way that the DAG was constructed. We // only want to get the predicates from the SELECT(*). ExprWalkerInfo childPreds = owi .getPrunedPreds((Operator<? extends OperatorDesc>) nd.getChildren() .get(0)); owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, childPreds); return null; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { super.process(nd, stack, procCtx, nodeOutputs); OpWalkerInfo owi = (OpWalkerInfo) procCtx; ExprWalkerInfo prunedPred = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd); if (prunedPred == null || !prunedPred.hasAnyCandidates()) { return null; } Map<String, List<ExprNodeDesc>> candidates = prunedPred.getFinalCandidates(); createFilter((Operator)nd, prunedPred, owi); candidates.clear(); return null; }
protected ExprWalkerInfo mergeChildrenPred(Node nd, OpWalkerInfo owi, Set<String> excludedAliases, boolean ignoreAliases) throws SemanticException { if (nd.getChildren() == null) { return null; } Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>)nd; ExprWalkerInfo ewi = new ExprWalkerInfo(); for (Operator<? extends OperatorDesc> child : op.getChildOperators()) { ExprWalkerInfo childPreds = owi.getPrunedPreds(child); if (childPreds == null) { continue; } for (Entry<String, List<ExprNodeDesc>> e : childPreds .getFinalCandidates().entrySet()) { if (ignoreAliases || excludedAliases == null || !excludedAliases.contains(e.getKey()) || e.getKey() == null) { ewi.addPushDowns(e.getKey(), e.getValue()); logExpr(nd, ewi); } } } return ewi; } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; // The lateral view forward operator has 2 children, a SELECT(*) and // a SELECT(cols) (for the UDTF operator) The child at index 0 is the // SELECT(*) because that's the way that the DAG was constructed. We // only want to get the predicates from the SELECT(*). ExprWalkerInfo childPreds = owi .getPrunedPreds((Operator<? extends OperatorDesc>) nd.getChildren() .get(0)); owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, childPreds); return null; }
private static ExprWalkerInfo getChildWalkerInfo(Operator<?> current, OpWalkerInfo owi) { if (current.getNumChild() == 0) { return null; } if (current.getNumChild() > 1) { // ppd for multi-insert query is not yet implemented // we assume that nothing can is pushed beyond this operator List<Operator<? extends OperatorDesc>> children = Lists.newArrayList(current.getChildOperators()); for (Operator<?> child : children) { ExprWalkerInfo childInfo = owi.getPrunedPreds(child); createFilter(child, childInfo, owi); } return null; } return owi.getPrunedPreds(current.getChildOperators().get(0)); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { super.process(nd, stack, procCtx, nodeOutputs); OpWalkerInfo owi = (OpWalkerInfo) procCtx; ExprWalkerInfo prunedPred = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd); if (prunedPred == null || !prunedPred.hasAnyCandidates()) { return null; } Map<String, List<ExprNodeDesc>> candidates = prunedPred.getFinalCandidates(); createFilter((Operator)nd, prunedPred, owi); candidates.clear(); return null; }
protected ExprWalkerInfo mergeChildrenPred(Node nd, OpWalkerInfo owi, Set<String> excludedAliases, boolean ignoreAliases) throws SemanticException { if (nd.getChildren() == null) { return null; } Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>)nd; ExprWalkerInfo ewi = new ExprWalkerInfo(); for (Operator<? extends OperatorDesc> child : op.getChildOperators()) { ExprWalkerInfo childPreds = owi.getPrunedPreds(child); if (childPreds == null) { continue; } for (Entry<String, List<ExprNodeDesc>> e : childPreds .getFinalCandidates().entrySet()) { if (ignoreAliases || excludedAliases == null || !excludedAliases.contains(e.getKey()) || e.getKey() == null) { ewi.addPushDowns(e.getKey(), e.getValue()); logExpr(nd, ewi); } } } return ewi; } }
throws SemanticException { ExprWalkerInfo joinPreds = owi.getPrunedPreds(join); if (joinPreds == null || !joinPreds.hasAnyCandidates()) { return; List<ExprNodeDesc> targetKeys = target.getConf().getKeyCols(); ExprWalkerInfo rsPreds = owi.getPrunedPreds(target); for (int sourcePos = 0; sourcePos < parentOperators.size(); sourcePos++) { ReduceSinkOperator source = (ReduceSinkOperator) parentOperators.get(sourcePos);
owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd); if (prunePreds != null) { Set<String> toRemove = new HashSet<String>();
throws SemanticException { ExprWalkerInfo joinPreds = owi.getPrunedPreds(join); if (joinPreds == null || !joinPreds.hasAnyCandidates()) { return; List<ExprNodeDesc> targetKeys = target.getConf().getKeyCols(); ExprWalkerInfo rsPreds = owi.getPrunedPreds(target); for (int sourcePos = 0; sourcePos < parentOperators.size(); sourcePos++) { ReduceSinkOperator source = (ReduceSinkOperator) parentOperators.get(sourcePos);
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; Set<String> includes = getQualifiedAliases((Operator<?>) nd, owi); boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, includes); if (hasUnpushedPredicates && HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { if (includes != null || nd instanceof ReduceSinkOperator) { owi.getCandidateFilterOps().clear(); } else { ExprWalkerInfo pruned = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd); Map<String, List<ExprNodeDesc>> residual = pruned.getResidualPredicates(true); if (residual != null && !residual.isEmpty()) { createFilter((Operator) nd, residual, owi); pruned.getNonFinalCandidates().clear(); } } } return null; }
owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd); if (prunePreds != null) { Set<String> toRemove = new HashSet<String>();
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; Set<String> includes = getQualifiedAliases((Operator<?>) nd, owi); boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, includes); if (hasUnpushedPredicates && HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { if (includes != null || nd instanceof ReduceSinkOperator) { owi.getCandidateFilterOps().clear(); } else { ExprWalkerInfo pruned = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd); Map<String, List<ExprNodeDesc>> residual = pruned.getResidualPredicates(true); if (residual != null && !residual.isEmpty()) { createFilter((Operator) nd, residual, owi); pruned.getNonFinalCandidates().clear(); } } } return null; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; TableScanOperator tsOp = (TableScanOperator) nd; mergeWithChildrenPred(tsOp, owi, null, null); if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { // remove all the candidate filter operators // when we get to the TS removeAllCandidates(owi); } ExprWalkerInfo pushDownPreds = owi.getPrunedPreds(tsOp); // nonFinalCandidates predicates should be empty assert pushDownPreds == null || !pushDownPreds.hasNonFinalCandidates(); return createFilter(tsOp, pushDownPreds, owi); }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; TableScanOperator tsOp = (TableScanOperator) nd; mergeWithChildrenPred(tsOp, owi, null, null); if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) { // remove all the candidate filter operators // when we get to the TS removeAllCandidates(owi); } ExprWalkerInfo pushDownPreds = owi.getPrunedPreds(tsOp); // nonFinalCandidates predicates should be empty assert pushDownPreds == null || !pushDownPreds.hasNonFinalCandidates(); return createFilter(tsOp, pushDownPreds, owi); }
ExprWalkerInfo walkerInfo = owi.getPrunedPreds(op); if (walkerInfo != null) { walkerInfo.getNonFinalCandidates().clear();
ExprWalkerInfo walkerInfo = owi.getPrunedPreds(op); if (walkerInfo != null) { walkerInfo.getNonFinalCandidates().clear();
ExprWalkerInfo ewi = owi.getPrunedPreds(op); .getPrunedPreds((Operator<? extends OperatorDesc>) (op .getChildren().get(0))), owi);
ExprWalkerInfo ewi = owi.getPrunedPreds(op); .getPrunedPreds((Operator<? extends OperatorDesc>) (op .getChildren().get(0))), owi);