public static SearchArgument create(Configuration conf, ExprNodeGenericFuncDesc expression) { return new ConvertAstToSearchArg(conf, expression).buildSearchArgument(); }
private static SearchArgument getSearchArgumentFromString(Configuration conf, String sargString) { try { return isSargsCacheEnabled(conf)? getSargsCache(conf).get(sargString, () -> create(sargString)) : create(sargString); } catch (ExecutionException exception) { throw new RuntimeException(exception); } }
private static Object[] getLiteralList(ExprNodeGenericFuncDesc expr, BoxType boxType, int start) { List<ExprNodeDesc> children = expr.getChildren(); Object[] result = new Object[children.size() - start]; // ignore the first child, since it is the variable int posn = 0; for(ExprNodeDesc child: children.subList(start, children.size())) { if (child instanceof ExprNodeConstantDesc) { result[posn++] = boxLiteral((ExprNodeConstantDesc) child, boxType); } else { // if we get some non-literals, we need to punt return null; } } return result; }
/** * Create a leaf expression when we aren't sure where the variable is * located. * @param operator the operator type that was found * @param expression the expression to check */ private void createLeaf(PredicateLeaf.Operator operator, ExprNodeGenericFuncDesc expression) { createLeaf(operator, expression, findVariable(expression)); }
private void createLeaf(PredicateLeaf.Operator operator, ExprNodeGenericFuncDesc expression, int variable) { String columnName = getColumnName(expression, variable); if (columnName == null) { builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; BoxType boxType = getType(expression.getChildren().get(variable)); if (boxType == null) { builder.literal(SearchArgument.TruthValue.YES_NO_NULL); break; case EQUALS: builder.equals(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case NULL_SAFE_EQUALS: builder.nullSafeEquals(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case LESS_THAN: builder.lessThan(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case LESS_THAN_EQUALS: builder.lessThanEquals(columnName, boxType.type, findLiteral(conf, expression, boxType)); break; case IN: builder.in(columnName, boxType.type, getLiteralList(expression, boxType, variable + 1)); break; case BETWEEN:
@Override public SearchArgument createSarg(byte[] expr) { return ConvertAstToSearchArg.create(expr); } }
public FilterCompat.Filter setFilter(final JobConf conf, MessageType schema) { SearchArgument sarg = ConvertAstToSearchArg.createFromConf(conf); if (sarg == null) { return null; } // Create the Parquet FilterPredicate without including columns that do not exist // on the schema (such as partition columns). FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); if (p != null) { // Filter may have sensitive information. Do not send to debug. LOG.debug("PARQUET predicate push down generated."); ParquetInputFormat.setFilterPredicate(conf, p); return FilterCompat.get(p); } else { // Filter may have sensitive information. Do not send to debug. LOG.debug("No PARQUET predicate push down is generated."); return null; } }
addChildren(expr); builder.end(); } else if (op == GenericUDFOPAnd.class) { builder.startAnd(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPNot.class) { builder.startNot(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPEqual.class) { createLeaf(PredicateLeaf.Operator.EQUALS, expr); } else if (op == GenericUDFOPNotEqual.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.EQUALS, expr); builder.end(); } else if (op == GenericUDFOPEqualNS.class) { createLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, expr); } else if (op == GenericUDFOPGreaterThan.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.LESS_THAN_EQUALS, expr); builder.end(); } else if (op == GenericUDFOPEqualOrGreaterThan.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.LESS_THAN, expr); builder.end(); } else if (op == GenericUDFOPLessThan.class) { createLeaf(PredicateLeaf.Operator.LESS_THAN, expr); } else if (op == GenericUDFOPEqualOrLessThan.class) {
static boolean canCreateSargFromConf(Configuration conf) { if (getNeededColumnNamesString(conf) == null) { LOG.debug("No ORC pushdown predicate - no column names"); return false; } if (!ConvertAstToSearchArg.canCreateFromConf(conf)) { LOG.debug("No ORC pushdown predicate"); return false; } return true; }
private void createLeaf(PredicateLeaf.Operator operator, ExprNodeGenericFuncDesc expression, int variable) { String columnName = getColumnName(expression, variable); if (columnName == null) { builder.literal(SearchArgument.TruthValue.YES_NO_NULL); return; PredicateLeaf.Type type = getType(expression.getChildren().get(variable)); if (type == null) { builder.literal(SearchArgument.TruthValue.YES_NO_NULL); break; case EQUALS: builder.equals(columnName, type, findLiteral(conf, expression, type)); break; case NULL_SAFE_EQUALS: builder.nullSafeEquals(columnName, type, findLiteral(conf, expression, type)); break; case LESS_THAN: builder.lessThan(columnName, type, findLiteral(conf, expression, type)); break; case LESS_THAN_EQUALS: builder.lessThanEquals(columnName, type, findLiteral(conf, expression, type)); break; case IN: builder.in(columnName, type, getLiteralList(expression, type, variable + 1)); break; case BETWEEN:
@Override public SearchArgument call() { return create(sargString); } })
public void configure(HiveConf queryConfig) { this.conf = queryConfig; this.sarg = ConvertAstToSearchArg.createFromConf(conf); this.isPpdEnabled = HiveConf.getBoolVar(conf, ConfVars.HIVEOPTINDEXFILTER) && HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_MS_FOOTER_CACHE_PPD); this.isInTest = HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST); this.sargIsOriginal = this.sargNotIsOriginal = null; }
/** * Create a leaf expression when we aren't sure where the variable is * located. * @param operator the operator type that was found * @param expression the expression to check */ private void createLeaf(PredicateLeaf.Operator operator, ExprNodeGenericFuncDesc expression) { createLeaf(operator, expression, findVariable(expression)); }
addChildren(expr); builder.end(); } else if (op == GenericUDFOPAnd.class) { builder.startAnd(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPNot.class) { builder.startNot(); addChildren(expr); builder.end(); } else if (op == GenericUDFOPEqual.class) { createLeaf(PredicateLeaf.Operator.EQUALS, expr); } else if (op == GenericUDFOPNotEqual.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.EQUALS, expr); builder.end(); } else if (op == GenericUDFOPEqualNS.class) { createLeaf(PredicateLeaf.Operator.NULL_SAFE_EQUALS, expr); } else if (op == GenericUDFOPGreaterThan.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.LESS_THAN_EQUALS, expr); builder.end(); } else if (op == GenericUDFOPEqualOrGreaterThan.class) { builder.startNot(); createLeaf(PredicateLeaf.Operator.LESS_THAN, expr); builder.end(); } else if (op == GenericUDFOPLessThan.class) { createLeaf(PredicateLeaf.Operator.LESS_THAN, expr); } else if (op == GenericUDFOPEqualOrLessThan.class) {
static boolean canCreateSargFromConf(Configuration conf) { if (getNeededColumnNamesString(conf) == null) { LOG.debug("No ORC pushdown predicate - no column names"); return false; } if (!ConvertAstToSearchArg.canCreateFromConf(conf)) { LOG.debug("No ORC pushdown predicate"); return false; } return true; }
public static SearchArgument create(Configuration conf, ExprNodeGenericFuncDesc expression) { return new ConvertAstToSearchArg(conf, expression).buildSearchArgument(); }
private static SearchArgument getSearchArgumentFromExpression(Configuration conf, String sargString) { try { return isSargsCacheEnabled(conf)? getSargsCache(conf).get(sargString, () -> create(conf, SerializationUtilities.deserializeExpression(sargString))) : create(conf, SerializationUtilities.deserializeExpression(sargString)); } catch (ExecutionException exception) { throw new RuntimeException(exception); } }
@Override public SearchArgument createSarg(byte[] expr) { return ConvertAstToSearchArg.create(expr); } }
public FilterCompat.Filter setFilter(final JobConf conf, MessageType schema) { SearchArgument sarg = ConvertAstToSearchArg.createFromConf(conf); if (sarg == null) { return null; } // Create the Parquet FilterPredicate without including columns that do not exist // on the schema (such as partition columns). FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); if (p != null) { // Filter may have sensitive information. Do not send to debug. LOG.debug("PARQUET predicate push down generated."); ParquetInputFormat.setFilterPredicate(conf, p); return FilterCompat.get(p); } else { // Filter may have sensitive information. Do not send to debug. LOG.debug("No PARQUET predicate push down is generated."); return null; } }
private static Object[] getLiteralList(ExprNodeGenericFuncDesc expr, PredicateLeaf.Type type, int start) { List<ExprNodeDesc> children = expr.getChildren(); Object[] result = new Object[children.size() - start]; // ignore the first child, since it is the variable int posn = 0; for(ExprNodeDesc child: children.subList(start, children.size())) { if (child instanceof ExprNodeConstantDesc) { result[posn++] = boxLiteral((ExprNodeConstantDesc) child, type); } else { // if we get some non-literals, we need to punt return null; } } return result; }