@Override public org.apache.spark.sql.catalyst.expressions.Expression[] pushCatalystFilters( org.apache.spark.sql.catalyst.expressions.Expression[] filters) { this.tasks = null; // invalidate cached tasks, if present List<Expression> expressions = Lists.newArrayListWithExpectedSize(filters.length); List<org.apache.spark.sql.catalyst.expressions.Expression> pushed = Lists.newArrayListWithExpectedSize(filters.length); for (org.apache.spark.sql.catalyst.expressions.Expression filter : filters) { Expression expr = SparkExpressions.convert(filter); if (expr != null) { expressions.add(expr); pushed.add(filter); } } this.filterExpressions = expressions; this.pushedExprs = pushed.toArray(new org.apache.spark.sql.catalyst.expressions.Expression[0]); // invalidate the schema that will be projected this.schema = null; this.type = null; // Spark doesn't support residuals per task, so return all filters // to get Spark to handle record-level filtering return filters; }
private static List<Object> convertLiterals(List<Expression> values) { List<Object> converted = Lists.newArrayListWithExpectedSize(values.size()); for (Expression value : values) { if (value instanceof Literal) { Literal lit = (Literal) value; converted.add(valueFromSpark(lit)); } else { return null; } } return converted; }
case NOT_EQ: BinaryExpression binary = (BinaryExpression) expr; return convert(op, binary.left(), binary.right()); case NOT: com.netflix.iceberg.expressions.Expression child = convert(((Not) expr).child()); if (child != null) { return not(child); case AND: And andExpr = (And) expr; com.netflix.iceberg.expressions.Expression andLeft = convert(andExpr.left()); com.netflix.iceberg.expressions.Expression andRight = convert(andExpr.right()); if (andLeft != null && andRight != null) { return and(convert(andExpr.left()), convert(andExpr.right())); com.netflix.iceberg.expressions.Expression orLeft = convert(orExpr.left()); com.netflix.iceberg.expressions.Expression orRight = convert(orExpr.right()); if (orLeft != null && orRight != null) { return or(orLeft, orRight); if (expr instanceof In) { In inExpr = (In) expr; List<Object> literals = convertLiterals(seqAsJavaListConverter(inExpr.list()).asJava()); if (literals != null) { return convertIn(inExpr.value(), literals); } else { return convertIn(inExpr.child(), literals);
private static com.netflix.iceberg.expressions.Expression convert(Operation op, Expression left, Expression right) { Pair<Transform, String> attrPair = null; Operation leftOperation = null; Literal lit = null; if (right instanceof Literal) { lit = (Literal) right; attrPair = convertAttr(left); leftOperation = op; } else if (left instanceof Literal) { lit = (Literal) left; attrPair = convertAttr(right); leftOperation = op.flipLR(); } if (attrPair != null) { switch (attrPair.first()) { case IDENTITY: return predicate(leftOperation, attrPair.second(), valueFromSpark(lit)); case YEAR: return filter(leftOperation, attrPair.second(), (int) lit.value(), SparkExpressions::yearToTimestampMicros); case DAY: return filter(leftOperation, attrPair.second(), (int) lit.value(), SparkExpressions::dayToTimestampMicros); default: } } return null; }
private static com.netflix.iceberg.expressions.Expression filter( Operation op, String name, int value, Function<Integer, Long> startTsMicros) { switch (op) { case LT: return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value))); case LT_EQ: return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1))); case GT: return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1))); case GT_EQ: return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value))); case EQ: return and( predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value))), predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1))) ); case NOT_EQ: return or( predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1))), predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value))) ); case IN: case NOT_IN: default: throw new IllegalArgumentException("Cannot convert operation to year filter: " + op); } }
private void pushFilters(DataSourceReader reader, com.netflix.iceberg.expressions.Expression... filters) { Expression[] expressions = new Expression[filters.length]; for (int i = 0; i < filters.length; i += 1) { expressions[i] = SparkExpressions.convert(filters[i], SCHEMA); } pushFilters(reader, expressions); }