@Override public org.apache.spark.sql.catalyst.expressions.Expression[] pushCatalystFilters( org.apache.spark.sql.catalyst.expressions.Expression[] filters) { this.tasks = null; // invalidate cached tasks, if present List<Expression> expressions = Lists.newArrayListWithExpectedSize(filters.length); List<org.apache.spark.sql.catalyst.expressions.Expression> pushed = Lists.newArrayListWithExpectedSize(filters.length); for (org.apache.spark.sql.catalyst.expressions.Expression filter : filters) { Expression expr = SparkExpressions.convert(filter); if (expr != null) { expressions.add(expr); pushed.add(filter); } } this.filterExpressions = expressions; this.pushedExprs = pushed.toArray(new org.apache.spark.sql.catalyst.expressions.Expression[0]); // invalidate the schema that will be projected this.schema = null; this.type = null; // Spark doesn't support residuals per task, so return all filters // to get Spark to handle record-level filtering return filters; }
case NOT_EQ: BinaryExpression binary = (BinaryExpression) expr; return convert(op, binary.left(), binary.right()); case NOT: com.netflix.iceberg.expressions.Expression child = convert(((Not) expr).child()); if (child != null) { return not(child); case AND: And andExpr = (And) expr; com.netflix.iceberg.expressions.Expression andLeft = convert(andExpr.left()); com.netflix.iceberg.expressions.Expression andRight = convert(andExpr.right()); if (andLeft != null && andRight != null) { return and(convert(andExpr.left()), convert(andExpr.right())); com.netflix.iceberg.expressions.Expression orLeft = convert(orExpr.left()); com.netflix.iceberg.expressions.Expression orRight = convert(orExpr.right()); if (orLeft != null && orRight != null) { return or(orLeft, orRight);
private void pushFilters(DataSourceReader reader, com.netflix.iceberg.expressions.Expression... filters) { Expression[] expressions = new Expression[filters.length]; for (int i = 0; i < filters.length; i += 1) { expressions[i] = SparkExpressions.convert(filters[i], SCHEMA); } pushFilters(reader, expressions); }