public Evaluator(Types.StructType struct, Expression unbound) { this.expr = Binder.bind(struct, unbound); }
@Override public TableScan select(Collection<String> columns) { Set<Integer> requiredFieldIds = Sets.newHashSet(); // all of the filter columns are required requiredFieldIds.addAll( Binder.boundReferences(table.schema().asStruct(), Collections.singletonList(rowFilter))); // all of the projection columns are required requiredFieldIds.addAll(TypeUtil.getProjectedIds(table.schema().select(columns))); Schema projection = TypeUtil.select(table.schema(), requiredFieldIds); return new BaseTableScan(ops, table, snapshotId, projection, rowFilter); }
public static Set<Integer> boundReferences(StructType struct, List<Expression> exprs) { if (exprs == null) { return ImmutableSet.of(); } ReferenceVisitor visitor = new ReferenceVisitor(); for (Expression expr : exprs) { ExpressionVisitors.visit(bind(struct, expr), visitor); } return visitor.references; }
/** * Prune columns from a {@link Schema} using a {@link StructType Spark type} projection. * <p> * This requires that the Spark type is a projection of the Schema. Nullability and types must * match. * <p> * The filters list of {@link Expression} is used to ensure that columns referenced by filters * are projected. * * @param schema a Schema * @param requestedType a projection of the Spark representation of the Schema * @param filter a filters * @return a Schema corresponding to the Spark projection * @throws IllegalArgumentException if the Spark type does not match the Schema */ public static Schema prune(Schema schema, StructType requestedType, Expression filter) { Set<Integer> filterRefs = Binder.boundReferences(schema.asStruct(), Collections.singletonList(filter)); return new Schema(visit(schema, new PruneColumnsWithoutReordering(requestedType, filterRefs)) .asNestedType() .asStructType() .fields()); }
public ParquetDictionaryRowGroupFilter(Schema schema, Expression unbound) { this.schema = schema; this.struct = schema.asStruct(); this.expr = Binder.bind(struct, rewriteNot(unbound)); }
/** * Prune columns from a {@link Schema} using a {@link StructType Spark type} projection. * <p> * This requires that the Spark type is a projection of the Schema. Nullability and types must * match. * <p> * The filters list of {@link Expression} is used to ensure that columns referenced by filters * are projected. * * @param schema a Schema * @param requestedType a projection of the Spark representation of the Schema * @param filters a list of filters * @return a Schema corresponding to the Spark projection * @throws IllegalArgumentException if the Spark type does not match the Schema */ public static Schema prune(Schema schema, StructType requestedType, List<Expression> filters) { Set<Integer> filterRefs = Binder.boundReferences(schema.asStruct(), filters); return new Schema(visit(schema, new PruneColumnsWithoutReordering(requestedType, filterRefs)) .asNestedType() .asStructType() .fields()); }
public ParquetMetricsRowGroupFilter(Schema schema, Expression unbound) { this.schema = schema; this.struct = schema.asStruct(); this.expr = Binder.bind(struct, rewriteNot(unbound)); }
public StrictMetricsEvaluator(Schema schema, Expression unbound) { this.schema = schema; this.struct = schema.asStruct(); this.expr = Binder.bind(struct, rewriteNot(unbound)); }
public InclusiveMetricsEvaluator(Schema schema, Expression unbound) { this.schema = schema; this.struct = schema.asStruct(); this.expr = Binder.bind(struct, rewriteNot(unbound)); }
public static Expression convert(com.netflix.iceberg.expressions.Expression filter, Schema schema) { return visit(Binder.bind(schema.asStruct(), filter), new ExpressionToSpark(schema)); }
public InclusiveManifestEvaluator(PartitionSpec spec, Expression rowFilter) { this.struct = spec.partitionType(); this.expr = Binder.bind(struct, rewriteNot(Projections.inclusive(spec).project(rowFilter))); }
@Test public void testSingleReference() { Expression expr = not(equal("x", 7)); TestHelpers.assertAllReferencesBound("Single reference", Binder.bind(STRUCT, expr)); }
@Test public void testAlwaysTrue() { Assert.assertEquals("Should not change alwaysTrue", alwaysTrue(), Binder.bind(STRUCT, alwaysTrue())); }
@Test public void testAlwaysFalse() { Assert.assertEquals("Should not change alwaysFalse", alwaysFalse(), Binder.bind(STRUCT, alwaysFalse())); }
@Test public void testMissingReference() { Expression expr = and(equal("t", 5), equal("x", 7)); try { Binder.bind(STRUCT, expr); Assert.fail("Should not successfully bind to struct without field 't'"); } catch (ValidationException e) { Assert.assertTrue("Should complain about missing field", e.getMessage().contains("Cannot find field 't' in struct:")); } }
@Test public void testMultipleReferences() { Expression expr = or(and(equal("x", 7), lessThan("y", 100)), greaterThan("z", -100)); TestHelpers.assertAllReferencesBound("Multiple references", Binder.bind(STRUCT, expr)); }
@Test public void testBasicSimplification() { // this tests that a basic simplification is done by calling the helpers in Expressions. those // are more thoroughly tested in TestExpressionHelpers. // the second predicate is always true once it is bound because z is an integer and the literal // is less than any 32-bit integer value Assert.assertEquals("Should simplify or expression to alwaysTrue", alwaysTrue(), Binder.bind(STRUCT, or(lessThan("y", 100), greaterThan("z", -9999999999L)))); // similarly, the second predicate is always false Assert.assertEquals("Should simplify and expression to predicate", alwaysFalse(), Binder.bind(STRUCT, and(lessThan("y", 100), lessThan("z", -9999999999L)))); Expression bound = Binder.bind(STRUCT, not(not(lessThan("y", 100)))); BoundPredicate<?> pred = TestHelpers.assertAndUnwrap(bound); Assert.assertEquals("Should have the correct bound field", 1, pred.ref().fieldId()); } }
@Test public void testNot() { Expression expr = not(equal("x", 7)); Expression boundExpr = Binder.bind(STRUCT, expr); TestHelpers.assertAllReferencesBound("Not", boundExpr); // make sure the result is a Not Not not = TestHelpers.assertAndUnwrap(boundExpr, Not.class); // make sure the refs are for the right fields BoundPredicate<?> child = TestHelpers.assertAndUnwrap(not.child()); Assert.assertEquals("Should bind x correctly", 0, child.ref().fieldId()); }
@Test public void testAnd() { Expression expr = and(equal("x", 7), lessThan("y", 100)); Expression boundExpr = Binder.bind(STRUCT, expr); TestHelpers.assertAllReferencesBound("And", boundExpr); // make sure the result is an And And and = TestHelpers.assertAndUnwrap(boundExpr, And.class); // make sure the refs are for the right fields BoundPredicate<?> left = TestHelpers.assertAndUnwrap(and.left()); Assert.assertEquals("Should bind x correctly", 0, left.ref().fieldId()); BoundPredicate<?> right = TestHelpers.assertAndUnwrap(and.right()); Assert.assertEquals("Should bind y correctly", 1, right.ref().fieldId()); }