static Expression joinFilters(List<Expression> expressions) { Expression result = Expressions.alwaysTrue(); for (Expression expression : expressions) { result = Expressions.and(result, expression); } return result; }
private com.netflix.iceberg.expressions.Expression convert(OpType op, Column col, Const constant) { String name = col.getName(); Object value = constant.getValue(); switch (op) { case OP_GE: return greaterThanOrEqual(name, value); case OP_GT: return greaterThan(name, value); case OP_LE: return lessThanOrEqual(name, value); case OP_LT: return lessThan(name, value); case OP_EQ: return equal(name, value); case OP_NE: return notEqual(name, value); } throw new RuntimeException(format("[%s]: Failed to pushdown expression: %s %s %s", signature, col, op, constant)); }
public static Expression and(Expression left, Expression right) { Preconditions.checkNotNull(left, "Left expression cannot be null."); Preconditions.checkNotNull(right, "Right expression cannot be null."); if (left == alwaysFalse() || right == alwaysFalse()) { return alwaysFalse(); } else if (left == alwaysTrue()) { return right; } else if (right == alwaysTrue()) { return left; } return new And(left, right); }
private static com.netflix.iceberg.expressions.Expression convertIn(Expression expr, Collection<Object> values) { if (expr instanceof Attribute) { Attribute attr = (Attribute) expr; com.netflix.iceberg.expressions.Expression converted = alwaysFalse(); for (Object item : values) { converted = or(converted, equal(attr.name(), item)); } return converted; } return null; }
@Test public void testMissingStats() { Expression[] exprs = new Expression[] { lessThan("id", 5), lessThanOrEqual("id", 30), equal("id", 70), greaterThan("id", 78), greaterThanOrEqual("id", 90), notEqual("id", 101), isNull("id"), notNull("id") }; for (Expression expr : exprs) { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, expr).eval(NO_STATS); Assert.assertTrue("Should read when missing stats for expr: " + expr, shouldRead); } }
@Test public void testExpressions() throws Exception { Schema schema = new Schema( Types.NestedField.optional(34, "a", Types.IntegerType.get()) ); Expression[] expressions = new Expression[] { Expressions.alwaysFalse(), Expressions.alwaysTrue(), Expressions.lessThan("x", 5), Expressions.lessThanOrEqual("y", -3), Expressions.greaterThan("z", 0), Expressions.greaterThanOrEqual("t", 129), Expressions.equal("col", "data"), Expressions.notEqual("col", "abc"), Expressions.notNull("maybeNull"), Expressions.isNull("maybeNull2"), Expressions.not(Expressions.greaterThan("a", 10)), Expressions.and(Expressions.greaterThanOrEqual("a", 0), Expressions.lessThan("a", 3)), Expressions.or(Expressions.lessThan("a", 0), Expressions.greaterThan("a", 10)), Expressions.equal("a", 5).bind(schema.asStruct()) }; for (Expression expression : expressions) { Expression copy = TestHelpers.roundTripSerialize(expression); Assert.assertTrue( "Expression should equal the deserialized copy: " + expression + " != " + copy, equals(expression, copy)); } }
@Test public void testAnd() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, and(lessThan("id", 5), greaterThanOrEqual("id", 0))) .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA); Assert.assertFalse("Should skip: and(false, false)", shouldRead); shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, and(greaterThan("id", 5), lessThanOrEqual("id", 30))) .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA); Assert.assertTrue("Should read: and(true, true)", shouldRead); }
@Test public void testBasicSimplification() { // this tests that a basic simplification is done by calling the helpers in Expressions. those // are more thoroughly tested in TestExpressionHelpers. // the second predicate is always true once it is bound because z is an integer and the literal // is less than any 32-bit integer value Assert.assertEquals("Should simplify or expression to alwaysTrue", alwaysTrue(), Binder.bind(STRUCT, or(lessThan("y", 100), greaterThan("z", -9999999999L)))); // similarly, the second predicate is always false Assert.assertEquals("Should simplify and expression to predicate", alwaysFalse(), Binder.bind(STRUCT, and(lessThan("y", 100), lessThan("z", -9999999999L)))); Expression bound = Binder.bind(STRUCT, not(not(lessThan("y", 100)))); BoundPredicate<?> pred = TestHelpers.assertAndUnwrap(bound); Assert.assertEquals("Should have the correct bound field", 1, pred.ref().fieldId()); } }
@Test public void testOr() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = new ParquetDictionaryRowGroupFilter(SCHEMA, or(lessThan("id", 5), greaterThanOrEqual("id", 80))) .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA, DICTIONARY_STORE); Assert.assertFalse("Should skip: or(false, false)", shouldRead); shouldRead = new ParquetDictionaryRowGroupFilter(SCHEMA, or(lessThan("id", 5), greaterThanOrEqual("id", 60))) .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA, DICTIONARY_STORE); Assert.assertTrue("Should read: or(false, true)", shouldRead); }
@Test public void testMultipleReferences() { Expression expr = or(and(equal("x", 7), lessThan("y", 100)), greaterThan("z", -100)); TestHelpers.assertAllReferencesBound("Multiple references", Binder.bind(STRUCT, expr)); }
@Test public void testNot() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, not(lessThan("id", 5))) .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA); Assert.assertTrue("Should read: not(false)", shouldRead); shouldRead = new ParquetMetricsRowGroupFilter(SCHEMA, not(greaterThan("id", 5))) .shouldRead(PARQUET_SCHEMA, ROW_GROUP_METADATA); Assert.assertFalse("Should skip: not(true)", shouldRead); }
@Test public void testOverwriteFailsDelete() { TableMetadata base = TestTables.readMetadata(TABLE_NAME); long baseId = base.currentSnapshot().snapshotId(); OverwriteFiles overwrite = table.newOverwrite() .overwriteByRowFilter(and(equal("date", "2018-06-09"), lessThan("id", 9))); AssertHelpers.assertThrows("Should reject commit with file not matching delete expression", ValidationException.class, "Cannot delete file where some, but not all, rows match filter", overwrite::commit); Assert.assertEquals("Should not create a new snapshot", baseId, table.currentSnapshot().snapshotId()); }
@Test public void testSimplifyAnd() { Assert.assertEquals("alwaysTrue and pred => pred", pred, and(alwaysTrue(), pred)); Assert.assertEquals("pred and alwaysTrue => pred", pred, and(pred, alwaysTrue())); Assert.assertEquals("alwaysFalse and pred => alwaysFalse", alwaysFalse(), and(alwaysFalse(), pred)); Assert.assertEquals("pred and alwaysFalse => alwaysFalse", alwaysFalse(), and(pred, alwaysFalse())); }
@Override public Object visit(final ASTBETWEEN node, final Object data) { final Object value = node.jjtGetChild(0).jjtAccept(this, data); final Object startValue = node.jjtGetChild(1).jjtAccept(this, data); final Object endValue = node.jjtGetChild(2).jjtAccept(this, data); final Expression compare1 = createIcebergExpression(value, startValue, node.not ? Compare.LT : Compare.GTE); final Expression compare2 = createIcebergExpression(value, endValue, node.not ? Compare.GT : Compare.LTE); return (node.not) ? Expressions.or(compare1, compare2) : Expressions.and(compare1, compare2); }
@Test public void testSimplifyOr() { Assert.assertEquals("alwaysTrue or pred => alwaysTrue", alwaysTrue(), or(alwaysTrue(), pred)); Assert.assertEquals("pred or alwaysTrue => alwaysTrue", alwaysTrue(), or(pred, alwaysTrue())); Assert.assertEquals("alwaysFalse or pred => pred", pred, or(alwaysFalse(), pred)); Assert.assertEquals("pred or alwaysTrue => pred", pred, or(pred, alwaysFalse())); }
@Test public void testOr() { Expression expr = or(greaterThan("z", -100), lessThan("y", 100)); Expression boundExpr = Binder.bind(STRUCT, expr); TestHelpers.assertAllReferencesBound("Or", boundExpr); // make sure the result is an Or Or or = TestHelpers.assertAndUnwrap(boundExpr, Or.class); // make sure the refs are for the right fields BoundPredicate<?> left = TestHelpers.assertAndUnwrap(or.left()); Assert.assertEquals("Should bind z correctly", 2, left.ref().fieldId()); BoundPredicate<?> right = TestHelpers.assertAndUnwrap(or.right()); Assert.assertEquals("Should bind y correctly", 1, right.ref().fieldId()); }
@Override public FilteredManifest filterPartitions(Expression expr) { return new FilteredManifest(reader, Expressions.and(partFilter, expr), rowFilter, columns); }
@Test public void testSimplifyNot() { Assert.assertEquals("not(alwaysTrue) => alwaysFalse", alwaysFalse(), not(alwaysTrue())); Assert.assertEquals("not(alwaysFalse) => alwaysTrue", alwaysTrue(), not(alwaysFalse())); Assert.assertEquals("not(not(pred)) => pred", pred, not(not(pred))); }
@Test public void testAnd() { Evaluator evaluator = new Evaluator(STRUCT, and(equal("x", 7), notNull("z"))); Assert.assertTrue("7, 3 => true", evaluator.eval(TestHelpers.Row.of(7, 0, 3))); Assert.assertFalse("8, 3 => false", evaluator.eval(TestHelpers.Row.of(8, 0, 3))); Assert.assertFalse("7, null => false", evaluator.eval(TestHelpers.Row.of(7, 0, null))); Assert.assertFalse("8, null => false", evaluator.eval(TestHelpers.Row.of(8, 0, null))); }