@Override public InclusiveManifestEvaluator load(Integer specId) { PartitionSpec spec = ops.current().spec(specId); return new InclusiveManifestEvaluator(spec, dataFilter); } });
/** * Test whether the file may contain records that match the expression. * * @param manifest a manifest file * @return false if the file cannot contain rows that match the expression, true otherwise. */ public boolean eval(ManifestFile manifest) { return visitor().eval(manifest); }
@Test public void testIntegerLt() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, lessThan("id", 5)).eval(FILE); Assert.assertFalse("Should not read: id range below lower bound (5 < 30)", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, lessThan("id", 30)).eval(FILE); Assert.assertFalse("Should not read: id range below lower bound (30 is not < 30)", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, lessThan("id", 31)).eval(FILE); Assert.assertTrue("Should read: one possible id", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, lessThan("id", 79)).eval(FILE); Assert.assertTrue("Should read: may possible ids", shouldRead); }
manifest -> EVAL_CACHE.getUnchecked(manifest.partitionSpecId()).eval(manifest));
@Test public void testIntegerGt() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, greaterThan("id", 85)).eval(FILE); Assert.assertFalse("Should not read: id range above upper bound (85 < 79)", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, greaterThan("id", 79)).eval(FILE); Assert.assertFalse("Should not read: id range above upper bound (79 is not > 79)", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, greaterThan("id", 78)).eval(FILE); Assert.assertTrue("Should read: one possible id", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, greaterThan("id", 75)).eval(FILE); Assert.assertTrue("Should read: may possible ids", shouldRead); }
manifest -> EVAL_CACHE.getUnchecked(manifest.partitionSpecId()).eval(manifest));
@Test public void testIntegerLtEq() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, lessThanOrEqual("id", 5)).eval(FILE); Assert.assertFalse("Should not read: id range below lower bound (5 < 30)", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, lessThanOrEqual("id", 29)).eval(FILE); Assert.assertFalse("Should not read: id range below lower bound (29 < 30)", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, lessThanOrEqual("id", 30)).eval(FILE); Assert.assertTrue("Should read: one possible id", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, lessThanOrEqual("id", 79)).eval(FILE); Assert.assertTrue("Should read: many possible ids", shouldRead); }
@Override public InclusiveManifestEvaluator load(Integer specId) { PartitionSpec spec = ops.current().spec(specId); return new InclusiveManifestEvaluator(spec, rowFilter); } });
@Test public void testAllNulls() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, notNull("all_nulls")).eval(FILE); Assert.assertFalse("Should skip: no non-null value in all null column", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notNull("some_nulls")).eval(FILE); Assert.assertTrue("Should read: column with some nulls contains a non-null value", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notNull("no_nulls")).eval(FILE); Assert.assertTrue("Should read: non-null column contains a non-null value", shouldRead); }
@Test public void testNoNulls() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, isNull("all_nulls")).eval(FILE); Assert.assertTrue("Should read: at least one null value in all null column", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, isNull("some_nulls")).eval(FILE); Assert.assertTrue("Should read: column with some nulls contains a null value", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, isNull("no_nulls")).eval(FILE); Assert.assertFalse("Should skip: non-null column contains no null values", shouldRead); }
@Test public void testIntegerEq() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, equal("id", 5)).eval(FILE); Assert.assertFalse("Should not read: id below lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, equal("id", 29)).eval(FILE); Assert.assertFalse("Should not read: id below lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, equal("id", 30)).eval(FILE); Assert.assertTrue("Should read: id equal to lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, equal("id", 75)).eval(FILE); Assert.assertTrue("Should read: id between lower and upper bounds", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, equal("id", 79)).eval(FILE); Assert.assertTrue("Should read: id equal to upper bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, equal("id", 80)).eval(FILE); Assert.assertFalse("Should not read: id above upper bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, equal("id", 85)).eval(FILE); Assert.assertFalse("Should not read: id above upper bound", shouldRead); }
@Test public void testIntegerGtEq() { boolean shouldRead = new InclusiveManifestEvaluator( SPEC, greaterThanOrEqual("id", 85)).eval(FILE); Assert.assertFalse("Should not read: id range above upper bound (85 < 79)", shouldRead); shouldRead = new InclusiveManifestEvaluator( SPEC, greaterThanOrEqual("id", 80)).eval(FILE); Assert.assertFalse("Should not read: id range above upper bound (80 > 79)", shouldRead); shouldRead = new InclusiveManifestEvaluator( SPEC, greaterThanOrEqual("id", 79)).eval(FILE); Assert.assertTrue("Should read: one possible id", shouldRead); shouldRead = new InclusiveManifestEvaluator( SPEC, greaterThanOrEqual("id", 75)).eval(FILE); Assert.assertTrue("Should read: may possible ids", shouldRead); }
@Test public void testIntegerNotEq() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, notEqual("id", 5)).eval(FILE); Assert.assertTrue("Should read: id below lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notEqual("id", 29)).eval(FILE); Assert.assertTrue("Should read: id below lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notEqual("id", 30)).eval(FILE); Assert.assertTrue("Should read: id equal to lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notEqual("id", 75)).eval(FILE); Assert.assertTrue("Should read: id between lower and upper bounds", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notEqual("id", 79)).eval(FILE); Assert.assertTrue("Should read: id equal to upper bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notEqual("id", 80)).eval(FILE); Assert.assertTrue("Should read: id above upper bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, notEqual("id", 85)).eval(FILE); Assert.assertTrue("Should read: id above upper bound", shouldRead); }
@Test public void testMissingColumn() { TestHelpers.assertThrows("Should complain about missing column in expression", ValidationException.class, "Cannot find field 'missing'", () -> new InclusiveManifestEvaluator(SPEC, lessThan("missing", 5)).eval(FILE)); }
@Test public void testNot() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = new InclusiveManifestEvaluator(SPEC, not(lessThan("id", 5))).eval(FILE); Assert.assertTrue("Should read: not(false)", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, not(greaterThan("id", 5))).eval(FILE); Assert.assertFalse("Should skip: not(true)", shouldRead); }
@Test public void testIntegerNotEqRewritten() { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, not(equal("id", 5))).eval(FILE); Assert.assertTrue("Should read: id below lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, not(equal("id", 29))).eval(FILE); Assert.assertTrue("Should read: id below lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, not(equal("id", 30))).eval(FILE); Assert.assertTrue("Should read: id equal to lower bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, not(equal("id", 75))).eval(FILE); Assert.assertTrue("Should read: id between lower and upper bounds", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, not(equal("id", 79))).eval(FILE); Assert.assertTrue("Should read: id equal to upper bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, not(equal("id", 80))).eval(FILE); Assert.assertTrue("Should read: id above upper bound", shouldRead); shouldRead = new InclusiveManifestEvaluator(SPEC, not(equal("id", 85))).eval(FILE); Assert.assertTrue("Should read: id above upper bound", shouldRead); } }
@Test public void testAnd() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = new InclusiveManifestEvaluator( SPEC, and(lessThan("id", 5), greaterThanOrEqual("id", 0))).eval(FILE); Assert.assertFalse("Should skip: and(false, false)", shouldRead); shouldRead = new InclusiveManifestEvaluator( SPEC, and(greaterThan("id", 5), lessThanOrEqual("id", 30))).eval(FILE); Assert.assertTrue("Should read: and(true, true)", shouldRead); }
@Test public void testOr() { // this test case must use a real predicate, not alwaysTrue(), or binding will simplify it out boolean shouldRead = new InclusiveManifestEvaluator( SPEC, or(lessThan("id", 5), greaterThanOrEqual("id", 80))).eval(FILE); Assert.assertFalse("Should skip: or(false, false)", shouldRead); shouldRead = new InclusiveManifestEvaluator( SPEC, or(lessThan("id", 5), greaterThanOrEqual("id", 60))).eval(FILE); Assert.assertTrue("Should read: or(false, true)", shouldRead); }
@Test public void testMissingStats() { Expression[] exprs = new Expression[] { lessThan("id", 5), lessThanOrEqual("id", 30), equal("id", 70), greaterThan("id", 78), greaterThanOrEqual("id", 90), notEqual("id", 101), isNull("id"), notNull("id") }; for (Expression expr : exprs) { boolean shouldRead = new InclusiveManifestEvaluator(SPEC, expr).eval(NO_STATS); Assert.assertTrue("Should read when missing stats for expr: " + expr, shouldRead); } }