@Test public void testTimestampSerialization() throws Exception { // There is a kryo which after serialize/deserialize, // Timestamp becomes Date. We get around this issue in // SearchArgumentImpl.getLiteral. Once kryo fixed the issue // We can simplify SearchArgumentImpl.getLiteral Timestamp now = new Timestamp(new java.util.Date().getTime()); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.TIMESTAMP, now) .end() .build(); String serializedSarg = TestInputOutputFormat.toKryo(sarg); SearchArgument sarg2 = ConvertAstToSearchArg.create(serializedSarg); Field literalField = PredicateLeafImpl.class.getDeclaredField("literal"); literalField.setAccessible(true); assertTrue(literalField.get(sarg2.getLeaves().get(0)) instanceof java.util.Date); Timestamp ts = (Timestamp)sarg2.getLeaves().get(0).getLiteral(); assertEquals(ts, now); }
@Test public void testFilterBetween() { MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 bCol; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 1L, 5L) .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(lteq(bCol, 5), not(lt(bCol, 1)))"; assertEquals(expected, p.toString()); sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 5L, 1L) .build(); p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); expected = "and(lteq(bCol, 1), not(lt(bCol, 5)))"; assertEquals(expected, p.toString()); sarg = SearchArgumentFactory.newBuilder() .between("bCol", PredicateLeaf.Type.LONG, 1L, 1L) .build(); p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); expected = "and(lteq(bCol, 1), not(lt(bCol, 1)))"; assertEquals(expected, p.toString()); } }
@Test public void testBuilderFloat() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, 22L) .lessThan("x1", PredicateLeaf.Type.LONG, 22L) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.FLOAT, new Double(0.22)) .equals("z1", PredicateLeaf.Type.FLOAT, new Double(0.22)) .end() .build(); assertEquals("leaf-0 = (LESS_THAN x 22), " + "leaf-1 = (LESS_THAN x1 22), " + "leaf-2 = (LESS_THAN_EQUALS y hi ), " + "leaf-3 = (EQUALS z 0.22), " + "leaf-4 = (EQUALS z1 0.22), " + "expr = (and leaf-0 leaf-1 leaf-2 leaf-3 leaf-4)", sarg.toString()); }
new HiveDecimalWritable("1.0")) .end() .build(); MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required int32 x; required binary y; required binary z;}"); .end() .end() .build(); schema = MessageTypeParser.parseMessageType("message test {" + " optional int32 x; required binary y; required int32 z;" +
.equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")) .end() .build(); MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required int32 x; required binary y; required binary z;}"); .end() .end() .build(); schema = MessageTypeParser.parseMessageType("message test {" + " optional int32 x; required binary y; required int32 z;" +
.lessThan("z", PredicateLeaf.Type.STRING, new String("foo")) .end() .build(); conf.set("sarg.pushdown", toKryo(sarg)); conf.set("hive.io.file.readcolumn.names", "z");
@Test public void testBuilder() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, 10L) .lessThanEquals("y", PredicateLeaf.Type.STRING, "hi") .equals("z", PredicateLeaf.Type.FLOAT, 1.0) .end() .build(); assertEquals("leaf-0 = (LESS_THAN x 10), " + "leaf-1 = (LESS_THAN_EQUALS y hi), " + "leaf-2 = (EQUALS z 1.0), " + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString()); sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.LONG, 10L, 20L) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() .build(); assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " + "leaf-2 = (IN z 1 2 3), " + "leaf-3 = (NULL_SAFE_EQUALS a stinger), " + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString()); }
@Test public void testBuilderFloat() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, 22L) .lessThan("x1", PredicateLeaf.Type.LONG, 22L) .lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()) .equals("z", PredicateLeaf.Type.FLOAT, new Double(0.22)) .equals("z1", PredicateLeaf.Type.FLOAT, new Double(0.22)) .end() .build(); MessageType schema = MessageTypeParser.parseMessageType("message test {" + " required int32 x; required int32 x1;" + " required binary y; required float z; required float z1;}"); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(and(and(lt(x, 22), lt(x1, 22))," + " lteq(y, Binary{\"hi \"})), eq(z, " + "0.22)), eq(z1, 0.22))"; assertEquals(expected, p.toString()); } }
@Test public void testBuilder() throws Exception { SearchArgument sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("x", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.LONG, 10L, 20L) .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() .build(); MessageType schema = MessageTypeParser.parseMessageType("message test {" + " optional int32 x; required int32 y; required int32 z;" + " optional binary a;}"); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(and(not(eq(x, null)), not(and(lteq(y, 20), not(lt(y, 10))))), not(or(or(eq(z, 1), " + "eq(z, 2)), eq(z, 3)))), not(eq(a, Binary{\"stinger\"})))"; assertEquals(expected, p.toString()); }
@Test public void testFilterColumnsThatDoNoExistOnSchemaHighOrder1() { MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 a; required int32 b; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startOr() .startAnd() .equals("a", PredicateLeaf.Type.LONG, 1L) .equals("none", PredicateLeaf.Type.LONG, 1L) .end() .startAnd() .equals("a", PredicateLeaf.Type.LONG, 999L) .equals("none", PredicateLeaf.Type.LONG, 999L) .end() .end() .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "or(eq(a, 1), eq(a, 999))"; assertEquals(expected, p.toString()); }
options.searchArgument(b.build(), new String[]{});
@Test public void testFilterColumnsThatDoNoExistOnSchemaHighOrder2() { MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 a; required int32 b; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startAnd() .startOr() .equals("a", PredicateLeaf.Type.LONG, 1L) .equals("b", PredicateLeaf.Type.LONG, 1L) .end() .startOr() .equals("a", PredicateLeaf.Type.LONG, 999L) .equals("none", PredicateLeaf.Type.LONG, 999L) .end() .end() .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "or(eq(a, 1), eq(b, 1))"; assertEquals(expected, p.toString()); }
@Test public void testFilterFloatColumns() { MessageType schema = MessageTypeParser.parseMessageType("message test { required float a; required int32 b; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("a", PredicateLeaf.Type.FLOAT) .between("a", PredicateLeaf.Type.FLOAT, 10.2, 20.3) .in("b", PredicateLeaf.Type.LONG, 1L, 2L, 3L) .end() .end() .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(and(not(eq(a, null)), not(and(lteq(a, 20.3), not(lt(a, 10.2))))), not(or(or(eq(b, 1), eq(b, 2)), eq(b, 3))))"; assertEquals(expected, p.toString()); }
@Test public void testFilterColumnsThatDoNoExistOnSchema() { MessageType schema = MessageTypeParser.parseMessageType("message test { required int32 a; required binary stinger; }"); SearchArgument sarg = SearchArgumentFactory.newBuilder() .startNot() .startOr() .isNull("a", PredicateLeaf.Type.LONG) .between("y", PredicateLeaf.Type.LONG, 10L, 20L) // Column will be removed from filter .in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L) // Column will be removed from filter .nullSafeEquals("a", PredicateLeaf.Type.STRING, "stinger") .end() .end() .build(); FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema); String expected = "and(not(eq(a, null)), not(eq(a, Binary{\"stinger\"})))"; assertEquals(expected, p.toString()); }
/** * Build the search argument from the expression. * @return the search argument */ public SearchArgument buildSearchArgument() { return builder.build(); }
@Test(expected = IllegalArgumentException.class) public void testBadLiteralList() throws Exception { SearchArgumentFactory.newBuilder() .startAnd() .in("x", PredicateLeaf.Type.STRING, "hi", 23) .end() .build(); } }
@Test(expected = IllegalArgumentException.class) public void testBadLiteral() throws Exception { SearchArgumentFactory.newBuilder() .startAnd() .lessThan("x", PredicateLeaf.Type.LONG, "hi") .end() .build(); }
/** * Build the search argument from the expression. * @return the search argument */ public SearchArgument buildSearchArgument() { return builder.build(); }