private static Types.StructType addFields(Types.StructType struct, Collection<Types.NestedField> adds) { List<Types.NestedField> newFields = Lists.newArrayList(struct.fields()); newFields.addAll(adds); return Types.StructType.of(newFields); } }
public static StructType of(NestedField... fields) { return of(Arrays.asList(fields)); }
@Test public void testIncompatibleStructAndPrimitive() { Schema write = new Schema(required(0, "nested", Types.StructType.of( required(1, "from_field", Types.StringType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("struct cannot be read as a string")); }
@Test @SuppressWarnings("unchecked") public void testIsNull() { StructType optional = StructType.of(optional(19, "s", Types.StringType.get())); UnboundPredicate<?> unbound = new UnboundPredicate<>(IS_NULL, ref("s")); Expression expr = unbound.bind(optional); BoundPredicate<?> bound = assertAndUnwrap(expr); Assert.assertEquals("Should use the same operation", IS_NULL, bound.op()); Assert.assertEquals("Should use the correct field", 19, bound.ref().fieldId()); Assert.assertNull("Should not have a literal value", bound.literal()); StructType required = StructType.of(required(20, "s", Types.StringType.get())); Assert.assertEquals("IsNull inclusive a required field should be alwaysFalse", Expressions.alwaysFalse(), unbound.bind(required)); }
public Schema(List<Types.NestedField> columns) { this.struct = Types.StructType.of(columns); }
@Test @SuppressWarnings("unchecked") public void testLiteralConversion() { StructType struct = StructType.of(required(15, "d", Types.DecimalType.of(9, 2))); for (Expression.Operation op : COMPARISONS) { UnboundPredicate<String> unbound = new UnboundPredicate<>(op, ref("d"), "12.40"); Expression expr = unbound.bind(struct); BoundPredicate<BigDecimal> bound = assertAndUnwrap(expr); Assert.assertEquals("Should convert literal value to decimal", new BigDecimal("12.40"), bound.literal().value()); Assert.assertEquals("Should reference correct field ID", 15, bound.ref().fieldId()); Assert.assertEquals("Should not change the comparison operation", op, bound.op()); } }
@Test public void testNotNull() { StructType optional = StructType.of(optional(21, "s", Types.StringType.get())); UnboundPredicate<?> unbound = new UnboundPredicate<>(NOT_NULL, ref("s")); Expression expr = unbound.bind(optional); BoundPredicate<?> bound = assertAndUnwrap(expr); Assert.assertEquals("Should use the same operation", NOT_NULL, bound.op()); Assert.assertEquals("Should use the correct field", 21, bound.ref().fieldId()); Assert.assertNull("Should not have a literal value", bound.literal()); StructType required = StructType.of(required(22, "s", Types.StringType.get())); Assert.assertEquals("NotNull inclusive a required field should be alwaysTrue", Expressions.alwaysTrue(), unbound.bind(required)); } }
@Test public void testMissingRequiredStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( optional(2, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.IntegerType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required field is missing", errors.get(0).contains("is required, but is missing")); }
@Test public void testIncompatibleStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( required(1, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.FloatType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to float")); }
@Test public void testRequiredStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( optional(1, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.IntegerType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required field is optional", errors.get(0).contains("should be required, but is optional")); }
@Test @SuppressWarnings("unchecked") public void testComparisonPredicateBinding() { StructType struct = StructType.of(required(14, "x", Types.IntegerType.get())); for (Expression.Operation op : COMPARISONS) { UnboundPredicate<Integer> unbound = new UnboundPredicate<>(op, ref("x"), 5); Expression expr = unbound.bind(struct); BoundPredicate<Integer> bound = assertAndUnwrap(expr); Assert.assertEquals("Should not alter literal value", Integer.valueOf(5), bound.literal().value()); Assert.assertEquals("Should reference correct field ID", 14, bound.ref().fieldId()); Assert.assertEquals("Should not change the comparison operation", op, bound.op()); } }
@Test public void testMissingField() { StructType struct = StructType.of( required(13, "x", Types.IntegerType.get()) ); UnboundPredicate<Integer> unbound = new UnboundPredicate<>(LT, ref("missing"), 6); try { unbound.bind(struct); Assert.fail("Binding a missing field should fail"); } catch (ValidationException e) { Assert.assertTrue("Validation should complain about missing field", e.getMessage().contains("Cannot find field 'missing' in struct:")); } }
@Test public void testMissingOptionalStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( required(2, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( optional(1, "to_field", Types.IntegerType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce no error messages", 0, errors.size()); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
public Schema(List<Types.NestedField> columns, Map<String, Integer> aliases) { this.struct = Types.StructType.of(columns); this.aliasToId = aliases != null ? ImmutableBiMap.copyOf(aliases) : null; }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testCharSeqValue() { StructType struct = StructType.of(required(34, "s", Types.StringType.get())); Evaluator evaluator = new Evaluator(struct, equal("s", "abc")); Assert.assertTrue("string(abc) == utf8(abc) => true", evaluator.eval(TestHelpers.Row.of(new Utf8("abc")))); Assert.assertFalse("string(abc) == utf8(abcd) => false", evaluator.eval(TestHelpers.Row.of(new Utf8("abcd")))); } }
@Test public void testComplex() throws IOException { convertToPigSchema( new Schema( optional(1, "bag", ListType.ofOptional(2, BooleanType.get())), optional(3, "map", MapType.ofOptional(4,5, StringType.get(), DoubleType.get())), optional(6, "tuple", StructType.of(optional(7, "i", IntegerType.get()), optional(8,"f", FloatType.get()))) ),"bag:{(boolean)},map:[double],tuple:(i:int,f:float)", null ); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }