@Test public void testMultipleErrors() { // required field is optional and cannot be promoted to the read type Schema write = new Schema(required(0, "nested", Types.StructType.of( optional(1, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.FloatType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 2, errors.size()); Assert.assertTrue("Should complain that a required field is optional", errors.get(0).contains("should be required, but is optional")); Assert.assertTrue("Should complain about incompatible types", errors.get(1).contains("cannot be promoted to float")); }
@Test public void testInvalidBinaryConversions() { testInvalidConversions(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }
@Test @SuppressWarnings("unchecked") public void testMultipleFields() { StructType struct = StructType.of( required(10, "x", Types.IntegerType.get()), required(11, "y", Types.IntegerType.get()), required(12, "z", Types.IntegerType.get()) ); UnboundPredicate<Integer> unbound = new UnboundPredicate<>(LT, ref("y"), 6); Expression expr = unbound.bind(struct); BoundPredicate<Integer> bound = assertAndUnwrap(expr); Assert.assertEquals("Should reference correct field ID", 11, bound.ref().fieldId()); Assert.assertEquals("Should not change the comparison operation", LT, bound.op()); Assert.assertEquals("Should not alter literal value", Integer.valueOf(6), bound.literal().value()); }
@Test public void testAddDeleteConflict() { AssertHelpers.assertThrows("Should reject add then delete", IllegalArgumentException.class, "missing column: col", () -> { UpdateSchema update = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID); update.addColumn("col", Types.IntegerType.get()).deleteColumn("col"); } ); AssertHelpers.assertThrows("Should reject add then delete", IllegalArgumentException.class, "column that has additions: preferences", () -> { UpdateSchema update = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID); update.addColumn("preferences", "feature3", Types.IntegerType.get()) .deleteColumn("preferences"); } ); }
@Test public void testInvalidDoubleConversions() { testInvalidConversions(Literal.of(34.11D), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidFloatConversions() { testInvalidConversions(Literal.of(34.11F), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testFailedCommit() throws Exception { // apply the change to metadata without committing UpdateSchema update = table.updateSchema().addColumn("n", Types.IntegerType.get()); update.apply(); Assert.assertTrue("Should create v1 metadata", version(1).exists() && version(1).isFile()); Assert.assertFalse("Should not create v2 or newer versions", version(2).exists()); version(2).createNewFile(); AssertHelpers.assertThrows("Should fail to commit change based on v1 when v2 exists", CommitFailedException.class, "Version 2 already exists", update::commit); List<File> manifests = listManifestFiles(); Assert.assertEquals("Should contain 0 Avro manifest files", 0, manifests.size()); }
@Test public void testMapOfStringToStructs() { Type map = Types.MapType.ofRequired(33, 34, Types.StringType.get(), Types.StructType.of( required(35, "a", Types.IntegerType.get()), optional(36, "b", Types.IntegerType.get()) )); Schema schema = addKeyId(33, addValueId(34, SchemaBuilder.map().values( record("r34", requiredField(35, "a", Schema.create(Schema.Type.INT)), optionalField(36, "b", Schema.create(Schema.Type.INT)))))); Assert.assertEquals("Avro schema to map", map, AvroSchemaUtil.convert(schema)); Assert.assertEquals("Map to Avro schema", schema, AvroSchemaUtil.convert(map)); }
@Test public void testTruncateInteger() { Truncate<Integer> trunc = Truncate.get(Types.IntegerType.get(), 10); Assert.assertEquals(0, (int) trunc.apply(0)); Assert.assertEquals(0, (int) trunc.apply(1)); Assert.assertEquals(0, (int) trunc.apply(5)); Assert.assertEquals(0, (int) trunc.apply(9)); Assert.assertEquals(10, (int) trunc.apply(10)); Assert.assertEquals(10, (int) trunc.apply(11)); Assert.assertEquals(-10, (int) trunc.apply(-1)); Assert.assertEquals(-10, (int) trunc.apply(-5)); Assert.assertEquals(-10, (int) trunc.apply(-10)); Assert.assertEquals(-20, (int) trunc.apply(-11)); }
@Test public void testPrimitive() throws IOException { Schema icebergSchema = new Schema( optional(1, "b", BooleanType.get()), optional(1, "i", IntegerType.get()), optional(2, "l", LongType.get()), optional(3, "f", FloatType.get()), optional(4, "d", DoubleType.get()), optional(5, "dec", DecimalType.of(0,2)), optional(5, "s", StringType.get()), optional(6,"bi", BinaryType.get()) ); ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema); assertEquals("b:boolean,i:int,l:long,f:float,d:double,dec:bigdecimal,s:chararray,bi:bytearray", pigSchema.toString()); }
@Test public void testStructReadReordering() { // reads should allow reordering Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "field_a", Types.IntegerType.get()), required(2, "field_b", Types.IntegerType.get()) ))); Schema write = new Schema(required(0, "nested", Types.StructType.of( required(2, "field_b", Types.IntegerType.get()), required(1, "field_a", Types.IntegerType.get()) ))); List<String> errors = CheckCompatibility.readCompatibilityErrors(read, write); Assert.assertEquals("Should produce no error messages", 0, errors.size()); } }
@Test @SuppressWarnings("unchecked") public void testComparisonPredicateBinding() { StructType struct = StructType.of(required(14, "x", Types.IntegerType.get())); for (Expression.Operation op : COMPARISONS) { UnboundPredicate<Integer> unbound = new UnboundPredicate<>(op, ref("x"), 5); Expression expr = unbound.bind(struct); BoundPredicate<Integer> bound = assertAndUnwrap(expr); Assert.assertEquals("Should not alter literal value", Integer.valueOf(5), bound.literal().value()); Assert.assertEquals("Should reference correct field ID", 14, bound.ref().fieldId()); Assert.assertEquals("Should not change the comparison operation", op, bound.op()); } }
@Test public void testMissingField() { StructType struct = StructType.of( required(13, "x", Types.IntegerType.get()) ); UnboundPredicate<Integer> unbound = new UnboundPredicate<>(LT, ref("missing"), 6); try { unbound.bind(struct); Assert.fail("Binding a missing field should fail"); } catch (ValidationException e) { Assert.assertTrue("Validation should complain about missing field", e.getMessage().contains("Cannot find field 'missing' in struct:")); } }
@Test public void testInvalidStringConversions() { // Strings can be used for types that are difficult to construct, like decimal or timestamp, // but are not intended to support parsing strings to any type testInvalidConversions(Literal.of("abc"), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testIncompatibleStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( required(1, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.FloatType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to float")); }
@Override public Type getResultType(Type sourceType) { return Types.IntegerType.get(); }
@Override public Type getResultType(Type sourceType) { return Types.IntegerType.get(); }
@Test public void testRequiredListElement() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.ListType.ofRequired( 1, Types.IntegerType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that elements are optional", errors.get(0).contains("elements should be required, but are optional")); }
@Test public void testRequiredStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( optional(1, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.IntegerType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required field is optional", errors.get(0).contains("should be required, but is optional")); }
@Override public Type getResultType(Type sourceType) { return Types.IntegerType.get(); }