@Test public void testMultipleErrors() { // required field is optional and cannot be promoted to the read type Schema write = new Schema(required(0, "nested", Types.StructType.of( optional(1, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.FloatType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 2, errors.size()); Assert.assertTrue("Should complain that a required field is optional", errors.get(0).contains("should be required, but is optional")); Assert.assertTrue("Should complain about incompatible types", errors.get(1).contains("cannot be promoted to float")); }
@Test public void testIncompatibleStructField() { Schema write = new Schema(required(0, "nested", Types.StructType.of( required(1, "from_field", Types.IntegerType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StructType.of( required(1, "to_field", Types.FloatType.get()) ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to float")); }
@Test public void testInvalidUUIDConversions() { testInvalidConversions(Literal.of(UUID.randomUUID()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidTimeConversions() { testInvalidConversions( Literal.of("14:21:01.919").to(Types.TimeType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testIdentityTypes() throws Exception { // these types make a strong guarantee than equality, instances are identical Type[] identityPrimitives = new Type[] { Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withoutZone(), Types.TimestampType.withZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get() }; for (Type type : identityPrimitives) { Assert.assertSame("Serialization result should be identical to starting type", type, TestHelpers.roundTripSerialize(type)); } }
@Test public void testInvalidBinaryConversions() { testInvalidConversions(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }
@Test public void testInvalidDecimalConversions() { testInvalidConversions(Literal.of(new BigDecimal("34.11")), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidFixedConversions() { testInvalidConversions(Literal.of(new byte[] {0, 1, 2}), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }
@Test public void testInvalidDateConversions() { testInvalidConversions(Literal.of("2017-08-18").to(Types.DateType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidBooleanConversions() { testInvalidConversions(Literal.of(true), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidTimestampConversions() { testInvalidConversions( Literal.of("2017-08-18T14:21:01.919").to(Types.TimestampType.withoutZone()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.TimeType.get(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void complexNested() throws IOException { convertToPigSchema(new Schema( optional(1,"t", StructType.of( optional(2, "b", ListType.ofOptional(3,StructType.of( optional(4, "i", IntegerType.get()), optional(5,"s", StringType.get()) ))) )), optional(6, "m1", MapType.ofOptional(7,8, StringType.get(), StructType.of( optional(9, "b", ListType.ofOptional(10, BinaryType.get()) ), optional(11, "m2", MapType.ofOptional(12,13, StringType.get(), IntegerType.get())) ))), optional(14, "b1", ListType.ofOptional(15, MapType.ofOptional(16,17, StringType.get(), ListType.ofOptional(18, FloatType.get())))) ), "t:(b:{(i:int,s:chararray)}),m1:[(b:{(bytearray)},m2:[int])],b1:{([{(float)}])}", ""); }
@Test public void testPrimitive() throws IOException { Schema icebergSchema = new Schema( optional(1, "b", BooleanType.get()), optional(1, "i", IntegerType.get()), optional(2, "l", LongType.get()), optional(3, "f", FloatType.get()), optional(4, "d", DoubleType.get()), optional(5, "dec", DecimalType.of(0,2)), optional(5, "s", StringType.get()), optional(6,"bi", BinaryType.get()) ); ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema); assertEquals("b:boolean,i:int,l:long,f:float,d:double,dec:bigdecimal,s:chararray,bi:bytearray", pigSchema.toString()); }
@Test public void testInvalidConversions() { StructType struct = StructType.of(required(16, "f", Types.FloatType.get())); for (Expression.Operation op : COMPARISONS) { UnboundPredicate<String> unbound = new UnboundPredicate<>(op, ref("f"), "12.40"); try { unbound.bind(struct); Assert.fail("Should not convert string to float"); } catch (ValidationException e) { Assert.assertEquals("Should ", e.getMessage(), "Invalid value for comparison inclusive type float: 12.40 (java.lang.String)"); } } }
@Test public void testListOfStructs() { Type list = Types.ListType.ofRequired(34, Types.StructType.of( required(35, "lat", Types.FloatType.get()), required(36, "long", Types.FloatType.get()) )); Schema schema = addElementId(34, SchemaBuilder.array().items( record("r34", requiredField(35, "lat", Schema.create(Schema.Type.FLOAT)), requiredField(36, "long", Schema.create(Schema.Type.FLOAT))) )); Assert.assertEquals("Avro schema to list", list, AvroSchemaUtil.convert(schema)); Assert.assertEquals("List to Avro schema", schema, AvroSchemaUtil.convert(list)); }
@Test public void testInvalidStringConversions() { // Strings can be used for types that are difficult to construct, like decimal or timestamp, // but are not intended to support parsing strings to any type testInvalidConversions(Literal.of("abc"), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testComplex() throws IOException { convertToPigSchema( new Schema( optional(1, "bag", ListType.ofOptional(2, BooleanType.get())), optional(3, "map", MapType.ofOptional(4,5, StringType.get(), DoubleType.get())), optional(6, "tuple", StructType.of(optional(7, "i", IntegerType.get()), optional(8,"f", FloatType.get()))) ),"bag:{(boolean)},map:[double],tuple:(i:int,f:float)", null ); }
@Test public void testLongToFloatConversion() { Literal<Long> lit = Literal.of(34L); Literal<Float> floatLit = lit.to(Types.FloatType.get()); Assert.assertEquals("Value should match", 34.0F, floatLit.value(), 0.0000000001D); }
@Test public void testDoubleToFloatConversion() { Literal<Double> lit = Literal.of(34.56D); Literal<Float> doubleLit = lit.to(Types.FloatType.get()); Assert.assertEquals("Value should match", 34.56F, doubleLit.value(), 0.001D); // this adjusts Float.MAX_VALUE using multipliers because most integer adjustments are lost by // floating point precision. Assert.assertEquals("Values above Float.MAX_VALUE should be Literals.aboveMax()", Literals.aboveMax(), Literal.of(2 * ((double) Float.MAX_VALUE)).to(Types.FloatType.get())); Assert.assertEquals("Values below Float.MIN_VALUE should be Literals.belowMin()", Literals.belowMin(), Literal.of(-2 * ((double) Float.MAX_VALUE)).to(Types.FloatType.get())); }
@Test public void testIntegerToFloatConversion() { Literal<Integer> lit = Literal.of(34); Literal<Float> floatLit = lit.to(Types.FloatType.get()); Assert.assertEquals("Value should match", 34.0F, floatLit.value(), 0.0000000001D); }