public void checkSparkDate(String dateString) { Literal<Integer> date = Literal.of(dateString).to(Types.DateType.get()); String sparkDate = DateTimeUtils.toJavaDate(date.value()).toString(); System.err.println(dateString + ": " + date.value()); Assert.assertEquals("Should be the same date (" + date.value() + ")", dateString, sparkDate); }
Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get(),
@Test public void testStringToDateLiteral() { Literal<CharSequence> dateStr = Literal.of("2017-08-18"); Literal<Integer> date = dateStr.to(Types.DateType.get()); // use Avro's date conversion to validate the result Schema avroSchema = LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)); TimeConversions.DateConversion avroConversion = new TimeConversions.DateConversion(); int avroValue = avroConversion.toInt( new LocalDate(2017, 8, 18), avroSchema, avroSchema.getLogicalType()); Assert.assertEquals("Date should match", avroValue, (int) date.value()); }
@Test public void testIdentityConversions() { List<Pair<Literal<?>, Type>> pairs = Arrays.asList( Pair.of(Literal.of(true), Types.BooleanType.get()), Pair.of(Literal.of(34), Types.IntegerType.get()), Pair.of(Literal.of(34L), Types.LongType.get()), Pair.of(Literal.of(34.11F), Types.FloatType.get()), Pair.of(Literal.of(34.55D), Types.DoubleType.get()), Pair.of(Literal.of("34.55"), Types.DecimalType.of(9, 2)), Pair.of(Literal.of("2017-08-18"), Types.DateType.get()), Pair.of(Literal.of("14:21:01.919"), Types.TimeType.get()), Pair.of(Literal.of("2017-08-18T14:21:01.919"), Types.TimestampType.withoutZone()), Pair.of(Literal.of("abc"), Types.StringType.get()), Pair.of(Literal.of(UUID.randomUUID()), Types.UUIDType.get()), Pair.of(Literal.of(new byte[] {0, 1, 2}), Types.FixedType.ofLength(3)), Pair.of(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BinaryType.get()) ); for (Pair<Literal<?>, Type> pair : pairs) { Literal<?> lit = pair.first(); Type type = pair.second(); // first, convert the literal to the target type (date/times start as strings) Literal<?> expected = lit.to(type); // then check that converting again to the same type results in an identical literal Assert.assertSame("Converting twice should produce identical values", expected, expected.to(type)); } }
@Test public void testLiterals() throws Exception { Literal[] literals = new Literal[] { Literal.of(false), Literal.of(34), Literal.of(35L), Literal.of(36.75F), Literal.of(8.75D), Literal.of("2017-11-29").to(Types.DateType.get()), Literal.of("11:30:07").to(Types.TimeType.get()), Literal.of("2017-11-29T11:30:07.123").to(Types.TimestampType.withoutZone()), Literal.of("2017-11-29T11:30:07.123+01:00").to(Types.TimestampType.withZone()), Literal.of("abc"), Literal.of(UUID.randomUUID()), Literal.of(new byte[] { 1, 2, 3 }).to(Types.FixedType.ofLength(3)), Literal.of(new byte[] { 3, 4, 5, 6 }).to(Types.BinaryType.get()), Literal.of(new BigDecimal("122.50")), }; for (Literal<?> lit : literals) { checkValue(lit); } }
@Test public void testInvalidDecimalConversions() { testInvalidConversions(Literal.of(new BigDecimal("34.11")), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidTimeConversions() { testInvalidConversions( Literal.of("14:21:01.919").to(Types.TimeType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testIdentityTypes() throws Exception { // these types make a strong guarantee than equality, instances are identical Type[] identityPrimitives = new Type[] { Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withoutZone(), Types.TimestampType.withZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get() }; for (Type type : identityPrimitives) { Assert.assertSame("Serialization result should be identical to starting type", type, TestHelpers.roundTripSerialize(type)); } }
@Test public void testInvalidBinaryConversions() { testInvalidConversions(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }
@Test public void testInvalidFixedConversions() { testInvalidConversions(Literal.of(new byte[] {0, 1, 2}), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }
@Test public void testInvalidDateConversions() { testInvalidConversions(Literal.of("2017-08-18").to(Types.DateType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidBooleanConversions() { testInvalidConversions(Literal.of(true), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidDoubleConversions() { testInvalidConversions(Literal.of(34.11D), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidFloatConversions() { testInvalidConversions(Literal.of(34.11F), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidUUIDConversions() { testInvalidConversions(Literal.of(UUID.randomUUID()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testDateToHumanString() { Types.DateType type = Types.DateType.get(); Literal<Integer> date = Literal.of("2017-12-01").to(type); Transform<Integer, Integer> year = Transforms.year(type); Assert.assertEquals("Should produce the correct Human string", "2017", year.toHumanString(year.apply(date.value()))); Transform<Integer, Integer> month = Transforms.month(type); Assert.assertEquals("Should produce the correct Human string", "2017-12", month.toHumanString(month.apply(date.value()))); Transform<Integer, Integer> day = Transforms.day(type); Assert.assertEquals("Should produce the correct Human string", "2017-12-01", day.toHumanString(day.apply(date.value()))); }
@Test public void testDateHumanString() { Types.DateType date = Types.DateType.get(); Transform<Integer, Integer> identity = Transforms.identity(date); String dateString = "2017-12-01"; Literal<Integer> d = Literal.of(dateString).to(date); Assert.assertEquals("Should produce identical date", dateString, identity.toHumanString(d.value())); }
@Test public void testNullHumanString() { Types.DateType type = Types.DateType.get(); Assert.assertEquals("Should produce \"null\" for null", "null", Transforms.year(type).toHumanString(null)); Assert.assertEquals("Should produce \"null\" for null", "null", Transforms.month(type).toHumanString(null)); Assert.assertEquals("Should produce \"null\" for null", "null", Transforms.day(type).toHumanString(null)); } }
@Test public void testUpdateMissingColumn() { AssertHelpers.assertThrows("Should reject rename missing column", IllegalArgumentException.class, "missing column: col", () -> { UpdateSchema update = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID); update.updateColumn("col", Types.DateType.get()); } ); }
@Test public void testInvalidLongConversions() { testInvalidConversions(Literal.of(34L), Types.BooleanType.get(), Types.DateType.get(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }