public void checkSparkTimestamp(String timestampString, String sparkRepr) { Literal<Long> ts = Literal.of(timestampString).to(Types.TimestampType.withZone()); String sparkTimestamp = DateTimeUtils.timestampToString(ts.value()); System.err.println(timestampString + ": " + ts.value()); Assert.assertEquals("Should be the same timestamp (" + ts.value() + ")", sparkRepr, sparkTimestamp); } }
@Test public void testLiterals() throws Exception { Literal[] literals = new Literal[] { Literal.of(false), Literal.of(34), Literal.of(35L), Literal.of(36.75F), Literal.of(8.75D), Literal.of("2017-11-29").to(Types.DateType.get()), Literal.of("11:30:07").to(Types.TimeType.get()), Literal.of("2017-11-29T11:30:07.123").to(Types.TimestampType.withoutZone()), Literal.of("2017-11-29T11:30:07.123+01:00").to(Types.TimestampType.withZone()), Literal.of("abc"), Literal.of(UUID.randomUUID()), Literal.of(new byte[] { 1, 2, 3 }).to(Types.FixedType.ofLength(3)), Literal.of(new byte[] { 3, 4, 5, 6 }).to(Types.BinaryType.get()), Literal.of(new BigDecimal("122.50")), }; for (Literal<?> lit : literals) { checkValue(lit); } }
@Test public void testInvalidUUIDConversions() { testInvalidConversions(Literal.of(UUID.randomUUID()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testStringToTimestampLiteral() { // use Avro's timestamp conversion to validate the result Schema avroSchema = LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)); TimeConversions.LossyTimestampMicrosConversion avroConversion = new TimeConversions.LossyTimestampMicrosConversion(); // Timestamp with explicit UTC offset, +00:00 Literal<CharSequence> timestampStr = Literal.of("2017-08-18T14:21:01.919+00:00"); Literal<Long> timestamp = timestampStr.to(Types.TimestampType.withZone()); long avroValue = avroConversion.toLong( new LocalDateTime(2017, 8, 18, 14, 21, 1, 919).toDateTime(DateTimeZone.UTC), avroSchema, avroSchema.getLogicalType()); Assert.assertEquals("Timestamp should match", avroValue, (long) timestamp.value()); // Timestamp without an explicit zone should be UTC (equal to the previous converted value) timestampStr = Literal.of("2017-08-18T14:21:01.919"); timestamp = timestampStr.to(Types.TimestampType.withoutZone()); Assert.assertEquals("Timestamp without zone should match UTC", avroValue, (long) timestamp.value()); // Timestamp with an explicit offset should be adjusted to UTC timestampStr = Literal.of("2017-08-18T14:21:01.919-07:00"); timestamp = timestampStr.to(Types.TimestampType.withZone()); avroValue = avroConversion.toLong( new LocalDateTime(2017, 8, 18, 21, 21, 1, 919).toDateTime(DateTimeZone.UTC), avroSchema, avroSchema.getLogicalType()); Assert.assertEquals("Timestamp without zone should match UTC", avroValue, (long) timestamp.value()); }
@Test public void testIdentityTypes() throws Exception { // these types make a strong guarantee than equality, instances are identical Type[] identityPrimitives = new Type[] { Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withoutZone(), Types.TimestampType.withZone(), Types.StringType.get(), Types.UUIDType.get(), Types.BinaryType.get() }; for (Type type : identityPrimitives) { Assert.assertSame("Serialization result should be identical to starting type", type, TestHelpers.roundTripSerialize(type)); } }
@BeforeClass public static void startSpark() { TestFilteredScan.spark = SparkSession.builder().master("local[2]").getOrCreate(); // define UDFs used by partition tests Transform<Long, Integer> bucket4 = Transforms.bucket(Types.LongType.get(), 4); spark.udf().register("bucket4", (UDF1<Long, Integer>) bucket4::apply, IntegerType$.MODULE$); Transform<Long, Integer> day = Transforms.day(Types.TimestampType.withZone()); spark.udf().register("ts_day", (UDF1<Timestamp, Integer>) timestamp -> day.apply(fromJavaTimestamp(timestamp)), IntegerType$.MODULE$); Transform<Long, Integer> hour = Transforms.hour(Types.TimestampType.withZone()); spark.udf().register("ts_hour", (UDF1<Timestamp, Integer>) timestamp -> hour.apply(fromJavaTimestamp(timestamp)), IntegerType$.MODULE$); Transform<CharSequence, CharSequence> trunc1 = Transforms.truncate(Types.StringType.get(), 1); spark.udf().register("trunc1", (UDF1<CharSequence, CharSequence>) str -> trunc1.apply(str.toString()), StringType$.MODULE$); }
@Test public void testInvalidTimeConversions() { testInvalidConversions( Literal.of("14:21:01.919").to(Types.TimeType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidFixedConversions() { testInvalidConversions(Literal.of(new byte[] {0, 1, 2}), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }
@Test public void testInvalidDateConversions() { testInvalidConversions(Literal.of("2017-08-18").to(Types.DateType.get()), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidBooleanConversions() { testInvalidConversions(Literal.of(true), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidDecimalConversions() { testInvalidConversions(Literal.of(new BigDecimal("34.11")), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 4), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidBinaryConversions() { testInvalidConversions(Literal.of(ByteBuffer.wrap(new byte[] {0, 1, 2})), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.DecimalType.of(9, 2), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1) ); }
@Test public void testTimestampWithZoneToHumanString() { Types.TimestampType type = Types.TimestampType.withZone(); Literal<Integer> date = Literal.of("2017-12-01T10:12:55.038194-08:00").to(type); Transform<Integer, Integer> year = Transforms.year(type); Assert.assertEquals("Should produce the correct Human string", "2017", year.toHumanString(year.apply(date.value()))); Transform<Integer, Integer> month = Transforms.month(type); Assert.assertEquals("Should produce the correct Human string", "2017-12", month.toHumanString(month.apply(date.value()))); Transform<Integer, Integer> day = Transforms.day(type); Assert.assertEquals("Should produce the correct Human string", "2017-12-01", day.toHumanString(day.apply(date.value()))); // the hour is 18 because the value is always UTC Transform<Integer, Integer> hour = Transforms.hour(type); Assert.assertEquals("Should produce the correct Human string", "2017-12-01-18", hour.toHumanString(hour.apply(date.value()))); }
@Test public void testInvalidDoubleConversions() { testInvalidConversions(Literal.of(34.11D), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testInvalidFloatConversions() { testInvalidConversions(Literal.of(34.11F), Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(), Types.DateType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
private static long timestamp(String timestamp) { return Literal.of(timestamp).<Long>to(Types.TimestampType.withZone()).value(); }
@Test public void testNullHumanString() { Types.TimestampType type = Types.TimestampType.withZone(); Assert.assertEquals("Should produce \"null\" for null", "null", Transforms.year(type).toHumanString(null)); Assert.assertEquals("Should produce \"null\" for null", "null", Transforms.month(type).toHumanString(null)); Assert.assertEquals("Should produce \"null\" for null", "null", Transforms.day(type).toHumanString(null)); Assert.assertEquals("Should produce \"null\" for null", "null", Transforms.hour(type).toHumanString(null)); } }
@Test(expected = DateTimeException.class) public void testTimestampWithZoneWithoutZoneInLiteral() { // Zone must be present in literals when converting to timestamp with zone Literal<CharSequence> timestampStr = Literal.of("2017-08-18T14:21:01.919"); timestampStr.to(Types.TimestampType.withZone()); }
@Test public void testInvalidIntegerConversions() { testInvalidConversions(Literal.of(34), Types.BooleanType.get(), Types.TimeType.get(), Types.TimestampType.withZone(), Types.TimestampType.withoutZone(), Types.StringType.get(), Types.UUIDType.get(), Types.FixedType.ofLength(1), Types.BinaryType.get() ); }
@Test public void testTimestampWithZoneHumanString() { Types.TimestampType timestamptz = Types.TimestampType.withZone(); Transform<Long, Long> identity = Transforms.identity(timestamptz); Literal<Long> ts = Literal.of("2017-12-01T10:12:55.038194-08:00").to(timestamptz); // value will always be in UTC Assert.assertEquals("Should produce timestamp with time zone adjusted to UTC", "2017-12-01T18:12:55.038194Z", identity.toHumanString(ts.value())); }