@Test public void testArrayTypeToJavaType() { assertEquals( TypeDescriptors.lists(TypeDescriptors.longs()), FieldTypeDescriptors.javaTypeForFieldType(FieldType.array(FieldType.INT64))); assertEquals( TypeDescriptors.lists(TypeDescriptors.lists(TypeDescriptors.longs())), FieldTypeDescriptors.javaTypeForFieldType( FieldType.array(FieldType.array(FieldType.INT64)))); }
@Test public void testMapTypeToJavaType() { assertEquals( TypeDescriptors.maps(TypeDescriptors.strings(), TypeDescriptors.longs()), FieldTypeDescriptors.javaTypeForFieldType( FieldType.map(FieldType.STRING, FieldType.INT64))); assertEquals( TypeDescriptors.maps( TypeDescriptors.strings(), TypeDescriptors.lists(TypeDescriptors.longs())), FieldTypeDescriptors.javaTypeForFieldType( FieldType.map(FieldType.STRING, FieldType.array(FieldType.INT64)))); }
@Test public void testMapTypeToFieldType() { assertEquals( FieldType.map(FieldType.STRING, FieldType.INT64), FieldTypeDescriptors.fieldTypeForJavaType( TypeDescriptors.maps(TypeDescriptors.strings(), TypeDescriptors.longs()))); assertEquals( FieldType.map(FieldType.STRING, FieldType.array(FieldType.INT64)), FieldTypeDescriptors.fieldTypeForJavaType( TypeDescriptors.maps( TypeDescriptors.strings(), TypeDescriptors.lists(TypeDescriptors.longs())))); } }
input .apply( MapElements.into(TypeDescriptors.longs()) .via(t -> TimeUnit.SECONDS.toMillis(t) + customTimestampStartMillis)) .apply("TimestampDiff", ParDo.of(new ElementValueDiff()))
@Test public void testUnboundedSourceCreateTimestamps() { // Same as testUnboundedSourceCustomTimestamps with create timestamp. final int numElements = 1000; final long createTimestampStartMillis = 50000L; PCollection<Long> input = p.apply( mkKafkaReadTransform(numElements, null) .withCreateTime(Duration.ZERO) .updateConsumerProperties( ImmutableMap.of( TIMESTAMP_TYPE_CONFIG, "CreateTime", TIMESTAMP_START_MILLIS_CONFIG, createTimestampStartMillis)) .withoutMetadata()) .apply(Values.create()); addCountingAsserts(input, numElements); PCollection<Long> diffs = input .apply( MapElements.into(TypeDescriptors.longs()) .via(t -> TimeUnit.SECONDS.toMillis(t) + createTimestampStartMillis)) .apply("TimestampDiff", ParDo.of(new ElementValueDiff())) .apply("DistinctTimestamps", Distinct.create()); // This assert also confirms that diff only has one unique value. PAssert.thatSingleton(diffs).isEqualTo(0L); p.run(); }
.apply( MapElements.into( TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.longs())) .via(v -> KV.of("key", v))) .apply(Sum.longsPerKey());
@Test public void testUnboundedSourceLogAppendTimestamps() { // LogAppendTime (server side timestamp) for records is set based on record index // in MockConsumer above. Ensure that those exact timestamps are set by the source. int numElements = 1000; PCollection<Long> input = p.apply(mkKafkaReadTransform(numElements, null).withLogAppendTime().withoutMetadata()) .apply(Values.create()); addCountingAsserts(input, numElements); PCollection<Long> diffs = input .apply( MapElements.into(TypeDescriptors.longs()) .via(t -> LOG_APPEND_START_TIME.plus(Duration.standardSeconds(t)).getMillis())) .apply("TimestampDiff", ParDo.of(new ElementValueDiff())) .apply("DistinctTimestamps", Distinct.create()); // This assert also confirms that diff only has one unique value. PAssert.thatSingleton(diffs).isEqualTo(0L); p.run(); }
@Test public void testPrimitiveTypeToFieldType() { assertEquals( FieldType.BYTE, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.bytes())); assertEquals( FieldType.INT16, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.shorts())); assertEquals( FieldType.INT32, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.integers())); assertEquals( FieldType.INT64, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.longs())); assertEquals( FieldType.DECIMAL, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.bigdecimals())); assertEquals( FieldType.FLOAT, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.floats())); assertEquals( FieldType.DOUBLE, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.doubles())); assertEquals( FieldType.STRING, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.strings())); assertEquals( FieldType.DATETIME, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptor.of(Instant.class))); assertEquals( FieldType.BOOLEAN, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptors.booleans())); assertEquals( FieldType.BYTES, FieldTypeDescriptors.fieldTypeForJavaType(TypeDescriptor.of(byte[].class))); }
@Test public void testPrimitiveTypeToJavaType() { assertEquals( TypeDescriptors.bytes(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.BYTE)); assertEquals( TypeDescriptors.shorts(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.INT16)); assertEquals( TypeDescriptors.integers(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.INT32)); assertEquals( TypeDescriptors.longs(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.INT64)); assertEquals( TypeDescriptors.bigdecimals(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.DECIMAL)); assertEquals( TypeDescriptors.floats(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.FLOAT)); assertEquals( TypeDescriptors.doubles(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.DOUBLE)); assertEquals( TypeDescriptors.strings(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.STRING)); assertEquals( TypeDescriptor.of(Instant.class), FieldTypeDescriptors.javaTypeForFieldType(FieldType.DATETIME)); assertEquals( TypeDescriptors.booleans(), FieldTypeDescriptors.javaTypeForFieldType(FieldType.BOOLEAN)); assertEquals( TypeDescriptor.of(byte[].class), FieldTypeDescriptors.javaTypeForFieldType(FieldType.BYTES)); }