Schema.builder() .addDateTimeField("event_timestamp") .addMapField("attributes", VARCHAR, VARCHAR) .addRowField("payload", payloadSchema) .build();
Schema.builder() .addDateTimeField("event_timestamp") .addMapField("attributes", VARCHAR, VARCHAR) .addRowField("payload", payloadSchema) .build();
@Test public void testEquivalent() { final Schema expectedNested1 = Schema.builder().addStringField("yard1").addInt64Field("yard2").build(); final Schema expectedSchema1 = Schema.builder() .addStringField("field1") .addInt64Field("field2") .addRowField("field3", expectedNested1) .addArrayField("field4", FieldType.row(expectedNested1)) .addMapField("field5", FieldType.STRING, FieldType.row(expectedNested1)) .build(); final Schema expectedNested2 = Schema.builder().addInt64Field("yard2").addStringField("yard1").build(); final Schema expectedSchema2 = Schema.builder() .addMapField("field5", FieldType.STRING, FieldType.row(expectedNested2)) .addArrayField("field4", FieldType.row(expectedNested2)) .addRowField("field3", expectedNested2) .addInt64Field("field2") .addStringField("field1") .build(); assertNotEquals(expectedSchema1, expectedSchema2); assertTrue(expectedSchema1.equivalent(expectedSchema2)); }
@Test public void testSelectMapField() { PCollection<Row> input = pCollectionOf2Elements(); Schema resultType = Schema.builder() .addInt32Field("f_int") .addMapField("f_intStringMap", Schema.FieldType.STRING, Schema.FieldType.INT32) .build(); PCollection<Row> result = input.apply( "sqlQuery", SqlTransform.query("SELECT 42, MAP['aa', 1] as `f_map` FROM PCOLLECTION")); PAssert.that(result) .containsInAnyOrder( Row.withSchema(resultType).addValues(42, ImmutableMap.of("aa", 1)).build(), Row.withSchema(resultType).addValues(42, ImmutableMap.of("aa", 1)).build()); pipeline.run(); }
@Test public void testThrowsIfExtraFieldsExist() { PubsubJsonTableProvider provider = new PubsubJsonTableProvider(); Schema messageSchema = Schema.builder() .addDateTimeField("event_timestamp") .addMapField("attributes", VARCHAR, VARCHAR) .addStringField("someField") .addRowField("payload", Schema.builder().build()) .build(); Table tableDefinition = tableDefinition().schema(messageSchema).build(); thrown.expectMessage("Unsupported"); thrown.expectMessage("'event_timestamp'"); provider.buildBeamSqlTable(tableDefinition); }
@Test public void testSelectMapFieldKeyValueSameType() { PCollection<Row> input = pCollectionOf2Elements(); Schema resultType = Schema.builder() .addInt32Field("f_int") .addMapField("f_intStringMap", Schema.FieldType.STRING, Schema.FieldType.STRING) .build(); PCollection<Row> result = input.apply( "sqlQuery", SqlTransform.query("SELECT 42, MAP['aa', '1'] as `f_map` FROM PCOLLECTION")); PAssert.that(result) .containsInAnyOrder( Row.withSchema(resultType).addValues(42, ImmutableMap.of("aa", "1")).build(), Row.withSchema(resultType).addValues(42, ImmutableMap.of("aa", "1")).build()); pipeline.run(); }
@Test public void testCreatesTable() { PubsubJsonTableProvider provider = new PubsubJsonTableProvider(); Schema messageSchema = Schema.builder() .addDateTimeField("event_timestamp") .addMapField("attributes", VARCHAR, VARCHAR) .addRowField("payload", Schema.builder().build()) .build(); Table tableDefinition = tableDefinition().schema(messageSchema).build(); BeamSqlTable pubsubTable = provider.buildBeamSqlTable(tableDefinition); assertNotNull(pubsubTable); assertEquals(messageSchema, pubsubTable.getSchema()); }
@Test public void testThrowsIfTimestampFieldNotProvided() { PubsubJsonTableProvider provider = new PubsubJsonTableProvider(); Schema messageSchema = Schema.builder() .addMapField("attributes", VARCHAR, VARCHAR) .addRowField("payload", Schema.builder().build()) .build(); Table tableDefinition = tableDefinition().schema(messageSchema).build(); thrown.expectMessage("Unsupported"); thrown.expectMessage("'event_timestamp'"); provider.buildBeamSqlTable(tableDefinition); }
@Test public void testThrowsIfPayloadFieldNotProvided() { PubsubJsonTableProvider provider = new PubsubJsonTableProvider(); Schema messageSchema = Schema.builder() .addDateTimeField("event_timestamp") .addMapField("attributes", VARCHAR, VARCHAR) .build(); Table tableDefinition = tableDefinition().schema(messageSchema).build(); thrown.expectMessage("Unsupported"); thrown.expectMessage("'payload'"); provider.buildBeamSqlTable(tableDefinition); }
@Test public void testNestedMapsNotEquivalent() { Schema nestedSchema1 = Schema.builder().addInt64Field("foo").build(); Schema nestedSchema2 = Schema.builder().addStringField("foo").build(); Schema schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.row(nestedSchema1)).build(); Schema schema2 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.row(nestedSchema2)).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); } }
@Test public void testMapNotEquivalent() { Schema schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.BOOLEAN).build(); Schema schema2 = Schema.builder().addMapField("foo", FieldType.DATETIME, FieldType.BOOLEAN).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.BOOLEAN).build(); schema2 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.STRING).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); }