private void runPartialFieldsInRows(PCollection<Row> input) throws Exception { String sql = "SELECT f_int, f_long FROM TABLE_A"; PCollection<Row> result = PCollectionTuple.of(new TupleTag<>("TABLE_A"), input) .apply("testPartialFieldsInRows", SqlTransform.query(sql)); Schema resultType = Schema.builder().addInt32Field("f_int").addInt64Field("f_long").build(); List<Row> expectedRows = IntStream.range(0, 4).mapToObj(i -> rowAtIndex(resultType, i)).collect(toList()); PAssert.that(result).containsInAnyOrder(expectedRows); pipeline.run().waitUntilFinish(); }
private void runPartialFieldsInMultipleRow(PCollection<Row> input) throws Exception { String sql = "SELECT f_int, f_long FROM TABLE_A"; PCollection<Row> result = PCollectionTuple.of(new TupleTag<>("TABLE_A"), input) .apply("testPartialFieldsInMultipleRow", SqlTransform.query(sql)); Schema resultType = Schema.builder().addInt32Field("f_int").addInt64Field("f_long").build(); List<Row> expectedRows = IntStream.range(0, 4).mapToObj(i -> rowAtIndex(resultType, i)).collect(toList()); PAssert.that(result).containsInAnyOrder(expectedRows); pipeline.run().waitUntilFinish(); }
@Test public void testParsesArrayField() throws Exception { Schema schema = Schema.builder() .addInt32Field("f_int32") .addArrayField("f_intArray", FieldType.INT32) .build(); String rowString = "{\n" + "\"f_int32\" : 32,\n" + "\"f_intArray\" : [ 1, 2, 3, 4, 5]\n" + "}"; RowJsonDeserializer deserializer = RowJsonDeserializer.forSchema(schema); Row parsedRow = newObjectMapperWith(deserializer).readValue(rowString, Row.class); Row expectedRow = Row.withSchema(schema).addValues(32, Arrays.asList(1, 2, 3, 4, 5)).build(); assertEquals(expectedRow, parsedRow); }
@Test public void testNestedMapsNotEquivalent() { Schema nestedSchema1 = Schema.builder().addInt64Field("foo").build(); Schema nestedSchema2 = Schema.builder().addStringField("foo").build(); Schema schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.row(nestedSchema1)).build(); Schema schema2 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.row(nestedSchema2)).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); } }
private Schema newSimpleSchemaWith(FieldType fieldType) { return Schema.builder().addNullableField("fieldName", fieldType).build(); }
public static Schema of(Field... fields) { return Schema.builder().addFields(fields).build(); }
@Test public void testNestedTypes() throws Exception { Schema nestedSchema = Schema.builder().addInt32Field("f1_int").addStringField("f1_str").build(); Schema schema = Schema.builder().addInt32Field("f_int").addRowField("nested", nestedSchema).build(); Row nestedRow = Row.withSchema(nestedSchema).addValues(18, "foobar").build(); Row row = Row.withSchema(schema).addValues(42, nestedRow).build(); CoderProperties.coderDecodeEncodeEqual(RowCoder.of(schema), row); }
@Test public void testLength() throws Exception { Schema resultType = Schema.builder().addInt64Field("field").build(); Row resultRow = Row.withSchema(resultType).addValues(10L).build(); Row resultRow2 = Row.withSchema(resultType).addValues(0L).build(); Row resultRow3 = Row.withSchema(resultType).addValues(2L).build(); String sql = "SELECT LENGTH(f_bytes) FROM PCOLLECTION WHERE f_func = 'LENGTH'"; PCollection<Row> result = boundedInputBytes.apply("testUdf", SqlTransform.query(sql)); PAssert.that(result).containsInAnyOrder(resultRow, resultRow2, resultRow3); pipeline.run().waitUntilFinish(); }
private static Table.Builder tableDefinition() { return Table.builder() .name("FakeTable") .comment("fake table") .location("projects/project/topics/topic") .schema(Schema.builder().build()) .type("pubsub") .properties(JSON.parseObject("{ \"timestampAttributeKey\" : \"ts_field\" }")); } }
private List<Row> rowsWithSingleIntField(String fieldName, List<Integer> values) { return TestUtils.rowsBuilderOf(Schema.builder().addInt32Field(fieldName).build()) .addRows(values) .getRows(); }
@Test(expected = NonDeterministicException.class) public void testVerifyDeterministic() throws NonDeterministicException { Schema schema = Schema.builder() .addField("f1", FieldType.DOUBLE) .addField("f2", FieldType.FLOAT) .addField("f3", FieldType.INT32) .build(); RowCoder coder = RowCoder.of(schema); coder.verifyDeterministic(); }
@Test public void testThrowsForUnsupportedType() throws Exception { Schema schema = Schema.builder().addDateTimeField("f_dateTime").build(); thrown.expect(UnsupportedRowJsonException.class); thrown.expectMessage("DATETIME is not supported"); RowJsonDeserializer.forSchema(schema); }
@Test public void testBuildBeamSqlTable() throws Exception { Table table = mockTable("hello"); store.createTable(table); BeamSqlTable actualSqlTable = store.buildBeamSqlTable(table); assertNotNull(actualSqlTable); assertEquals( Schema.builder() .addNullableField("id", Schema.FieldType.INT32) .addNullableField("name", Schema.FieldType.STRING) .build(), actualSqlTable.getSchema()); }
@Test public void testMapNotEquivalent() { Schema schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.BOOLEAN).build(); Schema schema2 = Schema.builder().addMapField("foo", FieldType.DATETIME, FieldType.BOOLEAN).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); schema1 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.BOOLEAN).build(); schema2 = Schema.builder().addMapField("foo", FieldType.STRING, FieldType.STRING).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); }
public void runLiteralField(PCollection<Row> input) throws Exception { String sql = "SELECT 1 as literal_field FROM TABLE_A"; PCollection<Row> result = PCollectionTuple.of(new TupleTag<>("TABLE_A"), input) .apply("testLiteralField", SqlTransform.query(sql)); Schema resultType = Schema.builder().addInt32Field("literal_field").build(); Row row = Row.withSchema(resultType).addValues(1).build(); PAssert.that(result).containsInAnyOrder(row); pipeline.run().waitUntilFinish(); }
private Schema getOutputSchema(List<FieldAggregation> fieldAggregations) { Schema.Builder outputSchema = Schema.builder(); for (FieldAggregation aggregation : fieldAggregations) { outputSchema.addField(aggregation.outputField); } return outputSchema.build(); }
@Test public void testThrowsForMissingNotNullableField() throws Exception { Schema schema = Schema.builder().addByteField("f_byte").addStringField("f_string").build(); String rowString = "{\n" + "\"f_byte\" : 12\n" + "}"; RowJsonDeserializer deserializer = RowJsonDeserializer.forSchema(schema); thrown.expect(UnsupportedRowJsonException.class); thrown.expectMessage("'f_string' is not present"); newObjectMapperWith(deserializer).readValue(rowString, Row.class); }
@Test public void testThrowsForUnsupportedArrayElementType() throws Exception { Schema schema = Schema.builder().addArrayField("f_dateTimeArray", FieldType.DATETIME).build(); thrown.expect(UnsupportedRowJsonException.class); thrown.expectMessage("DATETIME is not supported"); RowJsonDeserializer.forSchema(schema); }
@Test public void testNestedArraysNotEquivalent() { Schema nestedSchema1 = Schema.builder().addInt64Field("foo").build(); Schema nestedSchema2 = Schema.builder().addStringField("foo").build(); Schema schema1 = Schema.builder().addArrayField("foo", FieldType.row(nestedSchema1)).build(); Schema schema2 = Schema.builder().addArrayField("foo", FieldType.row(nestedSchema2)).build(); assertNotEquals(schema1, schema2); assertFalse(schema1.equivalent(schema2)); }