@Test @Category(NeedsRunner.class) public void testOutputCoders() { Schema keySchema = Schema.builder().addStringField("field1").build(); PCollection<KV<Row, Iterable<POJO>>> grouped = pipeline .apply(Create.of(new POJO("key1", 1, "value1"))) .apply(Group.byFieldNames("field1")); // Make sure that the key has the right schema. PCollection<Row> keys = grouped.apply(Keys.create()); assertTrue(keys.getSchema().equivalent(keySchema)); // Make sure that the value has the right schema. PCollection<POJO> values = grouped.apply(Values.create()).apply(Flatten.iterables()); assertTrue(values.getSchema().equivalent(POJO_SCHEMA)); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testPerKeyAggregation() { Collection<POJO> elements = ImmutableList.of( new POJO("key1", 1, "value1"), new POJO("key1", 1, "value2"), new POJO("key2", 2, "value3"), new POJO("key2", 2, "value4"), new POJO("key2", 2, "value4")); PCollection<KV<Row, Long>> count = pipeline .apply(Create.of(elements)) .apply(Group.<POJO>byFieldNames("field1").aggregate(Count.combineFn())); Schema keySchema = Schema.builder().addStringField("field1").build(); Collection<KV<Row, Long>> expectedCounts = ImmutableList.of( KV.of(Row.withSchema(keySchema).addValue("key1").build(), 2L), KV.of(Row.withSchema(keySchema).addValue("key2").build(), 3L)); PAssert.that(count).containsInAnyOrder(expectedCounts); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testGroupByOneField() { PCollection<KV<Row, Iterable<POJO>>> grouped = pipeline .apply( Create.of( new POJO("key1", 1, "value1"), new POJO("key1", 2, "value2"), new POJO("key2", 3, "value3"), new POJO("key2", 4, "value4"))) .apply(Group.byFieldNames("field1")); Schema keySchema = Schema.builder().addStringField("field1").build(); List<KV<Row, Collection<POJO>>> expected = ImmutableList.of( KV.of( Row.withSchema(keySchema).addValue("key1").build(), ImmutableList.of(new POJO("key1", 1L, "value1"), new POJO("key1", 2L, "value2"))), KV.of( Row.withSchema(keySchema).addValue("key2").build(), ImmutableList.of(new POJO("key2", 3L, "value3"), new POJO("key2", 4L, "value4")))); PAssert.that(grouped).satisfies(actual -> containsKIterableVs(expected, actual, new POJO[0])); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testGroupByMultiple() { PCollection<KV<Row, Iterable<POJO>>> grouped = pipeline .apply( Create.of( new POJO("key1", 1, "value1"), new POJO("key1", 1, "value2"), new POJO("key2", 2, "value3"), new POJO("key2", 2, "value4"))) .apply(Group.byFieldNames("field1", "field2")); Schema keySchema = Schema.builder().addStringField("field1").addInt64Field("field2").build(); List<KV<Row, Collection<POJO>>> expected = ImmutableList.of( KV.of( Row.withSchema(keySchema).addValues("key1", 1L).build(), ImmutableList.of(new POJO("key1", 1L, "value1"), new POJO("key1", 1L, "value2"))), KV.of( Row.withSchema(keySchema).addValues("key2", 2L).build(), ImmutableList.of(new POJO("key2", 2L, "value3"), new POJO("key2", 2L, "value4")))); PAssert.that(grouped).satisfies(actual -> containsKIterableVs(expected, actual, new POJO[0])); pipeline.run(); }
.apply(Create.of(elements)) .apply( Group.<AggregatePojos>byFieldNames("field2") .aggregateField("field1", Sum.ofLongs(), "field1_sum") .aggregateField("field3", Sum.ofIntegers(), "field3_sum")