@Test public void noInputUnkeyedOutput() { PCollection<KV<Integer, Iterable<Void>>> unkeyed = p.apply( Create.of(KV.<Integer, Iterable<Void>>of(-1, Collections.emptyList())) .withCoder(KvCoder.of(VarIntCoder.of(), IterableCoder.of(VoidCoder.of())))); p.traverseTopologically(visitor); assertThat(visitor.getKeyedPValues(), not(hasItem(unkeyed))); }
@Test public void testGloballyOutputCoder() { p.enableAbandonedNodeEnforcement(false); BigEndianLongCoder inputCoder = BigEndianLongCoder.of(); PCollection<Long> output = p.apply(Create.of(1L, 2L).withCoder(inputCoder)).apply(Latest.globally()); Coder<Long> outputCoder = output.getCoder(); assertThat(outputCoder, instanceOf(NullableCoder.class)); assertEquals(inputCoder, ((NullableCoder<?>) outputCoder).getValueCoder()); }
public PCollection<KV<String, Integer>> createInputTable(Pipeline p) { return p.apply( "CreateInputTable", Create.of(Arrays.asList(TABLE)) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); }
private static PCollection<KV<String, Integer>> createInput( Pipeline p, List<KV<String, Integer>> table) { return p.apply( Create.of(table).withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); }
@Test @Category(NeedsRunner.class) public void testSampleZero() { PCollection<Integer> input = pipeline.apply( Create.of(ImmutableList.copyOf(DATA)).withCoder(BigEndianIntegerCoder.of())); PCollection<Iterable<Integer>> output = input.apply(Sample.fixedSizeGlobally(0)); PAssert.thatSingletonIterable(output).satisfies(new VerifyCorrectSample<>(0, DATA)); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testSample() { PCollection<Integer> input = pipeline.apply( Create.of(ImmutableList.copyOf(DATA)).withCoder(BigEndianIntegerCoder.of())); PCollection<Iterable<Integer>> output = input.apply(Sample.fixedSizeGlobally(3)); PAssert.thatSingletonIterable(output).satisfies(new VerifyCorrectSample<>(3, DATA)); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testConstantVoidKeys() { PCollection<String> input = p.apply(Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of())); PCollection<KV<Void, String>> output = input.apply(WithKeys.of((Void) null)); PAssert.that(output).containsInAnyOrder(WITH_CONST_NULL_KEYS); p.run(); }
@Test public void testCreateDefaultOutputCoderUsingCoder() throws Exception { Coder<Record> coder = new RecordCoder(); assertThat( p.apply(Create.of(new Record(), new Record2()).withCoder(coder)).getCoder(), equalTo(coder)); }
@Test public void testWindowIntoSetWindowfn() { WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10)))) .getWindowingStrategy(); assertTrue(strategy.getWindowFn() instanceof FixedWindows); assertTrue(strategy.getTrigger() instanceof DefaultTrigger); assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, strategy.getMode()); }
@Test @Category(NeedsRunner.class) @SuppressWarnings("unchecked") public void testCountPerElementEmpty() { PCollection<String> input = p.apply(Create.of(NO_LINES).withCoder(StringUtf8Coder.of())); PCollection<KV<String, Long>> output = input.apply(Count.perElement()); PAssert.that(output).empty(); p.run(); }
@Test public void testPerKeySerializabilityRequirement() { p.enableAbandonedNodeEnforcement(false); p.apply( "CreateCollection", Create.of(Arrays.asList(COLLECTION)).withCoder(StringUtf8Coder.of())); PCollection<KV<String, Integer>> inputTable = createInputTable(p); inputTable.apply(Top.perKey(1, new IntegerComparator())); inputTable.apply("PerKey2", Top.perKey(1, new IntegerComparator2())); }
@Test @Category(NeedsRunner.class) public void testCountGloballyEmpty() { PCollection<String> input = p.apply(Create.of(NO_LINES).withCoder(StringUtf8Coder.of())); PCollection<Long> output = input.apply(Count.globally()); PAssert.that(output).containsInAnyOrder(0L); p.run(); }
@Test(expected = NullPointerException.class) public void testJoinRightCollectionNull() { p.enableAbandonedNodeEnforcement(false); Join.leftOuterJoin( p.apply( Create.of(leftListOfKv).withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), null, ""); }
@Test @Category(ValidatesRunner.class) public void testFlattenIterablesCollections() { Set<String> linesSet = ImmutableSet.copyOf(LINES); PCollection<Collection<String>> input = p.apply( Create.<Collection<String>>of(linesSet) .withCoder(CollectionCoder.of(StringUtf8Coder.of()))); PCollection<String> output = input.apply(Flatten.iterables()); PAssert.that(output).containsInAnyOrder(LINES_ARRAY); p.run(); }
@Test @Category(ValidatesRunner.class) public void testFlattenIterablesSets() { Set<String> linesSet = ImmutableSet.copyOf(LINES); PCollection<Set<String>> input = p.apply(Create.<Set<String>>of(linesSet).withCoder(SetCoder.of(StringUtf8Coder.of()))); PCollection<String> output = input.apply(Flatten.iterables()); PAssert.that(output).containsInAnyOrder(LINES_ARRAY); p.run(); }
@Test @Category(ValidatesRunner.class) public void testFlattenIterables() { PCollection<Iterable<String>> input = p.apply( Create.<Iterable<String>>of(LINES).withCoder(IterableCoder.of(StringUtf8Coder.of()))); PCollection<String> output = input.apply(Flatten.iterables()); PAssert.that(output).containsInAnyOrder(LINES_ARRAY); p.run(); }
@Test(expected = IllegalArgumentException.class) public void testSampleNegative() { pipeline.enableAbandonedNodeEnforcement(false); PCollection<Integer> input = pipeline.apply( Create.of(ImmutableList.copyOf(DATA)).withCoder(BigEndianIntegerCoder.of())); input.apply(Sample.fixedSizeGlobally(-1)); }
@Test @Category(ValidatesRunner.class) public void testKeysEmpty() { PCollection<KV<String, Integer>> input = p.apply( Create.of(Arrays.asList(EMPTY_TABLE)) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); PCollection<String> output = input.apply(Keys.create()); PAssert.that(output).empty(); p.run(); } }
@Test @Category(ValidatesRunner.class) public void testKeys() { PCollection<KV<String, Integer>> input = p.apply( Create.of(Arrays.asList(TABLE)) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); PCollection<String> output = input.apply(Keys.create()); PAssert.that(output).containsInAnyOrder("one", "two", "three", "dup", "dup"); p.run(); }
@Test public void testMissingLateness() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); Repeatedly trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(5)); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("requires that the allowed lateness"); pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply("Mode", Window.<String>configure().accumulatingFiredPanes()) .apply("Window", Window.into(fixed10)) .apply("Trigger", Window.<String>configure().triggering(trigger)); }