@Test @Category(NeedsRunner.class) public void testSampleAnyEmpty() { PCollection<Integer> input = pipeline.apply(Create.empty(BigEndianIntegerCoder.of())); PCollection<Integer> output = input .apply(Window.into(FixedWindows.of(Duration.standardSeconds(3)))) .apply(Sample.any(2)); PAssert.that(output).satisfies(new VerifyCorrectSample<>(0, EMPTY)); pipeline.run(); }
@Test public void testNonDeterministicWindowCoder() throws NonDeterministicException { FixedWindows mockWindowFn = Mockito.mock(FixedWindows.class); @SuppressWarnings({"unchecked", "rawtypes"}) Class<Coder<IntervalWindow>> coderClazz = (Class) Coder.class; Coder<IntervalWindow> mockCoder = Mockito.mock(coderClazz); when(mockWindowFn.windowCoder()).thenReturn(mockCoder); NonDeterministicException toBeThrown = new NonDeterministicException(mockCoder, "Its just not deterministic."); Mockito.doThrow(toBeThrown).when(mockCoder).verifyDeterministic(); thrown.expect(IllegalArgumentException.class); thrown.expectCause(Matchers.sameInstance(toBeThrown)); thrown.expectMessage("Window coders must be deterministic"); Window.into(mockWindowFn); }
/** Test a WriteFiles with a windowed PCollection. */ @Test @Category(NeedsRunner.class) public void testWriteWindowed() throws IOException { List<String> inputs = Arrays.asList( "Critical canary", "Apprehensive eagle", "Intimidating pigeon", "Pedantic gull", "Frisky finch"); runWrite( inputs, new WindowAndReshuffle<>(Window.into(FixedWindows.of(Duration.millis(2)))), getBaseOutputFilename(), WriteFiles.to(makeSimpleSink())); }
@Test public void testUnsupportedGlobalWindowWithDefaultTrigger() { exceptions.expect(UnsupportedOperationException.class); pipeline.enableAbandonedNodeEnforcement(false); PCollection<Row> input = unboundedInput1.apply( "unboundedInput1.globalWindow", Window.<Row>into(new GlobalWindows()).triggering(DefaultTrigger.of())); String sql = "SELECT f_int2, COUNT(*) AS `size` FROM PCOLLECTION GROUP BY f_int2"; input.apply("testUnsupportedGlobalWindows", SqlTransform.query(sql)); }
/** Test a WriteFiles with sessions. */ @Test @Category(NeedsRunner.class) public void testWriteWithSessions() throws IOException { List<String> inputs = Arrays.asList( "Critical canary", "Apprehensive eagle", "Intimidating pigeon", "Pedantic gull", "Frisky finch"); runWrite( inputs, new WindowAndReshuffle<>(Window.into(Sessions.withGapDuration(Duration.millis(1)))), getBaseOutputFilename(), WriteFiles.to(makeSimpleSink())); }
@Test @Category({ValidatesRunner.class, DataflowPortabilityApiUnsupported.class}) public void testWindowedCombineEmpty() { PCollection<Double> mean = pipeline .apply(Create.empty(BigEndianIntegerCoder.of())) .apply(Window.into(FixedWindows.of(Duration.millis(1)))) .apply(Combine.globally(new MeanInts()).withoutDefaults()); PAssert.that(mean).empty(); pipeline.run(); }
@Test public void classEqualToDoesNotMatchUnrelatedClass() { PTransformMatcher matcher = PTransformMatchers.classEqualTo(ParDo.SingleOutput.class); AppliedPTransform<?, ?, ?> application = getAppliedTransform(Window.<KV<String, Integer>>into(new GlobalWindows())); assertThat(matcher.matches(application), is(false)); }
@Test @Category(NeedsRunner.class) public void testWriteSpilling() throws IOException { List<String> inputs = Lists.newArrayList(); for (int i = 0; i < 100; ++i) { inputs.add("mambo_number_" + i); } runWrite( inputs, Window.into(FixedWindows.of(Duration.millis(2))), getBaseOutputFilename(), WriteFiles.to(makeSimpleSink()) .withMaxNumWritersPerBundle(2) .withWindowedWrites() .withNumShards(1)); }
@Test public void testInvalidWindowsService() { Pipeline p = createTestServiceRunner(); List<KV<String, Integer>> ungroupedPairs = Arrays.asList(); PCollection<KV<String, Integer>> input = p.apply( Create.of(ungroupedPairs) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))) .apply(Window.into(Sessions.withGapDuration(Duration.standardMinutes(1)))); thrown.expect(IllegalStateException.class); thrown.expectMessage("GroupByKey must have a valid Window merge function"); input.apply("GroupByKey", GroupByKey.create()).apply("GroupByKeyAgain", GroupByKey.create()); }
@Test public void testAssignDisplayDataUnchanged() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); Window<Object> original = Window.into(windowFn); WindowingStrategy<?, ?> updated = WindowingStrategy.globalDefault().withWindowFn(windowFn); DisplayData displayData = DisplayData.from(new Window.Assign<>(original, updated)); assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(displayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(displayData, not(hasDisplayItem("trigger"))); assertThat(displayData, not(hasDisplayItem("accumulationMode"))); assertThat(displayData, not(hasDisplayItem("allowedLateness"))); assertThat(displayData, not(hasDisplayItem("closingBehavior"))); assertThat(displayData, not(hasDisplayItem("timestampCombiner"))); }
@Test public void testIncompatibleWindowFnPropagationFailure() { p.enableAbandonedNodeEnforcement(false); PCollection<String> input1 = p.apply("CreateInput1", Create.of("Input1")) .apply("Window1", Window.into(FixedWindows.of(Duration.standardMinutes(1)))); PCollection<String> input2 = p.apply("CreateInput2", Create.of("Input2")) .apply("Window2", Window.into(FixedWindows.of(Duration.standardMinutes(2)))); try { PCollectionList.of(input1).and(input2).apply(Flatten.pCollections()); Assert.fail("Exception should have been thrown"); } catch (IllegalStateException e) { Assert.assertTrue( e.getMessage().startsWith("Inputs to Flatten had incompatible window windowFns")); } }
@Test public void testDisplayDataExcludesDefaults() { Window<?> window = Window.into(new GlobalWindows()) .triggering(DefaultTrigger.of()) .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis())); DisplayData data = DisplayData.from(window); assertThat(data, not(hasDisplayItem("trigger"))); assertThat(data, not(hasDisplayItem("allowedLateness"))); }
/** * Applies a window to the input collection if one hasn't already been specified. * * @return the input collection if it already has been windowed, otherwise a the same collection inside a default * window. */ public static <T> PCollection<T> ofDefaultWindow(PCollection<T> in) { if (in.getWindowingStrategy() != WindowingStrategy.globalDefault() && in.getWindowingStrategy() != null) return in; return in.apply("ApplyDefaultWindow", Window.<T> into(FixedWindows.of(DEFAULT_WINDOW_SIZE))); }