Refine search
@Test public void testDeserialization() Assert.assertEquals(new Duration(15 * 60 * 1000), config.getLoadTimeoutDelay()); Assert.assertNull(config.getConsoleStatic()); Assert.assertEquals(Duration.millis(50), config.getLoadQueuePeonRepeatDelay()); Assert.assertEquals(new Duration("PT1s"), config.getLoadTimeoutDelay()); Assert.assertEquals("test", config.getConsoleStatic()); Assert.assertEquals(Duration.millis(100), config.getLoadQueuePeonRepeatDelay());
/** Tests that the watermark that guarantees firing is that of the subtrigger. */ @Test public void testFireDeadline() throws Exception { setUp(FixedWindows.of(Duration.millis(10))); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); Instant arbitraryInstant = new Instant(34957849); when(mockTrigger.getWatermarkThatGuaranteesFiring(Mockito.<IntervalWindow>any())) .thenReturn(arbitraryInstant); assertThat( Repeatedly.forever(mockTrigger).getWatermarkThatGuaranteesFiring(window), equalTo(arbitraryInstant)); }
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testAssignWindowsWithSessions() { WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10)); WindowAssignTranslator.AssignWindows<String> assignWindows = new WindowAssignTranslator.AssignWindows(slidingWindows); String value = "v1"; Instant timestamp = new Instant(1); WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow(value, timestamp); ArrayList<WindowedValue<String>> expected = new ArrayList<>(); expected.add( WindowedValue.of( value, timestamp, new IntervalWindow(new Instant(1), new Instant(11)), PaneInfo.NO_FIRING)); Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue); assertThat(expected, equalTo(Lists.newArrayList(result))); }
private static void awaitWatermarksOrTimeout( TestSparkPipelineOptions testSparkPipelineOptions, SparkPipelineResult result) { Long timeoutMillis = Duration.standardSeconds(checkNotNull(testSparkPipelineOptions.getTestTimeoutSeconds())) .getMillis(); Long batchDurationMillis = testSparkPipelineOptions.getBatchIntervalMillis(); Instant stopPipelineWatermark = new Instant(testSparkPipelineOptions.getStopPipelineWatermark()); // we poll for pipeline status in batch-intervals. while this is not in-sync with Spark's // execution clock, this is good enough. // we break on timeout or end-of-time WM, which ever comes first. Instant globalWatermark; result.waitUntilFinish(Duration.millis(batchDurationMillis)); do { SparkTimerInternals sparkTimerInternals = SparkTimerInternals.global(GlobalWatermarkHolder.get(batchDurationMillis)); sparkTimerInternals.advanceWatermark(); globalWatermark = sparkTimerInternals.currentInputWatermarkTime(); // let another batch-interval period of execution, just to reason about WM propagation. Uninterruptibles.sleepUninterruptibly(batchDurationMillis, TimeUnit.MILLISECONDS); } while ((timeoutMillis -= batchDurationMillis) > 0 && globalWatermark.isBefore(stopPipelineWatermark)); } }
private void testWindowedSideInput(IsBounded bounded) { PCollection<Integer> mainInput = p.apply( "main", Create.timestamped( TimestampedValue.of(0, new Instant(0)), TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)), TimestampedValue.of(3, new Instant(3)), TimestampedValue.of(4, new Instant(4)), TimestampedValue.of(5, new Instant(5)), TimestampedValue.of(6, new Instant(6)), TimestampedValue.of(7, new Instant(7)))) .apply("window 2", Window.into(FixedWindows.of(Duration.millis(2)))); PCollectionView<String> sideInput = p.apply( "side", Create.timestamped( TimestampedValue.of("a", new Instant(0)), TimestampedValue.of("b", new Instant(4)))) .apply("window 4", Window.into(FixedWindows.of(Duration.millis(4)))) .apply("singleton", View.asSingleton()); PCollection<String> res = mainInput.apply(ParDo.of(sdfWithSideInput(bounded, sideInput)).withSideInputs(sideInput)); PAssert.that(res).containsInAnyOrder("a:0", "a:1", "a:2", "a:3", "b:4", "b:5", "b:6", "b:7"); p.run(); }
@Test public void testShouldFire() throws Exception { TriggerStateMachineTester<Integer, IntervalWindow> tester = TriggerStateMachineTester.forTrigger( ReshuffleTriggerStateMachine.create(), FixedWindows.of(Duration.millis(100))); IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(300), new Instant(400)); assertTrue(tester.shouldFire(arbitraryWindow)); }
@Test public void retryer_should_wait_after_failure_before_trying_again() throws Exception { Retryer timeRetryer = new Retryer(1, Duration.millis(100)); Stopwatch stopwatch = Stopwatch.createStarted(); when(operation.call()).thenThrow(new DockerExecutionException()).thenAnswer(i -> { assertThat(stopwatch.elapsed(TimeUnit.MILLISECONDS), greaterThan(100L)); return "success"; }); String result = timeRetryer.runWithRetries(operation); assertThat(result, is("success")); }
@Test public void testAfterProcessingTimeWithMergingWindow() throws Exception { SimpleTriggerStateMachineTester<IntervalWindow> tester = TriggerStateMachineTester.forTrigger( AfterProcessingTimeStateMachine.pastFirstElementInPane() .plusDelayOf(Duration.millis(5)), Sessions.withGapDuration(Duration.millis(10))); tester.advanceProcessingTime(new Instant(10)); tester.injectElements(1); // in [1, 11), timer for 15 IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); assertFalse(tester.shouldFire(firstWindow)); tester.advanceProcessingTime(new Instant(12)); tester.injectElements(3); // in [3, 13), timer for 17 IntervalWindow secondWindow = new IntervalWindow(new Instant(3), new Instant(13)); assertFalse(tester.shouldFire(secondWindow)); tester.mergeWindows(); IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(13)); tester.advanceProcessingTime(new Instant(16)); assertTrue(tester.shouldFire(mergedWindow)); }
/** Test a WriteFiles with a windowed PCollection. */ @Test @Category(NeedsRunner.class) public void testWriteWindowed() throws IOException { List<String> inputs = Arrays.asList( "Critical canary", "Apprehensive eagle", "Intimidating pigeon", "Pedantic gull", "Frisky finch"); runWrite( inputs, new WindowAndReshuffle<>(Window.into(FixedWindows.of(Duration.millis(2)))), getBaseOutputFilename(), WriteFiles.to(makeSimpleSink())); }
@Parameters(name = "{index}: {0}") public static Iterable<TestStream<?>> data() { return ImmutableList.of( TestStream.create(VarIntCoder.of()).advanceWatermarkToInfinity(), TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(new Instant(42)) .advanceWatermarkToInfinity(), TestStream.create(VarIntCoder.of()) .addElements(TimestampedValue.of(3, new Instant(17))) .advanceWatermarkToInfinity(), TestStream.create(StringUtf8Coder.of()) .advanceProcessingTime(Duration.millis(82)) .advanceWatermarkToInfinity()); }
/** Tests that the repeatedly is ready to fire whenever the subtrigger is ready. */ @Test public void testShouldFire() throws Exception { setUp(FixedWindows.of(Duration.millis(10))); when(mockTrigger.shouldFire(anyTriggerContext())).thenReturn(true); assertTrue(tester.shouldFire(new IntervalWindow(new Instant(0), new Instant(10)))); when(mockTrigger.shouldFire(Mockito.any())).thenReturn(false); assertFalse(tester.shouldFire(new IntervalWindow(new Instant(0), new Instant(10)))); }
/** Validates that the output timestamp for aggregate data falls within the acceptable range. */ @Test public void testValidOutputTimes() throws Exception { for (long timestamp : Arrays.asList(200, 800, 700)) { WindowFnTestUtils.validateGetOutputTimestamp( Sessions.withGapDuration(Duration.millis(500)), timestamp); } }
@Test public void testShouldFireAfterMerge() throws Exception { tester = TriggerStateMachineTester.forTrigger( RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(2)), Sessions.withGapDuration(Duration.millis(10))); tester.injectElements(1); IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); assertFalse(tester.shouldFire(firstWindow)); tester.injectElements(5); IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15)); assertFalse(tester.shouldFire(secondWindow)); // Merge them, if the merged window were on the second trigger, it would be ready tester.mergeWindows(); IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15)); assertTrue(tester.shouldFire(mergedWindow)); }
/** * With {@link #testWindowIntoNullWindowFnNoAssign()}, demonstrates that the expansions of the * {@link Window} transform depends on if it actually assigns elements to windows. */ @Test public void testWindowIntoWindowFnAssign() { pipeline .apply(Create.of(1, 2, 3)) .apply( Window.into(FixedWindows.of(Duration.standardMinutes(11L).plus(Duration.millis(1L))))); final AtomicBoolean foundAssign = new AtomicBoolean(false); pipeline.traverseTopologically( new PipelineVisitor.Defaults() { @Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { if (node.getTransform() instanceof Window.Assign) { foundAssign.set(true); } } }); assertThat(foundAssign.get(), is(true)); }
@Override public void populateDisplayData(Builder builder) { builder .add(DisplayData.item("string", stringValue)) .add(DisplayData.item("long", intValue)) .add(DisplayData.item("double", floatValue)) .add(DisplayData.item("boolean", boolValue)) .add(DisplayData.item("instant", new Instant(0))) .add(DisplayData.item("duration", Duration.millis(durationMillis))) .add( DisplayData.item("class", DisplayDataTest.class) .withLinkUrl("http://abc") .withLabel("baz")); } };
@Test public void testOnTimer() throws Exception { TriggerStateMachineTester<Integer, IntervalWindow> tester = TriggerStateMachineTester.forTrigger( ReshuffleTriggerStateMachine.create(), FixedWindows.of(Duration.millis(100))); IntervalWindow arbitraryWindow = new IntervalWindow(new Instant(100), new Instant(200)); tester.fireIfShouldFire(arbitraryWindow); assertFalse(tester.isMarkedFinished(arbitraryWindow)); }
@Test public void testAfterProcessingTimeWithMergingWindow() throws Exception { Duration windowDuration = Duration.millis(10); SimpleTriggerStateMachineTester<IntervalWindow> tester = TriggerStateMachineTester.forTrigger( AfterProcessingTimeStateMachine.pastFirstElementInPane() .plusDelayOf(Duration.millis(5)), Sessions.withGapDuration(windowDuration)); tester.advanceProcessingTime(new Instant(10)); tester.injectElements(1); // in [1, 11), timer for 15 IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11)); assertFalse(tester.shouldFire(firstWindow)); tester.advanceProcessingTime(new Instant(12)); tester.injectElements(3); // in [3, 13), timer for 17 IntervalWindow secondWindow = new IntervalWindow(new Instant(3), new Instant(13)); assertFalse(tester.shouldFire(secondWindow)); tester.mergeWindows(); IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(13)); tester.advanceProcessingTime(new Instant(16)); assertTrue(tester.shouldFire(mergedWindow)); } }
@Test @Category(NeedsRunner.class) public void testTextIoInput() throws Exception { File tmpFile = tmpFolder.newFile("file.txt"); String filename = tmpFile.getPath(); try (PrintStream writer = new PrintStream(new FileOutputStream(tmpFile))) { writer.println("a 1"); writer.println("b 2"); writer.println("b 3"); writer.println("c 11"); writer.println("d 11"); } PCollection<String> output = p.begin() .apply("ReadLines", TextIO.read().from(filename)) .apply(ParDo.of(new ExtractWordsWithTimestampsFn())) .apply(new WindowedCount(FixedWindows.of(Duration.millis(10)))); PAssert.that(output) .containsInAnyOrder( output("a", 1, 1, 0, 10), output("b", 2, 2, 0, 10), output("c", 1, 11, 10, 20), output("d", 1, 11, 10, 20)); p.run(); }