/** * @param p pipeline. * @return source. */ private static PCollection<Long> getSource(final Pipeline p) { return p.apply(GenerateSequence .from(1) .withRate(2, Duration.standardSeconds(1)) .withTimestampFn(num -> new Instant(num * 500))); // 0.5 second between subsequent elements } /**
@Test public void nonAdditionalInputsWithOneMainInputSucceeds() { PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L)); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "input-single", Collections.singletonMap(new TupleTag<Long>() {}, input), Collections.emptyMap(), new TestTransform(), pipeline); assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.containsInAnyOrder(input)); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testAssertionSiteIsCapturedWithMessage() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); assertThatCollectionIsEmptyWithMessage(vals); Throwable thrown = runExpectingAssertionFailure(pipeline); assertThat(thrown.getMessage(), containsString("Should be empty")); assertThat(thrown.getMessage(), containsString("Expected: iterable over [] in any order")); String stacktrace = Throwables.getStackTraceAsString(thrown); assertThat(stacktrace, containsString("testAssertionSiteIsCapturedWithMessage")); assertThat(stacktrace, containsString("assertThatCollectionIsEmptyWithMessage")); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testAssertionSiteIsCapturedWithoutMessage() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); assertThatCollectionIsEmptyWithoutMessage(vals); Throwable thrown = runExpectingAssertionFailure(pipeline); assertThat(thrown.getMessage(), containsString("Expected: iterable over [] in any order")); String stacktrace = Throwables.getStackTraceAsString(thrown); assertThat(stacktrace, containsString("testAssertionSiteIsCapturedWithoutMessage")); assertThat(stacktrace, containsString("assertThatCollectionIsEmptyWithoutMessage")); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testEmptyFalse() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); PAssert.that("Vals should have been empty", vals).empty(); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("Vals should have been empty")); assertThat(message, containsString("Expected: iterable over [] in any order")); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testEmptyFalseDefaultReasonString() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); PAssert.that(vals).empty(); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("GenerateSequence/Read(BoundedCountingSource).out")); assertThat(message, containsString("Expected: iterable over [] in any order")); }
@Test public void registerPCollection() throws IOException { PCollection<Long> pCollection = pipeline.apply(GenerateSequence.from(0)).setName("foo"); String id = components.registerPCollection(pCollection); assertThat(id, equalTo("foo")); components.toComponents().getPcollectionsOrThrow(id); }
@Test public void mapOutputsSucceeds() { PCollection<Long> original = pipeline.apply("Original", GenerateSequence.from(0)); PCollection<Long> replacement = pipeline.apply("Replacement", GenerateSequence.from(0)); Map<PValue, ReplacementOutput> mapping = factory.mapOutputs(original.expand(), replacement); assertThat( mapping, Matchers.hasEntry( replacement, ReplacementOutput.of( TaggedPValue.ofExpandedValue(original), TaggedPValue.ofExpandedValue(replacement)))); }
@Test @Category(NeedsRunner.class) public void testBoundedInput() { long numElements = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements)); addCountingAsserts(input, 0, numElements); p.run(); }
@Test @Category(NeedsRunner.class) public void testBoundedInputSubrange() { long start = 10; long end = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(start).to(end)); addCountingAsserts(input, start, end); p.run(); }
@Test public void testTranslationModeOverrideWithUnboundedSources() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(FlinkRunner.class); options.setStreaming(false); FlinkPipelineExecutionEnvironment flinkEnv = new FlinkPipelineExecutionEnvironment(options); Pipeline pipeline = Pipeline.create(options); pipeline.apply(GenerateSequence.from(0)); flinkEnv.translate(pipeline); assertThat(options.isStreaming(), is(true)); }
@Test @Category(NeedsRunner.class) public void testEmptyBoundedInput() { PCollection<Long> input = p.apply(GenerateSequence.from(0).to(0)); PAssert.that(input).empty(); p.run(); }
@Test @Category(NeedsRunner.class) public void testEmptyBoundedInputSubrange() { PCollection<Long> input = p.apply(GenerateSequence.from(42).to(42)); PAssert.that(input).empty(); p.run(); }
@Test public void testBoundedDisplayData() { PTransform<?, ?> input = GenerateSequence.from(0).to(1234); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("from", 0)); assertThat(displayData, hasDisplayItem("to", 1234)); }
@Test public void testBoundedDisplayDataSubrange() { PTransform<?, ?> input = GenerateSequence.from(12).to(1234); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("from", 12)); assertThat(displayData, hasDisplayItem("to", 1234)); }
@Category({ValidatesRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) @Test public void test() throws Exception { pipeline .apply( // Use maxReadTime to force unbounded mode. GenerateSequence.from(0).to(NUM_ELEMENTS).withMaxReadTime(Duration.standardDays(1))) .apply(ParDo.of(new CountingDoFn())); pipeline.run(); // give metrics pusher time to push Thread.sleep((pipeline.getOptions().getMetricsPushPeriod() + 1L) * 1000); assertThat(TestMetricsSink.getCounterValue(), is(NUM_ELEMENTS)); }
@Test public void testUnboundedDisplayData() { Duration maxReadTime = Duration.standardHours(5); SerializableFunction<Long, Instant> timestampFn = input -> Instant.now(); PTransform<?, ?> input = GenerateSequence.from(0).to(1234).withMaxReadTime(maxReadTime).withTimestampFn(timestampFn); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("maxReadTime", maxReadTime)); assertThat(displayData, hasDisplayItem("timestampFn", timestampFn.getClass())); }
@Test @Ignore("https://issues.apache.org/jira/browse/BEAM-4088 Test reliably fails.") public void testNoThreadsLeakInPipelineExecution() { pipeline.apply(GenerateSequence.from(0).to(NUM_ELEMENTS)).apply(ParDo.of(new CountingDoFn())); pipeline.run(); }
@Test public void testUnencodableOutputFromUnboundedRead() { Pipeline p = getPipeline(); p.apply(GenerateSequence.from(0)).setCoder(new LongNoDecodeCoder()); thrown.expectCause(isA(CoderException.class)); thrown.expectMessage("Cannot decode a long"); p.run(); }
@Test public void testUnencodableOutputFromBoundedRead() throws Exception { Pipeline p = getPipeline(); p.apply(GenerateSequence.from(0).to(10)).setCoder(new LongNoDecodeCoder()); thrown.expectCause(isA(CoderException.class)); thrown.expectMessage("Cannot decode a long"); p.run(); }