/** Tests that {@code containsInAnyOrder} is actually order-independent. */ @Test @Category(ValidatesRunner.class) public void testContainsInAnyOrder() throws Exception { PCollection<Integer> pcollection = pipeline.apply(Create.of(1, 2, 3, 4)); PAssert.that(pcollection).containsInAnyOrder(2, 1, 4, 3); pipeline.run(); }
/** The default {@link PipelineOptions} which will be used by {@link #create()}. */ public static PipelineOptions getDefaultOptions() { return TestPipeline.testingPipelineOptions(); }
@Test public void testStableUniqueNameWarning() { pipeline.enableAbandonedNodeEnforcement(false); pipeline.getOptions().setStableUniqueNames(CheckEnabled.WARNING); pipeline.apply(Create.of(5, 6, 7)); pipeline.apply(Create.of(5, 6, 7)); ((Pipeline) pipeline).validate(pipeline.getOptions()); logged.verifyWarn("do not have stable unique names"); }
/** * Runs this {@link TestPipeline}, unwrapping any {@code AssertionError} that is raised during * testing. */ @Override public PipelineResult run() { return run(getOptions()); }
/** * Creates and returns a new test pipeline. * * <p>Use {@link PAssert} to add tests, then call {@link Pipeline#run} to execute the pipeline and * check the tests. */ public static TestPipeline create() { return fromOptions(testingPipelineOptions()); }
@Test(expected = IllegalArgumentException.class) public void testOutputChecking() throws Exception { p.enableAbandonedNodeEnforcement(false); p.apply(new InvalidCompositeTransform()); p.traverseTopologically(new Pipeline.PipelineVisitor.Defaults() {}); }
@Test public void testRuntimeOptionsNotCalledInApply() throws Exception { p.enableAbandonedNodeEnforcement(false); RuntimeTestOptions options = PipelineOptionsFactory.as(RuntimeTestOptions.class); p.apply(TextIO.read().from(options.getInput())); }
@Test @Category(NeedsRunner.class) public void pipelineOptionsDisplayDataExceptionShouldFail() { Object brokenValueType = new Object() { @JsonValue public int getValue() { return 42; } @Override public String toString() { throw new RuntimeException("oh noes!!"); } }; p.getOptions().as(ObjectPipelineOptions.class).setValue(brokenValueType); p.apply(Create.of(1, 2, 3)); expectedException.expectMessage( ProxyInvocationHandler.PipelineOptionsDisplayData.class.getName()); expectedException.expectMessage("oh noes!!"); p.run(); }
/** Creates a simple pipeline with a {@link Combine.PerKey}. */ private static TestPipeline createCombinePerKeyPipeline() { TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false); PCollection<KV<String, Integer>> input = pipeline .apply(Create.of(KV.of("key", 1))) .setCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); input.apply(Combine.perKey(new SumCombineFn())); return pipeline; }
@Test public void testStableUniqueNameError() { pipeline.getOptions().setStableUniqueNames(CheckEnabled.ERROR); pipeline.apply(Create.of(5, 6, 7)); thrown.expectMessage("do not have stable unique names"); pipeline.apply(Create.of(5, 6, 7)); ((Pipeline) pipeline).validate(pipeline.getOptions()); }
@Test public void testRuntimeValueProviderTopic() { TestPipeline pipeline = TestPipeline.create(); ValueProvider<String> topic = pipeline.newProvider("projects/project/topics/topic"); Read<String> pubsubRead = PubsubIO.readStrings().fromTopic(topic); pipeline.apply(pubsubRead); assertThat(pubsubRead.getTopicProvider(), not(nullValue())); assertThat(pubsubRead.getTopicProvider().isAccessible(), is(false)); }
private <K, V> List<BoundedSource<KV<K, V>>> getBoundedSourceList( Class<?> inputFormatClass, Class<K> inputFormatKeyClass, Class<V> inputFormatValueClass, Coder<K> keyCoder, Coder<V> valueCoder) throws Exception { HadoopInputFormatBoundedSource<K, V> boundedSource = getTestHIFSource( inputFormatClass, inputFormatKeyClass, inputFormatValueClass, keyCoder, valueCoder); return boundedSource.split(0, p.getOptions()); } }
@Test public void getInputSucceeds() { TestPipeline p = TestPipeline.create(); PCollection<KV<String, Integer>> input = p.apply( Create.of(KV.of("foo", 1)) .withCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))); PCollection<KV<String, Iterable<Integer>>> grouped = input.apply(GroupByKey.create()); AppliedPTransform<?, ?, ?> producer = DirectGraphs.getProducer(grouped); PTransformReplacement< PCollection<KV<String, Integer>>, PCollection<KV<String, Iterable<Integer>>>> replacement = factory.getReplacementTransform((AppliedPTransform) producer); assertThat(replacement.getInput(), Matchers.<PCollection<?>>equalTo(input)); } }
@Before public void setup() { p = TestPipeline.create(); visitor = KeyedPValueTrackingVisitor.create(); }
@Category(NeedsRunner.class) @Test public void testMissingRunWithDisabledEnforcement() throws Exception { pipeline.enableAbandonedNodeEnforcement(false); addTransform(pCollection(pipeline)); // disable abandoned node detection }
@Test(expected = IllegalArgumentException.class) public void testSampleAnyNegative() { pipeline.enableAbandonedNodeEnforcement(false); pipeline.apply(Create.empty(BigEndianIntegerCoder.of())).apply(Sample.any(-10)); }
@Category({ValidatesRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) @Test public void test() throws Exception { pipeline .apply( // Use maxReadTime to force unbounded mode. GenerateSequence.from(0).to(NUM_ELEMENTS).withMaxReadTime(Duration.standardDays(1))) .apply(ParDo.of(new CountingDoFn())); pipeline.run(); // give metrics pusher time to push Thread.sleep((pipeline.getOptions().getMetricsPushPeriod() + 1L) * 1000); assertThat(TestMetricsSink.getCounterValue(), is(NUM_ELEMENTS)); }
/** Creates a simple pipeline with a {@link Combine.PerKey} with side inputs. */ private static TestPipeline createCombinePerKeyWithSideInputsPipeline() { TestPipeline pipeline = TestPipeline.create().enableAbandonedNodeEnforcement(false); PCollection<KV<String, Integer>> input = pipeline .apply(Create.of(KV.of("key", 1))) .setCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); PCollection<String> sideInput = pipeline.apply(Create.of("side input")); PCollectionView<String> sideInputView = sideInput.apply(View.asSingleton()); input.apply( Combine.<String, Integer, Integer>perKey(new SumCombineFnWithContext()) .withSideInputs(sideInputView)); return pipeline; }
/** A basic smoke test that ensures there is no crash at pipeline construction time. */ @Test public void testMinimalWordCount() throws Exception { p.getOptions().as(GcsOptions.class).setGcsUtil(buildMockGcsUtil()); p.apply(TextIO.read().from("gs://apache-beam-samples/shakespeare/*")) .apply( FlatMapElements.into(TypeDescriptors.strings()) .via((String word) -> Arrays.asList(word.split("[^a-zA-Z']+")))) .apply(Filter.by((String word) -> !word.isEmpty())) .apply(Count.perElement()) .apply( MapElements.into(TypeDescriptors.strings()) .via( (KV<String, Long> wordCount) -> wordCount.getKey() + ": " + wordCount.getValue())) .apply(TextIO.write().to("gs://your-output-bucket/and-output-prefix")); }