private static PCollection<Integer> makeUnboundedCollection(Pipeline pipeline) { return pipeline .apply("unbounded", Create.of(1, 2, 3)) .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED); }
private WriteResult writeResult(Pipeline p) { PCollection<TableRow> empty = p.apply("CreateEmptyFailedInserts", Create.empty(TypeDescriptor.of(TableRow.class))); return WriteResult.in(p, new TupleTag<>("failedInserts"), empty); }
@Test public void testCreateGetName() { assertEquals("Create.Values", Create.of(1, 2, 3).getName()); assertEquals("Create.TimestampedValues", Create.timestamped(Collections.emptyList()).getName()); }
@Test @Category(NeedsRunner.class) public void testCreateTimestampedEmpty() { PCollection<String> output = p.apply( Create.timestamped(new ArrayList<TimestampedValue<String>>()) .withCoder(StringUtf8Coder.of())); PAssert.that(output).empty(); p.run(); }
@Override public PCollection<MatchResult.Metadata> expand(PBegin input) { return input .apply("Create filepattern", Create.ofProvider(getFilepattern(), StringUtf8Coder.of())) .apply("Via MatchAll", matchAll().withConfiguration(getConfiguration())); } }
@Test public void testCreateTimestampedEmptyUnspecifiedCoder() { p.enableAbandonedNodeEnforcement(false); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("determine a default Coder"); thrown.expectMessage("Create.empty(Coder)"); thrown.expectMessage("Create.empty(TypeDescriptor)"); thrown.expectMessage("withCoder(Coder)"); thrown.expectMessage("withType(TypeDescriptor)"); p.apply(Create.timestamped(new ArrayList<>())); }
@Override public PCollection<Ddl> expand(PBegin p) { return p.apply("Create empty", Create.of((Void) null)) .apply( "Read Information Schema", ParDo.of(new ReadInformationSchemaFn(spannerConfig, tx)).withSideInputs(tx)); }
@Test public void parDoWithFnTypeNotParDo() { AppliedPTransform<?, ?, ?> notParDo = getAppliedTransform(Create.empty(VoidCoder.of())); PTransformMatcher matcher = PTransformMatchers.parDoWithFnType(doFnWithState.getClass()); assertThat(matcher.matches(notParDo), is(false)); }
@Test public void testCreateTimestampedDefaultOutputCoderUsingCoder() throws Exception { Coder<Record> coder = new RecordCoder(); Create.TimestampedValues<Record> values = Create.timestamped( TimestampedValue.of(new Record(), new Instant(0)), TimestampedValue.of(new Record2(), new Instant(0))) .withCoder(coder); assertThat(p.apply(values).getCoder(), equalTo(coder)); }
@Test public void testRunWithDummyEnvironmentVariableFails() { System.getProperties() .setProperty(TestPipeline.PROPERTY_USE_DEFAULT_DUMMY_RUNNER, Boolean.toString(true)); pipeline.apply(Create.of(1, 2, 3)); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Cannot call #run"); pipeline.run(); }
@Test(expected = NullPointerException.class) public void testJoinNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.rightOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), null); } }
@Override public PTransformReplacement<PBegin, PCollection<Long>> getReplacementTransform( AppliedPTransform<PBegin, PCollection<Long>, GenerateSequence> transform) { return PTransformReplacement.of(transform.getPipeline().begin(), Create.of(0L)); }
@Test(expected = NullPointerException.class) public void testJoinLeftNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.fullOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), null, ""); }
@Test public void testGetJobMetricsThatFailsForException() throws Exception { DataflowPipelineJob job = spy(new DataflowPipelineJob(mockClient, "test-job", options, null)); Pipeline p = TestPipeline.create(options); p.apply(Create.of(1, 2, 3)); when(mockClient.getJobMetrics(anyString())).thenThrow(new IOException()); TestDataflowRunner runner = TestDataflowRunner.fromOptionsAndClient(options, mockClient); assertNull(runner.getJobMetrics(job)); }
@Test(expected = NullPointerException.class) public void testJoinNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.leftOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), null); } }
public PCollection<KV<String, Integer>> createInputTable(Pipeline p) { return p.apply( "CreateInputTable", Create.of(Arrays.asList(TABLE)) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); }
@Test @Category(NeedsRunner.class) public void testEmptyInput() { PCollection<String> input = p.apply(Create.empty(StringUtf8Coder.of())); PCollection<String> output = input.apply(new WindowedCount(FixedWindows.of(new Duration(10)))); PAssert.that(output).empty(); p.run(); }
@Test public void testParDoWithNoOutputsErrorDoesNotMentionTupleTag() { PCollection<EmptyClass> input = p.apply(Create.of(1, 2, 3)).apply(ParDo.of(new EmptyClassDoFn())); thrown.expect(IllegalStateException.class); // Output specific to ParDo additional TupleTag outputs should not be present. thrown.expectMessage(not(containsString("erasure"))); thrown.expectMessage(not(containsString("see TupleTag Javadoc"))); // Instead, expect output suggesting other possible fixes. thrown.expectMessage("Building a Coder using a registered CoderProvider failed"); input.getCoder(); }
@Test(expected = IllegalArgumentException.class) public void testSampleAnyNegative() { pipeline.enableAbandonedNodeEnforcement(false); pipeline.apply(Create.empty(BigEndianIntegerCoder.of())).apply(Sample.any(-10)); }
@Test public void testStableUniqueNameWarning() { pipeline.enableAbandonedNodeEnforcement(false); pipeline.getOptions().setStableUniqueNames(CheckEnabled.WARNING); pipeline.apply(Create.of(5, 6, 7)); pipeline.apply(Create.of(5, 6, 7)); ((Pipeline) pipeline).validate(pipeline.getOptions()); logged.verifyWarn("do not have stable unique names"); }