protected PCollection<Row> getFloorCeilingTestPCollection() { try { return TestBoundedTable.of(ROW_TYPE_THREE) .addRows(parseTimestampWithUTCTimeZone("1986-02-15 11:35:26"), 1.4) .buildIOReader(pipeline.begin()) .setRowSchema(ROW_TYPE_THREE); } catch (Exception e) { throw new RuntimeException(e); } }
private PCollection<Row> queryFromOrderTables(String sql) { return tuple( "ORDER_DETAILS1", ORDER_DETAILS1.buildIOReader(pipeline.begin()).setRowSchema(SOURCE_ROW_TYPE), "ORDER_DETAILS2", ORDER_DETAILS2.buildIOReader(pipeline.begin()).setRowSchema(SOURCE_ROW_TYPE)) .apply("join", SqlTransform.query(sql)) .setRowSchema(RESULT_ROW_TYPE); } }
@Test public void registerTransformNoChildren() throws IOException { Create.Values<Integer> create = Create.of(1, 2, 3); PCollection<Integer> pt = pipeline.apply(create); String userName = "my_transform/my_nesting"; AppliedPTransform<?, ?, ?> transform = AppliedPTransform.of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline); String componentName = components.registerPTransform(transform, Collections.emptyList()); assertThat(componentName, equalTo(userName)); assertThat(components.getExistingPTransformId(transform), equalTo(componentName)); }
@Test @Category(NeedsRunner.class) public void testSimpleTransform() throws Exception { File tmpFile = tmpFolder.newFile("file.txt"); String filename = tmpFile.getPath(); p.begin().apply(new SimpleTransform(filename)); p.run(); } }
protected PCollection<Row> getTestPCollection() { try { return TestBoundedTable.of(ROW_TYPE) .addRows( parseTimestampWithUTCTimeZone("1986-02-15 11:35:26"), (byte) 1, (short) 1, 1, 1L, 1.0f, 1.0, BigDecimal.ONE, (byte) 127, (short) 32767, 2147483647, 9223372036854775807L) .buildIOReader(pipeline.begin()) .setRowSchema(ROW_TYPE); } catch (Exception e) { throw new RuntimeException(e); } }
/** Tests that trying to register a transform which has unregistered children throws. */ @Test public void registerTransformWithUnregisteredChildren() throws IOException { Create.Values<Long> create = Create.of(1L, 2L, 3L); GenerateSequence createChild = GenerateSequence.from(0); PCollection<Long> pt = pipeline.apply(create); String userName = "my_transform"; String childUserName = "my_transform/my_nesting"; AppliedPTransform<?, ?, ?> transform = AppliedPTransform.of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline); AppliedPTransform<?, ?, ?> childTransform = AppliedPTransform.of( childUserName, pipeline.begin().expand(), pt.expand(), createChild, pipeline); thrown.expect(IllegalArgumentException.class); thrown.expectMessage(childTransform.toString()); components.registerPTransform(transform, Collections.singletonList(childTransform)); }
@Ignore @Test @Category(ValidatesRunner.class) public void testEmptyTransform() { p.begin().apply(new EmptyTransform()); p.run(); }
@Test @Category(NeedsRunner.class) public void testTextIoInput() throws Exception { File tmpFile = tmpFolder.newFile("file.txt"); String filename = tmpFile.getPath(); try (PrintStream writer = new PrintStream(new FileOutputStream(tmpFile))) { writer.println("a 1"); writer.println("b 2"); writer.println("b 3"); writer.println("c 11"); writer.println("d 11"); } PCollection<String> output = p.begin() .apply("ReadLines", TextIO.read().from(filename)) .apply(ParDo.of(new ExtractWordsWithTimestampsFn())) .apply(new WindowedCount(FixedWindows.of(Duration.millis(10)))); PAssert.that(output) .containsInAnyOrder( output("a", 1, 1, 0, 10), output("b", 2, 2, 0, 10), output("c", 1, 11, 10, 20), output("d", 1, 11, 10, 20)); p.run(); }
@Test public void registerTransformEmptyFullName() throws IOException { Create.Values<Integer> create = Create.of(1, 2, 3); PCollection<Integer> pt = pipeline.apply(create); AppliedPTransform<?, ?, ?> transform = AppliedPTransform.of("", pipeline.begin().expand(), pt.expand(), create, pipeline); thrown.expect(IllegalArgumentException.class); thrown.expectMessage(transform.toString()); components.getExistingPTransformId(transform); }
@Test public void registerTransformAfterChildren() throws IOException { Create.Values<Long> create = Create.of(1L, 2L, 3L); GenerateSequence createChild = GenerateSequence.from(0); PCollection<Long> pt = pipeline.apply(create); String userName = "my_transform"; String childUserName = "my_transform/my_nesting"; AppliedPTransform<?, ?, ?> transform = AppliedPTransform.of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline); AppliedPTransform<?, ?, ?> childTransform = AppliedPTransform.of( childUserName, pipeline.begin().expand(), pt.expand(), createChild, pipeline); String childId = components.registerPTransform(childTransform, Collections.emptyList()); String parentId = components.registerPTransform(transform, Collections.singletonList(childTransform)); Components components = this.components.toComponents(); assertThat(components.getTransformsOrThrow(parentId).getSubtransforms(0), equalTo(childId)); assertThat(components.getTransformsOrThrow(childId).getSubtransformsCount(), equalTo(0)); }
BigDecimal.valueOf(3.0)) .addRows(null, null, null, null, null, null, null, null, null, null) .buildIOReader(pipeline.begin()) .setRowSchema(ROW_TYPE_TWO); } catch (Exception e) {
@Test @Category(NeedsRunner.class) public void testMultiGraphSetup() { PCollection<Integer> input = p.begin().apply(Create.of(1, 2, 3)); input.apply(new UnboundOutputCreator()); p.run(); } }
@Test public void registerTransformNullComponents() throws IOException { Create.Values<Integer> create = Create.of(1, 2, 3); PCollection<Integer> pt = pipeline.apply(create); String userName = "my_transform/my_nesting"; AppliedPTransform<?, ?, ?> transform = AppliedPTransform.of(userName, pipeline.begin().expand(), pt.expand(), create, pipeline); thrown.expect(NullPointerException.class); thrown.expectMessage("child nodes may not be null"); components.registerPTransform(transform, null); }
public Pipeline pipelineWith(CustomTestCoder coder) throws Exception { List<String> pipelineContents = Arrays.asList("String", "Testing", "Custom", "Coder", "In", "Beam"); // Create input. PCollection<String> customCoderPC = pipeline .begin() .apply("ReadStrings", Create.of(pipelineContents)) .setCoder(coder) .apply(Reshuffle.viaRandomKey()); PCollection<String> fixedCoderPC = customCoderPC.apply("Identity", ParDo.of(new IdentityDoFn())); fixedCoderPC.setCoder(StringUtf8Coder.of()); ContentReader r = ContentReader.elementsEqual(pipelineContents); // PAssert.that relies on the last coder added to the PCollection, so we // need to create an identity ParDo with a valid coder. PAssert.that(fixedCoderPC).satisfies(r); return pipeline; }
PCollection<Long> upstream = pipeline.apply(genUpstream); PCollection<Long> output = upstream.apply("Original", originalParDo); hierarchy.pushNode("Upstream", pipeline.begin(), genUpstream); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(upstream);
PCollection<Long> upstream = pipeline.apply(genUpstream); PCollection<Long> output = upstream.apply("Original", originalParDo); Node upstreamNode = hierarchy.pushNode("Upstream", pipeline.begin(), genUpstream); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(upstream);
"string_true_test", "string_false_test") .buildIOReader(pipeline.begin()) .setRowSchema(type); } catch (Exception e) {
pipeline.begin().apply(resultSignal.signalStart());