/** Test that we throw an error for false assertion on singleton. */ @Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testPAssertEqualsSingletonFalseDefaultReasonString() throws Exception { PCollection<Integer> pcollection = pipeline.apply(Create.of(42)); PAssert.thatSingleton(pcollection).isEqualTo(44); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("Create.Values/Read(CreateSource).out")); assertThat(message, containsString("Expected: <44>")); assertThat(message, containsString("but: was <42>")); }
/** Test that we throw an error for false assertion on singleton. */ @Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testPAssertEqualsSingletonFalse() throws Exception { PCollection<Integer> pcollection = pipeline.apply(Create.of(42)); PAssert.thatSingleton("The value was not equal to 44", pcollection).isEqualTo(44); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("The value was not equal to 44")); assertThat(message, containsString("Expected: <44>")); assertThat(message, containsString("but: was <42>")); }
@Test public void countAssertsMultipleCallsIndependent() { PCollection<Integer> create = pipeline.apply("FirstCreate", Create.of(1, 2, 3)); PAssert.that(create).containsInAnyOrder(1, 2, 3); PAssert.thatSingleton(create.apply(Sum.integersGlobally())).isEqualTo(6); assertThat(PAssert.countAsserts(pipeline), equalTo(2)); PAssert.thatMap(pipeline.apply("CreateMap", Create.of(KV.of(1, 2)))) .isEqualTo(Collections.singletonMap(1, 2)); assertThat(PAssert.countAsserts(pipeline), equalTo(3)); } }
/** Basic test for {@code isEqualTo}. */ @Test @Category(ValidatesRunner.class) public void testWindowedIsEqualTo() throws Exception { PCollection<Integer> pcollection = pipeline .apply( Create.timestamped( TimestampedValue.of(43, new Instant(250L)), TimestampedValue.of(22, new Instant(-250L)))) .apply(Window.into(FixedWindows.of(Duration.millis(500L)))); PAssert.thatSingleton(pcollection) .inOnlyPane(new IntervalWindow(new Instant(0L), new Instant(500L))) .isEqualTo(43); PAssert.thatSingleton(pcollection) .inOnlyPane(new IntervalWindow(new Instant(-500L), new Instant(0L))) .isEqualTo(22); pipeline.run(); }
private void runRead() { PCollection<Scientist> output = pipelineRead.apply( CassandraIO.<Scientist>read() .withHosts(options.getCassandraHost()) .withPort(options.getCassandraPort()) .withMinNumberOfSplits(20) .withKeyspace(KEYSPACE) .withTable(TABLE) .withEntity(Scientist.class) .withCoder(SerializableCoder.of(Scientist.class))); PCollection<String> consolidatedHashcode = output .apply(ParDo.of(new SelectNameFn())) .apply("Hash row contents", Combine.globally(new HashingFn()).withoutDefaults()); PAssert.thatSingleton(consolidatedHashcode) .isEqualTo(TestRow.getExpectedHashForRowCount(options.getNumberOfRecords())); pipelineRead.run().waitUntilFinish(); }
@Test public void testReadWithSingleStringParameter() throws Exception { PCollection<TestRow> rows = pipeline.apply( JdbcIO.<TestRow>read() .withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(dataSource)) .withQuery(String.format("select name,id from %s where name = ?", readTableName)) .withStatementPreparator( preparedStatement -> preparedStatement.setString(1, TestRow.getNameForSeed(1))) .withRowMapper(new JdbcTestHelper.CreateTestRowOfNameAndId()) .withCoder(SerializableCoder.of(TestRow.class))); PAssert.thatSingleton(rows.apply("Count All", Count.globally())).isEqualTo(1L); Iterable<TestRow> expectedValues = Collections.singletonList(TestRow.fromSeed(1)); PAssert.that(rows).containsInAnyOrder(expectedValues); pipeline.run(); }
@Test public void testRead() throws Exception { SpannerConfig spannerConfig = createSpannerConfig(); PCollectionView<Transaction> tx = p.apply( SpannerIO.createTransaction() .withSpannerConfig(spannerConfig) .withTimestampBound(TimestampBound.strong())); PCollection<Struct> output = p.apply( SpannerIO.read() .withSpannerConfig(spannerConfig) .withTable(options.getTable()) .withColumns("Key", "Value") .withTransaction(tx)); PAssert.thatSingleton(output.apply("Count rows", Count.<Struct>globally())).isEqualTo(5L); p.run(); }
@Test @Category(NeedsRunner.class) public void testUnboundedInputTimestamps() { long numElements = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements).withTimestampFn(new ValueAsTimestampFn())); addCountingAsserts(input, 0, numElements); PCollection<Long> diffs = input .apply("TimestampDiff", ParDo.of(new ElementValueDiff())) .apply("DistinctTimestamps", Distinct.create()); // This assert also confirms that diffs only has one unique value. PAssert.thatSingleton(diffs).isEqualTo(0L); p.run(); }
@Test public void testUnboundedSourceTimestamps() { int numElements = 1000; PCollection<Long> input = p.apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn()).withoutMetadata()) .apply(Values.create()); addCountingAsserts(input, numElements); PCollection<Long> diffs = input .apply("TimestampDiff", ParDo.of(new ElementValueDiff())) .apply("DistinctTimestamps", Distinct.create()); // This assert also confirms that diffs only has one unique value. PAssert.thatSingleton(diffs).isEqualTo(0L); p.run(); }
void testRead() throws Exception { if (!useAsITests) { ElasticSearchIOTestUtils.insertTestDocuments(connectionConfiguration, numDocs, restClient); } PCollection<String> output = pipeline.apply( ElasticsearchIO.read() .withConnectionConfiguration(connectionConfiguration) //set to default value, useful just to test parameter passing. .withScrollKeepalive("5m") //set to default value, useful just to test parameter passing. .withBatchSize(100L)); PAssert.thatSingleton(output.apply("Count", Count.globally())).isEqualTo(numDocs); pipeline.run(); }
private void runBigQueryIOReadPipeline() { Pipeline p = Pipeline.create(options); PCollection<Long> count = p.apply("Read", BigQueryIO.read().from(options.getInputTable())) .apply("Count", Count.globally()); PAssert.thatSingleton(count).isEqualTo(options.getNumRecords()); p.run().waitUntilFinish(); }
public static void addCountingAsserts(PCollection<Long> input, long start, long end) { // Count == numElements PAssert.thatSingleton(input.apply("Count", Count.globally())).isEqualTo(end - start); // Unique count == numElements PAssert.thatSingleton(input.apply(Distinct.create()).apply("UniqueCount", Count.globally())) .isEqualTo(end - start); // Min == start PAssert.thatSingleton(input.apply("Min", Min.globally())).isEqualTo(start); // Max == end-1 PAssert.thatSingleton(input.apply("Max", Max.globally())).isEqualTo(end - 1); }
public static void addCountingAsserts( PCollection<Long> input, long count, long uniqueCount, long min, long max) { PAssert.thatSingleton(input.apply("Count", Count.globally())).isEqualTo(count); PAssert.thatSingleton(input.apply(Distinct.create()).apply("UniqueCount", Count.globally())) .isEqualTo(uniqueCount); PAssert.thatSingleton(input.apply("Min", Min.globally())).isEqualTo(min); PAssert.thatSingleton(input.apply("Max", Max.globally())).isEqualTo(max); }
@Test public void testReadWithQuery() throws Exception { SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient); PCollection<SolrDocument> output = pipeline.apply( SolrIO.read() .withConnectionConfiguration(connectionConfiguration) .from(SOLR_COLLECTION) .withQuery("scientist:Franklin")); PAssert.thatSingleton(output.apply("Count", Count.globally())) .isEqualTo(NUM_DOCS / NUM_SCIENTISTS); pipeline.run(); }
public static void addCountingAsserts(PCollection<Long> input, long numElements) { // Count == numElements PAssert.thatSingleton(input.apply("Count", Count.globally())).isEqualTo(numElements); // Unique count == numElements PAssert.thatSingleton(input.apply(Distinct.create()).apply("UniqueCount", Count.globally())) .isEqualTo(numElements); // Min == 0 PAssert.thatSingleton(input.apply("Min", Min.globally())).isEqualTo(0L); // Max == numElements-1 PAssert.thatSingleton(input.apply("Max", Max.globally())).isEqualTo(numElements - 1); }
/** Basic test for {@code isEqualTo}. */ @Test @Category(ValidatesRunner.class) public void testIsEqualTo() throws Exception { PCollection<Integer> pcollection = pipeline.apply(Create.of(43)); PAssert.thatSingleton(pcollection).isEqualTo(43); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testApproximateUniqueWithSmallInput() { final PCollection<Integer> input = p.apply(Create.of(Arrays.asList(1, 2, 3, 3))); final PCollection<Long> estimate = input.apply(ApproximateUnique.globally(1000)); PAssert.thatSingleton(estimate).isEqualTo(3L); p.run(); }
private void testOutputAfterCheckpoint(IsBounded bounded) { PCollection<Integer> outputs = p.apply(Create.of("foo")) .apply(ParDo.of(sdfWithMultipleOutputsPerBlock(bounded, 3))) .apply(Window.<Integer>configure().triggering(Never.ever()).discardingFiredPanes()); PAssert.thatSingleton(outputs.apply(Count.globally())) .isEqualTo((long) SDFWithMultipleOutputsPerBlockBase.MAX_INDEX); p.run(); }
@Test public void testRead() throws Exception { SolrIOTestUtils.insertTestDocuments(SOLR_COLLECTION, NUM_DOCS, solrClient); PCollection<SolrDocument> output = pipeline.apply( SolrIO.read() .withConnectionConfiguration(connectionConfiguration) .from(SOLR_COLLECTION) .withBatchSize(101)); PAssert.thatSingleton(output.apply("Count", Count.globally())).isEqualTo(NUM_DOCS); pipeline.run(); }
@Test public void countAssertsSucceeds() { PCollection<Integer> create = pipeline.apply("FirstCreate", Create.of(1, 2, 3)); PAssert.that(create).containsInAnyOrder(1, 2, 3); PAssert.thatSingleton(create.apply(Sum.integersGlobally())).isEqualTo(6); PAssert.thatMap(pipeline.apply("CreateMap", Create.of(KV.of(1, 2)))) .isEqualTo(Collections.singletonMap(1, 2)); assertThat(PAssert.countAsserts(pipeline), equalTo(3)); }