@Override public PCollection<FeatureRow> expand(PInput input) { checkArgument(importSpec.getType().equals(PUBSUB_FEATURE_SOURCE_TYPE)); PubSubReadOptions options = OptionsParser.parse(importSpec.getOptionsMap(), PubSubReadOptions.class); PubsubIO.Read<FeatureRow> read = readProtos(); if (!Strings.isNullOrEmpty(options.subscription)) { read = read.fromSubscription(options.subscription); } else if (!Strings.isNullOrEmpty(options.topic)) { read = read.fromTopic(options.topic); } PCollection<FeatureRow> featureRow = input.getPipeline().apply(read); if (options.discardUnknownFeatures) { List<String> featureIds = new ArrayList<>(); for(Field field: importSpec.getSchema().getFieldsList()) { String featureId = field.getFeatureId(); if (!Strings.isNullOrEmpty(featureId)) { featureIds.add(featureId); } } return featureRow.apply(ParDo.of(new FilterFeatureRowDoFn(featureIds))); } return featureRow; }
streamData = pipeline.apply("ReadPubSub",PubsubIO.readStrings().fromSubscription(String.format("projects/%1$s/subscriptions/%2$s",projectId,pubSubTopicSub)));
} else {// normal PubsubIO.Read<PubsubMessage> pubsubRead = PubsubIO.readMessages().fromSubscription(String.format("projects/%s/subscriptions/%s", datastore.projectName.getValue(), dataset.subscription.getValue())); if (properties.idLabel.getValue() != null && !"".equals(properties.idLabel.getValue())) {
/** * Runs the pipeline with the supplied options. * * @param options The execution parameters to the pipeline. * @return The result of the pipeline execution. */ public static PipelineResult run(Options options) { // Create the pipeline Pipeline pipeline = Pipeline.create(options); /** * Steps: 1) Read PubSubMessage with attributes from input PubSub subscription. * 2) Apply any filters if an attribute=value pair is provided. * 3) Write each PubSubMessage to output PubSub topic. */ pipeline .apply( "Read PubSub Events", PubsubIO.readMessagesWithAttributes().fromSubscription(options.getInputSubscription())) .apply("Filter Events If Enabled", ParDo.of(new ExtractAndFilterEventsFn())) .apply("Write PubSub Events", PubsubIO.writeMessages().to(options.getOutputTopic())); // Execute the pipeline and return the result. return pipeline.run(); } }
@Test @Category({ValidatesRunner.class, UsesUnboundedPCollections.class}) public void testPrimitiveReadDisplayData() { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); Set<DisplayData> displayData; PubsubIO.Read<String> baseRead = PubsubIO.readStrings(); // Reading from a subscription. PubsubIO.Read<String> read = baseRead.fromSubscription("projects/project/subscriptions/subscription"); displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "PubsubIO.Read should include the subscription in its primitive display data", displayData, hasItem(hasDisplayItem("subscription"))); // Reading from a topic. read = baseRead.fromTopic("projects/project/topics/topic"); displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "PubsubIO.Read should include the topic in its primitive display data", displayData, hasItem(hasDisplayItem("topic"))); }
@Test public void testReadSubscriptionDisplayData() { String subscription = "projects/project/subscriptions/subscription"; PubsubIO.Read<String> read = PubsubIO.readStrings() .fromSubscription(StaticValueProvider.of(subscription)) .withTimestampAttribute("myTimestamp") .withIdAttribute("myId"); DisplayData displayData = DisplayData.from(read); assertThat(displayData, hasDisplayItem("subscription", subscription)); assertThat(displayData, hasDisplayItem("timestampAttribute", "myTimestamp")); assertThat(displayData, hasDisplayItem("idAttribute", "myId")); }
@Test public void testRuntimeValueProviderSubscription() { TestPipeline pipeline = TestPipeline.create(); ValueProvider<String> subscription = pipeline.newProvider("projects/project/subscriptions/subscription"); Read<String> pubsubRead = PubsubIO.readStrings().fromSubscription(subscription); pipeline.apply(pubsubRead); assertThat(pubsubRead.getSubscriptionProvider(), not(nullValue())); assertThat(pubsubRead.getSubscriptionProvider().isAccessible(), is(false)); }
@Test public void testNullTopic() { String subscription = "projects/project/subscriptions/subscription"; PubsubIO.Read<String> read = PubsubIO.readStrings().fromSubscription(StaticValueProvider.of(subscription)); assertNull(read.getTopicProvider()); assertNotNull(read.getSubscriptionProvider()); assertNotNull(DisplayData.from(read)); }
/** * Reads from the given subscription. * * <p>See {@link PubsubIO.PubsubSubscription#fromPath(String)} for more details on the format of * the {@code subscription} string. * * <p>Multiple readers reading from the same subscription will each receive some arbitrary * portion of the data. Most likely, separate readers should use their own subscriptions. */ public Read<T> fromSubscription(String subscription) { return fromSubscription(StaticValueProvider.of(subscription)); }
@Test public void testValueProviderSubscription() { StaticValueProvider<String> provider = StaticValueProvider.of("projects/project/subscriptions/subscription"); Read<String> pubsubRead = PubsubIO.readStrings().fromSubscription(provider); Pipeline.create().apply(pubsubRead); assertThat(pubsubRead.getSubscriptionProvider(), not(nullValue())); assertThat(pubsubRead.getSubscriptionProvider().isAccessible(), is(true)); assertThat(pubsubRead.getSubscriptionProvider().get().asPath(), equalTo(provider.get())); }