public static StormTopology buildTopology(LocalDRPC drpc) { TridentTopology topology = new TridentTopology(); TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB)); TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB)); topology.newDRPCStream("reach", drpc) .stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields("tweeters")) .each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle() .stateQuery(tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")) .each(new Fields("followers"), new ExpandList(), new Fields("follower")).groupBy(new Fields("follower")) .aggregate(new One(), new Fields("one")).aggregate(new Fields("one"), new Sum(), new Fields("reach")); return topology.build(); }
.shuffle() .groupBy(new Fields("tw_id"))
stream = stream.shuffle();
public StormTopology buildTopology(LocalDRPC drpc) { TridentKafkaConfig kafkaConfig = new TridentKafkaConfig(brokerHosts, "storm-sentence", "storm"); kafkaConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); TransactionalTridentKafkaSpout kafkaSpout = new TransactionalTridentKafkaSpout(kafkaConfig); TridentTopology topology = new TridentTopology(); TridentState wordCounts = topology.newStream("kafka", kafkaSpout).shuffle(). each(new Fields("str"), new WordSplit(), new Fields("word")). groupBy(new Fields("word")). persistentAggregate(new HazelCastStateFactory(), new Count(), new Fields("aggregates_words")).parallelismHint(2); topology.newDRPCStream("words", drpc) .each(new Fields("args"), new Split(), new Fields("word")) .groupBy(new Fields("word")) .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count")) .each(new Fields("count"), new FilterNull()) .aggregate(new Fields("count"), new Sum(), new Fields("sum")); return topology.build(); }
topology.newStream("parallel_and_partitioned", spout).parallelismHint(1).shuffle() .each(new Fields("text", "actor"), new PereTweetsFilter()).parallelismHint(5) .each(new Fields("text", "actor"), new Utils.PrintFilter()); .newStream("partial_aggregation", spout) .parallelismHint(1) .shuffle() .partitionAggregate(new Fields("location"), new LocationAggregator(), new Fields("aggregated_result")).parallelismHint(6)
.each(new Fields("count_map"), new Print("AFTER-HAS-SPAIN")) .parallelismHint(3) .shuffle() .each(new Fields("count_map"), new TimesTen(), new Fields("count_map_times_ten")) .each(new Fields("count_map_times_ten"), new Print("AFTER-TIMES-TEN"))