@Override public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, Fields fields) { declarer.fieldsGrouping(componentId, streamId, fields); } };
@Override public void assign(BoltDeclarer declarer, InputStream stream) { declarer.fieldsGrouping(stream.fromComponent, stream.id, new Fields("key")); } },
public static StormTopology buildTopology(String nameserverAddr, String topic){ Properties properties = new Properties(); properties.setProperty(SpoutConfig.NAME_SERVER_ADDR, nameserverAddr); properties.setProperty(SpoutConfig.CONSUMER_GROUP, CONSUMER_GROUP); properties.setProperty(SpoutConfig.CONSUMER_TOPIC, CONSUMER_TOPIC); RocketMqSpout spout = new RocketMqSpout(properties); WordCounter bolt = new WordCounter(); TupleToMessageMapper mapper = new FieldNameBasedTupleToMessageMapper("word", "count"); TopicSelector selector = new DefaultTopicSelector(topic); properties = new Properties(); properties.setProperty(RocketMqConfig.NAME_SERVER_ADDR, nameserverAddr); RocketMqBolt insertBolt = new RocketMqBolt() .withMapper(mapper) .withSelector(selector) .withProperties(properties); // wordSpout ==> countBolt ==> insertBolt TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(COUNT_BOLT, bolt, 1).fieldsGrouping(WORD_SPOUT, new Fields("str")); builder.setBolt(INSERT_BOLT, insertBolt, 1).shuffleGrouping(COUNT_BOLT); return builder.createTopology(); }
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int cntBoltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_COUNT_BOLT_NUM); final String inputFile = Helper.getStr(config, INPUT_FILE); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new WordGenSpout(inputFile), spoutNum); builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPOUT_ID, new Fields(WordGenSpout.FIELDS)); return builder.createTopology(); }
builder.setBolt(HBASE_BOLT, hbase, 1).fieldsGrouping(COUNT_BOLT, new Fields("word"));
builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); LOG.info("Topology name: " + topologyName);
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); Config conf = new Config(); conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class); String name = "in-order-test"; if (args != null && args.length > 0) { name = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology()); Map<String, Object> clusterConf = Utils.readStormConfig(); clusterConf.putAll(Utils.readCommandLineOpts()); Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient(); //Sleep for 50 mins for (int i = 0; i < 50; i++) { Thread.sleep(30 * 1000); printMetrics(client, name); } kill(client, name); }
protected int run(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); conf.setDebug(true); String topologyName = "word-count"; conf.setNumWorkers(3); if (args != null && args.length > 0) { topologyName = args[0]; } return submit(topologyName, conf, builder); }
public static void main(String[] args) throws Exception { Config config = new Config(); String host = TEST_REDIS_HOST; int port = TEST_REDIS_PORT; if (args.length >= 2) { host = args[0]; port = Integer.parseInt(args[1]); } JedisPoolConfig poolConfig = new JedisPoolConfig.Builder() .setHost(host).setPort(port).build(); WordSpout spout = new WordSpout(); WordCounter bolt = new WordCounter(); RedisStoreMapper storeMapper = setupStoreMapper(); RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper); // wordSpout ==> countBolt ==> RedisBolt TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(COUNT_BOLT, bolt, 1).fieldsGrouping(WORD_SPOUT, new Fields("word")); builder.setBolt(STORE_BOLT, storeBolt, 1).shuffleGrouping(COUNT_BOLT); String topoName = "test"; if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { System.out.println("Usage: PersistentWordCount <redis host> <redis port> (topology name)"); return; } StormSubmitter.submitTopology(topoName, config, builder.createTopology()); }
builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(LOOKUP_BOLT, hBaseLookupBolt, 1).shuffleGrouping(WORD_SPOUT); builder.setBolt(TOTAL_COUNT_BOLT, totalBolt, 1).fieldsGrouping(LOOKUP_BOLT, new Fields("columnName")); String topoName = "test"; if (args.length == 1) {
protected int run(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 4); builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setMaxTaskParallelism(3); String topologyName = "word-count"; conf.setNumWorkers(3); if (args != null && args.length > 0) { topologyName = args[0]; } return submit(topologyName, conf, builder); }
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int spBoltNum = Helper.getInt(config, SPLIT_NUM, DEFAULT_SPLIT_BOLT_NUM); final int cntBoltNum = Helper.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM); final String inputFile = Helper.getStr(config, INPUT_FILE); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum); builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID); builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS)); return builder.createTopology(); }
builder.setBolt(INSERT_BOLT, insertBolt, 1).fieldsGrouping(COUNT_BOLT, new Fields("word"));
public static void main(String[] args) throws Exception { if (!NimbusClient.isLocalOverride()) { throw new IllegalStateException("This example only works in local mode. " + "Run with storm local not storm jar"); } FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender")); FeederSpout ageSpout = new FeederSpout(new Fields("id", "age")); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("genderSpout", genderSpout); builder.setSpout("ageSpout", ageSpout); // inner join of 'age' and 'gender' records on 'id' field JoinBolt joiner = new JoinBolt("genderSpout", "id") .join("ageSpout", "id", "genderSpout") .select("genderSpout:id,ageSpout:id,gender,age") .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS)); builder.setBolt("joiner", joiner) .fieldsGrouping("genderSpout", new Fields("id")) .fieldsGrouping("ageSpout", new Fields("id")); builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner"); Config conf = new Config(); StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology()); generateGenderData(genderSpout); generateAgeData(ageSpout); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), 4); builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class); String name = "wc-test"; if (args != null && args.length > 0) { name = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology()); Map<String, Object> clusterConf = Utils.readStormConfig(); clusterConf.putAll(Utils.readCommandLineOpts()); Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient(); //Sleep for 5 mins for (int i = 0; i < 10; i++) { Thread.sleep(30 * 1000); printMetrics(client, name); } kill(client, name); }
builder.setSpout("gender", genderSpout); builder.setSpout("age", ageSpout); builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id")) .fieldsGrouping("age", new Fields("id"));
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentence(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); String topoName = "word-count"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
public static void main(String[] args) throws Exception { Config config = new Config(); String host = TEST_REDIS_HOST; int port = TEST_REDIS_PORT; if (args.length >= 2) { host = args[0]; port = Integer.parseInt(args[1]); } JedisPoolConfig poolConfig = new JedisPoolConfig.Builder() .setHost(host).setPort(port).build(); WordSpout spout = new WordSpout(); RedisFilterMapper filterMapper = setupWhitelistMapper(); RedisFilterBolt whitelistBolt = new RedisFilterBolt(poolConfig, filterMapper); WordCounter wordCounterBolt = new WordCounter(); PrintWordTotalCountBolt printBolt = new PrintWordTotalCountBolt(); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(WHITELIST_BOLT, whitelistBolt, 1).shuffleGrouping(WORD_SPOUT); builder.setBolt(COUNT_BOLT, wordCounterBolt, 1).fieldsGrouping(WHITELIST_BOLT, new Fields("word")); builder.setBolt(PRINT_BOLT, printBolt, 1).shuffleGrouping(COUNT_BOLT); String topoName = "test"; if (args.length == 3) { topoName = args[2]; } else if (args.length > 3) { System.out.println("Usage: WhitelistWordCount <redis host> <redis port> [topology name]"); return; } StormSubmitter.submitTopology(topoName, config, builder.createTopology()); }
builder.setSpout(WORD_SPOUT, spout, 1); builder.setBolt(LOOKUP_BOLT, lookupBolt, 1).shuffleGrouping(WORD_SPOUT); builder.setBolt(TOTAL_COUNT_BOLT, totalBolt, 1).fieldsGrouping(LOOKUP_BOLT, new Fields("word"));
builder.setBolt(UPDATE_BOLT, updateBolt, 1).fieldsGrouping(COUNT_BOLT, new Fields("word"));