.shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), numCounts).fieldsGrouping("split", new Fields("word"));
String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word")); builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); LOG.info("Topology name: " + topologyName);
static StormTopology getTopology(Map<String, Object> conf) { // 1 - Setup Spout -------- ConstSpout spout = new ConstSpout("some data").withOutputFields("str"); // 2 - Setup DevNull Bolt -------- DevNullBolt bolt = new DevNullBolt(); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); int numSpouts = Helper.getInt(conf, SPOUT_COUNT, 1); builder.setSpout(SPOUT_ID, spout, numSpouts); int numBolts = Helper.getInt(conf, BOLT_COUNT, 1); BoltDeclarer bd = builder.setBolt(BOLT_ID, bolt, numBolts); System.err.printf("====> Using : numSpouts = %d , numBolts = %d\n", numSpouts, numBolts); String groupingType = Helper.getStr(conf, GROUPING); if (groupingType == null || groupingType.equalsIgnoreCase(DEFAULT_GROUPING)) { bd.localOrShuffleGrouping(SPOUT_ID); } else if (groupingType.equalsIgnoreCase(SHUFFLE_GROUPING)) { bd.shuffleGrouping(SPOUT_ID); } return builder.createTopology(); }
builder.setSpout(SPOUT_ID, spout); builder.setBolt(PREPARE_ID, new PrepareRequest()) .noneGrouping(SPOUT_ID); int i = 0; for (; i < components.size(); i++) { declarer.addSharedMemory(request); declarer.addConfigurations(component.componentConf); declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request")); declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM); } else { String prevId; declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID); .fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0))) .fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request")); i++; builder.setBolt(boltId(i), new ReturnResults()) .noneGrouping(boltId(i - 1)); return builder.createTopology();
@Override public void assign(BoltDeclarer declarer, InputStream stream) { declarer.shuffleGrouping(stream.fromComponent, stream.id); } },
if (boltDef.getOnHeapMemoryLoad() > -1) { if (boltDef.getOffHeapMemoryLoad() > -1) { declarer.setMemoryLoad(boltDef.getOnHeapMemoryLoad(), boltDef.getOffHeapMemoryLoad()); } else { declarer.setMemoryLoad(boltDef.getOnHeapMemoryLoad()); declarer.setCPULoad(boltDef.getCpuLoad()); declarer.setNumTasks(boltDef.getNumTasks()); declarer.shuffleGrouping(stream.getFrom(), streamId); break; case FIELDS: declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(grouping.getArgs())); break; case ALL: declarer.allGrouping(stream.getFrom(), streamId); break; case DIRECT: declarer.directGrouping(stream.getFrom(), streamId); break; case GLOBAL: declarer.globalGrouping(stream.getFrom(), streamId); break; case LOCAL_OR_SHUFFLE: declarer.localOrShuffleGrouping(stream.getFrom(), streamId); break; case NONE:
@Override public void setBolts(TopologyBuilder builder) { BoltDeclarer boltDeclarer = builder.setBolt("split", new SplitStreamBolt(), config.boltThreads); if (config.localShuffle) { boltDeclarer.localOrShuffleGrouping("spout"); } else { boltDeclarer.shuffleGrouping("spout"); } builder.setBolt("count", new WordCountBolt(config), config.boltThreads).fieldsGrouping("split", new Fields("ip")); }
private static TopologyBuilder buildTopology() throws Exception { TopologyBuilder builder = new TopologyBuilder(); String topicName = Configuration.getConfig().getString("rtc.mq.spout.topic"); String groupName = Configuration.getConfig().getString("rtc.mq.spout.group"); BrokerHosts hosts = new ZkHosts(Configuration.getConfig().getString("rtc.zk.hosts")); SpoutConfig spoutConfig = new SpoutConfig(hosts, topicName, "/consumers", groupName); spoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime(); spoutConfig.zkServers = Arrays.asList(Configuration.getConfig().getString("rtc.storm.zkServers").split(",")); spoutConfig.zkPort = Configuration.getConfig().getInt("rtc.storm.zkPort"); spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig); builder.setSpout("MQSpout", kafkaSpout, Configuration.getConfig().getInt("rtc.storm.spout.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.spout.task")); builder.setBolt("ExtractBolt", new ExtractBolt(), Configuration.getConfig().getInt("rtc.storm.extract.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.extract.bolt.task")).shuffleGrouping("MQSpout"); builder.setBolt("Statistic", new StatisticBolt(), Configuration.getConfig().getInt("rtc.storm.statistic.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.statistic.bolt.task")).fieldsGrouping("ExtractBolt", new Fields(new String[]{"hashKeys"})); // builder.setBolt("Alarm", new AlarmBolt(), Configuration.getConfig().getInt("rtc.storm.alarm.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.alarm.bolt.task")).fieldsGrouping("Statistic", new Fields(new String[]{"EventName"})); return builder; }
.shuffleGrouping(recordComponent) .allGrouping(QUERY_COMPONENT, QUERY_STREAM) .allGrouping(QUERY_COMPONENT, METADATA_STREAM) .allGrouping(TICK_COMPONENT, TICK_STREAM) .setCPULoad(filterBoltCPULoad).setMemoryLoad(filterBoltMemoryOnheapLoad, filterBoltMemoryOffHeapLoad); .fieldsGrouping(QUERY_COMPONENT, QUERY_STREAM, new Fields(ID_FIELD)) .fieldsGrouping(QUERY_COMPONENT, METADATA_STREAM, new Fields(ID_FIELD)) .fieldsGrouping(FILTER_COMPONENT, DATA_STREAM, new Fields(ID_FIELD)) .allGrouping(TICK_COMPONENT, TICK_STREAM) .setCPULoad(joinBoltCPULoad).setMemoryLoad(joinBoltMemoryOnHeapLoad, joinBoltMemoryOffHeapLoad); .shuffleGrouping(JOIN_COMPONENT, RESULT_STREAM) .setCPULoad(resultBoltCPULoad).setMemoryLoad(resultBoltMemoryOnHeapLoad, resultBoltMemoryOffHeapLoad); } else { builder.setBolt(LOOP_COMPONENT, new LoopBolt(config), loopBoltParallelism) .shuffleGrouping(JOIN_COMPONENT, FEEDBACK_STREAM) .setCPULoad(loopBoltCPULoad).setMemoryLoad(loopBoltMemoryOnHeapLoad, loopBoltMemoryOffHeapLoad);
.shuffleGrouping("spout"); .fieldsGrouping("partitioner", new Fields("key")); .setNumTasks(2).localOrShuffleGrouping("fetch"); builder.setBolt("feed", new FeedParserBolt(), numWorkers).setNumTasks(4) .localOrShuffleGrouping("sitemap"); .localOrShuffleGrouping("feed"); builder.setBolt("warc", warcbolt).localOrShuffleGrouping("feed"); .localOrShuffleGrouping("fetch", Constants.StatusStreamName) .localOrShuffleGrouping("sitemap", Constants.StatusStreamName) .localOrShuffleGrouping("feed", Constants.StatusStreamName) .localOrShuffleGrouping("ssb", Constants.StatusStreamName) .setNumTasks(numShards);
builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0) .setMemoryLoad(500.0, 100.0); StormTopology stormTopology1 = builder1.createTopology();
builder.setSpout("1", new TestWordSpout(true), 1); builder.setBolt("2", new TestGlobalCount(), 1) .setNumTasks(10) .shuffleGrouping("1"); StormTopology stormTopology = builder.createTopology();
log.info("Setting bolt for "+getAdapterName()+". Workers:"+workers+" Executors:" + executors + " Tasks:" + tasks); builder.setBolt(LGConstants.LEMONGRENADE_COORDINATOR, new CoordinatorBolt(), executors) .setNumTasks(tasks) .fieldsGrouping("input", new Fields(LGConstants.LG_JOB_ID)); builder.setBolt("rabbitmq-sink", new RabbitMQBolt(new AdapterSinkScheme()), LGProperties.getInteger("rabbit.sink.threads", 1)) .addConfigurations(sinkConfig.asMap()) .shuffleGrouping(LGConstants.LEMONGRENADE_COORDINATOR); LGProperties.getInteger("coordinator.command.threads", 10)).fieldsGrouping("command-input", new Fields("job_id" )); builder.setBolt("cmd-rabbitmq-sink", new RabbitMQBolt(new CommandSinkScheme())) .addConfigurations(sinkConfig.asMap()) .shuffleGrouping(LGConstants.LEMONGRENADE_COORDINATORCMD);
.shuffleGrouping(kafkaSpoutBuilder.getId()); .fieldsGrouping(splitSentenceBoltBuilder.getId(),new Fields("word")); .shuffleGrouping(rollingWordCountBoltBuilder.getId()); .fieldsGrouping(splitSentenceBoltBuilder.getId(), new Fields("word")); .fieldsGrouping(wordCountBoltBuilder.getId(), new Fields("word")); .fieldsGrouping(rollingWordCountBoltBuilder.getId(),new Fields("obj")); .globalGrouping(intermediateRankingsWordCountBoltBuilder.getId()); .globalGrouping(totalRankingsWordCountBoltBuilder.getId());
.shuffleGrouping("word") .addSharedMemory(exclaimCache); .shuffleGrouping("exclaim1") .setMemoryLoad(100) .addSharedMemory(exclaimCache) .addSharedMemory(notImplementedButJustAnExample);
@Override public void declareGrouping(BoltDeclarer declarer, String componentId, String streamId, Fields fields) { declarer.fieldsGrouping(componentId, streamId, fields); } };
log.info("Setting bolt for "+getAdapterName()+". Workers:"+workers+" Executors:"+executors+" Tasks:"+tasks); .shuffleGrouping("input") .setNumTasks(tasks)//sets the total number of tasks .addConfigurations(sinkConfig.asMap()) .shuffleGrouping(getAdapterName());
private StormTopology buildTopolgy() { final TopologyBuilder topologyBuilder = new TopologyBuilder(); topologyBuilder.setSpout("kafka_spout", new KafkaSpout<>(getKafkaSpoutConfig()), 1); topologyBuilder.setBolt("rsvpSummarizer", new MeetupRSVPSummaryBolt(), 1).globalGrouping("kafka_spout", STREAM_NAME); topologyBuilder.setBolt("summarySerializer", new KafkaBolt(), 1).shuffleGrouping("rsvpSummarizer"); return topologyBuilder.createTopology(); }
String totalRankerId = "finalRanker"; builder.setSpout(spoutId, new TestWordSpout(), 5); builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word")); builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId, new Fields("obj")); builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId); LOG.info("Topology name: " + topologyName);
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int spBoltNum = Helper.getInt(config, SPLIT_NUM, DEFAULT_SPLIT_BOLT_NUM); final int cntBoltNum = Helper.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM); final String inputFile = Helper.getStr(config, INPUT_FILE); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum); builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID); builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS)); return builder.createTopology(); }