private static void buildSpouts(ExecutionContext context, TopologyBuilder builder) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, NoSuchFieldException { for (SpoutDef sd : context.getTopologyDef().getSpouts()) { IRichSpout spout = buildSpout(sd, context); SpoutDeclarer declarer = builder.setSpout(sd.getId(), spout, sd.getParallelism()); if (sd.getOnHeapMemoryLoad() > -1) { if (sd.getOffHeapMemoryLoad() > -1) { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad(), sd.getOffHeapMemoryLoad()); } else { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad()); } } if (sd.getCpuLoad() > -1) { declarer.setCPULoad(sd.getCpuLoad()); } if (sd.getNumTasks() > -1) { declarer.setNumTasks(sd.getNumTasks()); } context.addSpout(sd.getId(), spout); } }
String spoutId = sensorToParserConfigs.size() > 1 ? "kafkaSpout-" + entry.getKey() : "kafkaSpout"; builder.setSpout(spoutId, kafkaSpout, spoutParallelism.get(i)) .setNumTasks(spoutNumTasks.get(i)); spoutIds.add(spoutId); ++i;
private static TopologyBuilder buildTopology() throws Exception { TopologyBuilder builder = new TopologyBuilder(); String topicName = Configuration.getConfig().getString("rtc.mq.spout.topic"); String groupName = Configuration.getConfig().getString("rtc.mq.spout.group"); BrokerHosts hosts = new ZkHosts(Configuration.getConfig().getString("rtc.zk.hosts")); SpoutConfig spoutConfig = new SpoutConfig(hosts, topicName, "/consumers", groupName); spoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime(); spoutConfig.zkServers = Arrays.asList(Configuration.getConfig().getString("rtc.storm.zkServers").split(",")); spoutConfig.zkPort = Configuration.getConfig().getInt("rtc.storm.zkPort"); spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig); builder.setSpout("MQSpout", kafkaSpout, Configuration.getConfig().getInt("rtc.storm.spout.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.spout.task")); builder.setBolt("ExtractBolt", new ExtractBolt(), Configuration.getConfig().getInt("rtc.storm.extract.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.extract.bolt.task")).shuffleGrouping("MQSpout"); builder.setBolt("Statistic", new StatisticBolt(), Configuration.getConfig().getInt("rtc.storm.statistic.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.statistic.bolt.task")).fieldsGrouping("ExtractBolt", new Fields(new String[]{"hashKeys"})); // builder.setBolt("Alarm", new AlarmBolt(), Configuration.getConfig().getInt("rtc.storm.alarm.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.alarm.bolt.task")).fieldsGrouping("Statistic", new Fields(new String[]{"EventName"})); return builder; }
private static TopologyBuilder buildTopology() throws Exception { TopologyBuilder builder = new TopologyBuilder(); String topicName = Configuration.getConfig().getString("rtc.mq.spout.topic"); String groupName = Configuration.getConfig().getString("rtc.mq.spout.group"); BrokerHosts hosts = new ZkHosts(Configuration.getConfig().getString("rtc.zk.hosts")); SpoutConfig spoutConfig = new SpoutConfig(hosts, topicName, "/consumers", groupName); spoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime(); spoutConfig.zkServers = Arrays.asList(Configuration.getConfig().getString("rtc.storm.zkServers").split(",")); spoutConfig.zkPort = Configuration.getConfig().getInt("rtc.storm.zkPort"); spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig); builder.setSpout("MQSpout", kafkaSpout, Configuration.getConfig().getInt("rtc.storm.spout.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.spout.task")); builder.setBolt("ExtractBolt", new ExtractBolt(), Configuration.getConfig().getInt("rtc.storm.extract.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.extract.bolt.task")).shuffleGrouping("MQSpout"); builder.setBolt("Statistic", new StatisticBolt(), Configuration.getConfig().getInt("rtc.storm.statistic.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.statistic.bolt.task")).fieldsGrouping("ExtractBolt", new Fields(new String[]{"hashKeys"})); // builder.setBolt("Alarm", new AlarmBolt(), Configuration.getConfig().getInt("rtc.storm.alarm.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.alarm.bolt.task")).fieldsGrouping("Statistic", new Fields(new String[]{"EventName"})); return builder; }
public static int run(String[] args) throws Exception { CommandLineParser parser = new BasicParser(); Options options = getCommonRequiredOptions(); CommandLine cmd = parser.parse( options, args); Map<String, String> dsConf = getAccumuloDataStoreConf(cmd); String featureName = cmd.getOptionValue(FEATURE_NAME); SimpleFeatureType featureType = DataUtilities.createType(featureName, "geom:Point:srid=4326"); DataStore ds = DataStoreFinder.getDataStore(dsConf); ds.createSchema(featureType); TopologyBuilder topologyBuilder = new TopologyBuilder(); String topic = cmd.getOptionValue(TOPIC); String groupId = topic; dsConf.put(OSMIngest.FEATURE_NAME, featureName); OSMKafkaSpout OSMKafkaSpout = new OSMKafkaSpout(dsConf, groupId, topic); topologyBuilder.setSpout("Spout", OSMKafkaSpout, 10).setNumTasks(10); OSMKafkaBolt OSMKafkaBolt = new OSMKafkaBolt(dsConf, groupId, topic); topologyBuilder.setBolt("Bolt", OSMKafkaBolt, 20).shuffleGrouping("Spout"); Config stormConf = new Config(); stormConf.setNumWorkers(10); stormConf.setDebug(true); StormSubmitter.submitTopology(topic, stormConf, topologyBuilder.createTopology()); return 0; } }
private static void buildSpouts(ExecutionContext context, TopologyBuilder builder) throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, NoSuchFieldException { for (SpoutDef sd : context.getTopologyDef().getSpouts()) { IRichSpout spout = buildSpout(sd, context); SpoutDeclarer declarer = builder.setSpout(sd.getId(), spout, sd.getParallelism()); if (sd.getOnHeapMemoryLoad() > -1) { if (sd.getOffHeapMemoryLoad() > -1) { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad(), sd.getOffHeapMemoryLoad()); } else { declarer.setMemoryLoad(sd.getOnHeapMemoryLoad()); } } if (sd.getCpuLoad() > -1) { declarer.setCPULoad(sd.getCpuLoad()); } if (sd.getNumTasks() > -1) { declarer.setNumTasks(sd.getNumTasks()); } context.addSpout(sd.getId(), spout); } }