@Override boolean initializeParsingBolt(String topology_name, String name) { try { String messageUpstreamComponent = messageComponents.get(messageComponents.size()-1); System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); PcapParserBolt pcapParser = new PcapParserBolt().withTsPrecision(config.getString("bolt.parser.ts.precision")); builder.setBolt(name, pcapParser, config.getInt("bolt.parser.parallelism.hint")) .setNumTasks(config.getInt("bolt.parser.num.tasks")) .shuffleGrouping(messageUpstreamComponent); } catch (Exception e) { e.printStackTrace(); System.exit(0); } return true; } }
private boolean initializeKafkaBolt(String name) { try { String messageUpstreamComponent = messageComponents .get(messageComponents.size() - 1); System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); Map<String, String> kafka_broker_properties = new HashMap<String, String>(); kafka_broker_properties.put("zk.connect", config.getString("kafka.zk")); kafka_broker_properties.put("metadata.broker.list", config.getString("kafka.br")); kafka_broker_properties.put("serializer.class", "com.opensoc.json.serialization.JSONKafkaSerializer"); kafka_broker_properties.put("key.serializer.class", "kafka.serializer.StringEncoder"); String output_topic = config.getString("bolt.kafka.topic"); conf.put("kafka.broker.properties", kafka_broker_properties); conf.put("topic", output_topic); builder.setBolt(name, new KafkaBolt<String, JSONObject>(), config.getInt("bolt.kafka.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent, "message") .setNumTasks(config.getInt("bolt.kafka.num.tasks")); } catch (Exception e) { e.printStackTrace(); System.exit(0); } return true; }
config.getInt("bolt.parser.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent) .setNumTasks(config.getInt("bolt.parser.num.tasks"));
config.getInt("bolt.parser.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent) .setNumTasks(config.getInt("bolt.parser.num.tasks"));
config.getInt("bolt.parser.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent) .setNumTasks(config.getInt("bolt.parser.num.tasks"));
config.getInt("bolt.parser.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent) .setNumTasks(config.getInt("bolt.parser.num.tasks"));
config.getInt("bolt.parser.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent) .setNumTasks(config.getInt("bolt.parser.num.tasks"));
config.getInt("bolt.parser.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent) .setNumTasks(config.getInt("bolt.parser.num.tasks"));
config.getInt("bolt.parser.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent) .setNumTasks(config.getInt("bolt.parser.num.tasks"));
BoltDeclarer buildBoltDeclarer(TopologyBuilder builder, StormParallelismConfig stormParallelismConfig, String boltId, IComponent baseComponent) throws ConfigurationException { if (baseComponent instanceof IBasicBolt) { return builder .setBolt(boltId, (IBasicBolt) baseComponent, stormParallelismConfig.getParallelismHint()) .setNumTasks(stormParallelismConfig.getNumTasks()); } else if (baseComponent instanceof IRichBolt) { return builder .setBolt(boltId, (IRichBolt) baseComponent, stormParallelismConfig.getParallelismHint()) .setNumTasks(stormParallelismConfig.getNumTasks()); } else { throw new ConfigurationException("Unknown component type: " + baseComponent.getClass()); } }
.fieldsGrouping(messageUpstreamComponent, "message", new Fields("key")) .setNumTasks( config.getInt("bolt.enrichment.whois.num.tasks"));
topologyBuilder.setBolt(boltName, new AggregationBolt(aggregationConfig.getStormConfig(), new MRMetricsAggregateContainer(metrics, aggregationConfig)), tasks).setNumTasks(tasks).shuffleGrouping(spoutName);
.fieldsGrouping(messageUpstreamComponent, "message", new Fields("key")) .setNumTasks(config.getInt("bolt.enrichment.threat.num.tasks"));
.fieldsGrouping(messageUpstreamComponent, "message", new Fields("key")) .setNumTasks(config.getInt("bolt.alerts.num.tasks"));
indexing_bolt, config.getInt("bolt.error.indexing.parallelism.hint")) .setNumTasks(config.getInt("bolt.error.indexing.num.tasks"));
builder.setBolt(persistBoltName, bolt, numOfPersistTasks).setNumTasks(numOfPersistTasks).shuffleGrouping(spoutName); .setNumTasks(numOfSinkTasks).shuffleGrouping(persistBoltName);
public StormTopology buildTopology(String topic) { SpoutConfig kafkaConfig = new SpoutConfig(brokerHosts, topic, "", "xlog_"+topic); kafkaConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("KafkaSpout", new KafkaSpout(kafkaConfig), 2).setNumTasks(8); builder.setBolt("SplitBolt", new SplitSentence(), 1).setNumTasks(2).shuffleGrouping("KafkaSpout"); builder.setBolt("XlogBolt", new XlogBolt(), 4).setNumTasks(8).fieldsGrouping("SplitBolt", new Fields("ip")); return builder.createTopology(); }
@Override public StormTopology execute(Config config, StormEnvironment environment) { // 1. Init conf SparkHistoryJobAppConfig sparkHistoryJobAppConfig = SparkHistoryJobAppConfig.newInstance(config); final String jobFetchSpoutName = SparkHistoryJobAppConfig.SPARK_HISTORY_JOB_FETCH_SPOUT_NAME; final String jobParseBoltName = SparkHistoryJobAppConfig.SPARK_HISTORY_JOB_PARSE_BOLT_NAME; // 2. Config topology. TopologyBuilder topologyBuilder = new TopologyBuilder(); topologyBuilder.setSpout( jobFetchSpoutName, new SparkHistoryJobSpout(sparkHistoryJobAppConfig), sparkHistoryJobAppConfig.stormConfig.numOfSpoutExecutors ).setNumTasks(sparkHistoryJobAppConfig.stormConfig.numOfSpoutTasks); topologyBuilder.setBolt( jobParseBoltName, new SparkHistoryJobParseBolt(sparkHistoryJobAppConfig), sparkHistoryJobAppConfig.stormConfig.numOfParserBoltExecutors ).setNumTasks(sparkHistoryJobAppConfig.stormConfig.numOfParserBoltTasks).shuffleGrouping(jobFetchSpoutName); return topologyBuilder.createTopology(); } }
topologyBuilder.setSpout("blue-spout", new BlueSpout()); // parallelism hint topologyBuilder.setBolt("green-bolt-1", new GreenBolt(0)).setNumTasks(2) .customGrouping("blue-spout", new CustomStreamGrouping() { int count = 0;
sparkRunningJobAppConfig.getEndpointConfig(), sparkRunningJobAppConfig.getJobExtractorConfig()), parallelism).setNumTasks(tasks).fieldsGrouping(spoutName, new Fields("appId"));