/** * Define a new spout in this topology. * * @param id the id of this component. This id is referenced by other components that want to consume this spout's outputs. * @param spout the spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public SpoutDeclarer setSpout(String id, IRichSpout spout) throws IllegalArgumentException { return setSpout(id, spout, null); }
/** * Define a new spout in this topology. * * @param id the id of this component. This id is referenced by other components that want to consume this spout's outputs. * @param supplier lambda expression that implements tuple generating for this spout * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public SpoutDeclarer setSpout(String id, SerializableSupplier<?> supplier) throws IllegalArgumentException { return setSpout(id, supplier, null); }
/** * If the topology has at least one stateful bolt add a {@link CheckpointSpout} component to the topology. */ private void maybeAddCheckpointSpout() { if (hasStatefulBolt) { setSpout(CHECKPOINT_COMPONENT_ID, new CheckpointSpout(), 1); } }
@Test(expected = IllegalArgumentException.class) public void testSetSpout() { builder.setSpout("spout", mock(IRichSpout.class), 0); }
static StormTopology getTopology() { // 1 - Setup Const Spout -------- ConstSpout spout = new ConstSpout("some data").withOutputFields("str"); // 2 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, 1); return builder.createTopology(); }
@Override protected StormTopology getTopologyKafkaSpout(KafkaSpoutConfig<String, String> spoutConfig) { final TopologyBuilder tp = new TopologyBuilder(); tp.setSpout("kafka_spout", new KafkaSpout<>(spoutConfig), 1); tp.setBolt("kafka_bolt", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout", STREAM); return tp.createTopology(); }
static StormTopology getTopology(Map<String, Object> conf) { Long sleepMs = ObjectReader.getLong(conf.get(SLEEP_MS)); // 1 - Setup Spout -------- ConstSpout spout = new ConstSpout("some data").withOutputFields("string"); // 2 - Setup DevNull Bolt -------- ThrottledBolt bolt = new ThrottledBolt(sleepMs); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, Helper.getInt(conf, SPOUT_COUNT, 1)); BoltDeclarer bd = builder.setBolt(BOLT_ID, bolt, Helper.getInt(conf, BOLT_COUNT, 1)); bd.localOrShuffleGrouping(SPOUT_ID); return builder.createTopology(); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); DRPCSpout spout = new DRPCSpout("exclamation"); builder.setSpout("drpc", spout); builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc"); builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim"); Config conf = new Config(); StormSubmitter.submitTopology("exclaim", conf, builder.createTopology()); try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { System.out.println(drpc.execute("exclamation", "aaa")); System.out.println(drpc.execute("exclamation", "bbb")); } }
public static StormTopology getStormTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(WORD, new FixedOrderWordSpout(), SPOUT_EXECUTORS); builder.setBolt(EXCLAIM_1, new ExclamationTopology.ExclamationBolt(), 3).shuffleGrouping(WORD); builder.setBolt(EXCLAIM_2, new ExclamationTopology.ExclamationBolt(), EXCLAIM_2_EXECUTORS).shuffleGrouping(EXCLAIM_1); return builder.createTopology(); } }
protected StormTopology getTopology() throws IOException { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("SolrJsonSpout", new SolrJsonSpout()); builder.setBolt("SolrUpdateBolt", new SolrUpdateBolt(getSolrConfig(), getSolrMapper(), getSolrCommitStgy())) .shuffleGrouping("SolrJsonSpout"); return builder.createTopology(); } }
protected StormTopology getTopology() throws IOException { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("SolrFieldsSpout", new SolrFieldsSpout()); SolrConfig solrConfig = getSolrConfig(); builder.setBolt("SolrUpdateBolt", new SolrUpdateBolt(solrConfig, getSolrMapper(solrConfig), getSolrCommitStgy())) .shuffleGrouping("SolrFieldsSpout"); return builder.createTopology(); } }
protected StormTopology getTopologyKafkaSpout(KafkaSpoutConfig<String, String> spoutConfig) { final TopologyBuilder tp = new TopologyBuilder(); tp.setSpout("kafka_spout", new KafkaSpout<>(spoutConfig), 1); tp.setBolt("kafka_bolt", new KafkaSpoutTestBolt()) .shuffleGrouping("kafka_spout", TOPIC_0_1_STREAM) .shuffleGrouping("kafka_spout", TOPIC_2_STREAM); tp.setBolt("kafka_bolt_1", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout", TOPIC_2_STREAM); return tp.createTopology(); }
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int cntBoltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_COUNT_BOLT_NUM); final String inputFile = Helper.getStr(config, INPUT_FILE); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new WordGenSpout(inputFile), spoutNum); builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPOUT_ID, new Fields(WordGenSpout.FIELDS)); return builder.createTopology(); }
@Override public StormTopology newTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(getSpoutName(), new IncrementingSpout(), spoutExecutors); builder.setBolt(getBoltName(), new VerificationBolt() .withTumblingWindow(new BaseWindowedBolt.Count(tumbleSize)), boltExecutors) .shuffleGrouping(getSpoutName()); return builder.createTopology(); } }
protected int run(String[] args) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word", new TestWordSpout(), 10); builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word"); builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1"); conf.setDebug(true); String topologyName = "test"; conf.setNumWorkers(3); if (args != null && args.length > 0) { topologyName = args[0]; } return submit(topologyName, conf, builder); }
private StormTopology newTopology() throws Exception { final TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(RAW_INPUT_FROM_CSV_SPOUT, RawInputFromCSVSpout.newInstance(rawInputs)); builder.setBolt(PMML_PREDICTOR_BOLT, newBolt()).shuffleGrouping(RAW_INPUT_FROM_CSV_SPOUT); builder.setBolt(PRINT_BOLT_1, new PrinterBolt()).shuffleGrouping(PMML_PREDICTOR_BOLT); builder.setBolt(PRINT_BOLT_2, new PrinterBolt()).shuffleGrouping(PMML_PREDICTOR_BOLT, NON_DEFAULT_STREAM_ID); return builder.createTopology(); }
@Override public StormTopology newTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(getSpoutName(), new IncrementingSpout(), spoutExecutors); builder.setBolt(getBoltName(), new VerificationBolt() .withWindow(new BaseWindowedBolt.Count(windowSize), new BaseWindowedBolt.Count(slideSize)), boltExecutors) .shuffleGrouping(getSpoutName()); return builder.createTopology(); } }
protected int run(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); conf.setDebug(true); String topologyName = "word-count"; conf.setNumWorkers(3); if (args != null && args.length > 0) { topologyName = args[0]; } return submit(topologyName, conf, builder); }
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int spBoltNum = Helper.getInt(config, SPLIT_NUM, DEFAULT_SPLIT_BOLT_NUM); final int cntBoltNum = Helper.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM); final String inputFile = Helper.getStr(config, INPUT_FILE); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum); builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID); builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS)); return builder.createTopology(); }
public static StormTopology createTestTopology() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("words", new TestWordSpout(), generateParallelismHint()); builder.setBolt("count", new TestWordCounter(), generateParallelismHint()).shuffleGrouping("words"); builder.setBolt("globalCount", new TestGlobalCount(), generateParallelismHint()).shuffleGrouping("count"); return builder.createTopology(); }