/** * Submits a topology to run on the cluster with a progress bar. A topology runs forever or until explicitly killed. * * @param name the name of the storm. * @param topoConf the topology-specific configuration. See {@link Config}. * @param topology the processing to execute. * @throws AlreadyAliveException if a topology with this name is already running * @throws InvalidTopologyException if an invalid topology was submitted * @throws AuthorizationException if authorization is failed */ public static void submitTopologyWithProgressBar(String name, Map<String, Object> topoConf, StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException { submitTopologyWithProgressBar(name, topoConf, topology, null); }
public static void runOnClusterAndPrintMetrics(int durationSec, String topoName, Map<String, Object> topoConf, StormTopology topology) throws Exception { // submit topology StormSubmitter.submitTopologyWithProgressBar(topoName, topoConf, topology); setupShutdownHook(topoName); // handle Ctrl-C // poll metrics every minute, then kill topology after specified duration Integer pollIntervalSec = 60; collectMetricsAndKill(topoName, pollIntervalSec, durationSec); } }
public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(20); conf.put(Config.TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT, 100); // window-state table should already be created with cf:tuples column HBaseWindowsStoreFactory windowStoreFactory = new HBaseWindowsStoreFactory(new HashMap<String, Object>(), "window-state", "cf".getBytes("UTF-8"), "tuples".getBytes("UTF-8")); String topoName = "wordCounterWithWindowing"; if (args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(windowStoreFactory)); }
public static void main(String[] args) throws Exception { StormTopology topology = buildVehiclesTopology(); Config conf = new Config(); conf.setMaxSpoutPending(20); conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar("vehicles-topology", conf, topology); }
public static void main(String[] args) throws Exception { StormTopology topology = getStormTopology(); Config conf = new Config(); conf.setDebug(true); String topoName = "test"; if (args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, topology); }
public static void main(String[] args) throws Exception { StormTopology topology = buildDevicesTopology(); Config conf = new Config(); conf.setMaxSpoutPending(20); conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar("devices-topology", conf, topology); }
public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(20); String topoName = "wordCounter"; if (args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology()); try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { for (int i = 0; i < 10; i++) { System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped")); Thread.sleep(1000); } } }
public static void main(String[] args) throws Exception { Config conf = new Config(); conf.setMaxSpoutPending(20); String topoName = "wordCounter"; if (args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology()); try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { for (int i = 0; i < 10; i++) { System.out.println("DRPC RESULT: " + drpc.execute("words", "CAT THE DOG JUMPED")); Thread.sleep(1000); } } } }
public static void main(String[] args) throws Exception { Config conf = new Config(); WindowsStoreFactory mapState = new InMemoryWindowsStoreFactory(); String topoName = "wordCounter"; if (args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(mapState, SlidingCountWindow.of(1000, 100))); } }
public static void main(String[] args) throws Exception { LinearDRPCTopologyBuilder builder = construct(); Config conf = new Config(); conf.setNumWorkers(6); String topoName = "reach-drpc"; if (args.length > 0) { topoName = args[0]; } StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology()); try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" }; for (String url : urlsToTry) { System.out.println("Reach of " + url + ": " + drpc.execute("reach", url)); } } }
public static void main(String[] args) throws Exception { LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation"); builder.addBolt(new ExclaimBolt(), 3); Config conf = new Config(); String topoName = "DRPCExample"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology()); if (args.length > 1) { try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) { for (int i = 1; i < args.length; i++) { String word = args[i]; System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word)); } } } }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word", new TestWordSpout(), 10); builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word"); builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1"); Config conf = new Config(); conf.setDebug(true); String topoName = MultipleLoggerTopology.class.getName(); if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(2); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
public void buildAndLaunchWordCountTopology(String[] args) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("filter", new FilterWords(), 6).shuffleGrouping("split"); Config conf = new Config(); conf.setDebug(true); try { conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology()); } catch (InvalidTopologyException | AuthorizationException | AlreadyAliveException exp) { throw new RuntimeException(exp); } }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentence(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); String topoName = "word-count"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(3); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /** * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case. */ Stream<Tuple3<Integer, Long, Long>> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2)); PairStream<Long, Integer> pairs = stream.mapToPair(t -> Pair.of(t._2 / 10000, t._1)); pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print(); String topoName = "test"; if (args.length > 0) { topoName = args[0]; } Config config = new Config(); config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); } }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomIntegerSpout()); builder.setBolt("partialsum", new StatefulSumBolt("partial"), 1).shuffleGrouping("spout"); builder.setBolt("printer", new PrinterBolt(), 2).shuffleGrouping("partialsum"); builder.setBolt("total", new StatefulSumBolt("total"), 1).shuffleGrouping("printer"); Config conf = new Config(); conf.setDebug(false); String topoName = "test"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /** * Computes average of the stream of numbers emitted by the spout. Internally the per-partition * sum and counts are accumulated and emitted to a downstream task where the partially accumulated * results are merged and the final result is emitted. */ builder.newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0), 2) .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5))) .filter(x -> x > 0 && x < 500) .aggregate(new Avg()) .print(); Config config = new Config(); String topoName = "AGG_EXAMPLE"; if (args.length > 0) { topoName = args[0]; } config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomIntegerSpout()); builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3)) .withMessageIdField("msgid"), 1).shuffleGrouping("spout"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt"); Config conf = new Config(); conf.setDebug(false); //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider"); String topoName = "test"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); BaseWindowedBolt bolt = new SlidingWindowSumBolt() .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS)) .withTimestampField("ts") .withLag(new Duration(5, TimeUnit.SECONDS)); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum"); Config conf = new Config(); conf.setDebug(true); String topoName = "test"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); } }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1) .shuffleGrouping("integer"); builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1) .shuffleGrouping("slidingsum"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg"); Config conf = new Config(); conf.setDebug(true); String topoName = "test"; if (args != null && args.length > 0) { topoName = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology()); }