public static void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException { String streamName = (String) conf.get(Config.TOPOLOGY_NAME); if (streamName == null) { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); streamName = className[className.length - 1]; } TopologyBuilder builder = new TopologyBuilder(); int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2); builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint); BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint); // localFirstGrouping is only for jstorm // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME); boltDeclarer.shuffleGrouping("spout"); // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60); conf.put(Config.STORM_CLUSTER_MODE, "distributed"); StormSubmitter.submitTopology(streamName, conf, builder.createTopology()); }
new DeploySpoult(_messageSize, _ackEnabled), _spoutParallel); builder.setBolt("messageBolt1", new DeployBolt(), _boltParallel) .shuffleGrouping("messageSpout"); for (int levelNum = 2; levelNum <= _boltNum; levelNum++) { builder.setBolt("messageBolt" + levelNum, new DeployBolt(), _boltParallel) .shuffleGrouping("messageBolt" + (levelNum - 1));
public static void test() { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(new Count(30), new Count(10)), 1) .shuffleGrouping("integer"); builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(new Count(3)), 1) .shuffleGrouping("slidingsum"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg"); conf.setDebug(true); JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.fillInStackTrace(); Assert.fail("Failed to submit topology"); } }
builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");
public static void test() { String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { TopologyBuilder builder = new TopologyBuilder(); BaseWindowedBolt bolt = new SlidingWindowSumBolt() .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS)) .withTimestampField("ts").withLag(new Duration(5, TimeUnit.SECONDS)); builder.setSpout("integer", new RandomIntegerSpout(), 1); builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer"); builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum"); conf.setDebug(true); JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void test() { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.setDebug(true); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); Assert.fail("Failed"); } }
public static void testDrpc() { TopologyBuilder builder = new TopologyBuilder(); LocalDRPC drpc = new LocalDRPC(); DRPCSpout spout = new DRPCSpout("exclamation", drpc); builder.setSpout("drpc", spout); builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc"); builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim"); LocalCluster cluster = new LocalCluster(); Config conf = new Config(); cluster.submitTopology("exclaim", conf, builder.createTopology()); JStormUtils.sleepMs(30 * 1000); try { System.out.println(drpc.execute("exclamation", "aaa")); System.out.println(drpc.execute("exclamation", "bbb")); } catch (Exception e) { Assert.fail("Failed to test drpc"); } drpc.shutdown(); cluster.shutdown(); }
boltDeclarer.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME) .allGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, SequenceTopologyDef.CONTROL_STREAM_ID) .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3); .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID); builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint) .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
public static void test() throws Exception { int spout_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_parallelism_hint); builder.setBolt("split", new SplitSentence(), split_parallelism_hint).shuffleGrouping("spout"); builder.setBolt("count", new WordCount() .ingestionTimeWindow(Time.seconds(1L), Time.milliseconds(500L)) .withStateSize(Time.minutes(10)), count_parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); }
builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
public static TopologyBuilder setBuilder() { BatchTopologyBuilder topologyBuilder = new BatchTopologyBuilder(topologyName); int spoutParallel = JStormUtils.parseInt(conf.get("topology.spout.parallel"), 1); BoltDeclarer boltDeclarer = topologyBuilder.setSpout("Spout", new SimpleSpout(), spoutParallel); int boltParallel = JStormUtils.parseInt(conf.get("topology.bolt.parallel"), 2); topologyBuilder.setBolt("Bolt", new SimpleBolt(), boltParallel).shuffleGrouping("Spout"); return topologyBuilder.getTopologyBuilder(); }
builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).localFirstGrouping("spout"); else builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout"); int topN = 10; Time win = Time.seconds(10L); builder.setBolt("count", new WordCount(topN) .timeWindow(win) .withStateSize(Time.seconds(120L)), count_Parallelism_hint).fieldsGrouping("split", new Fields("word")); builder.setBolt("merge", new MergeTopN(topN).timeWindow(win), 1).allGrouping("count"); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); } catch (Exception e) { e.printStackTrace(); } }
public static void test() { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); boolean isLocalShuffle = JStormUtils.parseBoolean(conf.get("is.local.first.group"), false); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); if (isLocalShuffle) { builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).localFirstGrouping("spout"); } else { builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout"); } builder.setBolt("count", new WordCount() .timeWindow(Time.seconds(1L), Time.milliseconds(500L)) .withStateSize(Time.hours(2)), count_Parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; try { JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), true); } catch (Exception e) { e.printStackTrace(); } }
public static void test(Config conf) throws Exception { int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1); int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1); int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint); builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout"); long windowSize = JStormUtils.parseLong(conf.get("window.size.sec"), 10); long stateWindowSize = JStormUtils.parseLong(conf.get("state.window.size.sec"), 60); builder.setBolt("count", new WordCount() .timeWindow(Time.seconds(windowSize)) .withStateSize(Time.seconds(stateWindowSize)), count_Parallelism_hint).fieldsGrouping("split", new Fields("word")); String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\."); String topologyName = className[className.length - 1]; boolean isLocal = true; if (conf.containsKey("storm.cluster.mode")) { isLocal = StormConfig.local_mode(conf); } // RUN_LONG_TIME = true JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal); }
.shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID); builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint) .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
private void setDefaultBoltGrouping(BoltDeclarer bolt, String streanName) throws StreamingException { IRichOperator operator = getOperatorByOutputStreamName(streanName); if (operator == null) { StreamingException exception = new StreamingException(ErrorCode.PLATFORM_INVALID_TOPOLOGY); LOG.error("Can't find operator by stream name : {} .", streanName, exception); throw exception; } bolt.shuffleGrouping(operator.getOperatorId(), streanName); }
private void setDefaultBoltGrouping(BoltDeclarer bolt, String strname) throws StreamingException { IRichOperator operator = getOperatorByOutputStreamName(strname); if (operator == null) { StreamingException exception = new StreamingException(ErrorCode.PLATFORM_INVALID_TOPOLOGY); LOG.error("Can't find opertor by stream name : {} .", strname, exception); throw exception; } bolt.shuffleGrouping(operator.getOperatorId(), strname); }
public TransformedStream transformBy(TransformFunction function) { String componentId = generateId(function.getName()); topologyBuilder.setBolt(componentId, new TransformFunctionBolt(function)).shuffleGrouping(getId()); return new TransformedStream(componentId); } }
/** * Persist source data stream as metric. */ public BuilderContext saveAsMetric(MetricDescriptor metricDescriptor) { String metricDataID = generateId("MetricDataSink"); String metricSchemaID = generateId("MetricSchemaGenerator"); topologyBuilder.setBolt(metricDataID, new MetricStreamPersist(metricDescriptor, appConfig)).shuffleGrouping(getId()); topologyBuilder.setBolt(metricSchemaID, new MetricSchemaGenerator(metricDescriptor,appConfig)).fieldsGrouping(metricDataID,new Fields(MetricStreamPersist.METRIC_NAME_FIELD)); return this; }