@Override public void assign(BoltDeclarer declarer, InputStream stream) { declarer.localOrShuffleGrouping(stream.fromComponent, stream.id); } },
static StormTopology getTopology(Map<String, Object> conf) { // 1 - Setup Spout -------- ConstSpout spout = new ConstSpout("some data").withOutputFields("str"); // 2 - Setup IdBolt & DevNullBolt -------- IdBolt bolt1 = new IdBolt(); DevNullBolt bolt2 = new DevNullBolt(); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); int numSpouts = Helper.getInt(conf, SPOUT_COUNT, 1); builder.setSpout(SPOUT_ID, spout, numSpouts); int numBolt1 = Helper.getInt(conf, BOLT1_COUNT, 1); builder.setBolt(BOLT1_ID, bolt1, numBolt1) .localOrShuffleGrouping(SPOUT_ID); int numBolt2 = Helper.getInt(conf, BOLT2_COUNT, 1); builder.setBolt(BOLT2_ID, bolt2, numBolt2) .localOrShuffleGrouping(BOLT1_ID); System.err.printf("====> Using : numSpouts = %d , numBolt1 = %d, numBolt2=%d\n", numSpouts, numBolt1, numBolt2); return builder.createTopology(); }
static StormTopology getTopology(Map<String, Object> conf) { // 1 - Setup Spout -------- ConstSpout spout = new ConstSpout("some data").withOutputFields("str"); // 2 - Setup DevNull Bolt -------- DevNullBolt bolt = new DevNullBolt(); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); int numSpouts = Helper.getInt(conf, SPOUT_COUNT, 1); builder.setSpout(SPOUT_ID, spout, numSpouts); int numBolts = Helper.getInt(conf, BOLT_COUNT, 1); BoltDeclarer bd = builder.setBolt(BOLT_ID, bolt, numBolts); System.err.printf("====> Using : numSpouts = %d , numBolts = %d\n", numSpouts, numBolts); String groupingType = Helper.getStr(conf, GROUPING); if (groupingType == null || groupingType.equalsIgnoreCase(DEFAULT_GROUPING)) { bd.localOrShuffleGrouping(SPOUT_ID); } else if (groupingType.equalsIgnoreCase(SHUFFLE_GROUPING)) { bd.shuffleGrouping(SPOUT_ID); } return builder.createTopology(); }
static StormTopology getTopology(Map<String, Object> conf) { Long sleepMs = ObjectReader.getLong(conf.get(SLEEP_MS)); // 1 - Setup Spout -------- ConstSpout spout = new ConstSpout("some data").withOutputFields("string"); // 2 - Setup DevNull Bolt -------- ThrottledBolt bolt = new ThrottledBolt(sleepMs); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, Helper.getInt(conf, SPOUT_COUNT, 1)); BoltDeclarer bd = builder.setBolt(BOLT_ID, bolt, Helper.getInt(conf, BOLT_COUNT, 1)); bd.localOrShuffleGrouping(SPOUT_ID); return builder.createTopology(); }
static StormTopology getTopology(Map<String, Object> conf) { Long sleepMs = ObjectReader.getLong(conf.get(SLEEP_MS)); // 1 - Setup Spout -------- ThrottledSpout spout = new ThrottledSpout(sleepMs).withOutputFields(ThrottledSpout.DEFAULT_FIELD_NAME); // 2 - Setup DevNull Bolt -------- LatencyPrintBolt bolt = new LatencyPrintBolt(); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, Helper.getInt(conf, SPOUT_COUNT, 1)); BoltDeclarer bd = builder.setBolt(BOLT_ID, bolt, Helper.getInt(conf, BOLT_COUNT, 1)); bd.localOrShuffleGrouping(SPOUT_ID); // bd.shuffleGrouping(SPOUT_ID); return builder.createTopology(); }
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int spBoltNum = Helper.getInt(config, SPLIT_NUM, DEFAULT_SPLIT_BOLT_NUM); final int cntBoltNum = Helper.getInt(config, COUNT_NUM, DEFAULT_COUNT_BOLT_NUM); final String inputFile = Helper.getStr(config, INPUT_FILE); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, new FileReadSpout(inputFile), spoutNum); builder.setBolt(SPLIT_ID, new SplitSentenceBolt(), spBoltNum).localOrShuffleGrouping(SPOUT_ID); builder.setBolt(COUNT_ID, new CountBolt(), cntBoltNum).fieldsGrouping(SPLIT_ID, new Fields(SplitSentenceBolt.FIELDS)); return builder.createTopology(); }
break; case LOCAL_OR_SHUFFLE: declarer.localOrShuffleGrouping(stream.getFrom(), streamId); break; case NONE:
static StormTopology getTopology(Map<String, Object> config) { final int spoutNum = Helper.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int boltNum = Helper.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM); final String fileFormat = Helper.getStr(config, "text"); final String hdfsUri = Helper.getStr(config, HDFS_URI); final String sourceDir = Helper.getStr(config, SOURCE_DIR); final String archiveDir = Helper.getStr(config, ARCHIVE_DIR); final String badDir = Helper.getStr(config, BAD_DIR); // 1 - Setup Hdfs Spout -------- HdfsSpout spout = new HdfsSpout() .setReaderType(fileFormat) .setHdfsUri(hdfsUri) .setSourceDir(sourceDir) .setArchiveDir(archiveDir) .setBadFilesDir(badDir) .withOutputFields(TextFileReader.defaultFields); // 2 - DevNull Bolt -------- DevNullBolt bolt = new DevNullBolt(); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, spoutNum); builder.setBolt(BOLT_ID, bolt, boltNum) .localOrShuffleGrouping(SPOUT_ID); return builder.createTopology(); }
static StormTopology getTopology(Map<String, Object> topoConf) { final int hdfsBatch = Helper.getInt(topoConf, HDFS_BATCH, DEFAULT_HDFS_BATCH); // 1 - Setup StringGen Spout -------- StringGenSpout spout = new StringGenSpout(100).withFieldName("str"); // 2 - Setup HFS Bolt -------- String Hdfs_url = Helper.getStr(topoConf, HDFS_URI); RecordFormat format = new LineWriter("str"); SyncPolicy syncPolicy = new CountSyncPolicy(hdfsBatch); FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, FileSizeRotationPolicy.Units.GB); final int spoutNum = Helper.getInt(topoConf, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int boltNum = Helper.getInt(topoConf, BOLT_NUM, DEFAULT_BOLT_NUM); // Use default, Storm-generated file names FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(Helper.getStr(topoConf, HDFS_PATH)); // Instantiate the HdfsBolt HdfsBolt bolt = new HdfsBolt() .withFsUrl(Hdfs_url) .withFileNameFormat(fileNameFormat) .withRecordFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy); // 3 - Setup Topology -------- TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, spoutNum); builder.setBolt(BOLT_ID, bolt, boltNum) .localOrShuffleGrouping(SPOUT_ID); return builder.createTopology(); }
builder.setSpout(SPOUT_ID, spout, spoutNum); builder.setBolt(BOLT_ID, bolt, boltNum) .localOrShuffleGrouping(SPOUT_ID);
@Override public void setBolts(TopologyBuilder builder) { BoltDeclarer boltDeclarer = builder.setBolt("identity", new IdentityBolt(config), config.boltThreads); if (config.localShuffle) { boltDeclarer.localOrShuffleGrouping("spout"); } else { boltDeclarer.shuffleGrouping("spout"); } }
@Override public void setBolts(TopologyBuilder builder) { BoltDeclarer boltDeclarer = builder.setBolt("split", new SplitStreamBolt(), config.boltThreads); if (config.localShuffle) { boltDeclarer.localOrShuffleGrouping("spout"); } else { boltDeclarer.shuffleGrouping("spout"); } builder.setBolt("count", new WordCountBolt(config), config.boltThreads).fieldsGrouping("split", new Fields("ip")); }
boltDeclarer.localOrShuffleGrouping(spoutId); builder.setBolt("errorMessageWriter", errorBolt, errorWriterParallelism) .setNumTasks(errorWriterNumTasks) .localOrShuffleGrouping("parserBolt", Constants.ERROR_STREAM);
@Override public void setBolts(TopologyBuilder builder) { Duration windowDuration = new Duration((int) config.windowDuration, TimeUnit.MILLISECONDS); Duration windowSlide = new Duration((int) config.windowSlideStep, TimeUnit.MILLISECONDS); BoltDeclarer boltDeclarer = builder.setBolt("parser", new ParserBolt(), config.boltThreads); if (config.localShuffle) { boltDeclarer.localOrShuffleGrouping("spout"); } else { boltDeclarer.shuffleGrouping("spout"); } builder.setBolt("window", new SlidingWindowBolt(config) .withWindow(windowDuration, windowSlide), config.boltThreads).fieldsGrouping("parser", new Fields("ip")); }
.localOrShuffleGrouping("fetch"); builder.setBolt("parse", new JSoupParserBolt()).localOrShuffleGrouping( "sitemap"); builder.setBolt("indexer", new IndexerBolt()).localOrShuffleGrouping( "parse"); .localOrShuffleGrouping("fetch", Constants.StatusStreamName) .localOrShuffleGrouping("sitemap", Constants.StatusStreamName) .localOrShuffleGrouping("parse", Constants.StatusStreamName) .localOrShuffleGrouping("indexer", Constants.StatusStreamName);
.localOrShuffleGrouping("fetch"); .localOrShuffleGrouping("sitemap"); .localOrShuffleGrouping("feed"); .localOrShuffleGrouping("parse"); builder.setBolt("warc", warcbolt).localOrShuffleGrouping("fetch"); .localOrShuffleGrouping("fetch", Constants.StatusStreamName) .localOrShuffleGrouping("sitemap", Constants.StatusStreamName) .localOrShuffleGrouping("feed", Constants.StatusStreamName) .localOrShuffleGrouping("parse", Constants.StatusStreamName) .localOrShuffleGrouping("ssb", Constants.StatusStreamName) .setNumTasks(numShards);
.setNumTasks(2).localOrShuffleGrouping("fetch"); .localOrShuffleGrouping("sitemap"); .localOrShuffleGrouping("feed"); builder.setBolt("warc", warcbolt).localOrShuffleGrouping("feed"); .localOrShuffleGrouping("fetch", Constants.StatusStreamName) .localOrShuffleGrouping("sitemap", Constants.StatusStreamName) .localOrShuffleGrouping("feed", Constants.StatusStreamName) .localOrShuffleGrouping("ssb", Constants.StatusStreamName) .setNumTasks(numShards);
.localOrShuffleGrouping("fetch"); .localOrShuffleGrouping("sitemap"); .localOrShuffleGrouping("parse"); .localOrShuffleGrouping("status", Constants.DELETION_STREAM_NAME);
break; case LOCAL_OR_SHUFFLE: declarer.localOrShuffleGrouping(stream.getFrom(), streamId); break; case NONE: