public static Map<String, Object> readStormConfig() { Map<String, Object> ret = readDefaultConfig(); String confFile = System.getProperty("storm.conf.file"); Map<String, Object> storm; if (confFile == null || confFile.equals("")) { storm = findAndReadConfigFile("storm.yaml", false); } else { storm = findAndReadConfigFile(confFile, true); } ret.putAll(storm); ret.putAll(readCommandLineOpts()); return ret; }
/** * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming */ public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("args: runDurationSec topConfFile"); return; } Integer durationSec = Integer.parseInt(args[0]); String confFile = args[1]; Map<String, Object> topoConf = Utils.findAndReadConfigFile(confFile); topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.putAll(Utils.readCommandLineOpts()); // Submit topology to Storm cluster Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); }
public static void main(String[] args) throws Exception { int runTime = -1; Map<String, Object> topoConf = Utils.findAndReadConfigFile(args[1]); topoConf.put(Config.TOPOLOGY_SPOUT_RECVQ_SKIPS, 1); if (args.length > 0) { long sleepMs = Integer.parseInt(args[0]); topoConf.put(SLEEP_MS, sleepMs); } if (args.length > 1) { runTime = Integer.parseInt(args[1]); } if (args.length > 2) { System.err.println("args: spoutSleepMs [runDurationSec] "); return; } topoConf.putAll(Utils.readCommandLineOpts()); // Submit topology to storm cluster Helper.runOnClusterAndPrintMetrics(runTime, "LowThroughputTopo", topoConf, getTopology(topoConf)); }
/** * Spout generates random strings and HDFS bolt writes them to a text file */ public static void main(String[] args) throws Exception { String confFile = "conf/HdfsSpoutTopo.yaml"; int runTime = -1; //Run until Ctrl-C if (args.length > 0) { runTime = Integer.parseInt(args[0]); } if (args.length > 1) { confFile = args[1]; } // Submit to Storm cluster if (args.length > 2) { System.err.println("args: [runDurationSec] [confFile]"); return; } Map<String, Object> topoConf = Utils.findAndReadConfigFile(confFile); topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); topoConf.putAll(Utils.readCommandLineOpts()); Helper.runOnClusterAndPrintMetrics(runTime, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); }
fullConf.putAll(Utils.readCommandLineOpts()); fullConf.putAll(conf);
throws AuthorizationException, NotAliveException, InvalidTopologyException { topoConf = new HashMap(topoConf); topoConf.putAll(Utils.readCommandLineOpts()); Map<String, Object> conf = Utils.readStormConfig(); conf.putAll(topoConf);
public static void main(String[] args) throws Exception { int runTime = -1; Config topoConf = new Config(); topoConf.put(Config.TOPOLOGY_SPOUT_RECVQ_SKIPS, 1); topoConf.putAll(Utils.readCommandLineOpts()); if (args.length > 0) { long sleepMs = Integer.parseInt(args[0]); topoConf.put(SLEEP_MS, sleepMs); } if (args.length > 1) { runTime = Integer.parseInt(args[1]); } if (args.length > 2) { System.err.println("args: boltSleepMs [runDurationSec] "); return; } // Submit topology to storm cluster Helper.runOnClusterAndPrintMetrics(runTime, "BackPressureTopo", topoConf, getTopology(topoConf)); }
LOG.info("Using topology conf from {} as basis for getting new creds", topologyId); Map<String, Object> commandLine = Utils.readCommandLineOpts(); List<String> clCreds = (List<String>)commandLine.get(Config.TOPOLOGY_AUTO_CREDENTIALS); List<String> topoCreds = (List<String>)topologyConf.get(Config.TOPOLOGY_AUTO_CREDENTIALS);
fullConf.putAll(Utils.readCommandLineOpts()); fullConf.putAll(conf); conf = fullConf;
/** * ConstSpout only topology (No bolts) */ public static void main(String[] args) throws Exception { int runTime = -1; Config topoConf = new Config(); if (args.length > 0) { runTime = Integer.parseInt(args[0]); } if (args.length > 1) { topoConf.putAll(Utils.findAndReadConfigFile(args[1])); } topoConf.put(Config.TOPOLOGY_SPOUT_RECVQ_SKIPS, 8); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); topoConf.putAll(Utils.readCommandLineOpts()); if (args.length > 2) { System.err.println("args: [runDurationSec] [optionalConfFile]"); return; } // Submit topology to storm cluster Helper.runOnClusterAndPrintMetrics(runTime, TOPOLOGY_NAME, topoConf, getTopology()); } }
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("args: runDurationSec topConfFile"); return; } final Integer durationSec = Integer.parseInt(args[0]); Config topoConf = new Config(); topoConf.putAll(Utils.findAndReadConfigFile(args[1])); topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); topoConf.putAll(Utils.readCommandLineOpts()); // Submit to Storm cluster Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); } }
public static void main(String[] args) throws Exception { int runTime = -1; Config topoConf = new Config(); if (args.length > 0) { runTime = Integer.parseInt(args[0]); } if (args.length > 1) { topoConf.putAll(Utils.findAndReadConfigFile(args[1])); } topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); topoConf.putAll(Utils.readCommandLineOpts()); if (args.length > 2) { System.err.println("args: [runDurationSec] [optionalConfFile]"); return; } // Submit topology to storm cluster Helper.runOnClusterAndPrintMetrics(runTime, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); } }
public static void main(String[] args) throws Exception { int runTime = -1; Config topoConf = new Config(); if (args.length > 2) { String file = args[0]; runTime = Integer.parseInt(args[1]); topoConf.put(INPUT_FILE, file); topoConf.putAll(Utils.findAndReadConfigFile(args[1])); } if (args.length > 3 || args.length == 0) { System.err.println("args: file.txt [runDurationSec] [optionalConfFile]"); return; } topoConf.put(Config.TOPOLOGY_SPOUT_RECVQ_SKIPS, 8); topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 1000); topoConf.put(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING, true); topoConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 0.0005); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_STRATEGY, "org.apache.storm.policy.WaitStrategyPark"); topoConf.put(Config.TOPOLOGY_BOLT_WAIT_PARK_MICROSEC, 0); topoConf.putAll(Utils.readCommandLineOpts()); // Submit topology to storm cluster Helper.runOnClusterAndPrintMetrics(runTime, TOPOLOGY_NAME, topoConf, getTopology(topoConf)); } }
topoConf.putAll(Utils.readCommandLineOpts()); Map<String, Object> conf = Utils.readStormConfig(); conf.putAll(topoConf);
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new InOrderSpout(), 8); builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1")); Config conf = new Config(); conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class); String name = "in-order-test"; if (args != null && args.length > 0) { name = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology()); Map<String, Object> clusterConf = Utils.readStormConfig(); clusterConf.putAll(Utils.readCommandLineOpts()); Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient(); //Sleep for 50 mins for (int i = 0; i < 50; i++) { Thread.sleep(30 * 1000); printMetrics(client, name); } kill(client, name); }
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", new FastRandomSentenceSpout(), 4); builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word")); Config conf = new Config(); conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class); String name = "wc-test"; if (args != null && args.length > 0) { name = args[0]; } conf.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology()); Map<String, Object> clusterConf = Utils.readStormConfig(); clusterConf.putAll(Utils.readCommandLineOpts()); Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient(); //Sleep for 5 mins for (int i = 0; i < 10; i++) { Thread.sleep(30 * 1000); printMetrics(client, name); } kill(client, name); }
private void readGlobalTimeoutConfigs() { Map stormConf = readStormConfigWithoutCLI(); Map cliConf = Utils.readCommandLineOpts(); //parameter TOPOLOGY_MESSAGE_TIMEOUT_SECS is declared @isInteger and @NotNull in storm-core (org.apache.storm.Config) baseMessageTimeoutSecs = (Integer) stormConf.getOrDefault(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 0); cliMessageTimeoutSecs = (Integer) cliConf.getOrDefault(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 0); //parameter TOPOLOGY_TICK_TUPLE_FREQ_SECS is only declared @isInteger, and may in fact return null Object scratch; scratch = stormConf.getOrDefault(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 0); baseTickTupleFreqSecs = (scratch == null) ? 0 : (Integer) scratch; scratch = cliConf.getOrDefault(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 0); cliTickTupleFreqSecs = (scratch == null) ? 0 : (Integer) scratch; }
public static Map<String, Object> readStormConfig() { Map<String, Object> ret = readDefaultConfig(); String confFile = System.getProperty("storm.conf.file"); Map<String, Object> storm; if (confFile == null || confFile.equals("")) { storm = findAndReadConfigFile("storm.yaml", false); } else { storm = findAndReadConfigFile(confFile, true); } ret.putAll(storm); ret.putAll(readCommandLineOpts()); return ret; }
ret.putAll(Utils.readCommandLineOpts()); return ret;
throws AuthorizationException, NotAliveException, InvalidTopologyException { stormConf = new HashMap(stormConf); stormConf.putAll(Utils.readCommandLineOpts()); Map conf = Utils.readStormConfig(); conf.putAll(stormConf);