/** * This submits a topology after loading the configured Spout (and optionally, Bolt), which is either the {@link DSLSpout} * and {@link DSLBolt} or a custom Spout and Bolt. The topology is submitted with the given configuration as the source * of {@link com.yahoo.bullet.record.BulletRecord} using the given {@link TopologyBuilder} * * @param config The Storm settings for this Bullet topology. * @param builder The {@link TopologyBuilder} to use to add the topology to. * @throws Exception if there were any issues submitting the topology. */ public static void submit(BulletStormConfig config, TopologyBuilder builder) throws Exception { Boolean dslSpoutEnable = config.getAs(BulletStormConfig.DSL_SPOUT_ENABLE, Boolean.class); if (dslSpoutEnable) { addDSLSpout(config, builder); } else { addBulletSpout(config, builder); } submit(config, TopologyConstants.RECORD_COMPONENT, builder); } }
private void submitWithTopology(String recordComponent) { try { StormUtils.submit(config, recordComponent, builder); } catch (Exception ignored) { } }
@Test(expectedExceptions = ClassNotFoundException.class) public void testFailingSubmitOnMissingSpout() throws Exception { StormUtils utils = new StormUtils(); utils.submit("non.existant.spout", null, new BulletStormConfig(), null, null, null, null); }
private void submitWithSpout(String name, List<String> args, Number parallelism, Number cpu, Number on, Number off) { try { StormUtils.submit(builder, name, args, config, parallelism, cpu, on, off); } catch (Exception ignored) { } }
/** * This submits a topology after loading the given spout with the given configuration as the source of * {@link com.yahoo.bullet.record.BulletRecord}. * * @param spout The name of the instance of an {@link org.apache.storm.topology.IRichSpout} to load. * @param args The arguments to pass to the constructor of this spout (otherwise the default constructor is used). * @param config The Storm settings for this Bullet topology. * @param parallelism The parallelism of the spout component. * @param cpuLoad The CPU load for the Storm RAS scheduler. * @param onHeapMemoryLoad The on heap memory load for the Storm RAS scheduler. * @param offHeapMemoryLoad The off heap memory load for the Storm RAS scheduler. * @throws Exception if there were issues creating the topology. */ public static void submit(String spout, List<String> args, BulletStormConfig config, Number parallelism, Number cpuLoad, Number onHeapMemoryLoad, Number offHeapMemoryLoad) throws Exception { submit(new TopologyBuilder(), spout, args, config, parallelism, cpuLoad, onHeapMemoryLoad, offHeapMemoryLoad); } }
/** * This submits a topology after loading the given spout with the given configuration as the source of * {@link com.yahoo.bullet.record.BulletRecord} using the given {@link TopologyBuilder}. * * @param builder The {@link TopologyBuilder} to use to add the topology to. * @param spout The name of the instance of an {@link org.apache.storm.topology.IRichSpout} to load. * @param args The arguments to pass to the constructor of this spout (otherwise the default constructor is used). * @param config The Storm settings for this Bullet topology. * @param parallelism The parallelism of the spout component. * @param cpuLoad The CPU load for the Storm RAS scheduler. * @param onHeapMemoryLoad The on heap memory load for the Storm RAS scheduler. * @param offHeapMemoryLoad The off heap memory load for the Storm RAS scheduler. * @throws Exception if there were issues creating the topology. */ public static void submit(TopologyBuilder builder, String spout, List<String> args, BulletStormConfig config, Number parallelism, Number cpuLoad, Number onHeapMemoryLoad, Number offHeapMemoryLoad) throws Exception { builder.setSpout(TopologyConstants.RECORD_COMPONENT, ReflectionUtils.getSpout(spout, args), parallelism) .setCPULoad(cpuLoad).setMemoryLoad(onHeapMemoryLoad, offHeapMemoryLoad); log.info("Added spout {} with Parallelism {}, CPU load {}, On-heap memory {}, Off-heap memory {}", spout, parallelism, cpuLoad, onHeapMemoryLoad, offHeapMemoryLoad); submit(config, TopologyConstants.RECORD_COMPONENT, builder); }
/** * Main. Launches a remote Storm topology. * @param args The input args. * @throws Exception if any. */ public static void main(String[] args) throws Exception { OptionSet options = PARSER.parse(args); if (!options.hasOptions() || options.has(HELP_ARG) || !options.has(SPOUT_ARG)) { System.out.println("If you are looking to connect your existing topology to Bullet, you should compile"); System.out.println("in the Bullet jar and use the submit method in the StormUtils class to wire up Bullet"); System.out.println("to the tail end of your topology that produces BulletRecords. If you are simply"); System.out.println("looking to connect a Spout class that implements IRichSpout and emits BulletRecords,"); System.out.println("use the main class directly with the arguments below."); PARSER.printHelpOn(System.out); return; } String spoutClass = (String) options.valueOf(SPOUT_ARG); List<String> arguments = (List<String>) options.valuesOf(ARGUMENT_ARG); Integer parallelism = (Integer) options.valueOf(PARALLELISM_ARG); Double cpuLoad = (Double) options.valueOf(CPU_LOAD_ARG); Double onHeapMemoryLoad = (Double) options.valueOf(ON_HEAP_MEMORY_LOAD_ARG); Double offHeapMemoryLoad = (Double) options.valueOf(OFF_HEAP_MEMORY_LOAD_ARG); String yamlPath = (String) options.valueOf(CONFIGURATION_ARG); BulletStormConfig config = new BulletStormConfig(yamlPath); log.info(config.toString()); StormUtils.submit(spoutClass, arguments, config, parallelism, cpuLoad, onHeapMemoryLoad, offHeapMemoryLoad); } }
/** * Main. Launches a remote Storm topology. * @param args The input args. * @throws Exception if any. */ public static void main(String[] args) throws Exception { OptionSet options = PARSER.parse(args); if (options.has(HELP_ARG) || !options.has(CONFIGURATION_ARG)) { printHelp(); return; } String yamlPath = (String) options.valueOf(CONFIGURATION_ARG); BulletStormConfig config = new BulletStormConfig(yamlPath); log.info(config.toString()); StormUtils.submit(config, new TopologyBuilder()); }