@Override public LocalTopology submitTopology(String topologyName, Map<String, Object> conf, TrackedTopology topology) throws TException { return submitTopology(topologyName, conf, topology.getTopology()); }
@Override public void submitTopology(String name, String uploadedJarLocation, String jsonConf, StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException { try { @SuppressWarnings("unchecked") Map<String, Object> conf = (Map<String, Object>) JSONValue.parseWithException(jsonConf); submitTopology(name, conf, topology); } catch (ParseException e) { throw new RuntimeException(e); } }
public static void runStormTopology(LocalCluster cluster, final List<?> watchedList, final int expectedValueSize, AbstractStreamsProcessor proc, StormTopology topo) throws Exception { final Config conf = new Config(); conf.setMaxSpoutPending(20); conf.setDebug(true); if (proc.getClassLoaders() != null && proc.getClassLoaders().size() > 0) { CompilingClassLoader lastClassloader = proc.getClassLoaders().get(proc.getClassLoaders().size() - 1); Utils.setClassLoaderForJavaDeSerialize(lastClassloader); } try (LocalCluster.LocalTopology stormTopo = cluster.submitTopology("storm-sql", conf, topo)) { waitForCompletion(1000 * 1000, () -> watchedList.size() < expectedValueSize); } finally { while (cluster.getClusterInfo().get_topologies_size() > 0) { Thread.sleep(10); } Utils.resetClassLoaderForJavaDeSerialize(); } }
@Test public void testAckBranching() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withTracked() .build()) { AckTrackingFeeder feeder = new AckTrackingFeeder("num"); Map<String, SpoutDetails> spoutMap = new HashMap<>(); spoutMap.put("1", Thrift.prepareSpoutDetails(feeder.getSpout())); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); Map<GlobalStreamId, Grouping> aggregatorInputs = new HashMap<>(); aggregatorInputs.put(Utils.getGlobalStreamId("2", null), Thrift.prepareShuffleGrouping()); aggregatorInputs.put(Utils.getGlobalStreamId("3", null), Thrift.prepareShuffleGrouping()); boltMap.put("4", Thrift.prepareBoltDetails(aggregatorInputs, new AggBolt(4))); TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);; cluster.submitTopology("test-acking2", Collections.emptyMap(), tracked); cluster.advanceClusterTime(11); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(0); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(2); } }
@Test public void testWithTrackedCluster() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withTracked() .build()) { AckTrackingFeeder feeder = new AckTrackingFeeder("num"); Map<String, Thrift.SpoutDetails> spoutMap = new HashMap<>(); spoutMap.put("1", Thrift.prepareSpoutDetails(feeder.getSpout())); Map<String, Thrift.BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new IdentityBolt())); Map<GlobalStreamId, Grouping> aggregatorInputs = new HashMap<>(); aggregatorInputs.put(Utils.getGlobalStreamId("2", null), Thrift.prepareShuffleGrouping()); aggregatorInputs.put(Utils.getGlobalStreamId("3", null), Thrift.prepareShuffleGrouping()); boltMap.put("4", Thrift.prepareBoltDetails(aggregatorInputs, new AggBolt(4))); TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);; cluster.submitTopology("test-acking2", new Config(), tracked); cluster.advanceClusterTime(11); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(0); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(2); } }
@Test public void testAckingSelfAnchor() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withTracked() .build()) { AckTrackingFeeder feeder = new AckTrackingFeeder("num"); Map<String, SpoutDetails> spoutMap = new HashMap<>(); spoutMap.put("1", Thrift.prepareSpoutDetails(feeder.getSpout())); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("2", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareShuffleGrouping()), new DupAnchorBolt())); boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("2", null), Thrift.prepareShuffleGrouping()), new AckBolt())); TrackedTopology tracked = new TrackedTopology(Thrift.buildTopology(spoutMap, boltMap), cluster);; cluster.submitTopology("test", Collections.emptyMap(), tracked); cluster.advanceClusterTime(11); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 1); feeder.assertNumAcks(1); feeder.feed(new Values(1)); feeder.feed(new Values(1)); feeder.feed(new Values(1)); Testing.trackedWait(tracked, 3); feeder.assertNumAcks(3); } }
try (LocalCluster.LocalTopology stormTopo = localCluster.submitTopology("storm-sql", conf, topo)) { waitForCompletion(waitTimeoutMs, waitCondition); } finally {
StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); cluster.submitTopology("timeout-tester", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology);
stormConf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, false); cluster.submitTopology("disable-timeout-tester", stormConf, topology);
stormConf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10); cluster.submitTopology("timeout-tester", stormConf, topology);
cluster.submitTopology("acking-test1", Collections.emptyMap(), tracked);
StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); cluster.submitTopology("reset-timeout-tester", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology);
public static void main(String[] args) { try { Options options = new Options(); final CommandLine cmd = parse(options, args); if (cmd.hasOption("h")) { final HelpFormatter usageFormatter = new HelpFormatter(); usageFormatter.printHelp("ParserTopologyCLI", null, options, null, true); System.exit(0); } ParserTopologyCLI cli = new ParserTopologyCLI(); ParserTopologyBuilder.ParserTopology topology = cli.createParserTopology(cmd); String sensorTypes = ParserOptions.SENSOR_TYPES.get(cmd); if (ParserOptions.TEST.has(cmd)) { topology.getTopologyConfig().put(Config.TOPOLOGY_DEBUG, true); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(sensorTypes.replaceAll(",", STORM_JOB_SEPARATOR), topology.getTopologyConfig(), topology.getBuilder().createTopology()); Utils.sleep(300000); cluster.shutdown(); } else { StormSubmitter.submitTopology(sensorTypes.replaceAll(",", STORM_JOB_SEPARATOR), topology.getTopologyConfig(), topology.getBuilder().createTopology()); } } catch (Exception e) { e.printStackTrace(); System.exit(-1); } }
stormCluster.submitTopology(getTopologyName(), stormConf, topologyBuilder.getBuilder().createTopology()); } catch (Exception e) { throw new UnableToStartException("Unable to start parser topology for sensorTypes: " + sensorTypes, e);
LocalCluster cluster = new LocalCluster(); cluster.submitTopology("mytopology", conf, builder.createTopology());
private static void submitTopology(TopologyBuilder builder) { try { LocalCluster cluster = new LocalCluster(); config.put(Config.STORM_CLUSTER_MODE, "local"); cluster.submitTopology(Configuration.getConfig().getString("topology.name"), config, builder.createTopology()); } catch (Exception e) { LOG.error(e.getMessage(), e.getCause()); } } }
LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test", conf, builder.createTopology()); Utils.sleep(10000); cluster.killTopology("test"); cluster.shutdown();
private void runLocally() throws InterruptedException { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(this.TOPOLOGY_NAME, getConfig(), buildTopolgy()); stopWaitingForInput(); }
public void runLocal() throws Exception { LocalCluster cluster = new LocalCluster(); if(getConfig() == null) setConfig();//uses the default config cluster.submitTopology("lemongrenade-test", getConfig(), getTopology()); Utils.sleep(90000000); cluster.killTopology("lemongrenade-test"); cluster.shutdown(); }
import backtype.storm.Config; import backtype.storm.LocalCluster; import backtype.storm.topology.TopologyBuilder; public class SampleTopology { /** * @param args */ public static void main(String[] args) { TopologyBuilder topology=new TopologyBuilder(); topology.setSpout("sampleSpout",new SampleSpout()); topology.setBolt("sampleBolt",new SampleBolt()).shuffleGrouping("sampleSpout"); Config conf = new Config(); conf.setDebug(true); LocalCluster cluster=new LocalCluster(); cluster.submitTopology("test", conf, topology.createTopology()); } }