return new LocalCluster.Builder() .withSupervisors(supervisors.intValue()) .withDaemonConf(conf) .withPortsPerSupervisor(ports.intValue()) .withINimbus(inimbus)
/** * Run c with a local mode cluster overriding the NimbusClient and DRPCClient calls. NOTE local mode override happens by default now * unless netty is turned on for the local cluster. * * @param c the callable to run in this mode * @param ttlSec the number of seconds to let the cluster run after c has completed * @param daemonConf configs to set for the daemon processes. * @return the result of calling C * * @throws Exception on any Exception. */ public static <T> T withLocalModeOverride(Callable<T> c, long ttlSec, Map<String, Object> daemonConf) throws Exception { LOG.info("\n\n\t\tSTARTING LOCAL MODE CLUSTER\n\n"); Builder builder = new Builder(); if (daemonConf != null) { builder.withDaemonConf(daemonConf); } try (LocalCluster local = builder.build(); LocalDRPC drpc = new LocalDRPC(local.metricRegistry); DRPCClient.LocalOverride drpcOverride = new DRPCClient.LocalOverride(drpc)) { T ret = c.call(); LOG.info("\n\n\t\tRUNNING LOCAL CLUSTER for {} seconds.\n\n", ttlSec); Thread.sleep(ttlSec * 1000); LOG.info("\n\n\t\tSTOPPING LOCAL MODE CLUSTER\n\n"); return ret; } }
/** * Create a LocalCluster that connects to an existing Zookeeper instance. * * @param zkHost the host for ZK * @param zkPort the port for ZK * @throws Exception on any error */ public LocalCluster(String zkHost, Long zkPort) throws Exception { this(new Builder().withDaemonConf(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true) .withDaemonConf(Config.STORM_ZOOKEEPER_SERVERS, Arrays.asList(zkHost)) .withDaemonConf(Config.STORM_ZOOKEEPER_PORT, zkPort)); }
.withSimulatedTime() .withSupervisors(4) .withDaemonConf(Collections.singletonMap(Config.STORM_LOCAL_MODE_ZMQ, !useLocalMessaging)) .build()) { Map<String, SpoutDetails> spoutMap = Collections.singletonMap("1", Thrift.prepareSpoutDetails(new TestWordSpout(true), 3));
conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap); try (ILocalCluster cluster = new LocalCluster.Builder().withDaemonConf(conf).build()) {
try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withDaemonConf(daemonConf) .build()) { TopologyBuilder topologyBuilder = new TopologyBuilder();
try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withDaemonConf(Collections.singletonMap(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)) .build()) { FeederSpout feeder = new FeederSpout(new Fields("field1"));
.withSimulatedTime() .withSupervisors(4) .withDaemonConf(Collections.singletonMap(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)) .build()) { FeederSpout feeder = new FeederSpout(new Fields("field1"));
.withDaemonConf(topoConf).build()) {
@Test public void testSubmitInactiveTopology() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withDaemonConf(Collections.singletonMap(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)) .build()) { FeederSpout feeder = new FeederSpout(new Fields("field1")); AckFailMapTracker tracker = new AckFailMapTracker(); feeder.setAckFailDelegate(tracker); Map<String, SpoutDetails> spoutMap = new HashMap<>(); spoutMap.put("1", Thrift.prepareSpoutDetails(feeder)); spoutMap.put("2", Thrift.prepareSpoutDetails(new OpenTrackedSpout())); Map<String, BoltDetails> boltMap = new HashMap<>(); boltMap.put("3", Thrift.prepareBoltDetails(Collections.singletonMap(Utils.getGlobalStreamId("1", null), Thrift.prepareGlobalGrouping()), new PrepareTrackedBolt())); boltPrepared = false; spoutOpened = false; StormTopology topology = Thrift.buildTopology(spoutMap, boltMap); cluster.submitTopologyWithOpts("test", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology, new SubmitOptions(TopologyInitialStatus.INACTIVE)); cluster.advanceClusterTime(11); feeder.feed(new Values("a"), 1); cluster.advanceClusterTime(9); assertThat(boltPrepared, is(false)); assertThat(spoutOpened, is(false)); cluster.getNimbus().activate("test"); cluster.advanceClusterTime(12); assertAcked(tracker, 1); assertThat(boltPrepared, is(true)); assertThat(spoutOpened, is(true)); } }
daemonConf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, false); try (LocalCluster cluster = new LocalCluster.Builder() .withDaemonConf(daemonConf) .withSimulatedTime() .build()) {
daemonConf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true); try (LocalCluster cluster = new LocalCluster.Builder() .withDaemonConf(daemonConf) .withSimulatedTime() .build()) {
@Test public void testKryoDecoratorsConfig() throws Exception { Map<String, Object> daemonConf = new HashMap<>(); daemonConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true); daemonConf.put(Config.TOPOLOGY_KRYO_DECORATORS, "this-is-overridden"); try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withDaemonConf(daemonConf) .build()) { TopologyBuilder topologyBuilder = new TopologyBuilder(); topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf"))); topologyBuilder.setBolt("2", new TestConfBolt(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "two")))) .shuffleGrouping("1"); List<FixedTuple> testTuples = Arrays.asList(new Values(Config.TOPOLOGY_KRYO_DECORATORS)).stream() .map(value -> new FixedTuple(value)) .collect(Collectors.toList()); MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples)); CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam(); completeTopologyParams.setMockedSources(mockedSources); completeTopologyParams.setStormConf(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "three"))); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams); List<Object> concatValues = Testing.readTuples(results, "2").stream() .flatMap(values -> values.stream()) .collect(Collectors.toList()); assertThat(concatValues.get(0), is(Config.TOPOLOGY_KRYO_DECORATORS)); assertThat(concatValues.get(1), is(Arrays.asList("one", "two", "three"))); } }
@Test public void testLocalTransport() throws Exception { Config topoConf = new Config(); topoConf.put(Config.TOPOLOGY_WORKERS, 2); topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context"); try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime() .withSupervisors(1).withPortsPerSupervisor(2) .withDaemonConf(topoConf).build()) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("1", new TestWordSpout(true), 2); builder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1"); StormTopology stormTopology = builder.createTopology(); List<FixedTuple> fixedTuples = new ArrayList<>(); for (int i = 0; i < 12; i++) { fixedTuples.add(new FixedTuple(Collections.singletonList("a"))); fixedTuples.add(new FixedTuple(Collections.singletonList("b"))); } Map<String, List<FixedTuple>> data = new HashMap<>(); data.put("1", fixedTuples); MockedSources mockedSources = new MockedSources(data); CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam(); completeTopologyParam.setMockedSources(mockedSources); Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam); Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size()); } }
@Test public void testSubmitTopologyToLocalNimbus() throws Exception { int port = Utils.getAvailablePort(); try (ILocalCluster localCluster = new LocalCluster.Builder() .withNimbusDaemon(true) .withDaemonConf(Config.NIMBUS_THRIFT_PORT, port) .build()) { Config topoConf = new Config(); topoConf.putAll(Utils.readDefaultConfig()); topoConf.setDebug(true); topoConf.put("storm.cluster.mode", "local"); // default is aways "distributed" but here local cluster is being used. topoConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN, InmemoryTopologySubmitterHook.class.getName()); topoConf.put(Config.NIMBUS_THRIFT_PORT, port); List<TopologyDetails> topologyNames = new ArrayList<>(); for (int i = 0; i < 4; i++) { final String topologyName = "word-count-" + UUID.randomUUID().toString(); final StormTopology stormTopology = createTestTopology(); topologyNames.add(new TopologyDetails(topologyName, stormTopology)); localCluster.submitTopology(topologyName, topoConf, stormTopology); } Assert.assertEquals(InmemoryTopologySubmitterHook.submittedTopologies, topologyNames); } }
private static LocalCluster cluster(MkClusterParam param, String id, boolean simulated) throws Exception { Integer supervisors = param.getSupervisors(); if (supervisors == null) { supervisors = 2; } Integer ports = param.getPortsPerSupervisor(); if (ports == null) { ports = 3; } Map<String, Object> conf = param.getDaemonConf(); if (conf == null) { conf = new HashMap<>(); } return new LocalCluster.Builder() .withSupervisors(supervisors) .withPortsPerSupervisor(ports) .withDaemonConf(conf) .withNimbusDaemon(param.isNimbusDaemon()) .withTracked(id) .withSimulatedTime(simulated) .build(); }
@Test public void testValidateTopologystructure() throws Exception { try (LocalCluster cluster = new LocalCluster.Builder() .withSimulatedTime() .withDaemonConf(Collections.singletonMap(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)) .build()) { assertThat(tryCompleteWordCountTopology(cluster, mkValidateTopology()), is(false)); assertThat(tryCompleteWordCountTopology(cluster, mkInvalidateTopology1()), is(true)); assertThat(tryCompleteWordCountTopology(cluster, mkInvalidateTopology2()), is(true)); assertThat(tryCompleteWordCountTopology(cluster, mkInvalidateTopology3()), is(true)); } }
/** * Create a default LocalCluster. * * @throws Exception on any error */ public LocalCluster() throws Exception { this(new Builder().withDaemonConf(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)); }