private static String getExecutorType(WorkerTopologyContext workerTopologyContext, String componentId) { StormTopology topology = workerTopologyContext.getRawTopology(); Map<String, SpoutSpec> spouts = topology.get_spouts(); Map<String, Bolt> bolts = topology.get_bolts(); if (spouts.containsKey(componentId)) { return ClientStatsUtil.SPOUT; } else if (bolts.containsKey(componentId)) { return ClientStatsUtil.BOLT; } else { throw new RuntimeException("Could not find " + componentId + " in " + topology); } }
StormTopology stormTopology = new StormTopology(spoutSpecs, boltSpecs, new HashMap<>()); stormTopology.set_worker_hooks(_workerHooks); stormTopology.set_component_to_shared_memory(_componentToSharedMemory); stormTopology.set_shared_memory(_sharedMemory);
case SPOUTS: if (value == null) { unset_spouts(); } else { set_spouts((java.util.Map<java.lang.String,SpoutSpec>)value); unset_bolts(); } else { set_bolts((java.util.Map<java.lang.String,Bolt>)value); unset_state_spouts(); } else { set_state_spouts((java.util.Map<java.lang.String,StateSpoutSpec>)value); unset_worker_hooks(); } else { set_worker_hooks((java.util.List<java.nio.ByteBuffer>)value); unset_dependency_jars(); } else { set_dependency_jars((java.util.List<java.lang.String>)value); unset_dependency_artifacts(); } else { set_dependency_artifacts((java.util.List<java.lang.String>)value); unset_storm_version(); } else { set_storm_version((java.lang.String)value);
public static Map<String, Object> allComponents(StormTopology topology) { Map<String, Object> components = new HashMap<>(topology.get_bolts()); components.putAll(topology.get_spouts()); components.putAll(topology.get_state_spouts()); return components; }
public java.lang.Object getFieldValue(_Fields field) { switch (field) { case SPOUTS: return get_spouts(); case BOLTS: return get_bolts(); case STATE_SPOUTS: return get_state_spouts(); case WORKER_HOOKS: return get_worker_hooks(); case DEPENDENCY_JARS: return get_dependency_jars(); case DEPENDENCY_ARTIFACTS: return get_dependency_artifacts(); case STORM_VERSION: return get_storm_version(); case JDK_VERSION: return get_jdk_version(); case COMPONENT_TO_SHARED_MEMORY: return get_component_to_shared_memory(); case SHARED_MEMORY: return get_shared_memory(); } throw new java.lang.IllegalStateException(); }
@Test public void testSpoutToBolt() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); stream.to(newBolt()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); }
final String simpleCurrentLocalFile = localizerRoot + "/usercache/" + user + "/filecache/files/simple.current"; final StormTopology st = new StormTopology(); st.set_spouts(new HashMap<>()); st.set_bolts(new HashMap<>()); st.set_state_spouts(new HashMap<>());
public static NormalizedResourceRequest getSpoutResources(StormTopology topology, Map<String, Object> topologyConf, String componentId) { if (topology.get_spouts() != null) { SpoutSpec spout = topology.get_spouts().get(componentId); return new NormalizedResourceRequest(spout.get_common(), topologyConf, componentId); } return null; }
public static NormalizedResourceRequest getBoltResources(StormTopology topology, Map<String, Object> topologyConf, String componentId) { if (topology.get_bolts() != null) { Bolt bolt = topology.get_bolts().get(componentId); return new NormalizedResourceRequest(bolt.get_common(), topologyConf, componentId); } return null; }
bolts.put(operatorName, new Bolt(null, common)); stormTopology = new StormTopology(spouts, bolts, new HashMap<String, StateSpoutSpec>()); } else { Map<String, SpoutSpec> spouts = stormTopology.get_spouts(); Map<String, Bolt> bolts = stormTopology.get_bolts(); Map<String, StateSpoutSpec> stateSpouts = stormTopology.get_state_spouts();
topology = topology.deepCopy(); //Don't modify the original for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) { String id = spoutEntry.getKey(); for (Entry<String, StreamInfo> streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) { for (Entry<String, Bolt> boltEntry : topology.get_bolts().entrySet()) { String id = boltEntry.getKey(); for (Entry<String, StreamInfo> streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) { topology.put_to_bolts(Utils.uuid(), new Bolt(Thrift.serializeComponentObject(capturer), Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null))); return new CapturedTopology<>(topology, capturer);
/** * Create a new topology to be tracked. * @param origTopo the original topology. * @param cluster a cluster that should have been launched with tracking enabled. */ public TrackedTopology(StormTopology origTopo, ILocalCluster cluster) { LOG.warn("CLUSTER {} - {}", cluster, cluster.getTrackedId()); this.cluster = cluster; lastSpoutCommit = new AtomicInteger(0); String id = cluster.getTrackedId(); topology = origTopo.deepCopy(); for (Bolt bolt : topology.get_bolts().values()) { IRichBolt obj = (IRichBolt) Thrift.deserializeComponentObject(bolt.get_bolt_object()); bolt.set_bolt_object(Thrift.serializeComponentObject(new BoltTracker(obj, id))); } for (SpoutSpec spout : topology.get_spouts().values()) { IRichSpout obj = (IRichSpout) Thrift.deserializeComponentObject(spout.get_spout_object()); spout.set_spout_object(Thrift.serializeComponentObject(new SpoutTracker(obj, id))); } }
for (Bolt bolt : topology.get_bolts().values()) { ComponentCommon common = bolt.get_common(); common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val"))); for (SpoutSpec spout : topology.get_spouts().values()) { ComponentCommon common = spout.get_common(); Map<String, Object> spoutConf = componentConf(spout); topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
@Test public void testGlobalAggregate() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); stream.aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected1 = new HashMap<>(); expected1.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); Map<GlobalStreamId, Grouping> expected2 = new HashMap<>(); expected2.put(new GlobalStreamId("bolt1", "s1"), Grouping.fields(Collections.emptyList())); expected2.put(new GlobalStreamId("bolt1", "s1__punctuation"), Grouping.all(new NullStruct())); assertEquals(expected1, bolt1.get_common().get_inputs()); assertEquals(expected2, bolt2.get_common().get_inputs()); }
private List<LocalResource> getLocalResources(PortAndAssignment pna) throws IOException { String topologyId = pna.getToplogyId(); Map<String, Object> topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId); @SuppressWarnings("unchecked") Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP); List<LocalResource> ret = new ArrayList<>(); if (blobstoreMap != null) { List<LocalResource> tmp = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap); if (tmp != null) { ret.addAll(tmp); } } StormTopology stormCode = ConfigUtils.readSupervisorTopology(conf, topologyId, fsOps); List<String> dependencies = new ArrayList<>(); if (stormCode.is_set_dependency_jars()) { dependencies.addAll(stormCode.get_dependency_jars()); } if (stormCode.is_set_dependency_artifacts()) { dependencies.addAll(stormCode.get_dependency_artifacts()); } for (String dependency : dependencies) { ret.add(new LocalResource(dependency, false, true)); } return ret; }
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }
/** * Gets the Thrift object representing the topology. * * @return the Thrift definition representing the topology */ @SuppressWarnings("deprecation") public StormTopology getRawTopology() { StormTopology stormTopology = new StormTopology(); Map<String, SpoutSpec> spouts = new HashMap<>(); for (TopologyAPI.Spout spout : this.delegate.getRawTopology().getSpoutsList()) { spouts.put(spout.getComp().getName(), new SpoutSpec(spout)); } Map<String, Bolt> bolts = new HashMap<>(); for (TopologyAPI.Bolt bolt : this.delegate.getRawTopology().getBoltsList()) { bolts.put(bolt.getComp().getName(), new Bolt(bolt)); } stormTopology.set_spouts(spouts); stormTopology.set_bolts(bolts); return stormTopology; }
private synchronized void readData() throws IOException { final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops); final List<String> dependencyLocations = new ArrayList<>(); if (stormTopology.get_dependency_jars() != null) { for (String dependency : stormTopology.get_dependency_jars()) { dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath()); } } if (stormTopology.get_dependency_artifacts() != null) { for (String dependency : stormTopology.get_dependency_artifacts()) { dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath()); } } _depLocs = dependencyLocations; _stormVersion = stormTopology.get_storm_version(); _dataCached = true; }
public StormTopology deepCopy() { return new StormTopology(this); }
StormTopology stormTopology = new StormTopology(spoutSpecs, boltSpecs, new HashMap<String, StateSpoutSpec>()); stormTopology.set_worker_hooks(_workerHooks);