public static NormalizedResourceRequest getBoltResources(StormTopology topology, Map<String, Object> topologyConf, String componentId) { if (topology.get_bolts() != null) { Bolt bolt = topology.get_bolts().get(componentId); return new NormalizedResourceRequest(bolt.get_common(), topologyConf, componentId); } return null; }
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }
public static Set<String> getComponentIds(StormTopology topology) { Set<String> ret = new HashSet<>(); ret.addAll(topology.get_bolts().keySet()); ret.addAll(topology.get_spouts().keySet()); ret.addAll(topology.get_state_spouts().keySet()); return ret; }
@Test public void testSpoutToBolt() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); stream.to(newBolt()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); }
/** * Get the coponenet type for a give id. * @param topology the topology this is a part of. * @param compId the id of the component. * @return the type as a String "BOLT" or "SPOUT". */ public static String componentType(StormTopology topology, String compId) { if (compId == null) { return null; } Map<String, Bolt> bolts = topology.get_bolts(); if (Utils.isSystemId(compId) || bolts.containsKey(compId)) { return ClientStatsUtil.BOLT; } return ClientStatsUtil.SPOUT; }
@Test public void testGlobalAggregate() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); stream.aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected1 = new HashMap<>(); expected1.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); Map<GlobalStreamId, Grouping> expected2 = new HashMap<>(); expected2.put(new GlobalStreamId("bolt1", "s1"), Grouping.fields(Collections.emptyList())); expected2.put(new GlobalStreamId("bolt1", "s1__punctuation"), Grouping.all(new NullStruct())); assertEquals(expected1, bolt1.get_common().get_inputs()); assertEquals(expected2, bolt2.get_common().get_inputs()); }
public static Map<String, Object> allComponents(StormTopology topology) { Map<String, Object> components = new HashMap<>(topology.get_bolts()); components.putAll(topology.get_spouts()); components.putAll(topology.get_state_spouts()); return components; }
@Test public void testRepartition() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); SpoutSpec spout = topology.get_spouts().get("spout1"); assertEquals(4, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); Bolt bolt3 = topology.get_bolts().get("bolt3"); Bolt bolt4 = topology.get_bolts().get("bolt4"); assertEquals(1, spout.get_common().get_parallelism_hint()); assertEquals(1, bolt1.get_common().get_parallelism_hint()); assertEquals(3, bolt2.get_common().get_parallelism_hint()); assertEquals(2, bolt3.get_common().get_parallelism_hint()); assertEquals(2, bolt4.get_common().get_parallelism_hint()); }
public static Map<GlobalStreamId, Grouping> eventLoggerInputs(StormTopology topology) { Map<GlobalStreamId, Grouping> inputs = new HashMap<GlobalStreamId, Grouping>(); Set<String> allIds = new HashSet<String>(); allIds.addAll(topology.get_bolts().keySet()); allIds.addAll(topology.get_spouts().keySet()); for (String id : allIds) { inputs.put(Utils.getGlobalStreamId(id, EVENTLOGGER_STREAM_ID), Thrift.prepareFieldsGrouping(Arrays.asList("component-id"))); } return inputs; }
@Test public void testMultiPartitionByKeyWithRepartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections.singletonList("key"))); expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping.all(new NullStruct())); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) .repartition(10) .reduceByKey((x, y) -> 0) .print(); StormTopology topology = streamBuilder.build(); assertEquals(3, topology.get_bolts_size()); assertEquals(expected, topology.get_bolts().get("bolt3").get_common().get_inputs()); }
private static String getExecutorType(WorkerTopologyContext workerTopologyContext, String componentId) { StormTopology topology = workerTopologyContext.getRawTopology(); Map<String, SpoutSpec> spouts = topology.get_spouts(); Map<String, Bolt> bolts = topology.get_bolts(); if (spouts.containsKey(componentId)) { return ClientStatsUtil.SPOUT; } else if (bolts.containsKey(componentId)) { return ClientStatsUtil.BOLT; } else { throw new RuntimeException("Could not find " + componentId + " in " + topology); } }
public static Map<String, NormalizedResourceRequest> getBoltsResources(StormTopology topology, Map<String, Object> topologyConf) { Map<String, NormalizedResourceRequest> boltResources = new HashMap<>(); if (topology.get_bolts() != null) { for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) { NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(bolt.getValue().get_common(), topologyConf, bolt.getKey()); if (LOG.isTraceEnabled()) { LOG.trace("Turned {} into {}", bolt.getValue().get_common().get_json_conf(), topologyResources); } boltResources.put(bolt.getKey(), topologyResources); } } return boltResources; }
static Map<String, Map<String, Double>> getBoltsResources(StormTopology topology, Map<String, Object> topologyConf) { Map<String, Map<String, Double>> boltResources = new HashMap<>(); if (topology.get_bolts() != null) { for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) { Map<String, Double> topologyResources = parseResources(bolt.getValue().get_common().get_json_conf()); checkInitialization(topologyResources, bolt.getValue().toString(), topologyConf); boltResources.put(bolt.getKey(), topologyResources); } } return boltResources; }
public IdDictionary(StormTopology topology) { List<String> componentNames = new ArrayList<>(topology.get_spouts().keySet()); componentNames.addAll(topology.get_bolts().keySet()); componentNames.addAll(topology.get_state_spouts().keySet()); for (String name : componentNames) { ComponentCommon common = Utils.getComponentCommon(topology, name); List<String> streams = new ArrayList<>(common.get_streams().keySet()); streamNametoId.put(name, idify(streams)); streamIdToName.put(name, simpleReverseMap(streamNametoId.get(name))); } }
public static ComponentCommon getComponentCommon(StormTopology topology, String id) { if (topology.get_spouts().containsKey(id)) { return topology.get_spouts().get(id).get_common(); } if (topology.get_bolts().containsKey(id)) { return topology.get_bolts().get(id).get_common(); } if (topology.get_state_spouts().containsKey(id)) { return topology.get_state_spouts().get(id).get_common(); } throw new IllegalArgumentException("Could not find component with id " + id); }
private static void validateIds(StormTopology topology) throws InvalidTopologyException { List<String> componentIds = new ArrayList<>(); componentIds.addAll(validateIds(topology.get_bolts())); componentIds.addAll(validateIds(topology.get_spouts())); componentIds.addAll(validateIds(topology.get_state_spouts())); List<String> offending = Utils.getRepeat(componentIds); if (!offending.isEmpty()) { throw new WrappedInvalidTopologyException("Duplicate component ids: " + offending); } }
public static ComponentCommon getComponentCommon(StormTopology topology, String componentId) { Bolt b = topology.get_bolts().get(componentId); if (b != null) { return b.get_common(); } SpoutSpec s = topology.get_spouts().get(componentId); if (s != null) { return s.get_common(); } StateSpoutSpec ss = topology.get_state_spouts().get(componentId); if (ss != null) { return ss.get_common(); } throw new IllegalArgumentException("Could not find component common for " + componentId); } }
public static Map<GlobalStreamId, Grouping> ackerInputs(StormTopology topology) { Map<GlobalStreamId, Grouping> inputs = new HashMap<>(); Set<String> boltIds = topology.get_bolts().keySet(); Set<String> spoutIds = topology.get_spouts().keySet(); for (String id : spoutIds) { inputs.put(Utils.getGlobalStreamId(id, Acker.ACKER_INIT_STREAM_ID), Thrift.prepareFieldsGrouping(Arrays.asList("id"))); } for (String id : boltIds) { inputs.put(Utils.getGlobalStreamId(id, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareFieldsGrouping(Arrays.asList("id"))); inputs.put(Utils.getGlobalStreamId(id, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareFieldsGrouping(Arrays.asList("id"))); inputs.put(Utils.getGlobalStreamId(id, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareFieldsGrouping(Arrays.asList("id"))); } return inputs; }
@Test public void testGenBoltId() { Set<String> pre = null; for (int i = 0; i < 100; i++) { StormTopology topology = buildTopology(); Map<String, Bolt> cur = topology.get_bolts(); System.out.println(cur.keySet()); if (pre != null) { Assert.assertTrue("bold id not consistent with group name", pre.equals(cur.keySet())); } pre = cur.keySet(); } }
/** * Create a new topology to be tracked. * @param origTopo the original topology. * @param cluster a cluster that should have been launched with tracking enabled. */ public TrackedTopology(StormTopology origTopo, ILocalCluster cluster) { LOG.warn("CLUSTER {} - {}", cluster, cluster.getTrackedId()); this.cluster = cluster; lastSpoutCommit = new AtomicInteger(0); String id = cluster.getTrackedId(); topology = origTopo.deepCopy(); for (Bolt bolt : topology.get_bolts().values()) { IRichBolt obj = (IRichBolt) Thrift.deserializeComponentObject(bolt.get_bolt_object()); bolt.set_bolt_object(Thrift.serializeComponentObject(new BoltTracker(obj, id))); } for (SpoutSpec spout : topology.get_spouts().values()) { IRichSpout obj = (IRichSpout) Thrift.deserializeComponentObject(spout.get_spout_object()); spout.set_spout_object(Thrift.serializeComponentObject(new SpoutTracker(obj, id))); } }