public java.lang.Object getFieldValue(_Fields field) { switch (field) { case BOLT_OBJECT: return get_bolt_object(); case COMMON: return get_common(); } throw new java.lang.IllegalStateException(); }
public static Map<String, NormalizedResourceRequest> getBoltsResources(StormTopology topology, Map<String, Object> topologyConf) { Map<String, NormalizedResourceRequest> boltResources = new HashMap<>(); if (topology.get_bolts() != null) { for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) { NormalizedResourceRequest topologyResources = new NormalizedResourceRequest(bolt.getValue().get_common(), topologyConf, bolt.getKey()); if (LOG.isTraceEnabled()) { LOG.trace("Turned {} into {}", bolt.getValue().get_common().get_json_conf(), topologyResources); } boltResources.put(bolt.getKey(), topologyResources); } } return boltResources; }
public static ComponentCommon getComponentCommon(Object component) { ComponentCommon common = null; if (component instanceof StateSpoutSpec) { common = ((StateSpoutSpec) component).get_common(); } else if (component instanceof SpoutSpec) { common = ((SpoutSpec) component).get_common(); } else if (component instanceof Bolt) { common = ((Bolt) component).get_common(); } return common; }
static Map<String, Map<String, Double>> getBoltsResources(StormTopology topology, Map<String, Object> topologyConf) { Map<String, Map<String, Double>> boltResources = new HashMap<>(); if (topology.get_bolts() != null) { for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) { Map<String, Double> topologyResources = parseResources(bolt.getValue().get_common().get_json_conf()); checkInitialization(topologyResources, bolt.getValue().toString(), topologyConf); boltResources.put(bolt.getKey(), topologyResources); } } return boltResources; }
public static NormalizedResourceRequest getBoltResources(StormTopology topology, Map<String, Object> topologyConf, String componentId) { if (topology.get_bolts() != null) { Bolt bolt = topology.get_bolts().get(componentId); return new NormalizedResourceRequest(bolt.get_common(), topologyConf, componentId); } return null; }
public static ComponentCommon getComponentCommon(StormTopology topology, String id) { if (topology.get_spouts().containsKey(id)) { return topology.get_spouts().get(id).get_common(); } if (topology.get_bolts().containsKey(id)) { return topology.get_bolts().get(id).get_common(); } if (topology.get_state_spouts().containsKey(id)) { return topology.get_state_spouts().get(id).get_common(); } throw new IllegalArgumentException("Could not find component with id " + id); }
for (String streamName : bolt.get_common().get_streams().keySet()) { if (streamName.contains(".")) { LOG.warn("Metrics for stream name '{}' will be reported as '{}'.", streamName, streamName.replace('.', '_'));
public static ComponentCommon getComponentCommon(StormTopology topology, String componentId) { Bolt b = topology.get_bolts().get(componentId); if (b != null) { return b.get_common(); } SpoutSpec s = topology.get_spouts().get(componentId); if (s != null) { return s.get_common(); } StateSpoutSpec ss = topology.get_state_spouts().get(componentId); if (ss != null) { return ss.get_common(); } throw new IllegalArgumentException("Could not find component common for " + componentId); } }
for (String streamName : bolt.get_common().get_streams().keySet()) { if (streamName.contains(".")) { throw new WrappedInvalidTopologyException(String.format("Stream name '%s' contains illegal character '.'", streamName));
ComponentCommon common = bolt.get_common(); common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val"))); common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
public static Map<ExecutorDetails, String> genExecsAndComps(StormTopology topology) { Map<ExecutorDetails, String> retMap = new HashMap<>(); int startTask = 0; int endTask = 0; for (Map.Entry<String, SpoutSpec> entry : topology.get_spouts().entrySet()) { SpoutSpec spout = entry.getValue(); String spoutId = entry.getKey(); int spoutParallelism = spout.get_common().get_parallelism_hint(); for (int i = 0; i < spoutParallelism; i++) { retMap.put(new ExecutorDetails(startTask, endTask), spoutId); startTask++; endTask++; } } for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) { String boltId = entry.getKey(); Bolt bolt = entry.getValue(); int boltParallelism = bolt.get_common().get_parallelism_hint(); for (int i = 0; i < boltParallelism; i++) { retMap.put(new ExecutorDetails(startTask, endTask), boltId); startTask++; endTask++; } } return retMap; }
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }
@Test public void testSpoutToBolt() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); stream.to(newBolt()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); }
@Test public void testGlobalAggregate() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); stream.aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected1 = new HashMap<>(); expected1.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); Map<GlobalStreamId, Grouping> expected2 = new HashMap<>(); expected2.put(new GlobalStreamId("bolt1", "s1"), Grouping.fields(Collections.emptyList())); expected2.put(new GlobalStreamId("bolt1", "s1__punctuation"), Grouping.all(new NullStruct())); assertEquals(expected1, bolt1.get_common().get_inputs()); assertEquals(expected2, bolt2.get_common().get_inputs()); }
@Test public void testGetComponentCommonWithoutWorkerHook() { StormTopology stormTopology = genereateStormTopology(false); ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(stormTopology, "bolt-1"); Assert.assertEquals( "We expect to get bolt-1's common", new Bolt().get_common(), componentCommon); }
@Test public void testBranch() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); Stream<Tuple>[] streams = stream.branch(x -> true); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); String spoutId = topology.get_spouts().keySet().iterator().next(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); assertEquals(1, streams.length); assertEquals(1, streams[0].node.getOutputStreams().size()); String parentStream = streams[0].node.getOutputStreams().iterator().next() + "-branch"; assertEquals(1, streams[0].node.getParents(parentStream).size()); Node processorNdoe = streams[0].node.getParents(parentStream).iterator().next(); assertTrue(processorNdoe instanceof ProcessorNode); assertTrue(((ProcessorNode) processorNdoe).getProcessor() instanceof BranchProcessor); assertTrue(processorNdoe.getParents("default").iterator().next() instanceof SpoutNode); }
@Test public void testGetComponentCommonWithWorkerHook() { StormTopology stormTopology = genereateStormTopology(true); ComponentCommon componentCommon = ThriftTopologyUtils.getComponentCommon(stormTopology, "bolt-1"); Assert.assertEquals( "We expect to get bolt-1's common", new Bolt().get_common(), componentCommon); }
@Test public void testRepartition() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); SpoutSpec spout = topology.get_spouts().get("spout1"); assertEquals(4, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); Bolt bolt3 = topology.get_bolts().get("bolt3"); Bolt bolt4 = topology.get_bolts().get("bolt4"); assertEquals(1, spout.get_common().get_parallelism_hint()); assertEquals(1, bolt1.get_common().get_parallelism_hint()); assertEquals(3, bolt2.get_common().get_parallelism_hint()); assertEquals(2, bolt3.get_common().get_parallelism_hint()); assertEquals(2, bolt4.get_common().get_parallelism_hint()); }
@Test public void testMultiPartitionByKeyWithRepartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections.singletonList("key"))); expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping.all(new NullStruct())); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) .repartition(10) .reduceByKey((x, y) -> 0) .print(); StormTopology topology = streamBuilder.build(); assertEquals(3, topology.get_bolts_size()); assertEquals(expected, topology.get_bolts().get("bolt3").get_common().get_inputs()); }
@Test public void testStatefulTopology() { builder.setSpout("spout1", makeDummySpout()); builder.setSpout("spout2", makeDummySpout()); builder.setBolt("bolt1", makeDummyStatefulBolt(), 1) .shuffleGrouping("spout1").shuffleGrouping("spout2"); builder.setBolt("bolt2", makeDummyStatefulBolt(), 1).shuffleGrouping("spout1"); builder.setBolt("bolt3", makeDummyStatefulBolt(), 1) .shuffleGrouping("bolt1").shuffleGrouping("bolt2"); StormTopology topology = builder.createTopology(); Assert.assertNotNull(topology); Set<String> spouts = topology.get_spouts().keySet(); // checkpoint spout should 've been added Assert.assertEquals(ImmutableSet.of("spout1", "spout2", "$checkpointspout"), spouts); // bolt1, bolt2 should also receive from checkpoint spout Assert.assertEquals(ImmutableSet.of(new GlobalStreamId("spout1", "default"), new GlobalStreamId("spout2", "default"), new GlobalStreamId("$checkpointspout", "$checkpoint")), topology.get_bolts().get("bolt1").get_common().get_inputs().keySet()); Assert.assertEquals(ImmutableSet.of(new GlobalStreamId("spout1", "default"), new GlobalStreamId("$checkpointspout", "$checkpoint")), topology.get_bolts().get("bolt2").get_common().get_inputs().keySet()); // bolt3 should also receive from checkpoint streams of bolt1, bolt2 Assert.assertEquals(ImmutableSet.of(new GlobalStreamId("bolt1", "default"), new GlobalStreamId("bolt1", "$checkpoint"), new GlobalStreamId("bolt2", "default"), new GlobalStreamId("bolt2", "$checkpoint")), topology.get_bolts().get("bolt3").get_common().get_inputs().keySet()); }