public static Grouping prepareShuffleGrouping() { return Grouping.shuffle(new NullStruct()); }
/** * ## Repartitioning Operation * * Use random round robin algorithm to evenly redistribute tuples across all target partitions * * @return */ public Stream shuffle() { return partition(Grouping.shuffle(new NullStruct())); }
public BoltDeclarer shuffleGrouping(String componentId, String streamId) { return grouping(componentId, streamId, Grouping.shuffle(new NullStruct())); }
@Test public void testSpoutToBolt() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); stream.to(newBolt()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); }
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }
@Test public void testBranch() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); Stream<Tuple>[] streams = stream.branch(x -> true); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); String spoutId = topology.get_spouts().keySet().iterator().next(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); assertEquals(1, streams.length); assertEquals(1, streams[0].node.getOutputStreams().size()); String parentStream = streams[0].node.getOutputStreams().iterator().next() + "-branch"; assertEquals(1, streams[0].node.getParents(parentStream).size()); Node processorNdoe = streams[0].node.getParents(parentStream).iterator().next(); assertTrue(processorNdoe instanceof ProcessorNode); assertTrue(((ProcessorNode) processorNdoe).getProcessor() instanceof BranchProcessor); assertTrue(processorNdoe.getParents("default").iterator().next() instanceof SpoutNode); }
@Test public void testGlobalAggregate() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); stream.aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); String spoutId = topology.get_spouts().keySet().iterator().next(); Map<GlobalStreamId, Grouping> expected1 = new HashMap<>(); expected1.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); Map<GlobalStreamId, Grouping> expected2 = new HashMap<>(); expected2.put(new GlobalStreamId("bolt1", "s1"), Grouping.fields(Collections.emptyList())); expected2.put(new GlobalStreamId("bolt1", "s1__punctuation"), Grouping.all(new NullStruct())); assertEquals(expected1, bolt1.get_common().get_inputs()); assertEquals(expected2, bolt2.get_common().get_inputs()); }
.mkGrouper(mockContext(Lists.newArrayList(1, 2)), "comp", "stream", null, Grouping.shuffle(new NullStruct()), Lists.newArrayList(1, 2), Collections.emptyMap()); int numMessages = 100000;
/** * ## Repartitioning Operation * * Use random round robin algorithm to evenly redistribute tuples across all target partitions * * @return */ public Stream shuffle() { return partition(Grouping.shuffle(new NullStruct())); }
public BoltDeclarer shuffleGrouping(String componentId, String streamId) { return grouping(componentId, streamId, Grouping.shuffle(new NullStruct())); }