/** * Returns a new stream where the values that arrive within a window having the same key will be reduced by repeatedly applying the * reducer. * * @param reducer the reducer * @param window the window configuration * @return the new stream */ public PairStream<K, V> reduceByKeyAndWindow(Reducer<V> reducer, Window<?, ?> window) { return partitionByKey().window(window).reduceByKey(reducer); }
/** * Returns a new stream where the values are grouped by keys and the given window. The values that arrive within a window having the * same key will be merged together and returned as an Iterable of values mapped to the key. * * @param window the window configuration * @return the new stream */ public PairStream<K, Iterable<V>> groupByKeyAndWindow(Window<?, ?> window) { return partitionByKey().window(window).aggregatePartition(new MergeValues<>()); }
@Test public void testGroupBy() throws Exception { PairStream<String, String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new PairValueMapper<>(0, 1), 2); stream.window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))).aggregateByKey(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); assertEquals(Grouping.shuffle(new NullStruct()), bolt1.get_common().get_inputs().values().iterator().next()); assertEquals(Grouping.fields(Collections.singletonList("key")), bolt2.get_common().get_inputs().values().iterator().next()); }
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); // a stream of (number, square) pairs PairStream<Integer, Integer> squares = builder .newStream(new NumberSpout(x -> x * x), new PairValueMapper<>(0, 1)); // a stream of (number, cube) pairs PairStream<Integer, Integer> cubes = builder .newStream(new NumberSpout(x -> x * x * x), new PairValueMapper<>(0, 1)); // create a windowed stream of five seconds duration squares.window(TumblingWindows.of(Duration.seconds(5))) /* * Join the squares and the cubes stream within the window. * The values in the squares stream having the same key as that * of the cubes stream within the window will be joined together. */ .join(cubes) /** * The results should be of the form (number, (square, cube)) */ .print(); Config config = new Config(); String topoName = JoinExample.class.getName(); if (args.length > 0) { topoName = args[0]; } config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); }
@Test public void testMultiPartitionByKey() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) .reduceByKey((x, y) -> 0) .print(); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); }
@Test public void testMultiPartitionByKeyWithRepartition() { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); expected.put(new GlobalStreamId("bolt2", "s3"), Grouping.fields(Collections.singletonList("key"))); expected.put(new GlobalStreamId("bolt2", "s3__punctuation"), Grouping.all(new NullStruct())); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.mapToPair(x -> Pair.of(x, x)) .window(TumblingWindows.of(BaseWindowedBolt.Count.of(10))) .reduceByKey((x, y) -> x + y) .repartition(10) .reduceByKey((x, y) -> 0) .print(); StormTopology topology = streamBuilder.build(); assertEquals(3, topology.get_bolts_size()); assertEquals(expected, topology.get_bolts().get("bolt3").get_common().get_inputs()); }
public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /** * The spout emits sequences of (Integer, Long, Long). TupleValueMapper can be used to extract fields * from the values and produce a stream of typed tuple (Tuple3<Integer, Long, Long> in this case. */ Stream<Tuple3<Integer, Long, Long>> stream = builder.newStream(new RandomIntegerSpout(), TupleValueMappers.of(0, 1, 2)); PairStream<Long, Integer> pairs = stream.mapToPair(t -> Pair.of(t._2 / 10000, t._1)); pairs.window(TumblingWindows.of(Count.of(10))).groupByKey().print(); String topoName = "test"; if (args.length > 0) { topoName = args[0]; } Config config = new Config(); config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); } }