/** * {@inheritDoc} */ public PairStream<K, V> filter(Predicate<? super Pair<K, V>> predicate) { return toPairStream(super.filter(predicate)); }
/** * Creates a new {@link PairStream} of key-value pairs from the given {@link IRichSpout} by extracting key and value from tuples via the * supplied {@link PairValueMapper}. * * @param spout the spout * @param pairValueMapper the pair value mapper * @param <K> the key type * @param <V> the value type * @return the new stream of key-value pairs */ public <K, V> PairStream<K, V> newStream(IRichSpout spout, PairValueMapper<K, V> pairValueMapper) { return newStream(spout).mapToPair(pairValueMapper); }
/** * Counts the number of values in this stream. This does a global count of values across all partitions. * <p> * If the stream is windowed, the counts are emitted after each window activation and represents the count of elements that fall within * that window. If the stream is not windowed, the count is emitted as each new element in the stream is processed. * </p> * * @return the new stream */ public Stream<Long> count() { return aggregate(new Count<>()); }
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); /** * Computes average of the stream of numbers emitted by the spout. Internally the per-partition * sum and counts are accumulated and emitted to a downstream task where the partially accumulated * results are merged and the final result is emitted. */ builder.newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0), 2) .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5))) .filter(x -> x > 0 && x < 500) .aggregate(new Avg()) .print(); Config config = new Config(); String topoName = "AGG_EXAMPLE"; if (args.length > 0) { topoName = args[0]; } config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); }
@Test public void testJoin() throws Exception { Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); Stream<Integer>[] streams = stream.branch(x -> x % 2 == 0, x -> x % 3 == 0); PairStream<Integer, Integer> s1 = streams[0].mapToPair(x -> Pair.of(x, 1)); PairStream<Integer, Integer> s2 = streams[1].mapToPair(x -> Pair.of(x, 1)); PairStream<Integer, Pair<Integer, Integer>> sj = s1.join(s2); assertEquals(Collections.singleton(s1.node), sj.node.getParents(s1.stream)); assertEquals(Collections.singleton(s2.node), sj.node.getParents(s2.stream)); }
@Test public void testRepartition() throws Exception { Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0)); stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>()); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); SpoutSpec spout = topology.get_spouts().get("spout1"); assertEquals(4, topology.get_bolts_size()); Bolt bolt1 = topology.get_bolts().get("bolt1"); Bolt bolt2 = topology.get_bolts().get("bolt2"); Bolt bolt3 = topology.get_bolts().get("bolt3"); Bolt bolt4 = topology.get_bolts().get("bolt4"); assertEquals(1, spout.get_common().get_parallelism_hint()); assertEquals(1, bolt1.get_common().get_parallelism_hint()); assertEquals(3, bolt2.get_common().get_parallelism_hint()); assertEquals(2, bolt3.get_common().get_parallelism_hint()); assertEquals(2, bolt4.get_common().get_parallelism_hint()); }
/** * Sends the elements of this stream to a bolt. This could be used to plug in existing bolts as sinks in the stream, for e.g. a {@code * RedisStoreBolt}. The bolt would have a parallelism of 1. * <p> * <b>Note:</b> This would provide guarantees only based on what the bolt provides. * </p> * * @param bolt the bolt */ public void to(IRichBolt bolt) { to(bolt, 1); }
@SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { StreamBuilder builder = new StreamBuilder(); Stream<Integer>[] evenAndOdd = builder /* * Create a stream of random numbers from a spout that * emits random integers by extracting the tuple value at index 0. */ .newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0)) /* * Split the stream of numbers into streams of * even and odd numbers. The first stream contains even * and the second contains odd numbers. */ .branch(x -> (x % 2) == 0, x -> (x % 2) == 1); evenAndOdd[0].forEach(x -> LOG.info("EVEN> " + x)); evenAndOdd[1].forEach(x -> LOG.info("ODD > " + x)); Config config = new Config(); String topoName = "branchExample"; if (args.length > 0) { topoName = args[0]; } config.setNumWorkers(1); StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build()); }
.mapToPair(w -> Pair.of(w, 1)) .stateQuery(ss).print();
/** * {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public PairStream<K, V>[] branch(Predicate<? super Pair<K, V>>... predicates) { List<PairStream<K, V>> pairStreams = new ArrayList<>(); for (Stream<Pair<K, V>> stream : super.branch(predicates)) { pairStreams.add(toPairStream(stream)); } return pairStreams.toArray(new PairStream[pairStreams.size()]); }
/** * Creates a new {@link Stream} of values from the given {@link IRichSpout} by extracting field(s) from tuples via the supplied {@link * TupleValueMapper}. * * @param spout the spout * @param valueMapper the value mapper * @param <T> the type of values in the resultant stream * @return the new stream */ public <T> Stream<T> newStream(IRichSpout spout, TupleValueMapper<T> valueMapper) { return newStream(spout).map(valueMapper); }
/** * {@inheritDoc} */ @Override public PairStream<K, V> repartition(int parallelism) { return toPairStream(super.repartition(parallelism)); }
/** * Print the values in this stream. */ public void print() { forEach(new PrintConsumer<T>()); }
/** * {@inheritDoc} */ @Override public PairStream<K, V> window(Window<?, ?> window) { return toPairStream(super.window(window)); }
final Stream finalStream = inputStream.flatMap(evalCalc);
@Test public void testBranchAndJoin() throws Exception { TopologyContext mockContext = Mockito.mock(TopologyContext.class); OutputCollector mockCollector = Mockito.mock(OutputCollector.class); Stream<Integer> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0), 2); Stream<Integer>[] streams = stream.branch(x -> x % 2 == 0, x -> x % 2 == 1); PairStream<Integer, Pair<Integer, Integer>> joined = streams[0].mapToPair(x -> Pair.of(x, 1)).join(streams[1].mapToPair(x -> Pair.of(x, 1))); assertTrue(joined.getNode() instanceof ProcessorNode); StormTopology topology = streamBuilder.build(); assertEquals(2, topology.get_bolts_size()); }
/** * Sends the elements of this stream to a bolt. This could be used to plug in existing bolts as sinks in the stream, for e.g. a {@code * RedisStoreBolt}. The bolt would have a parallelism of 1. * <p> * <b>Note:</b> This would provide guarantees only based on what the bolt provides. * </p> * * @param bolt the bolt */ public void to(IBasicBolt bolt) { to(bolt, 1); }
@Test public void testBranch() throws Exception { Stream<Tuple> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID)); Stream<Tuple>[] streams = stream.branch(x -> true); StormTopology topology = streamBuilder.build(); assertEquals(1, topology.get_spouts_size()); assertEquals(1, topology.get_bolts_size()); Map<GlobalStreamId, Grouping> expected = new HashMap<>(); String spoutId = topology.get_spouts().keySet().iterator().next(); expected.put(new GlobalStreamId(spoutId, "default"), Grouping.shuffle(new NullStruct())); assertEquals(expected, topology.get_bolts().values().iterator().next().get_common().get_inputs()); assertEquals(1, streams.length); assertEquals(1, streams[0].node.getOutputStreams().size()); String parentStream = streams[0].node.getOutputStreams().iterator().next() + "-branch"; assertEquals(1, streams[0].node.getParents(parentStream).size()); Node processorNdoe = streams[0].node.getParents(parentStream).iterator().next(); assertTrue(processorNdoe instanceof ProcessorNode); assertTrue(((ProcessorNode) processorNdoe).getProcessor() instanceof BranchProcessor); assertTrue(processorNdoe.getParents("default").iterator().next() instanceof SpoutNode); }