/** * Returns a stream consisting of the elements of this stream that matches the given filter. * * @param predicate the predicate to apply to each element to determine if it should be included * @return the new stream */ public Stream<T> filter(Predicate<? super T> predicate) { return new Stream<>(streamBuilder, addProcessorNode(new FilterProcessor<>(predicate), VALUE, true)); }
private <R> Stream<R> merge(CombinerAggregator<?, T, ? extends R> aggregator) { return new Stream<>(streamBuilder, addProcessorNode(new MergeAggregateProcessor<>(aggregator), VALUE)); }
/** * Returns a stream consisting of the result of applying the given mapping function to the values of this stream. * * @param function a mapping function to be applied to each value in this stream. * @return the new stream */ public <R> Stream<R> map(Function<? super T, ? extends R> function) { return new Stream<>(streamBuilder, addProcessorNode(new MapProcessor<>(function), VALUE)); }
/** * Returns a stream consisting of the results of replacing each value of this stream with the contents produced by applying the provided * mapping function to each value. This has the effect of applying a one-to-many transformation to the values of the stream, and then * flattening the resulting elements into a new stream. * * @param function a mapping function to be applied to each value in this stream which produces new values. * @return the new stream */ public <R> Stream<R> flatMap(FlatMapFunction<? super T, ? extends R> function) { return new Stream<>(streamBuilder, addProcessorNode(new FlatMapProcessor<>(function), VALUE)); }
private <A, R> Stream<R> aggregatePartition(CombinerAggregator<? super T, A, ? extends R> aggregator) { return new Stream<>(streamBuilder, addProcessorNode(new AggregateProcessor<>(aggregator), VALUE)); }
private <A> Stream<A> combinePartition(CombinerAggregator<? super T, A, ?> aggregator) { return new Stream<>(streamBuilder, addProcessorNode(new AggregateProcessor<>(aggregator, true), VALUE, true)); }
private Stream<T> reducePartition(Reducer<T> reducer) { return new Stream<>(streamBuilder, addProcessorNode(new ReduceProcessor<>(reducer), VALUE)); }
/** * Returns a new stream with the given value of parallelism. Further operations on this stream would execute at this level of * parallelism. * * @param parallelism the parallelism value * @return the new stream */ public Stream<T> repartition(int parallelism) { if (parallelism < 1) { throw new IllegalArgumentException("Parallelism should be >= 1"); } if (node.getParallelism() == parallelism) { LOG.debug("Node's current parallelism {}, new parallelism {}", node.getParallelism(), parallelism); return this; } Node partitionNode = addNode(node, new PartitionNode(stream, node.getOutputFields()), parallelism); return new Stream<>(streamBuilder, partitionNode); }
private Stream<T> global() { Node partitionNode = addNode(new PartitionNode(stream, node.getOutputFields(), GroupingInfo.global())); return new Stream<>(streamBuilder, partitionNode); }
/** * Returns a stream consisting of the elements of this stream, additionally performing the provided action on each element as they are * consumed from the resulting stream. * * @param action the action to perform on the element as they are consumed from the stream * @return the new stream */ public Stream<T> peek(Consumer<? super T> action) { return new Stream<>(streamBuilder, addProcessorNode(new PeekProcessor<>(action), node.getOutputFields(), true)); }
protected Stream<T> partitionBy(Fields fields, int parallelism) { return new Stream<>( streamBuilder, addNode(node, new PartitionNode(stream, node.getOutputFields(), GroupingInfo.fields(fields)), parallelism)); }
/** * Returns an array of streams by splitting the given stream into multiple branches based on the given predicates. The predicates are * applied in the given order to the values of this stream and the result is forwarded to the corresponding (index based) result stream * based on the (index of) predicate that matches. * <p> * <b>Note:</b> If none of the predicates match a value, that value is dropped. * </p> * * @param predicates the predicates * @return an array of result streams (branches) corresponding to the given predicates */ @SuppressWarnings("unchecked") public Stream<T>[] branch(Predicate<? super T>... predicates) { List<Stream<T>> childStreams = new ArrayList<>(); if (predicates.length > 0) { BranchProcessor<T> branchProcessor = new BranchProcessor<>(); Node branchNode = addProcessorNode(branchProcessor, VALUE); for (Predicate<? super T> predicate : predicates) { // create a child node (identity) per branch ProcessorNode child = makeProcessorNode(new MapProcessor<>(new IdentityFunction<>()), node.getOutputFields()); String branchStream = child.getOutputStreams().iterator().next() + "-branch"; // branchStream is the parent stream that connects branch processor to this child branchNode.addOutputStream(branchStream); addNode(branchNode, child, branchStream); childStreams.add(new Stream<>(streamBuilder, child)); branchProcessor.addPredicate(predicate, branchStream); } } return childStreams.toArray((Stream<T>[]) new Stream[childStreams.size()]); }
/** * Returns a new stream consisting of the elements that fall within the window as specified by the window parameter. The {@link Window} * specification could be used to specify sliding or tumbling windows based on time duration or event count. For example, * <pre> * // time duration based sliding window * stream.window(SlidingWindows.of(Duration.minutes(10), Duration.minutes(1)); * * // count based sliding window * stream.window(SlidingWindows.of(Count.(10), Count.of(2))); * * // time duration based tumbling window * stream.window(TumblingWindows.of(Duration.seconds(10)); * </p> * * @see org.apache.storm.streams.windowing.SlidingWindows * @see org.apache.storm.streams.windowing.TumblingWindows * @param window the window configuration * @return the new stream */ public Stream<T> window(Window<?, ?> window) { return new Stream<>(streamBuilder, addNode(new WindowNode(window, stream, node.getOutputFields()))); }
/** * Creates a new {@link Stream} of tuples from the given {@link IRichSpout} with the given parallelism. * * @param spout the spout * @param parallelism the parallelism of the stream * @return the new stream */ public Stream<Tuple> newStream(IRichSpout spout, int parallelism) { SpoutNode spoutNode = new SpoutNode(spout); String spoutId = UniqueIdGen.getInstance().getUniqueSpoutId(); spoutNode.setComponentId(spoutId); spoutNode.setParallelism(parallelism); graph.addVertex(spoutNode); return new Stream<>(this, spoutNode); }