ProcessorNode makeProcessorNode(Processor<?> processor, Fields outputFields, boolean preservesKey) { return new ProcessorNode(processor, UniqueIdGen.getInstance().getUniqueStreamId(), outputFields, preservesKey); }
private void doProcessCurGroup(TopologyBuilder topologyBuilder, List<ProcessorNode> group) { String boltId = UniqueIdGen.getInstance().getUniqueBoltId(); for (ProcessorNode processorNode : group) { processorNode.setComponentId(boltId); processorNode.setWindowedParentStreams(getWindowedParentStreams(processorNode)); } final Set<ProcessorNode> initialProcessors = initialProcessors(group); Set<Window<?, ?>> windowParams = getWindowParams(initialProcessors); if (windowParams.isEmpty()) { if (hasStatefulProcessor(group)) { addStatefulBolt(topologyBuilder, boltId, initialProcessors, group); } else { addBolt(topologyBuilder, boltId, initialProcessors, group); } } else if (windowParams.size() == 1) { addWindowedBolt(topologyBuilder, boltId, initialProcessors, windowParams.iterator().next(), group); } else { throw new IllegalStateException("More than one window config for current group " + group); } }
private void addSinkNode(SinkNode sinkNode, int parallelism) { String boltId = UniqueIdGen.getInstance().getUniqueBoltId(); sinkNode.setComponentId(boltId); sinkNode.setParallelism(parallelism); if (node instanceof SpoutNode) { addNode(node, sinkNode, Utils.DEFAULT_STREAM_ID, parallelism); } else { addNode(node, sinkNode, parallelism); } }
/** * Creates a new {@link Stream} of tuples from the given {@link IRichSpout} with the given parallelism. * * @param spout the spout * @param parallelism the parallelism of the stream * @return the new stream */ public Stream<Tuple> newStream(IRichSpout spout, int parallelism) { SpoutNode spoutNode = new SpoutNode(spout); String spoutId = UniqueIdGen.getInstance().getUniqueSpoutId(); spoutNode.setComponentId(spoutId); spoutNode.setParallelism(parallelism); graph.addVertex(spoutNode); return new Stream<>(this, spoutNode); }
@Before public void setUp() throws Exception { streamBuilder = new StreamBuilder(); UniqueIdGen.getInstance().reset(); }
private void splitStatefulProcessor(ProcessorNode processorNode, TopologyBuilder topologyBuilder) { for (Node parent : StreamUtil.<Node>getParents(graph, processorNode)) { ProcessorNode identity = new ProcessorNode(new MapProcessor<>(new IdentityFunction<>()), UniqueIdGen.getInstance().getUniqueStreamId(), parent.getOutputFields()); addNode(parent, identity); graph.removeEdge(parent, processorNode); processorNode.removeParentStreams(parent); addNode(identity, processorNode); curGroup.add(identity); } processCurGroup(topologyBuilder); }