/** * Define a new bolt in this topology. This defines a windowed bolt, intended * for windowing operations. The {@link IWindowedBolt#execute(TupleWindow)} method * is triggered for each window interval with the list of current events in the window. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around the cluster. * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IWindowedBolt bolt, Number parallelism_hint) throws IllegalArgumentException { return setBolt(id, new backtype.storm.topology.WindowedBoltExecutor(bolt), parallelism_hint); }
private void validate(Map stormConf, Count windowLengthCount, Duration windowLengthDuration, Count slidingIntervalCount, Duration slidingIntervalDuration) { int topologyTimeout = getTopologyTimeoutMillis(stormConf); int maxSpoutPending = getMaxSpoutPending(stormConf); if (windowLengthCount == null && windowLengthDuration == null) { throw new IllegalArgumentException("Window length is not specified"); } if (windowLengthDuration != null && slidingIntervalDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value + slidingIntervalDuration.value, topologyTimeout); } else if (windowLengthDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value, topologyTimeout); } else if (slidingIntervalDuration != null) { ensureDurationLessThanTimeout(slidingIntervalDuration.value, topologyTimeout); } if (windowLengthCount != null && slidingIntervalCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value + slidingIntervalCount.value, maxSpoutPending); } else if (windowLengthCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value, maxSpoutPending); } else if (slidingIntervalCount != null) { ensureCountLessThanMaxPending(slidingIntervalCount.value, maxSpoutPending); } }
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.windowedOutputCollector = new WindowedOutputCollector(collector); bolt.prepare(stormConf, context, windowedOutputCollector); this.listener = newWindowLifecycleListener(); this.windowManager = initWindowManager(listener, stormConf, context); start(); LOG.debug("Initialized window manager {} ", this.windowManager); }
maxLagMs, getComponentStreams(context)); validate(stormConf, windowLengthCount, windowLengthDuration, slidingIntervalCount, slidingIntervalDuration); evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDuration, manager); triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDuration, manager, evictionPolicy); manager.setEvictionPolicy(evictionPolicy); manager.setTriggerPolicy(triggerPolicy);
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }
maxLagMs, getComponentStreams(context)); validate(stormConf, windowLengthCount, windowLengthDuration, slidingIntervalCount, slidingIntervalDuration); evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDuration, manager); triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDuration, manager, evictionPolicy); manager.setEvictionPolicy(evictionPolicy);
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.windowedOutputCollector = new WindowedOutputCollector(collector); bolt.prepare(stormConf, context, windowedOutputCollector); this.listener = newWindowLifecycleListener(); this.windowManager = initWindowManager(listener, stormConf, context); start(); LOG.debug("Initialized window manager {} ", this.windowManager); }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
private void validate(Map stormConf, Count windowLengthCount, Duration windowLengthDuration, Count slidingIntervalCount, Duration slidingIntervalDuration) { int topologyTimeout = getTopologyTimeoutMillis(stormConf); int maxSpoutPending = getMaxSpoutPending(stormConf); if (windowLengthCount == null && windowLengthDuration == null) { throw new IllegalArgumentException("Window length is not specified"); } if (windowLengthDuration != null && slidingIntervalDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value + slidingIntervalDuration.value, topologyTimeout); } else if (windowLengthDuration != null) { ensureDurationLessThanTimeout(windowLengthDuration.value, topologyTimeout); } else if (slidingIntervalDuration != null) { ensureDurationLessThanTimeout(slidingIntervalDuration.value, topologyTimeout); } if (windowLengthCount != null && slidingIntervalCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value + slidingIntervalCount.value, maxSpoutPending); } else if (windowLengthCount != null) { ensureCountLessThanMaxPending(windowLengthCount.value, maxSpoutPending); } else if (slidingIntervalCount != null) { ensureCountLessThanMaxPending(slidingIntervalCount.value, maxSpoutPending); } }
@Override public void execute(Tuple input) { if (!isStateInitialized()) { throw new IllegalStateException("execute invoked before initState with input tuple " + input); } else if (isRecovering()) { handleRecovery(input); } else { super.execute(input); } }
public BoltDeclarer setBolt(String id, IWindowedBolt bolt) throws IllegalArgumentException { return setBolt(id, new backtype.storm.topology.WindowedBoltExecutor(bolt), null); }
private void handleRecovery(Tuple input) { long msgId = getMsgId(input); TaskStream taskStream = TaskStream.fromTuple(input); WindowState state = recoveryStates.get(taskStream); LOG.debug("handleRecovery, recoveryStates {}", recoveryStates); if (state != null) { LOG.debug("Tuple msgid {}, saved state {}", msgId, state); if (msgId <= state.lastExpired) { LOG.debug("Ignoring tuple since msg id {} <= lastExpired id {}", msgId, state.lastExpired); outputCollector.ack(input); } else if (msgId <= state.lastEvaluated) { super.execute(input); } else { LOG.debug("Tuple msg id {} > lastEvaluated id {}, adding to pendingTuples and clearing recovery state " + "for taskStream {}", msgId, state.lastEvaluated, taskStream); pendingTuples.add(input); clearRecoveryState(taskStream); } } else { pendingTuples.add(input); } }
/** * Define a new bolt in this topology. This defines a windowed bolt, intended * for windowing operations. The {@link IWindowedBolt#execute(TupleWindow)} method * is triggered for each window interval with the list of current events in the window. * * @param id the id of this component. This id is referenced by other components that want to consume this bolt's outputs. * @param bolt the windowed bolt * @param parallelism_hint the number of tasks that should be assigned to execute this bolt. Each task will run on a thread in a process somwehere around the cluster. * @return use the returned object to declare the inputs to this component * @throws IllegalArgumentException if {@code parallelism_hint} is not positive */ public BoltDeclarer setBolt(String id, IWindowedBolt bolt, Number parallelism_hint) throws IllegalArgumentException { return setBolt(id, new WindowedBoltExecutor(bolt), parallelism_hint); } /**