@SuppressWarnings("rawtypes") @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector; lastModifiedTracker = new NthLastModifiedTimeTracker( deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds)); }
public NthLastModifiedTimeTracker(int numTimesToTrack) { if (numTimesToTrack < 1) { throw new IllegalArgumentException( "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")"); } lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack); initLastModifiedTimesMillis(); }
private void emitCurrentWindowCounts() { Map<Object, Long> counts = counter.getCountsThenAdvanceWindow(); int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification(); lastModifiedTracker.markAsModified(); if (actualWindowLengthInSeconds != windowLengthInSeconds) { LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds)); } emit(counts, actualWindowLengthInSeconds); }
public int secondsSinceOldestModification() { long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue(); return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); }
public void markAsModified() { updateLastModifiedTime(); }
private void updateLastModifiedTime() { lastModifiedTimesMillis.add(now()); }
public void markAsModified() { updateLastModifiedTime(); }
private void emitCurrentWindowCounts() { Map<Object, Long> counts = counter.getCountsThenAdvanceWindow(); int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification(); lastModifiedTracker.markAsModified(); if (actualWindowLengthInSeconds != windowLengthInSeconds) { LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds)); } emit(counts, actualWindowLengthInSeconds); }
public int secondsSinceOldestModification() { long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue(); return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); }
@Override public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) { this.collector = collector; lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds)); }
public NthLastModifiedTimeTracker(int numTimesToTrack) { if (numTimesToTrack < 1) { throw new IllegalArgumentException( "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")"); } lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack); initLastModifiedTimesMillis(); }
public void markAsModified() { updateLastModifiedTime(); }
private void emitCurrentWindowCounts() { /** * counter是SlidingWindowCounter,他的类结构如下: * * SlidingWindowCounter 滑动窗口计数器 * private SlotBasedCounter<T> objCounter; 基于槽的计数器 * private final Map<T, long[]> objToCounts = new HashMap<T, long[]>(); 构成一个二维矩阵纵轴为对象列,横轴为槽。表示某个对象在某个槽的计数 * private final int numSlots; 槽数量 即long[]的length * private int headSlot; 当前计数slot * private int tailSlot; 下一个计数slot * private int windowLengthInSlots; 用一组slot表示窗口长度。slot的个数便是windowLengthInSlots。也就是SlotBasedCounter的numSlots */ Map<Object, Long> counts = counter.getCountsThenAdvanceWindow(); /** * 根据追踪器获取上次最早时间到现在的时间长度,追踪器追踪了numSlots个时间的修改, * 这里返回的是第0个和当前时间之差,在这里表示的是一整个时间窗口(所有slot之和)。 */ int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification(); //记录当前时间为修改时间 lastModifiedTracker.markAsModified(); /** * 如果时间窗口长度和实际获取的时间窗口长度不一致, * 打印一个警告信息,然而并没有什么用。 */ if (actualWindowLengthInSeconds != windowLengthInSeconds) { LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds)); } //将实际的窗口长度和计数发送给下游 emit(counts, actualWindowLengthInSeconds); }
private void updateLastModifiedTime() { lastModifiedTimesMillis.add(now()); }
@SuppressWarnings("rawtypes") @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector; /** * new NthLastModifiedTimeTracker(numSlots) * 新建一个追踪器,使用CircularFifoBuffer实现,循环缓冲。追踪数量为numSlots。追踪内容为time-since-last-modify。 * 这个追踪器同样也是一个时间滑动窗口,并且和SlidingWindowCounter是一致的。 */ lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds)); }
public NthLastModifiedTimeTracker(int numTimesToTrack) { if (numTimesToTrack < 1) { throw new IllegalArgumentException( "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")"); } lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack); initLastModifiedTimesMillis(); }
private void initLastModifiedTimesMillis() { long nowCached = now(); for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) { lastModifiedTimesMillis.add(Long.valueOf(nowCached)); } }
private void initLastModifiedTimesMillis() { long nowCached = now(); for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) { lastModifiedTimesMillis.add(Long.valueOf(nowCached)); } }
public int secondsSinceOldestModification() { long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue(); return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); }
private void updateLastModifiedTime() { lastModifiedTimesMillis.add(now()); }