@Override public BatchResponse onNextBatch(BatchSource source, Collection<JetstreamEvent> events) throws EventException { initAutoAdvanceTime(source); incrementEventRecievedCounter(events.size()); if (m_config.isNoDuplication()) { cacheBatch(source, events); } else { BatchResponse ret = sendBatch(source, events); if (ret != null) return ret; } BatchResponse ret = autoAdvanceResponse(source); // check if the read rate exceeds the max, if exceeds, make the next // batch wait for a while String key = getKey(source); Long waitMs = waitForLastBatch.get(key); if (waitMs != null && waitMs > 0) { ret.setWaitTimeInMs(waitMs); } return ret; }
@Override public void onBatchProcessed(BatchSource source) { if (m_config.isNoDuplication()) { String key = getKey(source); List<JetstreamEvent> cached = cachedBatches.get(key); if (cached != null) { simplySendBatch(source, cached); cached.clear(); } } }
protected void simplySendBatch(BatchSource source, Collection<JetstreamEvent> events) { long start = System.currentTimeMillis(); // send to batch event sinks if (!getBatchEventSinks().isEmpty()) { try { simplySendEvents(events); } catch (Throwable e) { incrementEventDroppedCounter(events.size()); } } // send to non-batch event sinks if (!getEventSinks().isEmpty()) { Iterator<JetstreamEvent> it = events.iterator(); while (it.hasNext()) { try { JetstreamEvent event = it.next(); simplySendEvent(event); } catch (Throwable e) { super.incrementEventDroppedCounter(); } } } handleReadRate(start, source, events.size()); }
protected void initAutoAdvanceTime(BatchSource source) { String key = getKey(source); if (!m_lastAdvanceTimes.containsKey(key)) m_lastAdvanceTimes.put(key, System.currentTimeMillis()); }
protected void simplySendEvent(JetstreamEvent event) { super.fireSendEvent(event); incrementEventSentCounter(); }
@Override public BatchResponse onIdle(BatchSource source) { return autoAdvanceResponse(source); }
super.fireSendEvent(event);
if (!getBatchEventSinks().isEmpty()) { try { simplySendEvents(events); } catch (Throwable e) { incrementEventDroppedCounter(events.size()); return BatchResponse.getNextBatch().setOffset( source.getHeadOffset()); if (!getEventSinks().isEmpty()) { long startOffset = source.getHeadOffset(); int index = 0; simplySendEvent(event); index++; handleReadRate(start, source, events.size());
protected void cacheBatch(BatchSource source, Collection<JetstreamEvent> events) { String key = getKey(source); List<JetstreamEvent> cache = cachedBatches.get(key); if (cache == null) { cache = new ArrayList<JetstreamEvent>(); cachedBatches.put(key, cache); } cache.addAll(events); }
protected void simplySendEvents(Collection<JetstreamEvent> events) { EventMetaInfo meta = new EventMetaInfo(); super.fireSendEvents(events, meta); incrementEventSentCounter(events.size()); }
private void handleReadRate(long start, BatchSource source, int eventCount) { if (m_config.getMaxReadRate() > 0) { long end = System.currentTimeMillis(); int tasks; String key = getKey(source); if (waitForLastBatch.containsKey(key)) { tasks = waitForLastBatch.size(); } else { tasks = waitForLastBatch.size() + 1; } long durInRate = (1000 * eventCount * tasks) / m_config.getMaxReadRate(); long waitMs = durInRate - (end - start); waitForLastBatch.put(key, waitMs); } }