this.nbProcessedEvents = new AtomicLong(); final NotificationSqlDao sqlDao = (dbi != null) ? dbi.onDemand(NotificationSqlDao.class) : null; this.dao = new DBBackedQueue<NotificationEventModelDao>(clock, sqlDao, config, "notif-" + config.getTableName(), metricRegistry);
private List<NotificationEventModelDao> getReadyNotifications() { final List<NotificationEventModelDao> input = dao.getReadyEntries(); final List<NotificationEventModelDao> claimedNotifications = new ArrayList<NotificationEventModelDao>(); for (final NotificationEventModelDao cur : input) { // Skip non active queues... final NotificationQueue queue = queues.get(cur.getQueueName()); if (queue == null || !queue.isStarted()) { continue; } claimedNotifications.add(cur); } return claimedNotifications; }
@Override public void start() { if (isStarted.compareAndSet(false, true)) { dao.initialize(); startQueue(); } }
final List<T> entriesToClaim = fetchReadyEntries(config.getMaxEntriesClaimed()); if (entriesToClaim.size() > 0) { candidates = claimEntries(entriesToClaim); candidates = fetchReadyEntriesFromIds(); return claimEntries(candidates); List<T> prefetchedEntries = fetchReadyEntries(config.getPrefetchEntries()); if (removeInflightEventsWhenSwitchingToQueueOpenForRead(candidates)) { log.info(DB_QUEUE_LOG_ID + " Opening Q for read"); final boolean q = isQueueOpenForRead.compareAndSet(false, true); return claimEntries(candidates);
@Override public void removeNotification(final Long recordId) { final NotificationEventModelDao existing = dao.getSqlDao().getByRecordId(recordId, config.getTableName()); final NotificationEventModelDao removedEntry = new NotificationEventModelDao(existing, Hostname.get(), clock.getUTCNow(), PersistentQueueEntryLifecycleState.REMOVED); dao.moveEntryToHistory(removedEntry); }
public void testOnlyInflightQ() { final PersistentBusConfig config = createConfig(1, 10, 0); queue = new DBBackedQueue<BusEventModelDao>(clock, sqlDao, config, "default-bus_event", metricRegistry); queue.initialize(); for (int i = 0; i < 100; i++) { assertTrue(queue.isQueueOpenForRead()); assertTrue(queue.isQueueOpenForWrite()); queue.insertEntry(input); final List<BusEventModelDao> claimed = queue.getReadyEntries(); assertEquals(claimed.size(), 1); queue.moveEntryToHistory(historyInput); assertEquals(queue.getTotalInflightProcessed(), 100L); assertEquals(queue.getTotalProcessed(), 100L); assertEquals(queue.getTotalInflightWritten(), 100L); assertEquals(queue.getTotalWritten(), 100L);
queue = new DBBackedQueue<BusEventModelDao>(clock, sqlDao, config, "default-bus_event", metricRegistry); queue.initialize(); while (queue.isQueueOpenForWrite()) { try { Thread.sleep(10); assertEquals(ready.size(), 0); log.info("Got inflightProcessed = " + queue.getTotalInflightProcessed() + "/1000, inflightWritten = " + queue.getTotalInflightWritten() + "/1000"); assertEquals(queue.getTotalWritten(), 1000L);
@Override public int doProcessEvents() { final List<BusEventModelDao> events = dao.getReadyEntries(); if (events.size() == 0) { return 0; if (lastException == null) { BusEventModelDao processedEntry = new BusEventModelDao(cur, Hostname.get(), clock.getUTCNow(), PersistentQueueEntryLifecycleState.PROCESSED); dao.moveEntryToHistory(processedEntry); } else if (errorCount <= config.getMaxFailureRetries()) { log.info("Bus dispatch error, will attempt a retry ", lastException); BusEventModelDao retriedEntry = new BusEventModelDao(cur, Hostname.get(), clock.getUTCNow(), PersistentQueueEntryLifecycleState.AVAILABLE, errorCount); dao.updateOnError(retriedEntry); } else { log.error("Fatal Bus dispatch error, data corruption...", lastException); BusEventModelDao processedEntry = new BusEventModelDao(cur, Hostname.get(), clock.getUTCNow(), PersistentQueueEntryLifecycleState.FAILED); dao.moveEntryToHistory(processedEntry);
@Override public void run() { do { List<BusEventModelDao> entries = queue.getReadyEntries(); if (entries.size() == 0) { try { //log.info("Reader " + readerId + " sleeping for 10 ms got " + consumed.get()); Thread.sleep(10); } catch (InterruptedException e) { } } else { for (BusEventModelDao cur : entries) { search1.add(cur.getSearchKey1()); final BusEventModelDao history = new BusEventModelDao(cur, OWNER, clock.getUTCNow(), PersistentQueueEntryLifecycleState.PROCESSED); queue.moveEntryToHistory(history); } consumed.getAndAdd(entries.size()); } } while (consumed.get() < maxEntries); }
@Override public void recordFutureNotification(final DateTime futureNotificationTime, final NotificationEvent event, final UUID userToken, final Long searchKey1, final Long searchKey2) throws IOException { final String eventJson = objectMapper.writeValueAsString(event); final UUID futureUserToken = UUID.randomUUID(); final Long searchKey2WithNull = Objects.firstNonNull(searchKey2, new Long(0)); final NotificationEventModelDao notification = new NotificationEventModelDao(Hostname.get(), clock.getUTCNow(), event.getClass().getName(), eventJson, userToken, searchKey1, searchKey2WithNull, futureUserToken, futureNotificationTime, getFullQName()); dao.insertEntry(notification); }
private void clearFailedNotification(final NotificationEventModelDao cleared) { NotificationEventModelDao processedEntry = new NotificationEventModelDao(cleared, Hostname.get(), clock.getUTCNow(), PersistentQueueEntryLifecycleState.FAILED); dao.moveEntryToHistory(processedEntry); }
@Override public <T extends NotificationEvent> List<NotificationEventWithMetadata<T>> getFutureNotificationForSearchKey2(final Class<T> type, final Long searchKey2) { return getFutureNotificationsInternal(type, (NotificationSqlDao) dao.getSqlDao(), "search_key2", searchKey2); }
private List<T> fetchReadyEntriesFromIds() { final int size = config.getMaxEntriesClaimed() < inflightEvents.size() ? config.getMaxEntriesClaimed() : inflightEvents.size(); final List<Long> recordIds = new ArrayList<Long>(size); for (int i = 0; i < size; i++) { final Long entryId = inflightEvents.poll(); if (entryId != null) { totalInflightProcessed.inc(); totalProcessed.inc(); recordIds.add(entryId); } } // Before we return we filter on AVAILABLE entries for precaution; the case could potentially happen // at the time when we switch from !isQueueOpenForRead -> isQueueOpenForRead with two thread in parallel. // List<T> result = ImmutableList.<T>of(); if (recordIds.size() > 0) { if (log.isDebugEnabled()) { log.debug(DB_QUEUE_LOG_ID + "fetchReadyEntriesFromIds, size = " + size + ", ids = " + Joiner.on(", ").join(recordIds)); } final List<T> entriesFromIds = getEntriesFromIds(recordIds); result = ImmutableList.<T>copyOf(Collections2.filter(entriesFromIds, new Predicate<T>() { @Override public boolean apply(final T input) { return (input.getProcessingState() == PersistentQueueEntryLifecycleState.AVAILABLE); } })); } return result; }
public void initialize() { final List<T> entries = fetchReadyEntries(config.getPrefetchEntries()); if (entries.size() == 0) { isQueueOpenForWrite.set(true); isQueueOpenForRead.set(true); } else if (entries.size() < config.getPrefetchEntries()) { isQueueOpenForWrite.set(true); isQueueOpenForRead.set(false); } else { isQueueOpenForWrite.set(false); isQueueOpenForRead.set(false); } if (useInflightQueue) { inflightEvents.clear(); } // Lame, no more clear API totalInflightProcessed.dec(totalInflightProcessed.getCount()); totalProcessed.dec(totalProcessed.getCount()); totalInflightWritten.dec(totalInflightWritten.getCount()); totalWritten.dec(totalWritten.getCount()); log.info(DB_QUEUE_LOG_ID + "Initialized with isQueueOpenForWrite = " + isQueueOpenForWrite.get() + ", isQueueOpenForRead" + isQueueOpenForRead.get()); }
queue = new DBBackedQueue<BusEventModelDao>(clock, sqlDao, config, "default-bus_event", metricRegistry); queue.initialize(); assertFalse(queue.isQueueOpenForRead()); assertTrue(queue.isQueueOpenForWrite()); for (int i = 5; i < 105; i++) { final BusEventModelDao input = createEntry(new Long(i)); queue.insertEntry(input); assertFalse(queue.isQueueOpenForRead()); } else { assertTrue(queue.isQueueOpenForRead()); assertTrue(queue.isQueueOpenForWrite()); List<BusEventModelDao> claimed = queue.getReadyEntries(); final BusEventModelDao output = claimed.get(0); queue.moveEntryToHistory(historyInput); assertEquals(queue.getTotalInflightProcessed(), 99L); assertEquals(queue.getTotalProcessed(), 105L); assertEquals(queue.getTotalInflightWritten(), 100L); assertEquals(queue.getTotalWritten(), 100L);
queue = new DBBackedQueue<BusEventModelDao>(clock, sqlDao, config, "default-bus_event", metricRegistry); queue.initialize(); writers[1].start(); while (queue.isQueueOpenForWrite()) { try { Thread.sleep(10); assertEquals(ready.size(), 0); log.info("Got inflightProcessed = " + queue.getTotalInflightProcessed() + "/1000, inflightWritten = " + queue.getTotalInflightWritten() + "/1000"); assertEquals(queue.getTotalWritten(), 2000);
@Override public void post(final BusEvent event) throws EventBusException { try { if (isStarted.get()) { final String json = objectMapper.writeValueAsString(event); final BusEventModelDao entry = new BusEventModelDao(Hostname.get(), clock.getUTCNow(), event.getClass().getName(), json, event.getUserToken(), event.getSearchKey1(), event.getSearchKey2()); dao.insertEntry(entry); } else { log.warn("Attempting to post event " + event + " in a non initialized bus"); } } catch (Exception e) { log.error("Failed to post BusEvent " + event, e); } }
private void clearNotification(final NotificationEventModelDao cleared) { NotificationEventModelDao processedEntry = new NotificationEventModelDao(cleared, Hostname.get(), clock.getUTCNow(), PersistentQueueEntryLifecycleState.PROCESSED); dao.moveEntryToHistory(processedEntry); }
@Override public <T extends NotificationEvent> List<NotificationEventWithMetadata<T>> getFutureNotificationForSearchKey1(final Class<T> type, final Long searchKey1) { return getFutureNotificationsInternal(type, (NotificationSqlDao) dao.getSqlDao(), "search_key1", searchKey1); }
queue = new DBBackedQueue<BusEventModelDao>(clock, sqlDao, config, "default-bus_event", metricRegistry); queue.initialize(); assertFalse(queue.isQueueOpenForRead()); assertTrue(queue.isQueueOpenForWrite()); queue.insertEntry(input); if (i >= 100) { assertFalse(queue.isQueueOpenForWrite()); } else { assertTrue(queue.isQueueOpenForWrite()); for (int i = 0; i < 205; i++) { if (i <= 5) { assertFalse(queue.isQueueOpenForRead()); } else if (i < 106) { assertTrue(queue.isQueueOpenForRead()); } else { assertFalse(queue.isQueueOpenForRead()); List<BusEventModelDao> claimed = queue.getReadyEntries(); final BusEventModelDao output = claimed.get(0); queue.moveEntryToHistory(historyInput); assertEquals(queue.getTotalInflightProcessed(), 99L); assertEquals(queue.getTotalProcessed(), 205L); assertEquals(queue.getTotalInflightWritten(), 100L); assertEquals(queue.getTotalWritten(), 200L);