@Override boolean shouldAccept(final NotificationEvent event) { return eventFrom <= event.getEventId() && event.getEventId() <= eventTo; } }
@Override public long toEventId() { return event.getEventId(); } }
@Override public long fromEventId() { return event.getEventId(); }
@Override public NotificationEvent next() { eventCount++; NotificationEvent ev = batchIter.next(); pos = ev.getEventId(); return ev; }
/** * Process this notification by adding it to metastore DB. * * @param event NotificationEvent is the object written to the metastore DB. * @param listenerEvent ListenerEvent (from which NotificationEvent was based) used only to set the * DB_NOTIFICATION_EVENT_ID_KEY_NAME for future reference by other listeners. */ private void process(NotificationEvent event, ListenerEvent listenerEvent) throws MetaException { event.setMessageFormat(msgEncoder.getMessageFormat()); LOG.debug("DbNotificationListener: Processing : {}:{}", event.getEventId(), event.getMessage()); HMSHandler.getMSForConf(conf).addNotificationEvent(event); // Set the DB_NOTIFICATION_EVENT_ID for future reference by other listeners. if (event.isSetEventId()) { listenerEvent.putParameter( MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, Long.toString(event.getEventId())); } }
private void dumpEvent(NotificationEvent ev, Path evRoot, Path cmRoot, Hive db) throws Exception { EventHandler.Context context = new EventHandler.Context( evRoot, cmRoot, db, conf, getNewEventOnlyReplicationSpec(ev.getEventId()), work.dbNameOrPattern, work.tableNameOrPattern ); EventHandler eventHandler = EventHandlerFactory.handlerFor(ev); eventHandler.handle(context); replLogger.eventLog(String.valueOf(ev.getEventId()), eventHandler.dumpType().toString()); }
@Override public void run() { LOG.debug("Polling for notification events"); int eventsProcessed = 0; try { // Get any new notification events that have been since the last time we checked, // And pass them on to the event handlers. EventUtils.MSClientNotificationFetcher evFetcher = new EventUtils.MSClientNotificationFetcher(Hive.get()); EventUtils.NotificationEventIterator evIter = new EventUtils.NotificationEventIterator(evFetcher, lastCheckedEventId, 0, "*", null); while (evIter.hasNext()) { NotificationEvent event = evIter.next(); LOG.debug("Event: " + event); for (EventConsumer eventConsumer : eventConsumers) { try { eventConsumer.accept(event); } catch (Exception err) { LOG.error("Error processing notification event " + event, err); } } eventsProcessed++; lastCheckedEventId = event.getEventId(); } } catch (Exception err) { LOG.error("Error polling for notification events", err); } LOG.debug("Processed {} notification events", eventsProcessed); } }
@InterfaceAudience.LimitedPrivate({"HCatalog"}) @Override public NotificationEventResponse getNextNotification(long lastEventId, int maxEvents, NotificationFilter filter) throws TException { NotificationEventRequest rqst = new NotificationEventRequest(lastEventId); rqst.setMaxEvents(maxEvents); NotificationEventResponse rsp = client.get_next_notification(rqst); LOG.debug("Got back " + rsp.getEventsSize() + " events"); NotificationEventResponse filtered = new NotificationEventResponse(); if (rsp != null && rsp.getEvents() != null) { long nextEventId = lastEventId + 1; for (NotificationEvent e : rsp.getEvents()) { if (e.getEventId() != nextEventId) { LOG.error("Requested events are found missing in NOTIFICATION_LOG table. Expected: {}, Actual: {}. " + "Probably, cleaner would've cleaned it up. " + "Try setting higher value for hive.metastore.event.db.listener.timetolive. " + "Also, bootstrap the system again to get back the consistent replicated state.", nextEventId, e.getEventId()); throw new IllegalStateException(REPL_EVENTS_MISSING_IN_METASTORE); } if ((filter != null) && filter.accept(e)) { filtered.addToEvents(e); } nextEventId++; } } return (filter != null) ? filtered : rsp; }
public HCatNotificationEvent(NotificationEvent event) { eventId = event.getEventId(); eventTime = event.getEventTime(); eventType = event.getEventType(); dbName = event.getDbName(); tableName = event.getTableName(); message = event.getMessage(); }
listenerEvent.putParameter( MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, Long.toString(event.getEventId()));
public Object getFieldValue(_Fields field) { switch (field) { case EVENT_ID: return getEventId(); case EVENT_TIME: return getEventTime(); case EVENT_TYPE: return getEventType(); case DB_NAME: return getDbName(); case TABLE_NAME: return getTableName(); case MESSAGE: return getMessage(); case MESSAGE_FORMAT: return getMessageFormat(); case CAT_NAME: return getCatName(); } throw new IllegalStateException(); }
Path evRoot = new Path(dumpRoot, String.valueOf(ev.getEventId())); while (evIter.hasNext()){ NotificationEvent ev = evIter.next(); lastReplId = ev.getEventId(); Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId)); dumpEvent(ev, evRoot, cmRoot);
private void dumpEvent(NotificationEvent ev, Path evRoot, Path cmRoot) throws Exception { long evid = ev.getEventId(); String evidStr = String.valueOf(evid); ReplicationSpec replicationSpec = getNewEventOnlyReplicationSpec(evidStr); case MessageFactory.CREATE_TABLE_EVENT : { CreateTableMessage ctm = md.getCreateTableMessage(ev.getMessage()); LOG.info("Processing#{} CREATE_TABLE message : {}", ev.getEventId(), ev.getMessage()); org.apache.hadoop.hive.metastore.api.Table tobj = ctm.getTableObj(); LOG.info("Processing#{} ADD_PARTITION message : {}", ev.getEventId(), ev.getMessage()); Iterable<org.apache.hadoop.hive.metastore.api.Partition> ptns = apm.getPartitionObjs(); if ((ptns == null) || (!ptns.iterator().hasNext())) { LOG.info("Processing#{} DROP_TABLE message : {}", ev.getEventId(), ev.getMessage()); DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_DROP_TABLE, evid, evid, cmRoot); dmd.setPayload(ev.getMessage()); LOG.info("Processing#{} DROP_PARTITION message : {}", ev.getEventId(), ev.getMessage()); DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_DROP_PARTITION, evid, evid, cmRoot); dmd.setPayload(ev.getMessage()); LOG.info("Processing#{} ALTER_TABLE message : {}", ev.getEventId(), ev.getMessage()); AlterTableMessage atm = md.getAlterTableMessage(ev.getMessage()); org.apache.hadoop.hive.metastore.api.Table tobjBefore = atm.getTableObjBefore(); LOG.info("Processing#{} ALTER_PARTITION message : {}", ev.getEventId(), ev.getMessage()); AlterPartitionMessage apm = md.getAlterPartitionMessage(ev.getMessage()); org.apache.hadoop.hive.metastore.api.Table tblObj = apm.getTableObj();
while (evIter.hasNext()) { NotificationEvent ev = evIter.next(); lastReplId = ev.getEventId(); Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId)); dumpEvent(ev, evRoot, cmRoot, hiveDb);
@InterfaceAudience.LimitedPrivate({"HCatalog"}) @Override public NotificationEventResponse getNextNotification(long lastEventId, int maxEvents, NotificationFilter filter) throws TException { NotificationEventRequest rqst = new NotificationEventRequest(lastEventId); rqst.setMaxEvents(maxEvents); NotificationEventResponse rsp = client.get_next_notification(rqst); LOG.debug("Got back " + rsp.getEventsSize() + " events"); NotificationEventResponse filtered = new NotificationEventResponse(); if (rsp != null && rsp.getEvents() != null) { long nextEventId = lastEventId + 1; for (NotificationEvent e : rsp.getEvents()) { if (e.getEventId() != nextEventId) { LOG.error("Requested events are found missing in NOTIFICATION_LOG table. Expected: {}, Actual: {}. " + "Probably, cleaner would've cleaned it up. " + "Try setting higher value for hive.metastore.event.db.listener.timetolive. " + "Also, bootstrap the system again to get back the consistent replicated state.", nextEventId, e.getEventId()); throw new IllegalStateException("Notification events are missing."); } if ((filter != null) && filter.accept(e)) { filtered.addToEvents(e); } nextEventId++; } } return (filter != null) ? filtered : rsp; }
long previousId = 0; for (NotificationEvent event : eventResponse.getEvents()) { Assert.assertTrue("previous:" + previousId + " current:" + event.getEventId(), previousId < event.getEventId()); Assert.assertTrue(previousId + 1 == event.getEventId()); previousId = event.getEventId();
private MNotificationLog translateThriftToDb(NotificationEvent entry) { MNotificationLog dbEntry = new MNotificationLog(); dbEntry.setEventId(entry.getEventId()); dbEntry.setEventTime(entry.getEventTime()); dbEntry.setEventType(entry.getEventType()); dbEntry.setCatalogName(entry.isSetCatName() ? entry.getCatName() : getDefaultCatalog(conf)); dbEntry.setDbName(entry.getDbName()); dbEntry.setTableName(entry.getTableName()); dbEntry.setMessage(entry.getMessage()); dbEntry.setMessageFormat(entry.getMessageFormat()); return dbEntry; }
Assert.assertEquals(FIRST_EVENT_ID, event.getEventId()); objectStore.addNotificationEvent(event); Assert.assertEquals(SECOND_EVENT_ID, event.getEventId()); Assert.assertEquals(FIRST_EVENT_ID, eventResponse.getEvents().get(0).getEventId()); Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(1).getEventId()); Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(0).getEventId());
long eventId = event.getEventId(); if (eventId <= lastEventId) { LOG.error("Event id is not valid " + lastEventId + " : " + eventId);
private MNotificationLog translateThriftToDb(NotificationEvent entry) { MNotificationLog dbEntry = new MNotificationLog(); dbEntry.setEventId(entry.getEventId()); dbEntry.setEventTime(entry.getEventTime()); dbEntry.setEventType(entry.getEventType()); dbEntry.setDbName(entry.getDbName()); dbEntry.setTableName(entry.getTableName()); dbEntry.setMessage(entry.getMessage()); return dbEntry; }