public HCatNotificationEvent(NotificationEvent event) { eventId = event.getEventId(); eventTime = event.getEventTime(); eventType = event.getEventType(); dbName = event.getDbName(); tableName = event.getTableName(); message = event.getMessage(); }
/*** * @param addPrimaryKeyEvent add primary key event * @throws MetaException */ @Override public void onAddPrimaryKey(AddPrimaryKeyEvent addPrimaryKeyEvent) throws MetaException { List<SQLPrimaryKey> cols = addPrimaryKeyEvent.getPrimaryKeyCols(); if (cols.size() > 0) { AddPrimaryKeyMessage msg = MessageBuilder.getInstance() .buildAddPrimaryKeyMessage(addPrimaryKeyEvent.getPrimaryKeyCols()); NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_PRIMARYKEY.toString(), msgEncoder.getSerializer().serialize(msg)); event.setCatName(cols.get(0).isSetCatName() ? cols.get(0).getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(cols.get(0).getTable_db()); event.setTableName(cols.get(0).getTable_name()); process(event, addPrimaryKeyEvent); } }
public Object getFieldValue(_Fields field) { switch (field) { case EVENT_ID: return getEventId(); case EVENT_TIME: return getEventTime(); case EVENT_TYPE: return getEventType(); case DB_NAME: return getDbName(); case TABLE_NAME: return getTableName(); case MESSAGE: return getMessage(); case MESSAGE_FORMAT: return getMessageFormat(); case CAT_NAME: return getCatName(); } throw new IllegalStateException(); }
public static boolean shouldReplicate(NotificationEvent tableForEvent, ReplicationSpec replicationSpec, Hive db, HiveConf hiveConf) { Table table; try { table = db.getTable(tableForEvent.getDbName(), tableForEvent.getTableName()); } catch (HiveException e) { LOG.info( "error while getting table info for" + tableForEvent.getDbName() + "." + tableForEvent .getTableName(), e); return false; } return shouldReplicate(replicationSpec, table, hiveConf); }
private void addNotificationLog(NotificationEvent event, ListenerEvent listenerEvent, Connection dbConn, SQLGenerator sqlGenerator) throws MetaException, SQLException { LOG.debug("DbNotificationListener: adding notification log for : {}", event.getMessage()); if ((dbConn == null) || (sqlGenerator == null)) { LOG.info("connection or sql generator is not set so executing sql via DN"); try { stmt = dbConn.createStatement(); event.setMessageFormat(msgEncoder.getMessageFormat()); params.add(event.getEventType()); params.add(event.getMessage()); params.add(event.getMessageFormat()); String dbName = event.getDbName(); if (dbName != null) { assert dbName.equals(dbName.toLowerCase()); String tableName = event.getTableName(); if (tableName != null) { assert tableName.equals(tableName.toLowerCase()); String catName = event.getCatName(); if (catName != null) { assert catName.equals(catName.toLowerCase()); if (event.isSetEventId()) { listenerEvent.putParameter(
/*** * @param allocWriteIdEvent Alloc write id event * @throws MetaException */ @Override public void onAllocWriteId(AllocWriteIdEvent allocWriteIdEvent, Connection dbConn, SQLGenerator sqlGenerator) throws MetaException { String tableName = allocWriteIdEvent.getTableName(); String dbName = allocWriteIdEvent.getDbName(); AllocWriteIdMessage msg = MessageBuilder.getInstance() .buildAllocWriteIdMessage(allocWriteIdEvent.getTxnToWriteIdList(), dbName, tableName); NotificationEvent event = new NotificationEvent(0, now(), EventType.ALLOC_WRITE_ID.toString(), msgEncoder.getSerializer().serialize(msg) ); event.setTableName(tableName); event.setDbName(dbName); try { addNotificationLog(event, allocWriteIdEvent, dbConn, sqlGenerator); } catch (SQLException e) { throw new MetaException("Unable to execute direct SQL " + StringUtils.stringifyException(e)); } }
private NotificationEvent translateDbToThrift(MNotificationLog dbEvent) { NotificationEvent event = new NotificationEvent(); event.setEventId(dbEvent.getEventId()); event.setEventTime(dbEvent.getEventTime()); event.setEventType(dbEvent.getEventType()); event.setCatName(dbEvent.getCatalogName()); event.setDbName(dbEvent.getDbName()); event.setTableName(dbEvent.getTableName()); event.setMessage((dbEvent.getMessage())); event.setMessageFormat(dbEvent.getMessageFormat()); return event; }
try { stmt = dbConn.createStatement(); event.setMessageFormat(msgFactory.getMessageFormat()); quoteString(event.getEventType()) + "," + quoteString(event.getDbName()) + "," + quoteString(" ") + "," + quoteString(event.getMessage()) + "," + quoteString(event.getMessageFormat())); if (event.isSetEventId()) { listenerEvent.putParameter( MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, Long.toString(event.getEventId()));
String tableName; switch (event.getEventType()) { case MessageBuilder.ADD_PARTITION_EVENT: case MessageBuilder.ALTER_PARTITION_EVENT: case MessageBuilder.DROP_TABLE_EVENT: case MessageBuilder.INSERT_EVENT: dbName = event.getDbName(); tableName = event.getTableName(); break; default: LOG.debug("Handling event {} on table {}.{}", event.getEventType(), dbName, tableName); long eventTime = event.getEventTime() * 1000L; cache.notifyTableChanged(dbName, tableName, eventTime); } else {
@Test public void testGetAllNotificationsReturnedByHms() throws Exception { SentryStore store = Mockito.mock(SentryStore.class); HiveConnectionFactory hmsConnection = Mockito.mock(HiveConnectionFactory.class); HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class); Mockito.when(hmsConnection.connect()).thenReturn(new HMSClient(hmsClient)); try (HiveNotificationFetcher fetcher = new HiveNotificationFetcher(store, hmsConnection)) { List<NotificationEvent> events; Mockito.when(hmsClient.getNextNotification(0, Integer.MAX_VALUE, null)) .thenReturn(new NotificationEventResponse( Arrays.<NotificationEvent>asList( new NotificationEvent(1L, 0, "CREATE_DATABASE", ""), new NotificationEvent(2L, 0, "CREATE_TABLE", "") ) )); events = fetcher.fetchNotifications(0); assertEquals(2, events.size()); assertEquals(1, events.get(0).getEventId()); assertEquals("CREATE_DATABASE", events.get(0).getEventType()); assertEquals(2, events.get(1).getEventId()); assertEquals("CREATE_TABLE", events.get(1).getEventType()); } }
/** * Process this notification by adding it to metastore DB. * * @param event NotificationEvent is the object written to the metastore DB. * @param listenerEvent ListenerEvent (from which NotificationEvent was based) used only to set the * DB_NOTIFICATION_EVENT_ID_KEY_NAME for future reference by other listeners. */ private void process(NotificationEvent event, ListenerEvent listenerEvent) throws MetaException { event.setMessageFormat(msgEncoder.getMessageFormat()); LOG.debug("DbNotificationListener: Processing : {}:{}", event.getEventId(), event.getMessage()); HMSHandler.getMSForConf(conf).addNotificationEvent(event); // Set the DB_NOTIFICATION_EVENT_ID for future reference by other listeners. if (event.isSetEventId()) { listenerEvent.putParameter( MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, Long.toString(event.getEventId())); } }
/** * @param fnEvent function event * @throws MetaException */ @Override public void onDropFunction(DropFunctionEvent fnEvent) throws MetaException { Function fn = fnEvent.getFunction(); DropFunctionMessage msg = MessageBuilder.getInstance().buildDropFunctionMessage(fn); NotificationEvent event = new NotificationEvent(0, now(), EventType.DROP_FUNCTION.toString(), msgEncoder.getSerializer().serialize(msg)); event.setCatName(fn.isSetCatName() ? fn.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(fn.getDbName()); process(event, fnEvent); }
.debug("Processing event with id:{} and Type:{}", event.getEventId(), event.getEventType()); EventType eventType = EventType.valueOf(event.getEventType()); Timer timer = SentryMetrics .getInstance() return processAlterPartition(event); default: LOGGER.error("Notification with ID:{} has invalid event type: {}", event.getEventId(), event.getEventType()); return false;
@Override public void addNotificationEvent(NotificationEvent entry) throws MetaException { if (addNotificationEventModifier != null) { Boolean success = addNotificationEventModifier.apply(entry); if ((success != null) && !success) { throw new MetaException("InjectableBehaviourObjectStore: Invalid addNotificationEvent operation on DB: " + entry.getDbName() + " table: " + entry.getTableName() + " event : " + entry.getEventType()); } } super.addNotificationEvent(entry); }
@Override boolean shouldAccept(final NotificationEvent event) { return eventFrom <= event.getEventId() && event.getEventId() <= eventTo; } }
private void dumpEvent(NotificationEvent ev, Path evRoot, Path cmRoot) throws Exception { long evid = ev.getEventId(); String evidStr = String.valueOf(evid); ReplicationSpec replicationSpec = getNewEventOnlyReplicationSpec(evidStr); MessageDeserializer md = MessageFactory.getInstance().getDeserializer(); switch (ev.getEventType()){ case MessageFactory.CREATE_TABLE_EVENT : { CreateTableMessage ctm = md.getCreateTableMessage(ev.getMessage()); LOG.info("Processing#{} CREATE_TABLE message : {}", ev.getEventId(), ev.getMessage()); org.apache.hadoop.hive.metastore.api.Table tobj = ctm.getTableObj(); AddPartitionMessage apm = md.getAddPartitionMessage(ev.getMessage()); LOG.info("Processing#{} ADD_PARTITION message : {}", ev.getEventId(), ev.getMessage()); Iterable<org.apache.hadoop.hive.metastore.api.Partition> ptns = apm.getPartitionObjs(); if ((ptns == null) || (!ptns.iterator().hasNext())) { LOG.info("Processing#{} DROP_TABLE message : {}", ev.getEventId(), ev.getMessage()); DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_DROP_TABLE, evid, evid, cmRoot); dmd.setPayload(ev.getMessage()); dmd.write(); break; LOG.info("Processing#{} DROP_PARTITION message : {}", ev.getEventId(), ev.getMessage()); DumpMetaData dmd = new DumpMetaData(evRoot, DUMPTYPE.EVENT_DROP_PARTITION, evid, evid, cmRoot); dmd.setPayload(ev.getMessage()); dmd.write(); break; LOG.info("Processing#{} ALTER_TABLE message : {}", ev.getEventId(), ev.getMessage()); AlterTableMessage atm = md.getAlterTableMessage(ev.getMessage());
deserializer.getAlterTableMessage(event.getMessage()); String oldDbName = alterTableMessage.getDB(); String oldTableName = alterTableMessage.getTable(); String newDbName = event.getDbName(); String newTableName = event.getTableName(); String oldLocation = alterTableMessage.getOldLocation(); String newLocation = alterTableMessage.getNewLocation(); + " {}.{}", oldDbName, oldTableName); } catch (Exception e) { LOGGER.info("Could not process Alter table event. Event: {}", event.toString(), e); return false;
@Override public void onAcidWrite(AcidWriteEvent acidWriteEvent, Connection dbConn, SQLGenerator sqlGenerator) throws MetaException { AcidWriteMessage msg = MessageBuilder.getInstance().buildAcidWriteMessage(acidWriteEvent, new FileChksumIterator(acidWriteEvent.getFiles(), acidWriteEvent.getChecksums(), acidWriteEvent.getSubDirs())); NotificationEvent event = new NotificationEvent(0, now(), EventType.ACID_WRITE.toString(), msgEncoder.getSerializer().serialize(msg)); event.setMessageFormat(msgEncoder.getMessageFormat()); event.setDbName(acidWriteEvent.getDatabase()); event.setTableName(acidWriteEvent.getTable()); try { addWriteNotificationLog(event, acidWriteEvent, dbConn, sqlGenerator, msg); } catch (SQLException e) { throw new MetaException("Unable to add write notification log " + StringUtils.stringifyException(e)); } }
MessageDeserializer deserializer) { SentryJSONAlterTableMessage message = (SentryJSONAlterTableMessage) deserializer.getAlterTableMessage(event.getMessage()); String prevDbName = message.getDB(); if ((prevDbName == null) || prevDbName.isEmpty()) { String newDbName = event.getDbName(); if ((newDbName == null) || newDbName.isEmpty()) { LOGGER.error("Alter table event is missing new database name"); String newTableName = event.getTableName(); if ((newTableName == null) || newTableName.isEmpty()) { LOGGER.error("Alter table event is missing new table name");
new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), "CREATE DATABASE DB initial")); new NotificationEvent(0, 0, eventType, "CREATE DATABASE DB" + n); System.out.println("ADDING NOTIFICATION"); long previousId = 0; for (NotificationEvent event : eventResponse.getEvents()) { Assert.assertTrue("previous:" + previousId + " current:" + event.getEventId(), previousId < event.getEventId()); Assert.assertTrue(previousId + 1 == event.getEventId()); previousId = event.getEventId();