/** * Method to return HCatEventMessage contained in the JMS message. * @param message The JMS Message instance * @return The contained HCatEventMessage */ public static HCatEventMessage getMessage(Message message) { try { String messageBody = ((TextMessage)message).getText(); String eventType = message.getStringProperty(HCatConstants.HCAT_EVENT); String messageVersion = message.getStringProperty(HCatConstants.HCAT_MESSAGE_VERSION); String messageFormat = message.getStringProperty(HCatConstants.HCAT_MESSAGE_FORMAT); if (StringUtils.isEmpty(messageBody) || StringUtils.isEmpty(eventType)) throw new IllegalArgumentException("Could not extract HCatEventMessage. " + "EventType and/or MessageBody is null/empty."); return MessageFactory.getDeserializer(messageFormat, messageVersion).getHCatEventMessage(eventType, messageBody); } catch (JMSException exception) { throw new IllegalArgumentException("Could not extract HCatEventMessage. ", exception); } }
@Override public void onAddPartition(AddPartitionEvent partitionEvent) throws MetaException { // Subscriber can get notification of newly add partition in a // particular table by listening on a topic named "dbName.tableName" // and message selector string as "HCAT_EVENT = HCAT_ADD_PARTITION" if (partitionEvent.getStatus()) { Table table = partitionEvent.getTable(); String topicName = getTopicName(table); if (topicName != null && !topicName.equals("")) { send(messageFactory.buildAddPartitionMessage(table, partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " + partitionEvent.getTable().getDbName() + "." + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=<dbname>.<tablename>) or whatever you want topic name to be."); } } }
@Override public void onAlterPartition(AlterPartitionEvent ape) throws MetaException { if (ape.getStatus()) { Partition before = ape.getOldPartition(); Partition after = ape.getNewPartition(); String topicName = getTopicName(ape.getTable()); send(messageFactory.buildAlterPartitionMessage(ape.getTable(),before, after, ape.getWriteId()), topicName); } }
public abstract MessageDeserializer getDeserializer();
@Override public void onCreateDatabase(CreateDatabaseEvent dbEvent) throws MetaException { // Subscriber can get notification about addition of a database in HCAT // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_ADD_DATABASE" if (dbEvent.getStatus()) { String topicName = getTopicPrefix(dbEvent.getIHMSHandler().getConf()); send(messageFactory.buildCreateDatabaseMessage(dbEvent.getDatabase()), topicName); } }
@Override public void onCreateTable(CreateTableEvent tableEvent) throws MetaException { // Subscriber can get notification about addition of a table in HCAT // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_ADD_TABLE" if (tableEvent.getStatus()) { Table tbl = tableEvent.getTable(); IHMSHandler handler = tableEvent.getIHMSHandler(); Configuration conf = handler.getConf(); Table newTbl; try { newTbl = handler.get_table_core(tbl.getCatName(), tbl.getDbName(), tbl.getTableName()) .deepCopy(); newTbl.getParameters().put( HCatConstants.HCAT_MSGBUS_TOPIC_NAME, getTopicPrefix(conf) + "." + newTbl.getDbName().toLowerCase() + "." + newTbl.getTableName().toLowerCase()); handler.alter_table(newTbl.getDbName(), newTbl.getTableName(), newTbl); } catch (TException e) { MetaException me = new MetaException(e.toString()); me.initCause(e); throw me; } String topicName = getTopicPrefix(conf) + "." + newTbl.getDbName().toLowerCase(); send(messageFactory.buildCreateTableMessage(newTbl), topicName); } }
/** * Send dropped table notifications. Subscribers can receive these notifications for * dropped tables by listening on topic "HCAT" with message selector string * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_EVENT} = * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_DROP_TABLE_EVENT} * </br> * TODO: DataNucleus 2.0.3, currently used by the HiveMetaStore for persistence, has been * found to throw NPE when serializing objects that contain null. For this reason we override * some fields in the StorageDescriptor of this notification. This should be fixed after * HIVE-2084 "Upgrade datanucleus from 2.0.3 to 3.0.1" is resolved. */ @Override public void onDropTable(DropTableEvent tableEvent) throws MetaException { // Subscriber can get notification about drop of a table in HCAT // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_DROP_TABLE" // Datanucleus throws NPE when we try to serialize a table object // retrieved from metastore. To workaround that we reset following objects if (tableEvent.getStatus()) { Table table = tableEvent.getTable(); // I think this is wrong, the drop table statement should come on the table topic not the // DB topic - Alan. String topicName = getTopicPrefix(tableEvent.getIHMSHandler().getConf()) + "." + table.getDbName().toLowerCase(); send(messageFactory.buildDropTableMessage(table), topicName); } }
/** * Send altered table notifications. Subscribers can receive these notifications for * dropped tables by listening on topic "HCAT" with message selector string * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_EVENT} = * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_ALTER_TABLE_EVENT} */ @Override public void onAlterTable(AlterTableEvent tableEvent) throws MetaException { if (tableEvent.getStatus()) { Table before = tableEvent.getOldTable(); Table after = tableEvent.getNewTable(); // onCreateTable alters the table to add the topic name. Since this class is generating // that alter, we don't want to notify on that alter. So take a quick look and see if // that's what this this alter is, and if so swallow it. if (after.getParameters() != null && after.getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME) != null && (before.getParameters() == null || before.getParameters().get(HCatConstants.HCAT_MSGBUS_TOPIC_NAME) == null)) { return; } // I think this is wrong, the alter table statement should come on the table topic not the // DB topic - Alan. String topicName = getTopicPrefix(tableEvent.getIHMSHandler().getConf()) + "." + after.getDbName().toLowerCase(); send(messageFactory.buildAlterTableMessage(before, after, tableEvent.getWriteId()), topicName); } }
/** * Send dropped partition notifications. Subscribers can receive these notifications for a * particular table by listening on a topic named "dbName.tableName" with message selector * string {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_EVENT} = * {@value org.apache.hive.hcatalog.common.HCatConstants#HCAT_DROP_PARTITION_EVENT}. * </br> * TODO: DataNucleus 2.0.3, currently used by the HiveMetaStore for persistence, has been * found to throw NPE when serializing objects that contain null. For this reason we override * some fields in the StorageDescriptor of this notification. This should be fixed after * HIVE-2084 "Upgrade datanucleus from 2.0.3 to 3.0.1" is resolved. */ @Override public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException { if (partitionEvent.getStatus()) { String topicName = getTopicName(partitionEvent.getTable()); if (topicName != null && !topicName.equals("")) { send(messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " + partitionEvent.getTable().getDbName() + "." + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=<dbname>.<tablename>) or whatever you want topic name to be."); } } }
@Override public void onDropDatabase(DropDatabaseEvent dbEvent) throws MetaException { // Subscriber can get notification about drop of a database in HCAT // by listening on a topic named "HCAT" and message selector string // as "HCAT_EVENT = HCAT_DROP_DATABASE" if (dbEvent.getStatus()) { String topicName = getTopicPrefix(dbEvent.getIHMSHandler().getConf()); send(messageFactory.buildDropDatabaseMessage(dbEvent.getDatabase()), topicName); } }
@Test public void testInsert() throws HCatException { Table t = new Table(); t.setDbName("testdb"); t.setTableName("testtable"); List<FieldSchema> pkeys = HCatSchemaUtils.getFieldSchemas( HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields()); t.setPartitionKeys(pkeys); Partition p = createPtn(t, Arrays.asList("102", "lmn")); List<String> files = Arrays.asList("/tmp/test123"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_INSERT_EVENT, msgFactory.buildInsertMessage( t.getDbName(), t.getTableName(), getPtnDesc(t,p), files ).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyInsertReplicationTask(rtask, t, p); }
public abstract MessageDeserializer getDeserializer();
@Test public void testDropDb() throws IOException { Database db = new Database(); db.setName("testdb"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_DROP_DATABASE_EVENT, msgFactory.buildCreateDatabaseMessage(db).toString()); event.setDbName(db.getName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyDropDbReplicationTask(rtask); }
@Test public void testCreateTable() throws IOException { Table t = new Table(); t.setDbName("testdb"); t.setTableName("testtable"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_CREATE_TABLE_EVENT, msgFactory.buildCreateTableMessage(t).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyCreateTableReplicationTask(rtask); }
@Test public void testDropTable() throws IOException { Table t = new Table(); t.setDbName("testdb"); t.setTableName("testtable"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_DROP_TABLE_EVENT, msgFactory.buildDropTableMessage(t).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyDropTableReplicationTask(rtask); }
@Test public void testAlterTable() throws IOException { Table t = new Table(); t.setDbName("testdb"); t.setTableName("testtable"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_ALTER_TABLE_EVENT, msgFactory.buildAlterTableMessage(t, t, t.getWriteId()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyAlterTableReplicationTask(rtask); }
@Test public void testDropPartition() throws HCatException { Table t = new Table(); t.setDbName("testdb"); t.setTableName("testtable"); List<FieldSchema> pkeys = HCatSchemaUtils.getFieldSchemas( HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields()); t.setPartitionKeys(pkeys); Partition p = createPtn(t, Arrays.asList("102", "lmn")); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_DROP_PARTITION_EVENT, msgFactory.buildDropPartitionMessage( t, Collections.singletonList(p).iterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyDropPartitionReplicationTask(rtask, t, p); }
/** * @param dbEvent database event * @throws MetaException */ public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException { Database db = dbEvent.getDatabase(); NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_DROP_DATABASE_EVENT, msgFactory.buildDropDatabaseMessage(db).toString()); event.setDbName(db.getName()); enqueue(event); }
@Override public void onInsert(InsertEvent insertEvent) throws MetaException { NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_INSERT_EVENT, msgFactory.buildInsertMessage(insertEvent.getDb(), insertEvent.getTable(), insertEvent.getPartitionKeyValues(), insertEvent.getFiles()).toString()); event.setDbName(insertEvent.getDb()); event.setTableName(insertEvent.getTable()); enqueue(event); }
public AlterTableReplicationTask(HCatNotificationEvent event) { super(event); validateEventType(event, HCatConstants.HCAT_ALTER_TABLE_EVENT); alterTableMessage = messageFactory.getDeserializer().getAlterTableMessage(event.getMessage()); }