@Override public HCatNotificationEvent apply(@Nullable NotificationEvent notificationEvent) { return new HCatNotificationEvent(notificationEvent); } });
public NoopReplicationTask(HCatNotificationEvent event) { super(event); noopReturn = new ArrayList<Command>(); noopReturn.add(new NoopCommand(event.getEventId())); }
protected void validateEventType(HCatNotificationEvent event, String allowedEventType) { if (event == null || !allowedEventType.equals(event.getEventType())){ throw new IllegalStateException(this.getClass().getName() + " valid only for " + allowedEventType + " events."); } } }
private static ExportCommand getExpectedExportCommand(ReplicationTask rtask, Map<String, String> ptnDesc, boolean isMetadataOnly) { String dbName = rtask.getEvent().getDbName(); String tableName = rtask.getEvent().getTableName(); return new ExportCommand( dbName, tableName, ptnDesc, stagingDirectoryProvider.getStagingDirectory( ReplicationUtils.getUniqueKey( rtask.getEvent().getEventId(), dbName, tableName, ptnDesc) ), isMetadataOnly, rtask.getEvent().getEventId() ); }
public DropDatabaseReplicationTask(HCatNotificationEvent event) { super(event); validateEventType(event, HCatConstants.HCAT_DROP_DATABASE_EVENT); dropDatabaseMessage = messageFactory.getDeserializer().getDropDatabaseMessage(event.getMessage()); }
LOG.info("notif from dblistener:" + n.getEventId() + ":" + n.getEventTime() + ",t:" + n.getEventType() + ",o:" + n.getDbName() + "." + n.getTableName()); ReplicationTask task = taskIter.next(); HCatNotificationEvent n = task.getEvent(); LOG.info("notif from tasks:" + n.getEventId() + ":" + n.getEventTime() + ",t:" + n.getEventType() + ",o:" + n.getDbName() + "." + n.getTableName() + ",s:" + n.getEventScope()); LOG.info("task :" + task.getClass().getName()); if (task.needsStagingDirs()){
@Test public void testDropDb() throws IOException { Database db = new Database(); db.setName("testdb"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_DROP_DATABASE_EVENT, msgFactory.buildCreateDatabaseMessage(db).toString()); event.setDbName(db.getName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyDropDbReplicationTask(rtask); }
private static void verifyDropDbReplicationTask(ReplicationTask rtask) throws IOException { assertEquals(DropDatabaseReplicationTask.class, rtask.getClass()); assertEquals(false, rtask.needsStagingDirs()); assertEquals(true, rtask.isActionable()); rtask .withDbNameMapping(debugMapping) .withTableNameMapping(debugMapping); for (Command c : rtask.getSrcWhCommands()){ assertEquals(NoopCommand.class, c.getClass()); } List<? extends Command> dstCommands = Lists.newArrayList(rtask.getDstWhCommands()); assertEquals(1,dstCommands.size()); assertEquals(DropDatabaseCommand.class, dstCommands.get(0).getClass()); DropDatabaseCommand dropDatabaseCommand = new DropDatabaseCommand( debugMapping.apply(rtask.getEvent().getDbName()), rtask.getEvent().getEventId()); assertEquals(ReplicationUtils.serializeCommand(dropDatabaseCommand), ReplicationUtils.serializeCommand(dstCommands.get(0))); }
public CreateTableReplicationTask(HCatNotificationEvent event) { super(event); validateEventType(event, HCatConstants.HCAT_CREATE_TABLE_EVENT); createTableMessage = messageFactory.getDeserializer().getCreateTableMessage(event.getMessage()); }
@Test public void testCreateDb(){ Database db = new Database(); db.setName("testdb"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_CREATE_DATABASE_EVENT, msgFactory.buildCreateDatabaseMessage(db).toString()); event.setDbName(db.getName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyCreateDbReplicationTask(rtask); // CREATE DB currently replicated as Noop. }
private static ImportCommand getExpectedImportCommand(ReplicationTask rtask, Map<String,String> ptnDesc, boolean isMetadataOnly) { String dbName = rtask.getEvent().getDbName(); String tableName = rtask.getEvent().getTableName(); return new ImportCommand( ReplicationUtils.mapIfMapAvailable(dbName, debugMapping), ReplicationUtils.mapIfMapAvailable(tableName, debugMapping), ptnDesc, stagingDirectoryProvider.getStagingDirectory( ReplicationUtils.getUniqueKey( rtask.getEvent().getEventId(), dbName, tableName, ptnDesc) ), isMetadataOnly, rtask.getEvent().getEventId() ); }
@Override public Command apply(@Nullable Map<String, String> ptnDesc) { return new DropPartitionCommand( dstDbName, dstTableName, ptnDesc, true, event.getEventId() ); } });
public AlterPartitionReplicationTask(HCatNotificationEvent event) { super(event); validateEventType(event,HCatConstants.HCAT_ALTER_PARTITION_EVENT); alterPartitionMessage = messageFactory.getDeserializer().getAlterPartitionMessage(event.getMessage()); }
@Test public void testDropTable() throws IOException { Table t = new Table(); t.setDbName("testdb"); t.setTableName("testtable"); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_DROP_TABLE_EVENT, msgFactory.buildDropTableMessage(t).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyDropTableReplicationTask(rtask); }
private static void verifyDropTableReplicationTask(ReplicationTask rtask) throws IOException { assertEquals(DropTableReplicationTask.class, rtask.getClass()); assertEquals(false, rtask.needsStagingDirs()); assertEquals(true, rtask.isActionable()); rtask .withDbNameMapping(debugMapping) .withTableNameMapping(debugMapping); for (Command c : rtask.getSrcWhCommands()){ assertEquals(NoopCommand.class, c.getClass()); } List<? extends Command> dstCommands = Lists.newArrayList(rtask.getDstWhCommands()); assertEquals(1,dstCommands.size()); assertEquals(DropTableCommand.class, dstCommands.get(0).getClass()); DropTableCommand dropTableCommand = new DropTableCommand( debugMapping.apply(rtask.getEvent().getDbName()), debugMapping.apply(rtask.getEvent().getTableName()), true, rtask.getEvent().getEventId()); assertEquals(ReplicationUtils.serializeCommand(dropTableCommand), ReplicationUtils.serializeCommand(dstCommands.get(0))); }
public ReplicationTask create(HCatClient client, HCatNotificationEvent event){ // TODO : Java 1.7+ support using String with switches, but IDEs don't all seem to know that. // If casing is fine for now. But we should eventually remove this. Also, I didn't want to // create another enum just for this. if (event.getEventType().equals(HCatConstants.HCAT_CREATE_DATABASE_EVENT)) { return new CreateDatabaseReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_DROP_DATABASE_EVENT)) { return new DropDatabaseReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_CREATE_TABLE_EVENT)) { return new CreateTableReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_DROP_TABLE_EVENT)) { return new DropTableReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_ADD_PARTITION_EVENT)) { return new AddPartitionReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_DROP_PARTITION_EVENT)) { return new DropPartitionReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_ALTER_TABLE_EVENT)) { return new AlterTableReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_ALTER_PARTITION_EVENT)) { return new AlterPartitionReplicationTask(event); } else if (event.getEventType().equals(HCatConstants.HCAT_INSERT_EVENT)) { return new InsertReplicationTask(event); } else { return new ErroredReplicationTask(event, new IllegalStateException( "Unrecognized Event type, no replication task available")); } } }
ReplicationTask rtask = ReplicationTask.create(HCatClient.create(hconf), new HCatNotificationEvent(ev)); if (rtask instanceof ErroredReplicationTask) { unhandledTasks.put(ev, ((ErroredReplicationTask) rtask).getCause());
@Override public ReplicationTask next() { eventCount++; HCatNotificationEvent ev = batchIter.next(); pos = ev.getEventId(); return ReplicationTask.create(hcatClient,ev); }
public DropPartitionReplicationTask(HCatNotificationEvent event) { super(event); validateEventType(event, HCatConstants.HCAT_DROP_PARTITION_EVENT); dropPartitionMessage = messageFactory.getDeserializer().getDropPartitionMessage(event.getMessage()); }
@Test public void testAddPartition() throws IOException { Table t = new Table(); t.setDbName("testdb"); t.setTableName("testtable"); List<FieldSchema> pkeys = HCatSchemaUtils.getFieldSchemas( HCatSchemaUtils.getHCatSchema("a:int,b:string").getFields()); t.setPartitionKeys(pkeys); List<Partition> addedPtns = new ArrayList<Partition>(); addedPtns.add(createPtn(t, Arrays.asList("120", "abc"))); addedPtns.add(createPtn(t, Arrays.asList("201", "xyz"))); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), HCatConstants.HCAT_ADD_PARTITION_EVENT, msgFactory.buildAddPartitionMessage(t, addedPtns.iterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); HCatNotificationEvent hev = new HCatNotificationEvent(event); ReplicationTask rtask = ReplicationTask.create(client,hev); assertEquals(hev.toString(), rtask.getEvent().toString()); verifyAddPartitionReplicationTask(rtask, t, addedPtns); }