/** * Delete and reassign workers for provisioners that have not sent a heartbeat since the given timestamp in * milliseconds. * * @param timeoutTs Timestamp in milliseconds to use as a cut off for deleting provisioners. * @throws IOException */ public void timeoutProvisioners(long timeoutTs) throws IOException { tenantLock.lock(); try { Set<String> affectedTenants = Sets.newHashSet(); for (Provisioner provisioner : provisionerStore.getTimedOutProvisioners(timeoutTs)) { String provisionerId = provisioner.getId(); LOG.error("provisioner {} has not sent a heartbeat in over {} seconds, deleting it...", provisionerId, provisionerTimeoutSecs); provisionerStore.deleteProvisioner(provisioner.getId()); affectedTenants.addAll(provisioner.getAssignedTenants()); } for (String affectedTenant : affectedTenants) { balanceQueue.add(new Element(affectedTenant)); } } finally { tenantLock.unlock(); } }
@Test public void testGetIdleProvisioners() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); store.setHeartbeat(provisioner1.getId(), 100L); store.setHeartbeat(provisioner2.getId(), 1000L); Assert.assertTrue(store.getTimedOutProvisioners(99L).isEmpty()); Assert.assertEquals(ImmutableSet.of(provisioner1), ImmutableSet.copyOf(store.getTimedOutProvisioners(101L))); Assert.assertEquals(ImmutableSet.of(provisioner1, provisioner2), ImmutableSet.copyOf(store.getTimedOutProvisioners(1001L))); }