/** * Delete and reassign workers for provisioners that have not sent a heartbeat since the given timestamp in * milliseconds. * * @param timeoutTs Timestamp in milliseconds to use as a cut off for deleting provisioners. * @throws IOException */ public void timeoutProvisioners(long timeoutTs) throws IOException { tenantLock.lock(); try { Set<String> affectedTenants = Sets.newHashSet(); for (Provisioner provisioner : provisionerStore.getTimedOutProvisioners(timeoutTs)) { String provisionerId = provisioner.getId(); LOG.error("provisioner {} has not sent a heartbeat in over {} seconds, deleting it...", provisionerId, provisionerTimeoutSecs); provisionerStore.deleteProvisioner(provisioner.getId()); affectedTenants.addAll(provisioner.getAssignedTenants()); } for (String affectedTenant : affectedTenants) { balanceQueue.add(new Element(affectedTenant)); } } finally { tenantLock.unlock(); } }
@Test public void testGetAllProvisioners() throws IOException { ProvisionerStore store = getProvisionerStore(); Assert.assertTrue(store.getAllProvisioners().isEmpty()); store.writeProvisioner(provisioner1); Assert.assertEquals(ImmutableSet.of(provisioner1), ImmutableSet.copyOf(store.getAllProvisioners())); store.writeProvisioner(provisioner2); Assert.assertEquals(ImmutableSet.of(provisioner1, provisioner2), ImmutableSet.copyOf(store.getAllProvisioners())); }
private void checkCapacity(int diff) throws IOException, CapacityException { if (diff > provisionerStore.getFreeCapacity()) { throw new CapacityException("Not enough capacity."); } } }
/** * Handle the heartbeat of a provisioner, updating the last heartbeat time of the provisioner and updating the number * of live workers running on the provisioner for each tenant it is responsible for. * * @param provisionerId Id of the provisioner that sent the heartbeat * @param heartbeat The heartbeat containing live worker information * @throws IOException if there was an exception persisting the data * @throws MissingEntityException if there is no provisioner for the given id */ public void handleHeartbeat(String provisionerId, ProvisionerHeartbeat heartbeat) throws IOException, MissingEntityException { // no lock required here. Simply getting a provisioner and writing worker usage. Would only expect one provisioner // to be calling this at a time, and even if it is calling it concurrently for some reason, only the usage can // change and for that its ok for one of them to win. Provisioner provisioner = provisionerStore.getProvisioner(provisionerId); if (provisioner == null) { throw new MissingEntityException("Provisioner " + provisionerId + " not found."); } if (!provisioner.getUsage().equals(heartbeat.getUsage())) { provisioner.setUsage(heartbeat.getUsage()); provisionerStore.writeProvisioner(provisioner); } provisionerStore.setHeartbeat(provisionerId, System.currentTimeMillis()); }
@Test public void testGetIdleProvisioners() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); store.setHeartbeat(provisioner1.getId(), 100L); store.setHeartbeat(provisioner2.getId(), 1000L); Assert.assertTrue(store.getTimedOutProvisioners(99L).isEmpty()); Assert.assertEquals(ImmutableSet.of(provisioner1), ImmutableSet.copyOf(store.getTimedOutProvisioners(101L))); Assert.assertEquals(ImmutableSet.of(provisioner1, provisioner2), ImmutableSet.copyOf(store.getTimedOutProvisioners(1001L))); }
@Test public void testWriteWithTenantUpdate() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); Assert.assertEquals(provisioner1, store.getProvisioner(provisioner1.getId())); Provisioner updatedProvisioner1 = new Provisioner( provisioner1.getId(), provisioner1.getHost(), provisioner1.getPort(), 100, ImmutableMap.<String, Integer>of("tenantA", 50, "tenantY", 5, "tenantZ", 0), ImmutableMap.<String, Integer>of("tenantA", 50, "tenantY", 5, "tenantZ", 1) ); store.writeProvisioner(updatedProvisioner1); Assert.assertEquals(updatedProvisioner1, store.getProvisioner(provisioner1.getId())); Assert.assertEquals(0, store.getNumAssignedWorkers("tenantX")); Assert.assertEquals(44, store.getFreeCapacity()); for (Provisioner provisioner : store.getTenantProvisioners("tenantX")) { Assert.assertFalse(provisioner.getId().equals(provisioner1.getId())); } }
@Test public void testWriteGetDeleteProvisioner() throws IOException { ProvisionerStore store = getProvisionerStore(); String id = provisioner1.getId(); Assert.assertNull(store.getProvisioner(id)); store.writeProvisioner(provisioner1); Assert.assertEquals(provisioner1, store.getProvisioner(id)); store.deleteProvisioner(id); Assert.assertNull(store.getProvisioner(id)); }
Assert.assertEquals(0, store.getNumAssignedWorkers("tenantA")); Assert.assertEquals(0, store.getNumAssignedWorkers("tenantX")); Assert.assertEquals(0, store.getNumAssignedWorkers("tenantY")); Assert.assertEquals(0, store.getNumAssignedWorkers("tenantZ")); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); Assert.assertEquals(5, store.getNumAssignedWorkers("tenantA")); Assert.assertEquals(5, store.getNumAssignedWorkers("tenantX")); Assert.assertEquals(10, store.getNumAssignedWorkers("tenantY")); Assert.assertEquals(2, store.getNumAssignedWorkers("tenantZ")); ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 3, "tenantZ", 2), ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 50, "tenantZ", 20)); store.writeProvisioner(provisioner3); store.writeProvisioner(updatedProvisioner2); Assert.assertEquals(5, store.getNumAssignedWorkers("tenantA")); Assert.assertEquals(40, store.getNumAssignedWorkers("tenantB")); Assert.assertEquals(15, store.getNumAssignedWorkers("tenantX")); Assert.assertEquals(55, store.getNumAssignedWorkers("tenantY")); Assert.assertEquals(20, store.getNumAssignedWorkers("tenantZ")); store.deleteProvisioner(provisioner2.getId()); Assert.assertEquals(0, store.getNumAssignedWorkers("tenantA")); Assert.assertEquals(40, store.getNumAssignedWorkers("tenantB")); Assert.assertEquals(15, store.getNumAssignedWorkers("tenantX")); Assert.assertEquals(5, store.getNumAssignedWorkers("tenantY"));
@Test public void testGetFreeCapacity() throws IOException { ProvisionerStore store = getProvisionerStore(); Assert.assertEquals(0, store.getFreeCapacity()); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); Assert.assertEquals(88, store.getFreeCapacity()); ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 3, "tenantZ", 2), ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 50, "tenantZ", 20)); store.writeProvisioner(updatedProvisioner2); Assert.assertEquals(25, store.getFreeCapacity()); ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 3, "tenantZ", 2), ImmutableMap.<String, Integer>of("tenantA", 5, "tenantZ", 20)); store.writeProvisioner(updatedProvisioner2); Assert.assertEquals(75, store.getFreeCapacity()); store.deleteProvisioner(provisioner2.getId()); Assert.assertEquals(0, store.getFreeCapacity()); ImmutableMap.<String, Integer>of("tenantB", 10), ImmutableMap.<String, Integer>of("tenantB", 40)); store.writeProvisioner(provisioner3); Assert.assertEquals(10, store.getFreeCapacity());
@Test public void testGetTenantProvisioners() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); Assert.assertTrue(store.getTenantProvisioners("tenantB").isEmpty()); Assert.assertEquals(ImmutableSet.of(provisioner2), ImmutableSet.copyOf(store.getTenantProvisioners("tenantA"))); Assert.assertEquals(ImmutableSet.of(provisioner1), ImmutableSet.copyOf(store.getTenantProvisioners("tenantX"))); Assert.assertEquals(ImmutableSet.of(provisioner2), ImmutableSet.copyOf(store.getTenantProvisioners("tenantZ"))); Assert.assertEquals(ImmutableSet.of(provisioner1, provisioner2), ImmutableSet.copyOf(store.getTenantProvisioners("tenantY"))); }
@Test public void testGetProvisionersWithFreeCapacity() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); Assert.assertTrue(store.getProvisionersWithFreeCapacity().isEmpty()); store.writeProvisioner(provisioner2); Assert.assertEquals(ImmutableSet.of(provisioner2), ImmutableSet.copyOf(store.getProvisionersWithFreeCapacity())); Provisioner updatedProvisioner1 = new Provisioner( provisioner1.getId(), provisioner1.getHost(), provisioner1.getPort(), provisioner1.getCapacityTotal(), ImmutableMap.<String, Integer>of("tenantX", 5), ImmutableMap.<String, Integer>of("tenantX", 5)); store.writeProvisioner(updatedProvisioner1); Assert.assertEquals(ImmutableSet.of(updatedProvisioner1, provisioner2), ImmutableSet.copyOf(store.getProvisionersWithFreeCapacity())); }
/** * Get the provisioner for the given id for external display, with tenant ids mapped to tenant names, * or null if none exists. * * @param provisionerId Id of the provisioner to get * @return Provisioner for the given id with tenant ids mapped to tenant names, or null if none exists * @throws IOException */ public Provisioner getProvisioner(String provisionerId) throws IOException { return createExternalProvisioner(provisionerStore.getProvisioner(provisionerId)); }
/** * Delete the given tenant. A tenant must not have any assigned workers in order for deletion to be allowed. * * @param name Name of the tenant to delete * @throws IllegalStateException if the tenant has one or more assigned workers * @throws IOException if there was an exception persisting the deletion */ public void deleteTenantByName(String name) throws IllegalStateException, IOException { tenantLock.lock(); try { Tenant tenant = tenantStore.getTenantByName(name); if (tenant == null) { return; } int numAssignedWorkers = provisionerStore.getNumAssignedWorkers(tenant.getId()); if (numAssignedWorkers > 0) { throw new IllegalStateException("Tenant " + name + " still has " + numAssignedWorkers + " workers. " + "Cannot delete it until workers are set to 0."); } tenantStore.deleteTenantByName(name); for (QueueGroup queueGroup : queueService.getAllQueueGroups().values()) { queueGroup.removeAll(tenant.getId()); } } finally { tenantLock.unlock(); } }
entityStoreService.startAndWait(); provisionerStore = injector.getInstance(ProvisionerStore.class); provisionerStore.startAndWait(); resourceService = injector.getInstance(ResourceService.class); resourceService.startAndWait();
/** * Get an immutable collection of all provisioners for external display, with tenant ids mapped to tenant names. * * @return Immutable collection of all provisioners for external display, with tenant ids mapped to tenant names * @throws IOException */ public Collection<Provisioner> getAllProvisioners() throws IOException { Collection<Provisioner> provisioners = provisionerStore.getAllProvisioners(); List<Provisioner> externalProvisioners = Lists.newArrayListWithCapacity(provisioners.size()); for (Provisioner provisioner : provisioners) { externalProvisioners.add(createExternalProvisioner(provisioner)); } return externalProvisioners; }
private void syncProvisionerResources(String tenantId, ResourceCollection resourceCollection) throws IOException { for (Provisioner provisioner : provisionerStore.getTenantProvisioners(tenantId)) { if (!provisionerRequestService.putTenant(provisioner, tenantId, resourceCollection)) { LOG.error("Could not write resource metadata for tenant {} to provisioner {}. " + "The provisioner appears broken, deleting it and rebalancing its tenant workers", tenantId, provisioner.getId()); deleteProvisioner(provisioner); } } }
private void deleteProvisioner(Provisioner provisioner) throws IOException { for (String tenant : provisioner.getAssignedTenants()) { balanceQueue.add(new Element(tenant)); } provisionerStore.deleteProvisioner(provisioner.getId()); }
Provisioner provisioner = store.getProvisioner(provisionerId); Assert.assertTrue(provisioner.equals(pState1) || provisioner.equals(pState2) || provisioner.equals(pState3)); if (provisioner.equals(pState1)) { Assert.assertEquals(97, store.getFreeCapacity()); for (String tenant : pState1.getAssignedTenants()) { Collection<Provisioner> tenantProvisioners = store.getTenantProvisioners(tenant); Assert.assertEquals(1, tenantProvisioners.size()); Assert.assertEquals(provisioner, tenantProvisioners.iterator().next()); Assert.assertEquals(1, store.getNumAssignedWorkers(tenant)); Assert.assertEquals(94, store.getFreeCapacity()); for (String tenant : pState2.getAssignedTenants()) { Collection<Provisioner> tenantProvisioners = store.getTenantProvisioners(tenant); Assert.assertEquals(1, tenantProvisioners.size()); Assert.assertEquals(provisioner, tenantProvisioners.iterator().next()); Assert.assertEquals(2, store.getNumAssignedWorkers(tenant)); Assert.assertEquals(91, store.getFreeCapacity()); for (String tenant : pState3.getAssignedTenants()) { Collection<Provisioner> tenantProvisioners = store.getTenantProvisioners(tenant); Assert.assertEquals(1, tenantProvisioners.size()); Assert.assertEquals(provisioner, tenantProvisioners.iterator().next()); Assert.assertEquals(3, store.getNumAssignedWorkers(tenant));
private void removeWorkers(String tenantId, int numToRemove, ResourceCollection resources) throws IOException { // go through each provisioner, removing workers for the tenant until we've removed enough. for (Provisioner provisioner : provisionerStore.getTenantProvisioners(tenantId)) { int numRemoved = provisioner.tryRemoveTenantAssignments(tenantId, numToRemove); if (numRemoved > 0) { provisionerStore.writeProvisioner(provisioner); LOG.debug("Requesting provisioner {} to set workers to {} for tenant {} (removing {})", provisioner.getId(), provisioner.getAssignedWorkers(tenantId), tenantId, numRemoved); if (provisionerRequestService.putTenant(provisioner, tenantId, resources)) { numToRemove -= numRemoved; } else { // request failed with retries. something is wrong with the provisioner, delete it and rebalance its workers // TODO: what if this fails? LOG.error("Could not set workers for tenant {} to provisioner {}. " + "The provisioner appears broken, deleting it and rebalancing its tenant workers", tenantId, provisioner.getId()); deleteProvisioner(provisioner); } } } }
private void addWorkers(String tenantId, int numToAdd, ResourceCollection resources) throws CapacityException, IOException { for (Provisioner provisioner : provisionerStore.getProvisionersWithFreeCapacity()) { if (numToAdd <= 0) { break; provisionerStore.writeProvisioner(provisioner); LOG.debug("Requesting provisioner {} to set workers to {} for tenant {} (adding {})", provisioner.getId(), provisioner.getAssignedWorkers(tenantId), tenantId, numAdded);