@Override public void run() { try { barrier.await(); } catch (Exception e) { Throwables.propagate(e); } for (int j = 0; j < writesPerThread; j++) { try { store.writeProvisioner(provisioner); } catch (IOException e) { // some conflicts expected } } latch.countDown(); } });
/** * Write a provisioner, and queue jobs to rebalance workers for all tenants in the system. * * @param provisioner Provisioner to write * @throws IOException */ public void writeProvisioner(Provisioner provisioner) throws IOException { tenantLock.lock(); try { provisionerStore.writeProvisioner(provisioner); // rebalance tenants every time a provisioner registers itself for (Tenant tenant : tenantStore.getAllTenants()) { balanceQueue.add(new Element(tenant.getId())); } } finally { tenantLock.unlock(); } }
private void removeWorkers(String tenantId, int numToRemove, ResourceCollection resources) throws IOException { // go through each provisioner, removing workers for the tenant until we've removed enough. for (Provisioner provisioner : provisionerStore.getTenantProvisioners(tenantId)) { int numRemoved = provisioner.tryRemoveTenantAssignments(tenantId, numToRemove); if (numRemoved > 0) { provisionerStore.writeProvisioner(provisioner); LOG.debug("Requesting provisioner {} to set workers to {} for tenant {} (removing {})", provisioner.getId(), provisioner.getAssignedWorkers(tenantId), tenantId, numRemoved); if (provisionerRequestService.putTenant(provisioner, tenantId, resources)) { numToRemove -= numRemoved; } else { // request failed with retries. something is wrong with the provisioner, delete it and rebalance its workers // TODO: what if this fails? LOG.error("Could not set workers for tenant {} to provisioner {}. " + "The provisioner appears broken, deleting it and rebalancing its tenant workers", tenantId, provisioner.getId()); deleteProvisioner(provisioner); } } } }
@Test public void testGetAllProvisioners() throws IOException { ProvisionerStore store = getProvisionerStore(); Assert.assertTrue(store.getAllProvisioners().isEmpty()); store.writeProvisioner(provisioner1); Assert.assertEquals(ImmutableSet.of(provisioner1), ImmutableSet.copyOf(store.getAllProvisioners())); store.writeProvisioner(provisioner2); Assert.assertEquals(ImmutableSet.of(provisioner1, provisioner2), ImmutableSet.copyOf(store.getAllProvisioners())); }
/** * Handle the heartbeat of a provisioner, updating the last heartbeat time of the provisioner and updating the number * of live workers running on the provisioner for each tenant it is responsible for. * * @param provisionerId Id of the provisioner that sent the heartbeat * @param heartbeat The heartbeat containing live worker information * @throws IOException if there was an exception persisting the data * @throws MissingEntityException if there is no provisioner for the given id */ public void handleHeartbeat(String provisionerId, ProvisionerHeartbeat heartbeat) throws IOException, MissingEntityException { // no lock required here. Simply getting a provisioner and writing worker usage. Would only expect one provisioner // to be calling this at a time, and even if it is calling it concurrently for some reason, only the usage can // change and for that its ok for one of them to win. Provisioner provisioner = provisionerStore.getProvisioner(provisionerId); if (provisioner == null) { throw new MissingEntityException("Provisioner " + provisionerId + " not found."); } if (!provisioner.getUsage().equals(heartbeat.getUsage())) { provisioner.setUsage(heartbeat.getUsage()); provisionerStore.writeProvisioner(provisioner); } provisionerStore.setHeartbeat(provisionerId, System.currentTimeMillis()); }
@Test public void testGetTenantProvisioners() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); Assert.assertTrue(store.getTenantProvisioners("tenantB").isEmpty()); Assert.assertEquals(ImmutableSet.of(provisioner2), ImmutableSet.copyOf(store.getTenantProvisioners("tenantA"))); Assert.assertEquals(ImmutableSet.of(provisioner1), ImmutableSet.copyOf(store.getTenantProvisioners("tenantX"))); Assert.assertEquals(ImmutableSet.of(provisioner2), ImmutableSet.copyOf(store.getTenantProvisioners("tenantZ"))); Assert.assertEquals(ImmutableSet.of(provisioner1, provisioner2), ImmutableSet.copyOf(store.getTenantProvisioners("tenantY"))); }
@Test public void testGetProvisionersWithFreeCapacity() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); Assert.assertTrue(store.getProvisionersWithFreeCapacity().isEmpty()); store.writeProvisioner(provisioner2); Assert.assertEquals(ImmutableSet.of(provisioner2), ImmutableSet.copyOf(store.getProvisionersWithFreeCapacity())); Provisioner updatedProvisioner1 = new Provisioner( provisioner1.getId(), provisioner1.getHost(), provisioner1.getPort(), provisioner1.getCapacityTotal(), ImmutableMap.<String, Integer>of("tenantX", 5), ImmutableMap.<String, Integer>of("tenantX", 5)); store.writeProvisioner(updatedProvisioner1); Assert.assertEquals(ImmutableSet.of(updatedProvisioner1, provisioner2), ImmutableSet.copyOf(store.getProvisionersWithFreeCapacity())); }
provisionerStore.writeProvisioner(provisioner); LOG.debug("Requesting provisioner {} to set workers to {} for tenant {} (adding {})", provisioner.getId(), provisioner.getAssignedWorkers(tenantId), tenantId, numAdded);
@Test public void testGetIdleProvisioners() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); store.setHeartbeat(provisioner1.getId(), 100L); store.setHeartbeat(provisioner2.getId(), 1000L); Assert.assertTrue(store.getTimedOutProvisioners(99L).isEmpty()); Assert.assertEquals(ImmutableSet.of(provisioner1), ImmutableSet.copyOf(store.getTimedOutProvisioners(101L))); Assert.assertEquals(ImmutableSet.of(provisioner1, provisioner2), ImmutableSet.copyOf(store.getTimedOutProvisioners(1001L))); }
Assert.assertEquals(0, store.getFreeCapacity()); store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); Assert.assertEquals(88, store.getFreeCapacity()); ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 3, "tenantZ", 2), ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 50, "tenantZ", 20)); store.writeProvisioner(updatedProvisioner2); Assert.assertEquals(25, store.getFreeCapacity()); ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 3, "tenantZ", 2), ImmutableMap.<String, Integer>of("tenantA", 5, "tenantZ", 20)); store.writeProvisioner(updatedProvisioner2); Assert.assertEquals(75, store.getFreeCapacity()); ImmutableMap.<String, Integer>of("tenantB", 10), ImmutableMap.<String, Integer>of("tenantB", 40)); store.writeProvisioner(provisioner3); Assert.assertEquals(10, store.getFreeCapacity());
@Test public void testWriteWithTenantUpdate() throws IOException { ProvisionerStore store = getProvisionerStore(); store.writeProvisioner(provisioner1); Assert.assertEquals(provisioner1, store.getProvisioner(provisioner1.getId())); Provisioner updatedProvisioner1 = new Provisioner( provisioner1.getId(), provisioner1.getHost(), provisioner1.getPort(), 100, ImmutableMap.<String, Integer>of("tenantA", 50, "tenantY", 5, "tenantZ", 0), ImmutableMap.<String, Integer>of("tenantA", 50, "tenantY", 5, "tenantZ", 1) ); store.writeProvisioner(updatedProvisioner1); Assert.assertEquals(updatedProvisioner1, store.getProvisioner(provisioner1.getId())); Assert.assertEquals(0, store.getNumAssignedWorkers("tenantX")); Assert.assertEquals(44, store.getFreeCapacity()); for (Provisioner provisioner : store.getTenantProvisioners("tenantX")) { Assert.assertFalse(provisioner.getId().equals(provisioner1.getId())); } }
@Test public void testWriteGetDeleteProvisioner() throws IOException { ProvisionerStore store = getProvisionerStore(); String id = provisioner1.getId(); Assert.assertNull(store.getProvisioner(id)); store.writeProvisioner(provisioner1); Assert.assertEquals(provisioner1, store.getProvisioner(id)); store.deleteProvisioner(id); Assert.assertNull(store.getProvisioner(id)); }
store.writeProvisioner(provisioner1); store.writeProvisioner(provisioner2); ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 3, "tenantZ", 2), ImmutableMap.<String, Integer>of("tenantA", 5, "tenantY", 50, "tenantZ", 20)); store.writeProvisioner(provisioner3); store.writeProvisioner(updatedProvisioner2);