public SingularityBounceRequest build() { return new SingularityBounceRequest(incremental, skipHealthchecks, durationMillis, actionId, message, runShellCommandBeforeKill); }
public static SingularityBounceRequest defaultRequest() { return new SingularityBounceRequest(Optional.<Boolean>absent(), Optional.<Boolean>absent(), Optional.<Long>absent(), Optional.of(UUID.randomUUID().toString()), Optional.<String>absent(), Optional.<SingularityShellCommand>absent()); }
validator.checkRequestForPriorityFreeze(newRequest); SingularityBounceRequest bounceRequest = new SingularityBounceRequest(Optional.of(isIncrementalBounce), scaleRequest.getSkipHealthchecks(), Optional.<Long>absent(), Optional.of(UUID.randomUUID().toString()), Optional.<String>absent(), Optional.<SingularityShellCommand>absent());
@Test public void testExpiringBounceGoesAway() { initRequest(); initFirstDeploy(); startTask(firstDeploy, 1); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.of(false), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); resourceOffers(); runLaunchedTasks(); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(!requestManager.getExpiringBounce(requestId).isPresent()); }
@Test public void testBounceReleasesLockWithAlternateCleanupType() { initRequest(); initFirstDeploy(); startTask(firstDeploy, 1); List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds(); Assert.assertEquals(1, activeTaskIds.size()); SingularityTaskId firstTaskId = activeTaskIds.get(0); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); Assert.assertTrue(requestManager.isBouncing(requestId)); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); // Save a new cleanup type over the old one, and make sure the bounce lock still releases taskManager.saveTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.USER_REQUESTED, System.currentTimeMillis(), firstTaskId, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } Assert.assertTrue(requestManager.isBouncing(requestId)); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertFalse(requestManager.isBouncing(requestId)); }
@Test public void testRunCommandBeforeBounceKill() { setShellCommandsConfiguration(); initRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(new SingularityShellCommand("d1", Optional.of(Arrays.asList("o1", "o2")), user, Optional.absent())))), singularityUser); cleaner.drainCleanupQueue(); List<SingularityTaskCleanup> taskCleanups = taskManager.getCleanupTasks(); Assert.assertTrue(taskCleanups.get(0).getRunBeforeKillId().isPresent()); SingularityTaskShellCommandRequestId shellCommandRequestId = taskCleanups.get(0).getRunBeforeKillId().get(); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getCleanupTaskIds().size()); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size()); cleaner.drainCleanupQueue(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size()); taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), UpdateType.ACKED)); cleaner.drainCleanupQueue(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size()); taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), UpdateType.FINISHED)); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getKilledTaskIdRecords().size()); }
@Test public void testIncrementalBounceShutsDownOldTasksPerNewHealthyTask() { initRequest(); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()), singularityUser); initFirstDeploy(); startTask(firstDeploy, 1); startTask(firstDeploy, 2); startTask(firstDeploy, 3); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.of(true), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser); Assert.assertTrue(requestManager.cleanupRequestExists(requestId)); cleaner.drainCleanupQueue(); Assert.assertTrue(!requestManager.cleanupRequestExists(requestId)); Assert.assertEquals(3, taskManager.getCleanupTaskIds().size()); SingularityTask newTask = launchTask(request, firstDeploy, 5, TaskState.TASK_STARTING); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getKilledTaskIdRecords().size()); Assert.assertEquals(4, taskManager.getActiveTaskIds().size()); statusUpdate(newTask, TaskState.TASK_RUNNING); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getKilledTaskIdRecords().size()); Assert.assertEquals(4, taskManager.getActiveTaskIds().size()); }
@Test public void testBounceOnPendingInstancesReleasesLock() { initRequest(); initFirstDeploy(); SingularityTask task = startTask(firstDeploy, 1); statusUpdate(task, TaskState.TASK_FAILED); killKilledTasks(); Assert.assertEquals("Bounce starts when tasks have not yet been launched", 0, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); // It acquires a lock on the bounce Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } cleaner.drainCleanupQueue(); killKilledTasks(); // It finishes with one task running and the bounce released Assert.assertEquals("Should end bounce with target number of tasks", 1, taskManager.getActiveTaskIds().size()); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); } Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent()); }
Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.absent(), Optional.of(1L), Optional.of("aid"), Optional.absent(), Optional.absent())), singularityUser);
Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); Assert.assertTrue(requestManager.isBouncing(requestId)); cleaner.drainCleanupQueue();
Optional.of(new SingularityBounceRequest(Optional.of(true), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser);
@Test public void testSkipHealthchecksDuringBounce() { try { initRequest(); initHCDeploy(); SingularityTask firstTask = startTask(firstDeploy, 1); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); setConfigurationForNoDelay(); cleaner.drainCleanupQueue(); resourceOffers(); List<SingularityTaskId> taskIds = taskManager.getAllTaskIds(); taskIds.remove(firstTask.getTaskId()); SingularityTaskId secondTaskId = taskIds.get(0); SingularityTask secondTask = taskManager.getTask(secondTaskId).get(); statusUpdate(secondTask, TaskState.TASK_RUNNING); Assert.assertTrue(healthchecker.cancelHealthcheck(firstTask.getTaskId().getId())); newTaskChecker.cancelNewTaskCheck(firstTask.getTaskId().getId()); finishHealthchecks(); finishNewTaskChecks(); Assert.assertTrue(!taskManager.getLastHealthcheck(secondTask.getTaskId()).isPresent()); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(0, taskManager.getNumCleanupTasks()); Assert.assertEquals(1, taskManager.getNumActiveTasks()); } finally { unsetConfigurationForNoDelay(); } }
@Test public void testExpiringBounceGoesAway() { initRequest(); initFirstDeploy(); startTask(firstDeploy, 1); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.of(false), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); resourceOffers(); runLaunchedTasks(); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(!requestManager.getExpiringBounce(requestId).isPresent()); }
@Test public void testBounceReleasesLockWithAlternateCleanupType() { initRequest(); initFirstDeploy(); startTask(firstDeploy, 1); List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds(); Assert.assertEquals(1, activeTaskIds.size()); SingularityTaskId firstTaskId = activeTaskIds.get(0); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); Assert.assertTrue(requestManager.isBouncing(requestId)); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); // Save a new cleanup type over the old one, and make sure the bounce lock still releases taskManager.saveTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.USER_REQUESTED, System.currentTimeMillis(), firstTaskId, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } Assert.assertTrue(requestManager.isBouncing(requestId)); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertFalse(requestManager.isBouncing(requestId)); }
@Test public void testRunCommandBeforeBounceKill() { setShellCommandsConfiguration(); initRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(new SingularityShellCommand("d1", Optional.of(Arrays.asList("o1", "o2")), user, Optional.absent())))), singularityUser); cleaner.drainCleanupQueue(); List<SingularityTaskCleanup> taskCleanups = taskManager.getCleanupTasks(); Assert.assertTrue(taskCleanups.get(0).getRunBeforeKillId().isPresent()); SingularityTaskShellCommandRequestId shellCommandRequestId = taskCleanups.get(0).getRunBeforeKillId().get(); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getCleanupTaskIds().size()); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size()); cleaner.drainCleanupQueue(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size()); taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), UpdateType.ACKED)); cleaner.drainCleanupQueue(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForRequest(requestId).size()); taskManager.saveTaskShellCommandUpdate(new SingularityTaskShellCommandUpdate(shellCommandRequestId, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), UpdateType.FINISHED)); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getKilledTaskIdRecords().size()); }
@Test public void testIncrementalBounceShutsDownOldTasksPerNewHealthyTask() { initRequest(); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()), singularityUser); initFirstDeploy(); startTask(firstDeploy, 1); startTask(firstDeploy, 2); startTask(firstDeploy, 3); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.of(true), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser); Assert.assertTrue(requestManager.cleanupRequestExists(requestId)); cleaner.drainCleanupQueue(); Assert.assertTrue(!requestManager.cleanupRequestExists(requestId)); Assert.assertEquals(3, taskManager.getCleanupTaskIds().size()); SingularityTask newTask = launchTask(request, firstDeploy, 5, TaskState.TASK_STARTING); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getKilledTaskIdRecords().size()); Assert.assertEquals(4, taskManager.getActiveTaskIds().size()); statusUpdate(newTask, TaskState.TASK_RUNNING); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getKilledTaskIdRecords().size()); Assert.assertEquals(4, taskManager.getActiveTaskIds().size()); }
@Test public void testBounceOnPendingInstancesReleasesLock() { initRequest(); initFirstDeploy(); SingularityTask task = startTask(firstDeploy, 1); statusUpdate(task, TaskState.TASK_FAILED); killKilledTasks(); Assert.assertEquals("Bounce starts when tasks have not yet been launched", 0, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); // It acquires a lock on the bounce Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } cleaner.drainCleanupQueue(); killKilledTasks(); // It finishes with one task running and the bounce released Assert.assertEquals("Should end bounce with target number of tasks", 1, taskManager.getActiveTaskIds().size()); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); } Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent()); }
Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.absent(), Optional.of(1L), Optional.of("aid"), Optional.absent(), Optional.absent())), singularityUser);
Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); Assert.assertTrue(requestManager.isBouncing(requestId)); cleaner.drainCleanupQueue();
@Test public void testSkipHealthchecksDuringBounce() { try { initRequest(); initHCDeploy(); SingularityTask firstTask = startTask(firstDeploy, 1); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); setConfigurationForNoDelay(); cleaner.drainCleanupQueue(); resourceOffers(); List<SingularityTaskId> taskIds = taskManager.getAllTaskIds(); taskIds.remove(firstTask.getTaskId()); SingularityTaskId secondTaskId = taskIds.get(0); SingularityTask secondTask = taskManager.getTask(secondTaskId).get(); statusUpdate(secondTask, TaskState.TASK_RUNNING); Assert.assertTrue(healthchecker.cancelHealthcheck(firstTask.getTaskId().getId())); newTaskChecker.cancelNewTaskCheck(firstTask.getTaskId().getId()); finishHealthchecks(); finishNewTaskChecks(); Assert.assertTrue(!taskManager.getLastHealthcheck(secondTask.getTaskId()).isPresent()); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(0, taskManager.getNumCleanupTasks()); Assert.assertEquals(1, taskManager.getNumActiveTasks()); } finally { unsetConfigurationForNoDelay(); } }