private static Optional<TaskCleanupType> getTaskCleanupTypefromSingularityTaskHistoryUpdate(SingularityTaskHistoryUpdate taskHistoryUpdate) { if (!taskHistoryUpdate.getStatusMessage().isPresent()) { return Optional.absent(); } String taskCleanupTypeMsg = taskHistoryUpdate.getStatusMessage().get(); Matcher matcher = TASK_STATUS_BY_PATTERN.matcher(taskCleanupTypeMsg); if (matcher.find()) { taskCleanupTypeMsg = matcher.group(1); } try { return Optional.of(TaskCleanupType.valueOf(taskCleanupTypeMsg.toUpperCase())); } catch (IllegalArgumentException iae) { LOG.warn("Couldn't parse TaskCleanupType from update {}", taskHistoryUpdate); return Optional.absent(); } }
private boolean isTaskAlreadyCleanedUpForShuffle(SingularityTaskHistoryUpdate taskHistoryUpdate) { if (taskHistoryUpdate.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { return true; } for (SingularityTaskHistoryUpdate previous : taskHistoryUpdate.getPrevious()) { if (previous.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { return true; } } return false; }
private Optional<SingularityDeployFailure> getNonHealthcheckedTaskFailure(Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates, SingularityTaskId taskId) { List<SingularityTaskHistoryUpdate> updates = taskUpdates.get(taskId); SingularityTaskHistoryUpdate lastUpdate = Iterables.getLast(updates); if (lastUpdate.getTaskState().isSuccess()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_EXPECTED_RUNNING_FINISHED, Optional.of(taskId), Optional.of(String.format("Task was expected to maintain TASK_RUNNING state but finished. (%s)", lastUpdate.getStatusMessage().or(""))))); } else if (lastUpdate.getTaskState().isDone()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_FAILED_ON_STARTUP, Optional.of(taskId), lastUpdate.getStatusMessage())); } else if (SingularityTaskHistoryUpdate.getCurrentState(updates) == SimplifiedTaskState.WAITING) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_NEVER_ENTERED_RUNNING, Optional.of(taskId), Optional.of(String.format("Task never entered running state, last state was %s (%s)", lastUpdate.getTaskState().getDisplayName(), lastUpdate.getStatusMessage().or(""))))); } return Optional.absent(); } }
public List<SingularityMailTaskHistoryUpdate> getJadeTaskHistory(Collection<SingularityTaskHistoryUpdate> taskHistory) { List<SingularityMailTaskHistoryUpdate> output = Lists.newArrayListWithCapacity(taskHistory.size()); for (SingularityTaskHistoryUpdate taskUpdate : taskHistory) { output.add( new SingularityMailTaskHistoryUpdate( humanizeTimestamp(taskUpdate.getTimestamp()), WordUtils.capitalize(taskUpdate.getTaskState().getDisplayName()), taskUpdate.getStatusMessage().or(""))); } return output; }
if (lastUpdate.getTaskState().isSuccess()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_EXPECTED_RUNNING_FINISHED, Optional.of(taskId), Optional.of(String.format("Task was expected to maintain TASK_RUNNING state but finished. (%s)", lastUpdate.getStatusMessage().or(""))))); } else { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_FAILED_ON_STARTUP, Optional.of(taskId), lastUpdate.getStatusMessage()));
} else if (!maybeCleaningUpdate.get().getPrevious().isEmpty()) { for (SingularityTaskHistoryUpdate previousUpdate : maybeCleaningUpdate.get().getPrevious()) { if (previousUpdate.getStatusMessage().or("").contains("BOUNCE")) { LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId); foundBouncingTask = true;
@Test public void testBounceOnPendingInstancesReleasesLock() { initRequest(); initFirstDeploy(); SingularityTask task = startTask(firstDeploy, 1); statusUpdate(task, TaskState.TASK_FAILED); killKilledTasks(); Assert.assertEquals("Bounce starts when tasks have not yet been launched", 0, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); // It acquires a lock on the bounce Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } cleaner.drainCleanupQueue(); killKilledTasks(); // It finishes with one task running and the bounce released Assert.assertEquals("Should end bounce with target number of tasks", 1, taskManager.getActiveTaskIds().size()); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); } Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent()); }
String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE"));
private static Optional<TaskCleanupType> getTaskCleanupTypefromSingularityTaskHistoryUpdate(SingularityTaskHistoryUpdate taskHistoryUpdate) { if (!taskHistoryUpdate.getStatusMessage().isPresent()) { return Optional.absent(); } String taskCleanupTypeMsg = taskHistoryUpdate.getStatusMessage().get(); Matcher matcher = TASK_STATUS_BY_PATTERN.matcher(taskCleanupTypeMsg); if (matcher.find()) { taskCleanupTypeMsg = matcher.group(1); } try { return Optional.of(TaskCleanupType.valueOf(taskCleanupTypeMsg.toUpperCase())); } catch (IllegalArgumentException iae) { LOG.warn("Couldn't parse TaskCleanupType from update {}", taskHistoryUpdate); return Optional.absent(); } }
private boolean isTaskAlreadyCleanedUpForShuffle(SingularityTaskHistoryUpdate taskHistoryUpdate) { if (taskHistoryUpdate.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { return true; } for (SingularityTaskHistoryUpdate previous : taskHistoryUpdate.getPrevious()) { if (previous.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { return true; } } return false; }
private Optional<SingularityDeployFailure> getNonHealthcheckedTaskFailure(Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates, SingularityTaskId taskId) { List<SingularityTaskHistoryUpdate> updates = taskUpdates.get(taskId); SingularityTaskHistoryUpdate lastUpdate = Iterables.getLast(updates); if (lastUpdate.getTaskState().isSuccess()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_EXPECTED_RUNNING_FINISHED, Optional.of(taskId), Optional.of(String.format("Task was expected to maintain TASK_RUNNING state but finished. (%s)", lastUpdate.getStatusMessage().or(""))))); } else if (lastUpdate.getTaskState().isDone()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_FAILED_ON_STARTUP, Optional.of(taskId), lastUpdate.getStatusMessage())); } else if (SingularityTaskHistoryUpdate.getCurrentState(updates) == SimplifiedTaskState.WAITING) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_NEVER_ENTERED_RUNNING, Optional.of(taskId), Optional.of(String.format("Task never entered running state, last state was %s (%s)", lastUpdate.getTaskState().getDisplayName(), lastUpdate.getStatusMessage().or(""))))); } return Optional.absent(); } }
public List<SingularityMailTaskHistoryUpdate> getJadeTaskHistory(Collection<SingularityTaskHistoryUpdate> taskHistory) { List<SingularityMailTaskHistoryUpdate> output = Lists.newArrayListWithCapacity(taskHistory.size()); for (SingularityTaskHistoryUpdate taskUpdate : taskHistory) { output.add( new SingularityMailTaskHistoryUpdate( humanizeTimestamp(taskUpdate.getTimestamp()), WordUtils.capitalize(taskUpdate.getTaskState().getDisplayName()), taskUpdate.getStatusMessage().or(""))); } return output; }
if (lastUpdate.getTaskState().isSuccess()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_EXPECTED_RUNNING_FINISHED, Optional.of(taskId), Optional.of(String.format("Task was expected to maintain TASK_RUNNING state but finished. (%s)", lastUpdate.getStatusMessage().or(""))))); } else { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_FAILED_ON_STARTUP, Optional.of(taskId), lastUpdate.getStatusMessage()));
} else if (!maybeCleaningUpdate.get().getPrevious().isEmpty()) { for (SingularityTaskHistoryUpdate previousUpdate : maybeCleaningUpdate.get().getPrevious()) { if (previousUpdate.getStatusMessage().or("").contains("BOUNCE")) { LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId); foundBouncingTask = true;
@Test public void testBounceOnPendingInstancesReleasesLock() { initRequest(); initFirstDeploy(); SingularityTask task = startTask(firstDeploy, 1); statusUpdate(task, TaskState.TASK_FAILED); killKilledTasks(); Assert.assertEquals("Bounce starts when tasks have not yet been launched", 0, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); // It acquires a lock on the bounce Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } cleaner.drainCleanupQueue(); killKilledTasks(); // It finishes with one task running and the bounce released Assert.assertEquals("Should end bounce with target number of tasks", 1, taskManager.getActiveTaskIds().size()); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); } Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent()); }
String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE"));