private Optional<Long> getRunningAt(Collection<SingularityTaskHistoryUpdate> updates) { for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { return Optional.of(update.getTimestamp()); } } return Optional.absent(); }
@Test public void testTaskOrdering() { final SingularityTaskId taskId = new SingularityTaskId("r", "d", System.currentTimeMillis(), 1, "h", "r"); final Optional<String> msg = Optional.absent(); SingularityTaskHistoryUpdate update1 = new SingularityTaskHistoryUpdate(taskId, 1L, ExtendedTaskState.TASK_LAUNCHED, msg, Optional.<String>absent()); SingularityTaskHistoryUpdate update2 = new SingularityTaskHistoryUpdate(taskId, 2L, ExtendedTaskState.TASK_RUNNING, msg, Optional.<String>absent()); SingularityTaskHistoryUpdate update3 = new SingularityTaskHistoryUpdate(taskId, 2L, ExtendedTaskState.TASK_FAILED, msg, Optional.<String>absent()); List<SingularityTaskHistoryUpdate> list = Arrays.asList(update2, update1, update3); Collections.sort(list); Assert.assertTrue(list.get(0).getTaskState() == ExtendedTaskState.TASK_LAUNCHED); Assert.assertTrue(list.get(1).getTaskState() == ExtendedTaskState.TASK_RUNNING); Assert.assertTrue(list.get(2).getTaskState() == ExtendedTaskState.TASK_FAILED); }
private Optional<SingularityDeployFailure> getNonHealthcheckedTaskFailure(Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates, SingularityTaskId taskId) { List<SingularityTaskHistoryUpdate> updates = taskUpdates.get(taskId); SingularityTaskHistoryUpdate lastUpdate = Iterables.getLast(updates); if (lastUpdate.getTaskState().isSuccess()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_EXPECTED_RUNNING_FINISHED, Optional.of(taskId), Optional.of(String.format("Task was expected to maintain TASK_RUNNING state but finished. (%s)", lastUpdate.getStatusMessage().or(""))))); } else if (lastUpdate.getTaskState().isDone()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_FAILED_ON_STARTUP, Optional.of(taskId), lastUpdate.getStatusMessage())); } else if (SingularityTaskHistoryUpdate.getCurrentState(updates) == SimplifiedTaskState.WAITING) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_NEVER_ENTERED_RUNNING, Optional.of(taskId), Optional.of(String.format("Task never entered running state, last state was %s (%s)", lastUpdate.getTaskState().getDisplayName(), lastUpdate.getStatusMessage().or(""))))); } return Optional.absent(); } }
public SingularityTaskHistoryUpdate withPrevious(SingularityTaskHistoryUpdate previousUpdate) { Set<SingularityTaskHistoryUpdate> newPreviousUpdates = getFlattenedPreviousUpdates(this); newPreviousUpdates.add(previousUpdate.withoutPrevious()); newPreviousUpdates.addAll(getFlattenedPreviousUpdates(previousUpdate)); return new SingularityTaskHistoryUpdate(getTaskId(), timestamp, taskState, statusMessage, statusReason, newPreviousUpdates); }
public SingularityTaskHistoryUpdate withoutPrevious() { return new SingularityTaskHistoryUpdate(getTaskId(), timestamp, taskState, statusMessage, statusReason, Collections.<SingularityTaskHistoryUpdate>emptySet()); }
@Override public boolean apply(@Nonnull SingularityTaskHistoryUpdate input) { return input.getTaskState() == taskState; } });
public List<SingularityMailTaskHistoryUpdate> getJadeTaskHistory(Collection<SingularityTaskHistoryUpdate> taskHistory) { List<SingularityMailTaskHistoryUpdate> output = Lists.newArrayListWithCapacity(taskHistory.size()); for (SingularityTaskHistoryUpdate taskUpdate : taskHistory) { output.add( new SingularityMailTaskHistoryUpdate( humanizeTimestamp(taskUpdate.getTimestamp()), WordUtils.capitalize(taskUpdate.getTaskState().getDisplayName()), taskUpdate.getStatusMessage().or(""))); } return output; }
public void saveTaskHistoryUpdate(SingularityTaskHistoryUpdate taskHistoryUpdate, boolean overwrite) { if (!active) { LOG.warn("saveTaskHistoryUpdate {}, but not active", taskHistoryUpdate); return; } historyUpdates.putIfAbsent(taskHistoryUpdate.getTaskId(), new ConcurrentHashMap<>()); if (overwrite) { historyUpdates.get(taskHistoryUpdate.getTaskId()).put(taskHistoryUpdate.getTaskState(), taskHistoryUpdate); } else { historyUpdates.get(taskHistoryUpdate.getTaskId()).putIfAbsent(taskHistoryUpdate.getTaskState(), taskHistoryUpdate); } }
@Override public int compareTo(SingularityTaskHistoryUpdate o) { return ComparisonChain.start() .compare(taskState.ordinal(), o.getTaskState().ordinal()) .compare(timestamp, o.getTimestamp()) .compare(o.getTaskId().getId(), getTaskId().getId()) .result(); }
@Test public void testBounceOnPendingInstancesReleasesLock() { initRequest(); initFirstDeploy(); SingularityTask task = startTask(firstDeploy, 1); statusUpdate(task, TaskState.TASK_FAILED); killKilledTasks(); Assert.assertEquals("Bounce starts when tasks have not yet been launched", 0, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); // It acquires a lock on the bounce Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } cleaner.drainCleanupQueue(); killKilledTasks(); // It finishes with one task running and the bounce released Assert.assertEquals("Should end bounce with target number of tasks", 1, taskManager.getActiveTaskIds().size()); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); } Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent()); }
@Timed public SingularityCreateResult saveTaskHistoryUpdate(SingularityTaskHistoryUpdate taskHistoryUpdate, boolean overwriteExisting) { singularityEventListener.taskHistoryUpdateEvent(taskHistoryUpdate); if (overwriteExisting) { Optional<SingularityTaskHistoryUpdate> maybeExisting = getTaskHistoryUpdate(taskHistoryUpdate.getTaskId(), taskHistoryUpdate.getTaskState()); LOG.info("Found existing history {}", maybeExisting); SingularityTaskHistoryUpdate updateWithPrevious; if (maybeExisting.isPresent()) { updateWithPrevious = taskHistoryUpdate.withPrevious(maybeExisting.get()); LOG.info("Will save new update {}", updateWithPrevious); } else { updateWithPrevious = taskHistoryUpdate; } if (leaderCache.active()) { leaderCache.saveTaskHistoryUpdate(updateWithPrevious, overwriteExisting); } return save(getUpdatePath(taskHistoryUpdate.getTaskId(), taskHistoryUpdate.getTaskState()), updateWithPrevious, taskHistoryUpdateTranscoder); } else { if (leaderCache.active()) { leaderCache.saveTaskHistoryUpdate(taskHistoryUpdate, overwriteExisting); } return create(getUpdatePath(taskHistoryUpdate.getTaskId(), taskHistoryUpdate.getTaskState()), taskHistoryUpdate, taskHistoryUpdateTranscoder); } }
private void saveTaskHistoryUpdate(SingularityTaskCleanup cleanup) { StringBuilder msg = new StringBuilder(cleanup.getCleanupType().name()); if (cleanup.getUser().isPresent()) { msg.append(" by "); msg.append(cleanup.getUser().get()); } if (cleanup.getMessage().isPresent()) { msg.append(" - "); msg.append(cleanup.getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(cleanup.getTaskId(), cleanup.getTimestamp(), ExtendedTaskState.TASK_CLEANING, Optional.of(msg.toString()), Optional.<String>absent()), true); }
private boolean isEligibleForShuffle(SingularityTaskId task) { Optional<SingularityTaskHistoryUpdate> taskRunning = taskManager.getTaskHistoryUpdate(task, ExtendedTaskState.TASK_RUNNING); return ( !configuration.getDoNotShuffleRequests().contains(task.getRequestId()) && isLongRunning(task) && ( configuration.getMinutesBeforeNewTaskEligibleForShuffle() == 0 // Shuffle delay is disabled entirely || (taskRunning.isPresent() && TimeUnit.MILLISECONDS.toMinutes(System.currentTimeMillis() - taskRunning.get() .getTimestamp()) >= configuration.getMinutesBeforeNewTaskEligibleForShuffle()) ) ); }
if (lastUpdate.getTaskState().isDone()) { if (lastUpdate.getTaskState().isSuccess()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_EXPECTED_RUNNING_FINISHED, Optional.of(taskId), Optional.of(String.format("Task was expected to maintain TASK_RUNNING state but finished. (%s)", lastUpdate.getStatusMessage().or(""))))); } else { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_FAILED_ON_STARTUP, Optional.of(taskId), lastUpdate.getStatusMessage()));
private int checkTaskUpdates(SingularityWebhook webhook, List<CompletableFuture<Response>> webhookFutures) { final List<SingularityTaskHistoryUpdate> taskUpdates = webhookManager.getQueuedTaskUpdatesForHook(webhook.getId()); int numTaskUpdates = 0; for (SingularityTaskHistoryUpdate taskUpdate : taskUpdates) { Optional<SingularityTask> task = taskHistoryHelper.getTask(taskUpdate.getTaskId()); // TODO compress if (!task.isPresent()) { LOG.warn("Couldn't find task for taskUpdate {}", taskUpdate); webhookManager.deleteTaskUpdate(webhook, taskUpdate); continue; } String concreteUri = applyPlaceholders(webhook.getUri(), taskUpdate); webhookFutures.add(webhookSemaphore.call(() -> executeWebhookAsync( concreteUri, new SingularityTaskWebhook(task.get(), taskUpdate), new SingularityTaskWebhookAsyncHandler(webhookManager, webhook, taskUpdate, shouldDeleteUpdateOnFailure(numTaskUpdates, taskUpdate.getTimestamp()))) )); } return taskUpdates.size(); }
SimplifiedTaskState taskState = SingularityTaskHistoryUpdate.getCurrentState(history.get().getTaskUpdates()); end = Iterables.getLast(history.get().getTaskUpdates()).getTimestamp(); } else { end = System.currentTimeMillis();
Optional<SingularityTaskHistoryUpdate> maybeCleaningUpdate = taskManager.getTaskHistoryUpdate(activeTaskId, ExtendedTaskState.TASK_CLEANING); if (maybeCleaningUpdate.isPresent()) { if (maybeCleaningUpdate.get().getStatusReason().or("").contains("BOUNCE")) { // TaskCleanupType enum is included in status message LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId); foundBouncingTask = true; break; } else if (!maybeCleaningUpdate.get().getPrevious().isEmpty()) { for (SingularityTaskHistoryUpdate previousUpdate : maybeCleaningUpdate.get().getPrevious()) { if (previousUpdate.getStatusMessage().or("").contains("BOUNCE")) { LOG.debug("Found task {} still waiting for bounce to complete", activeTaskId); foundBouncingTask = true;
public boolean didTaskRun(Collection<SingularityTaskHistoryUpdate> history) { SingularityTaskHistoryUpdate.SimplifiedTaskState simplifiedTaskState = SingularityTaskHistoryUpdate.getCurrentState(history); return simplifiedTaskState == SingularityTaskHistoryUpdate.SimplifiedTaskState.DONE || simplifiedTaskState == SingularityTaskHistoryUpdate.SimplifiedTaskState.RUNNING; } }
private boolean isTaskAlreadyCleanedUpForShuffle(SingularityTaskHistoryUpdate taskHistoryUpdate) { if (taskHistoryUpdate.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { return true; } for (SingularityTaskHistoryUpdate previous : taskHistoryUpdate.getPrevious()) { if (previous.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { return true; } } return false; }
private boolean isRunningTaskHealthy(final Optional<SingularityDeploy> deploy, Collection<SingularityTaskHistoryUpdate> updates, SingularityTaskId taskId) { long runningThreshold = configuration.getConsiderTaskHealthyAfterRunningForSeconds(); if (deploy.isPresent()) { runningThreshold = deploy.get().getConsiderHealthyAfterRunningForSeconds().or(runningThreshold); } if (runningThreshold < 1) { return true; } Optional<SingularityTaskHistoryUpdate> runningUpdate = SingularityTaskHistoryUpdate.getUpdate(updates, ExtendedTaskState.TASK_RUNNING); long taskDuration = System.currentTimeMillis() - runningUpdate.get().getTimestamp(); long runningThresholdMillis = TimeUnit.SECONDS.toMillis(runningThreshold); if (taskDuration < runningThresholdMillis) { LOG.debug("Task {} has been running for {}, has not yet reached running threshold of {}", taskId, JavaUtils.durationFromMillis(taskDuration), JavaUtils.durationFromMillis(runningThresholdMillis)); return false; } return true; }