private Optional<Long> getRunningAt(Collection<SingularityTaskHistoryUpdate> updates) { for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { return Optional.of(update.getTimestamp()); } } return Optional.absent(); }
private Optional<Long> getRunningAt(Collection<SingularityTaskHistoryUpdate> updates) { for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { return Optional.of(update.getTimestamp()); } } return Optional.absent(); }
private long getTaskRunningStartTime(SingularityTaskId task) { Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(task); if (taskHistory.isPresent()) { java.util.Optional<SingularityTaskHistoryUpdate> taskRunningState = taskHistory.get().getTaskUpdates().stream().filter(h -> h.getTaskState().equals(ExtendedTaskState.TASK_RUNNING)).findFirst(); if (taskRunningState.isPresent()) { return taskRunningState.get().getTimestamp(); } LOG.error("Could not find time when task {} reached TASK_RUNNING state", task); } else { LOG.error("Could not find task history for {}", task); } return System.currentTimeMillis(); }
private boolean isEligibleForShuffle(SingularityTaskId task) { Optional<SingularityTaskHistoryUpdate> taskRunning = taskManager.getTaskHistoryUpdate(task, ExtendedTaskState.TASK_RUNNING); return ( !configuration.getDoNotShuffleRequests().contains(task.getRequestId()) && isLongRunning(task) && ( configuration.getMinutesBeforeNewTaskEligibleForShuffle() == 0 // Shuffle delay is disabled entirely || (taskRunning.isPresent() && TimeUnit.MILLISECONDS.toMinutes(System.currentTimeMillis() - taskRunning.get() .getTimestamp()) >= configuration.getMinutesBeforeNewTaskEligibleForShuffle()) ) ); }
private boolean isRunningTaskHealthy(final Optional<SingularityDeploy> deploy, Collection<SingularityTaskHistoryUpdate> updates, SingularityTaskId taskId) { long runningThreshold = configuration.getConsiderTaskHealthyAfterRunningForSeconds(); if (deploy.isPresent()) { runningThreshold = deploy.get().getConsiderHealthyAfterRunningForSeconds().or(runningThreshold); } if (runningThreshold < 1) { return true; } Optional<SingularityTaskHistoryUpdate> runningUpdate = SingularityTaskHistoryUpdate.getUpdate(updates, ExtendedTaskState.TASK_RUNNING); long taskDuration = System.currentTimeMillis() - runningUpdate.get().getTimestamp(); long runningThresholdMillis = TimeUnit.SECONDS.toMillis(runningThreshold); if (taskDuration < runningThresholdMillis) { LOG.debug("Task {} has been running for {}, has not yet reached running threshold of {}", taskId, JavaUtils.durationFromMillis(taskDuration), JavaUtils.durationFromMillis(runningThresholdMillis)); return false; } return true; }
public List<SingularityMailTaskHistoryUpdate> getJadeTaskHistory(Collection<SingularityTaskHistoryUpdate> taskHistory) { List<SingularityMailTaskHistoryUpdate> output = Lists.newArrayListWithCapacity(taskHistory.size()); for (SingularityTaskHistoryUpdate taskUpdate : taskHistory) { output.add( new SingularityMailTaskHistoryUpdate( humanizeTimestamp(taskUpdate.getTimestamp()), WordUtils.capitalize(taskUpdate.getTaskState().getDisplayName()), taskUpdate.getStatusMessage().or(""))); } return output; }
@Override public int compareTo(SingularityTaskHistoryUpdate o) { return ComparisonChain.start() .compare(taskState.ordinal(), o.getTaskState().ordinal()) .compare(timestamp, o.getTimestamp()) .compare(o.getTaskId().getId(), getTaskId().getId()) .result(); }
public static SingularityTaskIdHistory fromTaskIdAndTaskAndUpdates(SingularityTaskId taskId, SingularityTask task, List<SingularityTaskHistoryUpdate> updates) { ExtendedTaskState lastTaskState = null; long updatedAt = taskId.getStartedAt(); if (updates != null && !updates.isEmpty()) { SingularityTaskHistoryUpdate lastUpdate = Collections.max(updates); lastTaskState = lastUpdate.getTaskState(); updatedAt = lastUpdate.getTimestamp(); } return new SingularityTaskIdHistory(taskId, updatedAt, Optional.fromNullable(lastTaskState), task.getTaskRequest().getPendingTask().getRunId()); }
public Optional<SingularityTaskIdHistory> getMostRecentTask(SingularityRequest request) { List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(request.getId()); if (!activeTaskIds.isEmpty()) { SingularityTaskId lastTaskId = activeTaskIds.get(0); List<SingularityTaskHistoryUpdate> historyUpdates = taskManager.getTaskHistoryUpdates(lastTaskId); if (!historyUpdates.isEmpty()) { SingularityTaskHistoryUpdate lastUpdate = historyUpdates.get(historyUpdates.size() - 1); return Optional.of(new SingularityTaskIdHistory( lastTaskId, lastUpdate.getTimestamp(), Optional.of(lastUpdate.getTaskState()), Optional.absent() // runId not currently provided here, grabbing the full task data for this is a more expensive call )); } } List<SingularityTaskIdHistory> maybeRecentTasks = taskHistoryHelper.getBlendedHistory(new SingularityTaskHistoryQuery(request.getId()), 0 , 1); if (!maybeRecentTasks.isEmpty()) { return Optional.of(maybeRecentTasks.get(0)); } return Optional.absent(); }
@Override protected boolean moveToHistory(SingularityTaskId object) { final Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(object); if (taskHistory.isPresent()) { if (!taskHistory.get().getTaskUpdates().isEmpty()) { final long lastUpdateAt = taskHistory.get().getLastTaskUpdate().get().getTimestamp(); final long timeSinceLastUpdate = System.currentTimeMillis() - lastUpdateAt; if (timeSinceLastUpdate < taskMetadataConfiguration.getTaskPersistAfterFinishBufferMillis()) { LOG.debug("Not persisting {} yet - lastUpdate only happened {} ago, buffer {}", JavaUtils.durationFromMillis(timeSinceLastUpdate), JavaUtils.durationFromMillis(taskMetadataConfiguration.getTaskPersistAfterFinishBufferMillis())); return false; } } LOG.debug("Moving {} to history", object); try { historyManager.saveTaskHistory(taskHistory.get()); } catch (Throwable t) { LOG.warn("Failed to persist task into History for task {}", object, t); return false; } } else { LOG.warn("Inactive task {} did not have a task to persist", object); } return true; }
final long timeSinceLastUpdate = System.currentTimeMillis() - lastUpdate.get().getTimestamp();
end = Iterables.getLast(history.get().getTaskUpdates()).getTimestamp(); } else { end = System.currentTimeMillis();
private TaskCleanupResult cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional<SingularityTaskHistory> taskHistory) { SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, LOG, jsonObjectFileHelper, false); SingularityExecutorTaskCleanup taskCleanup = new SingularityExecutorTaskCleanup(logManager, executorConfiguration, taskDefinition, LOG, dockerUtils); boolean cleanupTaskAppDirectory = !taskDefinition.getExecutorData().getPreserveTaskSandboxAfterFinish().or(Boolean.FALSE); if (taskDefinition.shouldLogrotateLogFile()) { checkForUncompressedLogrotatedFile(taskDefinition); } if (taskHistory.isPresent()) { final Optional<SingularityTaskHistoryUpdate> lastUpdate = JavaUtils.getLast(taskHistory.get().getTaskUpdates()); if (lastUpdate.isPresent()) { if (lastUpdate.get().getTaskState().isDone() && System.currentTimeMillis() - lastUpdate.get().getTimestamp() > TimeUnit.MINUTES.toMillis(15)) { LOG.info("Task {} is done for > 15 minutes, removing logrotate files"); taskCleanup.cleanUpLogs(); } if (lastUpdate.get().getTaskState().isFailed()) { final long delta = System.currentTimeMillis() - lastUpdate.get().getTimestamp(); if (delta < cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis()) { LOG.info("Not cleaning up task app directory of {} because only {} has elapsed since it failed (will cleanup after {})", taskDefinition.getTaskId(), JavaUtils.durationFromMillis(delta), JavaUtils.durationFromMillis(cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis())); cleanupTaskAppDirectory = false; } } } } boolean isDocker = (taskHistory.isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().get().getType() == SingularityContainerType.DOCKER); return taskCleanup.cleanup(cleanupTaskAppDirectory, isDocker); }
private int checkTaskUpdates(SingularityWebhook webhook, List<CompletableFuture<Response>> webhookFutures) { final List<SingularityTaskHistoryUpdate> taskUpdates = webhookManager.getQueuedTaskUpdatesForHook(webhook.getId()); int numTaskUpdates = 0; for (SingularityTaskHistoryUpdate taskUpdate : taskUpdates) { Optional<SingularityTask> task = taskHistoryHelper.getTask(taskUpdate.getTaskId()); // TODO compress if (!task.isPresent()) { LOG.warn("Couldn't find task for taskUpdate {}", taskUpdate); webhookManager.deleteTaskUpdate(webhook, taskUpdate); continue; } String concreteUri = applyPlaceholders(webhook.getUri(), taskUpdate); webhookFutures.add(webhookSemaphore.call(() -> executeWebhookAsync( concreteUri, new SingularityTaskWebhook(task.get(), taskUpdate), new SingularityTaskWebhookAsyncHandler(webhookManager, webhook, taskUpdate, shouldDeleteUpdateOnFailure(numTaskUpdates, taskUpdate.getTimestamp()))) )); } return taskUpdates.size(); }
Optional<SingularityTaskHistoryUpdate> maybeStartingUpdate = taskManager.getTaskHistoryUpdate(task, ExtendedTaskState.TASK_STARTING); if (maybeStartingUpdate.isPresent()) { long startTimestampSeconds = TimeUnit.MILLISECONDS.toSeconds(maybeStartingUpdate.get().getTimestamp()); double usedCpusSinceStart = latestUsage.getCpuSeconds() / (latestUsage.getTimestamp() - startTimestampSeconds); currentUsage = new SingularityTaskCurrentUsage(latestUsage.getMemoryTotalBytes(), now, usedCpusSinceStart, latestUsage.getDiskTotalBytes());
private Optional<Long> getRunningAt(Collection<SingularityTaskHistoryUpdate> updates) { for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { return Optional.of(update.getTimestamp()); } } return Optional.absent(); }
private Optional<Long> getRunningAt(Collection<SingularityTaskHistoryUpdate> updates) { for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { return Optional.of(update.getTimestamp()); } } return Optional.absent(); }
private long getTaskRunningStartTime(SingularityTaskId task) { Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(task); if (taskHistory.isPresent()) { java.util.Optional<SingularityTaskHistoryUpdate> taskRunningState = taskHistory.get().getTaskUpdates().stream().filter(h -> h.getTaskState().equals(ExtendedTaskState.TASK_RUNNING)).findFirst(); if (taskRunningState.isPresent()) { return taskRunningState.get().getTimestamp(); } LOG.error("Could not find time when task {} reached TASK_RUNNING state", task); } else { LOG.error("Could not find task history for {}", task); } return System.currentTimeMillis(); }
private boolean isEligibleForShuffle(SingularityTaskId task) { Optional<SingularityTaskHistoryUpdate> taskRunning = taskManager.getTaskHistoryUpdate(task, ExtendedTaskState.TASK_RUNNING); return ( !configuration.getDoNotShuffleRequests().contains(task.getRequestId()) && isLongRunning(task) && ( configuration.getMinutesBeforeNewTaskEligibleForShuffle() == 0 // Shuffle delay is disabled entirely || (taskRunning.isPresent() && TimeUnit.MILLISECONDS.toMinutes(System.currentTimeMillis() - taskRunning.get() .getTimestamp()) >= configuration.getMinutesBeforeNewTaskEligibleForShuffle()) ) ); }
public List<SingularityMailTaskHistoryUpdate> getJadeTaskHistory(Collection<SingularityTaskHistoryUpdate> taskHistory) { List<SingularityMailTaskHistoryUpdate> output = Lists.newArrayListWithCapacity(taskHistory.size()); for (SingularityTaskHistoryUpdate taskUpdate : taskHistory) { output.add( new SingularityMailTaskHistoryUpdate( humanizeTimestamp(taskUpdate.getTimestamp()), WordUtils.capitalize(taskUpdate.getTaskState().getDisplayName()), taskUpdate.getStatusMessage().or(""))); } return output; }