private static List<DaemonTask> aggregateTasks(DaemonTask task) { List<DaemonTask> result = new ArrayList<>(); task.consumeTaskTree((t) -> result.add((DaemonTask) t)); return result; }
ShallowTaskInfo(DaemonTask task) { this.uuid = task.getUuid(); this.name = task.getName(); this.state = task.getState(); this.fatalException = task.getFatalError(); this.jobs = task.getRunningJobs(); if (!task.getEvents().isEmpty()) { this.lastEvent = task.getEvents().get(task.getEvents().size() - 1).toString(); } this.children = (List<String>) task.getChildren() .stream() .map(c -> c.toString()) .collect(Collectors.toList()); } }
<P> DaemonResponse<P> reapChild(DaemonTask task) throws InterruptedException { DaemonTask childTask = children.stream() .filter(c -> c.getUuid().equals(task.getUuid())) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Child " + task + " does not belong to this task " + this + "")); // Spin due to spurious wakeups while (!childTask.getState().isTerminal()) { try { synchronized (childTask) { childTask.wait(); } } catch (InterruptedException e) { throw e; } } TaskRepository.getTask(childTask.getUuid()); log.info(this + "Collected child task " + childTask + " with state " + childTask.getState()); if (childTask.getResponse() == null) { throw new RuntimeException("Child response may not be null."); } return childTask.getResponse(); }
@Override public void run() { long sleepTime = endTime - System.currentTimeMillis(); while (sleepTime > 0) { try { Thread.sleep(sleepTime); } catch (InterruptedException ignored) { } sleepTime = endTime - System.currentTimeMillis(); } switch (target.getState()) { case NOT_STARTED: case RUNNING: log.warn("Interrupting task " + target + " that timed out after " + (endTime - startTime) + " millis."); target.timeout(); break; case TIMED_OUT: case INTERRUPTED: case FAILED: case SUCCEEDED: log.info("Interrupter has no work to do, " + target + " already completed."); break; } try { Thread.sleep(DELETE_TASK_INFO_WINDOW); } catch (InterruptedException ignored) { } deleteTaskInfo(target.getUuid()); } }
public static <C, T> T get(DaemonTask<C, T> task) { int lastTaskCount = 0; task = Daemon.getTask(task.getUuid()); Set<String> loggedEvents = new HashSet<>(); while (!task.getState().isTerminal()) { updateCycle(); if (interrupted) { Daemon.interruptTask(task.getUuid()); throw TaskKilledException.interrupted(new InterruptedException("Interrupted by user")); task = Daemon.getTask(task.getUuid()); logTasks(aggregatedTasks, loggedEvents); DaemonResponse<T> response = task.getResponse(); switch (task.getState()) { case TIMED_OUT: throw TaskKilledException.timeout(); case INTERRUPTED: throw TaskKilledException.interrupted(task.getFatalError()); case FAILED: Exception fatal = task.getFatalError(); if (fatal == null) { throw new RuntimeException("Task failed without reason. This is a bug.");
DaemonResponse<U> response = new DaemonResponse<>(responseBody, problemSet); return (DaemonResponse) task.getChildren().stream().reduce(response, (o, t) -> { DaemonResponse<U> collector = (DaemonResponse<U>) o; DaemonResponse<T> childResponse; try { childResponse = task.reapChild(child); } catch (InterruptedException e) { throw new DaemonTaskInterrupted("Interrupted during reap", e); DaemonTask.State state = child.getState(); if (!state.isTerminal()) { throw new IllegalStateException("Child task " + child + " reaped but non-terminal."); throw new HalException(childResponse.getProblemSet().getProblems()); case INTERRUPTED: task.interrupt(); throw new DaemonTaskInterrupted(child.getFatalError()); case TIMED_OUT: task.timeout(); throw new DaemonTaskInterrupted("Child task timed out"); case SUCCEEDED:
static public <C, T> DaemonTask<C, T> submitTask(Supplier<DaemonResponse<T>> runner, String name, long timeout) { DaemonTask<C, T> task = new DaemonTask<>(name, timeout); String uuid = task.getUuid(); log.info("Scheduling task " + task); Runnable r = () -> { log.info("Starting task " + task); DaemonTaskHandler.setTask(task); task.setState(State.RUNNING); try { task.success(runner.get()); } catch (HalException e) { log.info("Task " + task + " failed with HalException: ", e); task.failure(e); } catch (Exception e) { log.warn("Task " + task + " failed with unexpected reason: ", e); task.failure(e); } finally { task.cleanupResources(); tasks.put(uuid, task.setRunner(t)); t.start();
private static String formatLoggedDaemonTask(DaemonTask task, DaemonEvent event) { return "Message from task " + task.getName() + ": " + event.getStage() + " - " + event.getMessage(); }
void cleanupResources() { log.info(this + " killing all jobs created by this task " + String.join(", ", runningJobs)); DaemonTaskHandler.getJobExecutor().cancelJobs(new ArrayList<>(runningJobs)); for (DaemonTask child : children) { if (child != null) { log.info(this + " interrupting child " + child); if (timedOut) { child.timeout(); } else { child.interrupt(); } } } }
void timeout() { timedOut = true; interrupt(); }
public static <C, T> T get(DaemonTask<C, T> task) { int lastTaskCount = 0; task = Daemon.getTask(task.getUuid()); Set<String> loggedEvents = new HashSet<>(); while (!task.getState().isTerminal()) { updateCycle(); if (interrupted) { Daemon.interruptTask(task.getUuid()); throw TaskKilledException.interrupted(new InterruptedException("Interrupted by user")); task = Daemon.getTask(task.getUuid()); logTasks(aggregatedTasks, loggedEvents); DaemonResponse<T> response = task.getResponse(); switch (task.getState()) { case TIMED_OUT: throw TaskKilledException.timeout(); case INTERRUPTED: throw TaskKilledException.interrupted(task.getFatalError()); case FAILED: Exception fatal = task.getFatalError(); if (fatal == null) { throw new RuntimeException("Task failed without reason. This is a bug.");
private static String formatLoggedDaemonTask(DaemonTask task, DaemonEvent event) { return "Message from task " + task.getName() + ": " + event.getStage() + " - " + event.getMessage(); }
@RequestMapping(value = "/{uuid:.+}/interrupt", method = RequestMethod.PUT) void interruptTask(@PathVariable String uuid, @Body String ignored) { DaemonTask task = TaskRepository.getTask(uuid); if (task == null) { throw new TaskNotFoundException("No such task with UUID " + uuid); } task.interrupt(); }
private static void logTasks(List<DaemonTask> tasks, Set<String> loggedEvents) { // This is expensive, so don't check all tasks to log unless it's necessary. if (GlobalOptions.getGlobalOptions().getLog() == Level.OFF) { return; } for (DaemonTask task : tasks) { for (Object oEvent : task.getEvents()) { DaemonEvent event = (DaemonEvent) oEvent; String loggedEvent = formatLoggedDaemonTask(task, event); if (!loggedEvents.contains(loggedEvent)) { loggedEvents.add(loggedEvent); log.info(loggedEvent); } } } }
private static List<DaemonTask> aggregateTasks(DaemonTask task) { List<DaemonTask> result = new ArrayList<>(); task.consumeTaskTree((t) -> result.add((DaemonTask) t)); return result; }
private static void logTasks(List<DaemonTask> tasks, Set<String> loggedEvents) { // This is expensive, so don't check all tasks to log unless it's necessary. if (GlobalOptions.getGlobalOptions().getLog() == Level.OFF) { return; } for (DaemonTask task : tasks) { for (Object oEvent : task.getEvents()) { DaemonEvent event = (DaemonEvent) oEvent; String loggedEvent = formatLoggedDaemonTask(task, event); if (!loggedEvents.contains(loggedEvent)) { loggedEvents.add(loggedEvent); log.info(loggedEvent); } } } }