public void killAndRecord(SingularityTaskId taskId, Optional<RequestCleanupType> requestCleanupType, Optional<TaskCleanupType> taskCleanupType, Optional<Long> originalTimestamp, Optional<Integer> retries, Optional<String> user) { Preconditions.checkState(isRunning()); Optional<TaskCleanupType> maybeCleanupFromRequestAndTask = getTaskCleanupType(requestCleanupType, taskCleanupType); if (maybeCleanupFromRequestAndTask.isPresent() && (maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY || maybeCleanupFromRequestAndTask.get() == TaskCleanupType.REQUEST_DELETING)) { Optional<SingularityTask> task = taskManager.getTask(taskId); if (task.isPresent()) { if (task.get().getTaskRequest().getDeploy().getCustomExecutorCmd().isPresent()) { byte[] messageBytes = transcoder.toBytes(new SingularityTaskDestroyFrameworkMessage(taskId, user)); mesosSchedulerClient.frameworkMessage( MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()), MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()), messageBytes ); } else { LOG.warn("Not using custom executor, will not send framework message to destroy task"); } } else { String message = String.format("No task data available to build kill task framework message for task %s", taskId); exceptionNotifier.notify(message); LOG.error(message); } } mesosSchedulerClient.kill(TaskID.newBuilder().setValue(taskId.toString()).build()); taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskId, System.currentTimeMillis(), originalTimestamp.or(System.currentTimeMillis()), requestCleanupType, taskCleanupType, retries.or(-1) + 1)); }
@POST @Path("/scheduler/statusUpdate/{taskId}/{taskState}") @Operation( summary = "Force an update for a specific task", responses = { @ApiResponse(responseCode = "403", description = "Test resource calls are currently not enabled, set `allowTestResourceCalls` to `true` in config yaml to enable") } ) public void statusUpdate(@PathParam("taskId") String taskId, @PathParam("taskState") String taskState) { checkForbidden(configuration.isAllowTestResourceCalls(), "Test resource calls are disabled (set isAllowTestResourceCalls to true in configuration)"); scheduler.statusUpdate(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId)) .setState(TaskState.valueOf(taskState)) .build()).join(); }
} else { TaskStatus.Builder fakeTaskStatusBuilder = TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskStatusHolder.getTaskId().getId())) .setState(TaskState.TASK_STARTING);
} else { TaskStatus.Builder fakeTaskStatusBuilder = TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskStatusHolder.getTaskId().getId())) .setState(TaskState.TASK_STARTING);
org.apache.mesos.v1.Protos.AgentID.newBuilder().setValue(agentId).build(), org.apache.mesos.v1.Protos.ExecutorID.newBuilder().setValue(executorId).build(), org.apache.mesos.v1.Protos.TaskID.newBuilder().setValue(taskId).build(), state, uuid
public void killAndRecord(SingularityTaskId taskId, Optional<RequestCleanupType> requestCleanupType, Optional<TaskCleanupType> taskCleanupType, Optional<Long> originalTimestamp, Optional<Integer> retries, Optional<String> user) { Preconditions.checkState(isRunning()); Optional<TaskCleanupType> maybeCleanupFromRequestAndTask = getTaskCleanupType(requestCleanupType, taskCleanupType); if (maybeCleanupFromRequestAndTask.isPresent() && (maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY || maybeCleanupFromRequestAndTask.get() == TaskCleanupType.REQUEST_DELETING)) { Optional<SingularityTask> task = taskManager.getTask(taskId); if (task.isPresent()) { if (task.get().getTaskRequest().getDeploy().getCustomExecutorCmd().isPresent()) { byte[] messageBytes = transcoder.toBytes(new SingularityTaskDestroyFrameworkMessage(taskId, user)); mesosSchedulerClient.frameworkMessage( MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()), MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()), messageBytes ); } else { LOG.warn("Not using custom executor, will not send framework message to destroy task"); } } else { String message = String.format("No task data available to build kill task framework message for task %s", taskId); exceptionNotifier.notify(message); LOG.error(message); } } mesosSchedulerClient.kill(TaskID.newBuilder().setValue(taskId.toString()).build()); taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskId, System.currentTimeMillis(), originalTimestamp.or(System.currentTimeMillis()), requestCleanupType, taskCleanupType, retries.or(-1) + 1)); }
private void runTest(RequestType requestType, Reason reason, boolean shouldRetry) { initRequestWithType(requestType, false); initFirstDeploy(); SingularityTask task = startTask(firstDeploy); Assert.assertEquals(taskManager.getPendingTaskIds().size(), 0); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); try { updateHandler.processStatusUpdateAsync(TaskStatus.newBuilder() .setState(TaskState.TASK_LOST) .setReason(reason) .setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())) .build()).get(); } catch (InterruptedException | ExecutionException e) { Assert.assertTrue(false); } if (shouldRetry) { Assert.assertEquals(requestManager.getPendingRequests().size(), 1); Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.RETRY); } else { if (requestManager.getPendingRequests().size() > 0) { Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.TASK_DONE); } } scheduler.drainPendingQueue(); } }
protected SingularityTask prepTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, boolean separateHosts, Optional<String> runId) { SingularityPendingTask pendingTask = buildPendingTask(request, deploy, launchTime, instanceNo, runId); SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); Offer offer; if (separateHosts) { offer = createOffer(125, 1024, 2048, String.format("slave%s", instanceNo), String.format("host%s", instanceNo)); } else { offer = createOffer(125, 1024, 2048); } SingularityTaskId taskId = new SingularityTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, offer.getHostname(), "rack1"); TaskID taskIdProto = TaskID.newBuilder().setValue(taskId.toString()).build(); TaskInfo taskInfo = TaskInfo.newBuilder() .setAgentId(offer.getAgentId()) .setExecutor(ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("executorID"))) .setTaskId(taskIdProto) .setName("name") .build(); SingularityTask task = new SingularityTask(taskRequest, taskId, Collections.singletonList(mesosProtosUtils.offerFromProtos(offer)), mesosProtosUtils.taskFromProtos(taskInfo), Optional.of("rack1")); taskManager.savePendingTask(pendingTask); return task; }
@Override public void applyMigration() { final long start = System.currentTimeMillis(); final List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds(); for (SingularityTaskId taskId : taskIds) { List<SingularityTaskHistoryUpdate> updates = Lists.reverse(taskManager.getTaskHistoryUpdates(taskId)); Optional<MesosTaskStatusObject> taskStatus = Optional.absent(); for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState().toTaskState().isPresent()) { Optional<SingularityTask> task = taskManager.getTask(taskId); taskStatus = Optional.of(mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.getId())) .setAgentId(MesosProtosUtils.toAgentId(task.get().getAgentId())) .setState(MesosProtosUtils.toTaskState(update.getTaskState())) .build())); break; } } SingularityTaskStatusHolder taskStatusHolder = new SingularityTaskStatusHolder(taskId, taskStatus, start, serverId, Optional.absent()); taskManager.saveLastActiveTaskStatus(taskStatusHolder); } }
@Test public void testTaskOddities() { // test unparseable status update TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue("task")) .setAgentId(AgentID.newBuilder().setValue("slave1")) .setState(TaskState.TASK_RUNNING); // should not throw exception: sms.statusUpdate(bldr.build()).join(); initRequest(); initFirstDeploy(); SingularityTask taskOne = launchTask(request, firstDeploy, 1, TaskState.TASK_STARTING); taskManager.deleteTaskHistory(taskOne.getTaskId()); Assert.assertTrue(taskManager.isActiveTask(taskOne.getTaskId().getId())); statusUpdate(taskOne, TaskState.TASK_RUNNING); statusUpdate(taskOne, TaskState.TASK_FAILED); Assert.assertTrue(!taskManager.isActiveTask(taskOne.getTaskId().getId())); Assert.assertEquals(2, taskManager.getTaskHistoryUpdates(taskOne.getTaskId()).size()); }
@POST @Path("/scheduler/statusUpdate/{taskId}/{taskState}") @Operation( summary = "Force an update for a specific task", responses = { @ApiResponse(responseCode = "403", description = "Test resource calls are currently not enabled, set `allowTestResourceCalls` to `true` in config yaml to enable") } ) public void statusUpdate(@PathParam("taskId") String taskId, @PathParam("taskState") String taskState) { checkForbidden(configuration.isAllowTestResourceCalls(), "Test resource calls are disabled (set isAllowTestResourceCalls to true in configuration)"); scheduler.statusUpdate(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId)) .setState(TaskState.valueOf(taskState)) .build()).join(); }
public static TaskID toTaskId(MesosStringValue stringValue) { return TaskID.newBuilder().setValue(stringValue.getValue()).build(); }
protected MesosTaskStatusObject buildTaskStatus(SingularityTask task) { return mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder().setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())).setState(TaskState.TASK_RUNNING).build()); }
.setTaskId(TaskID.newBuilder().setValue(taskId.toString()));
protected SingularityTask prepTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, boolean separateHosts, Optional<String> runId) { SingularityPendingTask pendingTask = buildPendingTask(request, deploy, launchTime, instanceNo, runId); SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); Offer offer; if (separateHosts) { offer = createOffer(125, 1024, 2048, String.format("slave%s", instanceNo), String.format("host%s", instanceNo)); } else { offer = createOffer(125, 1024, 2048); } SingularityTaskId taskId = new SingularityTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, offer.getHostname(), "rack1"); TaskID taskIdProto = TaskID.newBuilder().setValue(taskId.toString()).build(); TaskInfo taskInfo = TaskInfo.newBuilder() .setAgentId(offer.getAgentId()) .setExecutor(ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("executorID"))) .setTaskId(taskIdProto) .setName("name") .build(); SingularityTask task = new SingularityTask(taskRequest, taskId, Collections.singletonList(mesosProtosUtils.offerFromProtos(offer)), mesosProtosUtils.taskFromProtos(taskInfo), Optional.of("rack1")); taskManager.savePendingTask(pendingTask); return task; }
@Override public void applyMigration() { final long start = System.currentTimeMillis(); final List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds(); for (SingularityTaskId taskId : taskIds) { List<SingularityTaskHistoryUpdate> updates = Lists.reverse(taskManager.getTaskHistoryUpdates(taskId)); Optional<MesosTaskStatusObject> taskStatus = Optional.absent(); for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState().toTaskState().isPresent()) { Optional<SingularityTask> task = taskManager.getTask(taskId); taskStatus = Optional.of(mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.getId())) .setAgentId(MesosProtosUtils.toAgentId(task.get().getAgentId())) .setState(MesosProtosUtils.toTaskState(update.getTaskState())) .build())); break; } } SingularityTaskStatusHolder taskStatusHolder = new SingularityTaskStatusHolder(taskId, taskStatus, start, serverId, Optional.absent()); taskManager.saveLastActiveTaskStatus(taskStatusHolder); } }
private void runTest(RequestType requestType, Reason reason, boolean shouldRetry) { initRequestWithType(requestType, false); initFirstDeploy(); SingularityTask task = startTask(firstDeploy); Assert.assertEquals(taskManager.getPendingTaskIds().size(), 0); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); try { updateHandler.processStatusUpdateAsync(TaskStatus.newBuilder() .setState(TaskState.TASK_LOST) .setReason(reason) .setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())) .build()).get(); } catch (InterruptedException | ExecutionException e) { Assert.assertTrue(false); } if (shouldRetry) { Assert.assertEquals(requestManager.getPendingRequests().size(), 1); Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.RETRY); } else { if (requestManager.getPendingRequests().size() > 0) { Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.TASK_DONE); } } scheduler.drainPendingQueue(); } }
@Test public void testTaskOddities() { // test unparseable status update TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue("task")) .setAgentId(AgentID.newBuilder().setValue("slave1")) .setState(TaskState.TASK_RUNNING); // should not throw exception: sms.statusUpdate(bldr.build()).join(); initRequest(); initFirstDeploy(); SingularityTask taskOne = launchTask(request, firstDeploy, 1, TaskState.TASK_STARTING); taskManager.deleteTaskHistory(taskOne.getTaskId()); Assert.assertTrue(taskManager.isActiveTask(taskOne.getTaskId().getId())); statusUpdate(taskOne, TaskState.TASK_RUNNING); statusUpdate(taskOne, TaskState.TASK_FAILED); Assert.assertTrue(!taskManager.isActiveTask(taskOne.getTaskId().getId())); Assert.assertEquals(2, taskManager.getTaskHistoryUpdates(taskOne.getTaskId()).size()); }
public static TaskID toTaskId(MesosStringValue stringValue) { return TaskID.newBuilder().setValue(stringValue.getValue()).build(); }
protected MesosTaskStatusObject buildTaskStatus(SingularityTask task) { return mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder().setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())).setState(TaskState.TASK_RUNNING).build()); }