private boolean shouldScheduleTasks(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy, Optional<SingularityRequestDeployState> maybeRequestDeployState) { if (request.isDeployable() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY && !maybePendingDeploy.isPresent()) { return false; } if (request.getRequestType() == RequestType.RUN_ONCE && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { return true; } return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false); }
private int getNumMissingInstances(List<SingularityTaskId> matchingTaskIds, SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { PendingType pendingType = pendingRequest.getPendingType(); if (request.isOneOff()) { if (pendingType == PendingType.ONEOFF || pendingType == PendingType.RETRY) { return 1; } else { return 0; } } else if (request.getRequestType() == RequestType.RUN_ONCE && pendingType == PendingType.NEW_DEPLOY) { return 1; } return numInstancesExpected(request, pendingRequest, maybePendingDeploy) - matchingTaskIds.size(); }
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
SingularityPendingRequest pendingRequest = requestTranscoder.fromBytes(curator.getData() .forPath(String.format("%s/%s", basePath, childPath))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { String rewrittenPath = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()) .getId();
public SingularityCreateResult addToPendingQueue(SingularityPendingRequest pendingRequest) { SingularityCreateResult result = create(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); if (result == SingularityCreateResult.EXISTED) { Optional<SingularityPendingRequest> existingPendingRequest = getPendingRequest(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (!existingPendingRequest.isPresent() || (existingPendingRequest.get().getPendingType() == PendingType.STARTUP)) { // Fresh pending requests take priority over STARTUP-type pending requests (or if the pending request we originally found is gone, we'll go ahead and do the write now) set(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); LOG.info("{} added to pending queue, overwriting an existing SingularityPendingRequest of type STARTUP. Previous pending request was {}", existingPendingRequest, result); return result; } } LOG.info("{} added to pending queue with result: {}", pendingRequest, result); return result; }
SingularityPendingRequest pendingRequest = requestTranscoder.fromBytes(curator.getData() .forPath(String.format("%s/%s", basePath, originalBasename))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE || pendingRequest.getPendingType() == PendingType.ONEOFF) { String deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()).getId(); String rewrittenBasename = String.format("%s%s%s", deployKey, pendingRequest.getTimestamp(), pendingRequest.getRunId().or(""));
private Optional<Long> getNextRunAt(SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { PendingType pendingType = pendingRequest.getPendingType(); final long now = System.currentTimeMillis();
new SingularityPendingTaskId( request.getId(), deployId, nextRunAt.get(), nextInstanceNumber, pendingRequest.getPendingType(), pendingRequest.getTimestamp()), pendingRequest.getCmdLineArgsList(), pendingRequest.getUser(),
if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest); heldForScheduledActiveTask.getAndIncrement(); deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy); requestManager.deletePendingRequest(pendingRequest); } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { effectivePendingRequests.add(pendingRequest); RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics); deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy); requestManager.deletePendingRequest(pendingRequest); } else if (pendingRequest.getPendingType() == PendingType.ONEOFF) { effectivePendingRequests.add(pendingRequest); RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics); requestManager.deletePendingRequest(pendingRequest); } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
@Test public void testOnDemandDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND).build()); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().get(0).getPendingType() == PendingType.ONEOFF); }
@Test public void testSchedulerDropsMultipleScheduledTaskInstances() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, Instant.now().plus(3, ChronoUnit.DAYS).toEpochMilli(), Optional.absent(), PendingType.NEW_DEPLOY, Optional.absent(), Optional.absent())); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder().build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); Assert.assertEquals("Both requests make it into the pending queue", 2, requestManager.getPendingRequests().size()); Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(0).getPendingType()); Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(1).getPendingType()); scheduler.drainPendingQueue(); Assertions.assertThat(taskManager.getPendingTaskIds()) .describedAs("Only the immediate request gets run") .hasSize(1) .extracting(SingularityPendingTaskId::getPendingType) .containsExactly(PendingType.IMMEDIATE); Assertions.assertThat(requestManager.getPendingRequests()) .describedAs("The scheduled request is dropped from the pending queue") .hasSize(0); }
Assert.assertEquals(r.getDeployId(), p1.getDeployId()); Assert.assertEquals(r.getTimestamp(), p1.getTimestamp()); Assert.assertEquals(r.getPendingType(), p1.getPendingType()); Assert.assertTrue(!r.getCmdLineArgsList().isPresent()); Assert.assertEquals(r.getUser(), p1.getUser()); Assert.assertEquals(r.getDeployId(), p2.getDeployId()); Assert.assertEquals(r.getTimestamp(), p2.getTimestamp()); Assert.assertEquals(r.getPendingType(), p2.getPendingType()); Assert.assertTrue(r.getCmdLineArgsList().get().size() == 1); Assert.assertTrue(r.getCmdLineArgsList().get().get(0).equals("cmd line args"));
private void runTest(RequestType requestType, Reason reason, boolean shouldRetry) { initRequestWithType(requestType, false); initFirstDeploy(); SingularityTask task = startTask(firstDeploy); Assert.assertEquals(taskManager.getPendingTaskIds().size(), 0); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); try { updateHandler.processStatusUpdateAsync(TaskStatus.newBuilder() .setState(TaskState.TASK_LOST) .setReason(reason) .setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())) .build()).get(); } catch (InterruptedException | ExecutionException e) { Assert.assertTrue(false); } if (shouldRetry) { Assert.assertEquals(requestManager.getPendingRequests().size(), 1); Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.RETRY); } else { if (requestManager.getPendingRequests().size() > 0) { Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.TASK_DONE); } } scheduler.drainPendingQueue(); } }
@Test public void testScheduledTasksDontGetRescheduled() { initScheduledRequest(); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertEquals("NEW_DEPLOY is first", PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); scheduler.drainPendingQueue(); Assert.assertTrue("Pending queue is cleared", requestManager.getPendingRequests().isEmpty()); List<SingularityPendingTask> pending = taskManager.getPendingTasks(); Assert.assertEquals("One task is started", 1, taskManager.getPendingTaskIds().size()); Assert.assertEquals("First request takes precedence", PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); startup.checkSchedulerForInconsistentState(); scheduler.drainPendingQueue(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().size() == 1); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); Assert.assertTrue(pending.equals(taskManager.getPendingTasks())); taskManager.deletePendingTask(pending.get(0).getPendingTaskId()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().size() == 1); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); }
private boolean shouldScheduleTasks(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy, Optional<SingularityRequestDeployState> maybeRequestDeployState) { if (request.isDeployable() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY && !maybePendingDeploy.isPresent()) { return false; } if (request.getRequestType() == RequestType.RUN_ONCE && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { return true; } return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false); }
private int getNumMissingInstances(List<SingularityTaskId> matchingTaskIds, SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { PendingType pendingType = pendingRequest.getPendingType(); if (request.isOneOff()) { if (pendingType == PendingType.ONEOFF || pendingType == PendingType.RETRY) { return 1; } else { return 0; } } else if (request.getRequestType() == RequestType.RUN_ONCE && pendingType == PendingType.NEW_DEPLOY) { return 1; } return numInstancesExpected(request, pendingRequest, maybePendingDeploy) - matchingTaskIds.size(); }
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
public SingularityCreateResult addToPendingQueue(SingularityPendingRequest pendingRequest) { SingularityCreateResult result = create(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); if (result == SingularityCreateResult.EXISTED) { Optional<SingularityPendingRequest> existingPendingRequest = getPendingRequest(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (!existingPendingRequest.isPresent() || (existingPendingRequest.get().getPendingType() == PendingType.STARTUP)) { // Fresh pending requests take priority over STARTUP-type pending requests (or if the pending request we originally found is gone, we'll go ahead and do the write now) set(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); LOG.info("{} added to pending queue, overwriting an existing SingularityPendingRequest of type STARTUP. Previous pending request was {}", existingPendingRequest, result); return result; } } LOG.info("{} added to pending queue with result: {}", pendingRequest, result); return result; }
@Test public void testOnDemandDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND).build()); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().get(0).getPendingType() == PendingType.ONEOFF); }