private boolean shouldReschedule(SingularityRequest newRequest, SingularityRequest oldRequest) { if (newRequest.getInstancesSafe() != oldRequest.getInstancesSafe()) { return true; } if (newRequest.isScheduled() && oldRequest.isScheduled()) { if (!newRequest.getQuartzScheduleSafe().equals(oldRequest.getQuartzScheduleSafe())) { return true; } } return false; }
private void populateRequestEmailProperties(Map<String, Object> templateProperties, SingularityRequest request, SingularityEmailType emailType) { templateProperties.put("requestId", request.getId()); templateProperties.put("singularityRequestLink", mailTemplateHelpers.getSingularityRequestLink(request.getId())); templateProperties.put("requestAlwaysRunning", request.isAlwaysRunning()); templateProperties.put("requestRunOnce", request.getRequestType() == RequestType.RUN_ONCE); templateProperties.put("requestScheduled", request.isScheduled()); templateProperties.put("requestOneOff", request.isOneOff()); templateProperties.put("taskWillRetry", request.getNumRetriesOnFailure().or(0) > 0); templateProperties.put("numRetries", request.getNumRetriesOnFailure().or(0)); templateProperties.put("color", emailType.getColor()); }
private void checkForOverdueScheduledJob(long runtime, SingularityTaskId taskId, SingularityRequest request) { if (request.isScheduled() && !taskManager.hasNotifiedOverdue(taskId) && runtime >= configuration.getWarnIfScheduledJobIsRunningForAtLeastMillis()) { final Optional<Long> expectedRuntime = getExpectedRuntime(request, taskId); if (expectedRuntime.isPresent()) { final int overDuePct = (int) (100 * (runtime / (float) expectedRuntime.get())); if (overDuePct > configuration.getWarnIfScheduledJobIsRunningPastNextRunPct()) { LOG.info("{} is overdue by {}% (expectedRunTime: {}, warnIfScheduledJobIsRunningPastNextRunPct: {})", taskId, overDuePct, expectedRuntime.get(), configuration.getWarnIfScheduledJobIsRunningPastNextRunPct()); mailer.sendTaskOverdueMail(taskManager.getTask(taskId), taskId, request, runtime, expectedRuntime.get()); taskManager.saveNotifiedOverdue(taskId); } else { LOG.trace("{} is not overdue yet - runtime {}, expected {}, ({}% < {}%)", taskId, runtime, expectedRuntime.get(), overDuePct, configuration.getWarnIfScheduledJobIsRunningPastNextRunPct()); } } } }
private SingularityDeploy checkImmediateRunDeploy(SingularityRequest request, SingularityDeploy deploy, SingularityRunNowRequest runNowRequest, List<SingularityTaskId> activeTasks, List<SingularityPendingTaskId> pendingTasks) { if (!request.isScheduled() && !request.isOneOff()) { throw badRequest("Can not request an immediate run of a non-scheduled / always running request (%s)", request); } return deploy.toBuilder() .setRunImmediately(Optional.of(fillRunNowRequest(Optional.of(runNowRequest)))) .build(); }
private void checkActiveRequest(SingularityRequestWithState requestWithState, Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId, final long timestamp) { final SingularityRequest request = requestWithState.getRequest(); if (request.getRequestType() == RequestType.ON_DEMAND || request.getRequestType() == RequestType.RUN_ONCE) { return; // There's no situation where we'd want to schedule an On Demand or Run Once request at startup, so don't even bother with them. } Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId()); if (!requestDeployState.isPresent() || !requestDeployState.get().getActiveDeploy().isPresent()) { LOG.debug("No active deploy for {} - not scheduling on startup", request.getId()); return; } final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId(); if (request.isScheduled()) { SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId); SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey); if (pendingTaskId != null && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) { LOG.info("Not rescheduling {} because {} is newer than {}", request.getId(), pendingTaskId, requestWithState.getTimestamp()); return; } } requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), activeDeployId, timestamp, Optional.<String> absent(), PendingType.STARTUP, Optional.<Boolean> absent(), Optional.<String> absent())); }
if (request.isScheduled()) { if (pendingType == PendingType.IMMEDIATE || pendingType == PendingType.RETRY) { LOG.info("Scheduling requested immediate run of {}", request.getId());
if (requestWithState.getRequest().isScheduled()) { Optional<String> schedule = requestWithState.getRequest().getSchedule(); Optional<String> quartzSchedule = requestWithState.getRequest().getQuartzSchedule();
SingularityRunNowRequest runNowRequest = fillRunNowRequest(maybeRunNowRequest); PendingType pendingType; if (request.isScheduled()) { pendingType = PendingType.IMMEDIATE; checkConflict(activeTasks.isEmpty(), "Cannot request immediate run of a scheduled job which is currently running (%s)", activeTasks);
private Map<String, Long> getNumTasks(List<SingularityRequestWithState> requests) { final CounterMap<String> numTasks = new CounterMap<>(requests.size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) { numTasks.incr(taskId.getRequestId()); } for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) { numTasks.incr(pendingTaskId.getRequestId()); } for (SingularityTaskId cleaningTaskId : taskManager.getCleanupTaskIds()) { Optional<SingularityRequestWithState> request = requestManager.getRequest(cleaningTaskId.getRequestId()); if (request.isPresent() && request.get().getRequest().isScheduled()) { continue; } numTasks.decr(cleaningTaskId.getRequestId()); } return numTasks.toCountMap(); }
if (request.isScheduled()) { checkBadRequest(request.getQuartzSchedule().isPresent() || request.getSchedule().isPresent(), "Specify at least one of schedule or quartzSchedule"); if (request.isScheduled()) { checkBadRequest(request.getInstances().or(1) == 1, "Scheduler requests can not be ran on more than one instance");
if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest); heldForScheduledActiveTask.getAndIncrement(); deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy); requestManager.deletePendingRequest(pendingRequest); } else if (updatedRequest.isScheduled() && (pendingRequest.getPendingType() == PendingType.NEW_DEPLOY || pendingRequest.getPendingType() == PendingType.TASK_DONE)) {
if (request.isScheduled()) { if (activeTasks.isEmpty()) { pendingType = PendingType.IMMEDIATE;
private boolean shouldReschedule(SingularityRequest newRequest, SingularityRequest oldRequest) { if (newRequest.getInstancesSafe() != oldRequest.getInstancesSafe()) { return true; } if (newRequest.isScheduled() && oldRequest.isScheduled()) { if (!newRequest.getQuartzScheduleSafe().equals(oldRequest.getQuartzScheduleSafe())) { return true; } } return false; }
private void populateRequestEmailProperties(Map<String, Object> templateProperties, SingularityRequest request, SingularityEmailType emailType) { templateProperties.put("requestId", request.getId()); templateProperties.put("singularityRequestLink", mailTemplateHelpers.getSingularityRequestLink(request.getId())); templateProperties.put("requestAlwaysRunning", request.isAlwaysRunning()); templateProperties.put("requestRunOnce", request.getRequestType() == RequestType.RUN_ONCE); templateProperties.put("requestScheduled", request.isScheduled()); templateProperties.put("requestOneOff", request.isOneOff()); templateProperties.put("taskWillRetry", request.getNumRetriesOnFailure().or(0) > 0); templateProperties.put("numRetries", request.getNumRetriesOnFailure().or(0)); templateProperties.put("color", emailType.getColor()); }
private void checkForOverdueScheduledJob(long runtime, SingularityTaskId taskId, SingularityRequest request) { if (request.isScheduled() && !taskManager.hasNotifiedOverdue(taskId) && runtime >= configuration.getWarnIfScheduledJobIsRunningForAtLeastMillis()) { final Optional<Long> expectedRuntime = getExpectedRuntime(request, taskId); if (expectedRuntime.isPresent()) { final int overDuePct = (int) (100 * (runtime / (float) expectedRuntime.get())); if (overDuePct > configuration.getWarnIfScheduledJobIsRunningPastNextRunPct()) { LOG.info("{} is overdue by {}% (expectedRunTime: {}, warnIfScheduledJobIsRunningPastNextRunPct: {})", taskId, overDuePct, expectedRuntime.get(), configuration.getWarnIfScheduledJobIsRunningPastNextRunPct()); mailer.sendTaskOverdueMail(taskManager.getTask(taskId), taskId, request, runtime, expectedRuntime.get()); taskManager.saveNotifiedOverdue(taskId); } else { LOG.trace("{} is not overdue yet - runtime {}, expected {}, ({}% < {}%)", taskId, runtime, expectedRuntime.get(), overDuePct, configuration.getWarnIfScheduledJobIsRunningPastNextRunPct()); } } } }
private SingularityDeploy checkImmediateRunDeploy(SingularityRequest request, SingularityDeploy deploy, SingularityRunNowRequest runNowRequest, List<SingularityTaskId> activeTasks, List<SingularityPendingTaskId> pendingTasks) { if (!request.isScheduled() && !request.isOneOff()) { throw badRequest("Can not request an immediate run of a non-scheduled / always running request (%s)", request); } return deploy.toBuilder() .setRunImmediately(Optional.of(fillRunNowRequest(Optional.of(runNowRequest)))) .build(); }
private void checkActiveRequest(SingularityRequestWithState requestWithState, Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId, final long timestamp) { final SingularityRequest request = requestWithState.getRequest(); if (request.getRequestType() == RequestType.ON_DEMAND || request.getRequestType() == RequestType.RUN_ONCE) { return; // There's no situation where we'd want to schedule an On Demand or Run Once request at startup, so don't even bother with them. } Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId()); if (!requestDeployState.isPresent() || !requestDeployState.get().getActiveDeploy().isPresent()) { LOG.debug("No active deploy for {} - not scheduling on startup", request.getId()); return; } final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId(); if (request.isScheduled()) { SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId); SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey); if (pendingTaskId != null && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) { LOG.info("Not rescheduling {} because {} is newer than {}", request.getId(), pendingTaskId, requestWithState.getTimestamp()); return; } } requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), activeDeployId, timestamp, Optional.<String> absent(), PendingType.STARTUP, Optional.<Boolean> absent(), Optional.<String> absent())); }
if (requestWithState.getRequest().isScheduled()) { Optional<String> schedule = requestWithState.getRequest().getSchedule(); Optional<String> quartzSchedule = requestWithState.getRequest().getQuartzSchedule();
SingularityRunNowRequest runNowRequest = fillRunNowRequest(maybeRunNowRequest); PendingType pendingType; if (request.isScheduled()) { pendingType = PendingType.IMMEDIATE; checkConflict(activeTasks.isEmpty(), "Cannot request immediate run of a scheduled job which is currently running (%s)", activeTasks);
private Map<String, Long> getNumTasks(List<SingularityRequestWithState> requests) { final CounterMap<String> numTasks = new CounterMap<>(requests.size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIds()) { numTasks.incr(taskId.getRequestId()); } for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) { numTasks.incr(pendingTaskId.getRequestId()); } for (SingularityTaskId cleaningTaskId : taskManager.getCleanupTaskIds()) { Optional<SingularityRequestWithState> request = requestManager.getRequest(cleaningTaskId.getRequestId()); if (request.isPresent() && request.get().getRequest().isScheduled()) { continue; } numTasks.decr(cleaningTaskId.getRequestId()); } return numTasks.toCountMap(); }