@Override protected List<SingularityTaskIdHistory> getFromHistory(final SingularityDeployKey deployKey, int historyStart, int numFromHistory) { return historyManager.getTaskIdHistory(Optional.of(deployKey.getRequestId()), Optional.of(deployKey.getDeployId()), Optional.<String>absent(), Optional.<String> absent(), Optional.<ExtendedTaskState> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<OrderDirection> absent(), Optional.of(historyStart), numFromHistory); }
@Override protected Optional<Integer> getTotalCount(SingularityDeployKey deployKey) { final int numFromZk = taskManager.getInactiveTaskIdsForDeploy(deployKey.getRequestId(), deployKey.getDeployId()).size(); final int numFromHistory = historyManager.getTaskIdHistoryCount(Optional.of(deployKey.getRequestId()), Optional.of(deployKey.getDeployId()), Optional.<String>absent(), Optional.<String> absent(), Optional.<ExtendedTaskState> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent()); return Optional.of(numFromZk + numFromHistory); }
@Override protected List<SingularityDeployHistory> getFromZk(String requestId) { final List<SingularityDeployKey> deployKeys = deployManager.getDeployIdsFor(requestId); final List<SingularityDeployHistory> histories = Lists.newArrayListWithCapacity(deployKeys.size()); for (SingularityDeployKey key : deployKeys) { Optional<SingularityDeployHistory> deployHistory = deployManager.getDeployHistory(key.getRequestId(), key.getDeployId(), false); if (deployHistory.isPresent()) { histories.add(deployHistory.get()); } } Collections.sort(histories); return histories; }
public Map<SingularityDeployKey, SingularityDeploy> getDeploysForKeys(Collection<SingularityDeployKey> deployKeys) { final List<String> paths = Lists.newArrayListWithCapacity(deployKeys.size()); for (SingularityDeployKey deployKey : deployKeys) { paths.add(getDeployDataPath(deployKey.getRequestId(), deployKey.getDeployId())); } final List<SingularityDeploy> deploys = getAsync("getDeploysForKeys", paths, deployTranscoder, deploysCache); final Map<SingularityDeployKey, SingularityDeploy> deployKeyToDeploy = Maps.uniqueIndex(deploys, new Function<SingularityDeploy, SingularityDeployKey>() { @Override public SingularityDeployKey apply(SingularityDeploy input) { return SingularityDeployKey.fromDeploy(input); } }); return deployKeyToDeploy; }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
private List<SingularityTaskId> getMatchingTaskIds(SingularityRequest request, SingularityDeployKey deployKey) { List<SingularityTaskId> activeTaskIdsForRequest = leaderCache.getActiveTaskIdsForRequest(deployKey.getRequestId()); if (request.isLongRunning()) { Set<SingularityTaskId> killedTaskIds = leaderCache.getKilledTasks().stream() .map(SingularityKilledTaskIdRecord::getTaskId) .collect(Collectors.toSet()); List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId taskId : activeTaskIdsForRequest) { if (!taskId.getDeployId().equals(deployKey.getDeployId())) { continue; } if (leaderCache.getCleanupTaskIds().contains(taskId)) { continue; } if (killedTaskIds.contains(taskId)) { continue; } matchingTaskIds.add(taskId); } return matchingTaskIds; } else { return new ArrayList<>(activeTaskIdsForRequest); } }
@Override protected List<SingularityTaskIdHistory> getFromZk(final SingularityDeployKey deployKey) { List<SingularityTaskId> deployTaskIds = taskManager.getInactiveTaskIdsForDeploy(deployKey.getRequestId(), deployKey.getDeployId()); return getTaskHistoriesFor(taskManager, deployTaskIds); }
public SingularityDeleteResult deleteDeployHistory(SingularityDeployKey deployKey) { return delete(getDeployParentPath(deployKey.getRequestId(), deployKey.getDeployId())); }
SingularityRequestDeployState deployState = byRequestId.get(deployKey.getRequestId()); Optional<SingularityDeployHistory> deployHistory = deployManager.getDeployHistory(deployKey.getRequestId(), deployKey.getDeployId(), true); deployHistoryByRequestId.put(deployKey.getRequestId(), deployHistory.get()); } else { LOG.info("Deploy history for key {} not found", deployKey);
private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) { final String requestId = deployKey.getRequestId(); final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId); final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());
@Override protected List<SingularityTaskIdHistory> getFromHistory(final SingularityDeployKey deployKey, int historyStart, int numFromHistory) { return historyManager.getTaskIdHistory(Optional.of(deployKey.getRequestId()), Optional.of(deployKey.getDeployId()), Optional.<String>absent(), Optional.<String> absent(), Optional.<ExtendedTaskState> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<OrderDirection> absent(), Optional.of(historyStart), numFromHistory); }
@Override protected Optional<Integer> getTotalCount(SingularityDeployKey deployKey) { final int numFromZk = taskManager.getInactiveTaskIdsForDeploy(deployKey.getRequestId(), deployKey.getDeployId()).size(); final int numFromHistory = historyManager.getTaskIdHistoryCount(Optional.of(deployKey.getRequestId()), Optional.of(deployKey.getDeployId()), Optional.<String>absent(), Optional.<String> absent(), Optional.<ExtendedTaskState> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent(), Optional.<Long> absent()); return Optional.of(numFromZk + numFromHistory); }
@Override protected List<SingularityDeployHistory> getFromZk(String requestId) { final List<SingularityDeployKey> deployKeys = deployManager.getDeployIdsFor(requestId); final List<SingularityDeployHistory> histories = Lists.newArrayListWithCapacity(deployKeys.size()); for (SingularityDeployKey key : deployKeys) { Optional<SingularityDeployHistory> deployHistory = deployManager.getDeployHistory(key.getRequestId(), key.getDeployId(), false); if (deployHistory.isPresent()) { histories.add(deployHistory.get()); } } Collections.sort(histories); return histories; }
public Map<SingularityDeployKey, SingularityDeploy> getDeploysForKeys(Collection<SingularityDeployKey> deployKeys) { final List<String> paths = Lists.newArrayListWithCapacity(deployKeys.size()); for (SingularityDeployKey deployKey : deployKeys) { paths.add(getDeployDataPath(deployKey.getRequestId(), deployKey.getDeployId())); } final List<SingularityDeploy> deploys = getAsync("getDeploysForKeys", paths, deployTranscoder, deploysCache); final Map<SingularityDeployKey, SingularityDeploy> deployKeyToDeploy = Maps.uniqueIndex(deploys, new Function<SingularityDeploy, SingularityDeployKey>() { @Override public SingularityDeployKey apply(SingularityDeploy input) { return SingularityDeployKey.fromDeploy(input); } }); return deployKeyToDeploy; }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
private List<SingularityTaskId> getMatchingTaskIds(SingularityRequest request, SingularityDeployKey deployKey) { List<SingularityTaskId> activeTaskIdsForRequest = leaderCache.getActiveTaskIdsForRequest(deployKey.getRequestId()); if (request.isLongRunning()) { Set<SingularityTaskId> killedTaskIds = leaderCache.getKilledTasks().stream() .map(SingularityKilledTaskIdRecord::getTaskId) .collect(Collectors.toSet()); List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId taskId : activeTaskIdsForRequest) { if (!taskId.getDeployId().equals(deployKey.getDeployId())) { continue; } if (leaderCache.getCleanupTaskIds().contains(taskId)) { continue; } if (killedTaskIds.contains(taskId)) { continue; } matchingTaskIds.add(taskId); } return matchingTaskIds; } else { return new ArrayList<>(activeTaskIdsForRequest); } }
@Override protected List<SingularityTaskIdHistory> getFromZk(final SingularityDeployKey deployKey) { List<SingularityTaskId> deployTaskIds = taskManager.getInactiveTaskIdsForDeploy(deployKey.getRequestId(), deployKey.getDeployId()); return getTaskHistoriesFor(taskManager, deployTaskIds); }
public SingularityDeleteResult deleteDeployHistory(SingularityDeployKey deployKey) { return delete(getDeployParentPath(deployKey.getRequestId(), deployKey.getDeployId())); }
SingularityRequestDeployState deployState = byRequestId.get(deployKey.getRequestId()); Optional<SingularityDeployHistory> deployHistory = deployManager.getDeployHistory(deployKey.getRequestId(), deployKey.getDeployId(), true); deployHistoryByRequestId.put(deployKey.getRequestId(), deployHistory.get()); } else { LOG.info("Deploy history for key {} not found", deployKey);
private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) { final String requestId = deployKey.getRequestId(); final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId); final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId());