@Override public CompletableFuture<Collection<SlotOffer>> offerSlots( TaskManagerLocation taskManagerLocation, TaskManagerGateway taskManagerGateway, Collection<SlotOffer> offers) { validateRunsInMainThread(); List<CompletableFuture<Optional<SlotOffer>>> acceptedSlotOffers = offers.stream().map( offer -> offerSlot( taskManagerLocation, taskManagerGateway, offer) .<Optional<SlotOffer>>thenApply( (acceptedSlot) -> acceptedSlot ? Optional.of(offer) : Optional.empty() ) ).collect(Collectors.toList()); CompletableFuture<Collection<Optional<SlotOffer>>> optionalSlotOffers = FutureUtils.combineAll(acceptedSlotOffers); return optionalSlotOffers.thenApply(collection -> collection.stream() .flatMap(opt -> opt.map(Stream::of).orElseGet(Stream::empty)) .collect(Collectors.toList())); }
@Override public CompletableFuture<Collection<SlotOffer>> offerSlots( TaskManagerLocation taskManagerLocation, TaskManagerGateway taskManagerGateway, Collection<SlotOffer> offers) { validateRunsInMainThread(); List<CompletableFuture<Optional<SlotOffer>>> acceptedSlotOffers = offers.stream().map( offer -> offerSlot( taskManagerLocation, taskManagerGateway, offer) .<Optional<SlotOffer>>thenApply( (acceptedSlot) -> acceptedSlot ? Optional.of(offer) : Optional.empty() ) ).collect(Collectors.toList()); CompletableFuture<Collection<Optional<SlotOffer>>> optionalSlotOffers = FutureUtils.combineAll(acceptedSlotOffers); return optionalSlotOffers.thenApply(collection -> collection.stream() .flatMap(opt -> opt.map(Stream::of).orElseGet(Stream::empty)) .collect(Collectors.toList())); }
@Override public CompletableFuture<Collection<SlotOffer>> offerSlots( TaskManagerLocation taskManagerLocation, TaskManagerGateway taskManagerGateway, Collection<SlotOffer> offers) { validateRunsInMainThread(); List<CompletableFuture<Optional<SlotOffer>>> acceptedSlotOffers = offers.stream().map( offer -> { CompletableFuture<Optional<SlotOffer>> acceptedSlotOffer = offerSlot( taskManagerLocation, taskManagerGateway, offer) .thenApply( (acceptedSlot) -> acceptedSlot ? Optional.of(offer) : Optional.empty() ); return acceptedSlotOffer; } ).collect(Collectors.toList()); CompletableFuture<Collection<Optional<SlotOffer>>> optionalSlotOffers = FutureUtils.combineAll(acceptedSlotOffers); return optionalSlotOffers.thenApply(collection -> collection.stream() .flatMap(opt -> opt.map(Stream::of).orElseGet(Stream::empty)) .collect(Collectors.toList())); }
@Override public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) { List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout)); CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll( individualOptionalJobDetails); CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection); final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails(); return combinedJobDetails.thenApply( (Collection<JobDetails> runningJobDetails) -> { final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size()); allJobDetails.addAll(runningJobDetails); allJobDetails.addAll(completedJobDetails); return new MultipleJobsDetails(allJobDetails); }); }
@Override public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) { List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout)); CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll( individualOptionalJobDetails); CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection); final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails(); return combinedJobDetails.thenApply( (Collection<JobDetails> runningJobDetails) -> { final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size()); allJobDetails.addAll(runningJobDetails); allJobDetails.addAll(completedJobDetails); return new MultipleJobsDetails(allJobDetails); }); }
@Override public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) { List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout)); CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll( individualOptionalJobDetails); CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection); final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails(); return combinedJobDetails.thenApply( (Collection<JobDetails> runningJobDetails) -> { final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size()); allJobDetails.addAll(runningJobDetails); allJobDetails.addAll(completedJobDetails); return new MultipleJobsDetails(allJobDetails); }); }
@Override public CompletableFuture<Collection<Tuple2<ResourceID, String>>> requestTaskManagerMetricQueryServicePaths(Time timeout) { final ArrayList<CompletableFuture<Optional<Tuple2<ResourceID, String>>>> metricQueryServicePathFutures = new ArrayList<>(taskExecutors.size()); for (Map.Entry<ResourceID, WorkerRegistration<WorkerType>> workerRegistrationEntry : taskExecutors.entrySet()) { final ResourceID tmResourceId = workerRegistrationEntry.getKey(); final WorkerRegistration<WorkerType> workerRegistration = workerRegistrationEntry.getValue(); final TaskExecutorGateway taskExecutorGateway = workerRegistration.getTaskExecutorGateway(); final CompletableFuture<Optional<Tuple2<ResourceID, String>>> metricQueryServicePathFuture = taskExecutorGateway .requestMetricQueryServiceAddress(timeout) .thenApply(optional -> optional.map(path -> Tuple2.of(tmResourceId, path))); metricQueryServicePathFutures.add(metricQueryServicePathFuture); } return FutureUtils.combineAll(metricQueryServicePathFutures).thenApply( collection -> collection .stream() .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList())); }
@Override public CompletableFuture<Collection<Tuple2<ResourceID, String>>> requestTaskManagerMetricQueryServicePaths(Time timeout) { final ArrayList<CompletableFuture<Optional<Tuple2<ResourceID, String>>>> metricQueryServicePathFutures = new ArrayList<>(taskExecutors.size()); for (Map.Entry<ResourceID, WorkerRegistration<WorkerType>> workerRegistrationEntry : taskExecutors.entrySet()) { final ResourceID tmResourceId = workerRegistrationEntry.getKey(); final WorkerRegistration<WorkerType> workerRegistration = workerRegistrationEntry.getValue(); final TaskExecutorGateway taskExecutorGateway = workerRegistration.getTaskExecutorGateway(); final CompletableFuture<Optional<Tuple2<ResourceID, String>>> metricQueryServicePathFuture = taskExecutorGateway .requestMetricQueryServiceAddress(timeout) .thenApply(optional -> optional.map(path -> Tuple2.of(tmResourceId, path))); metricQueryServicePathFutures.add(metricQueryServicePathFuture); } return FutureUtils.combineAll(metricQueryServicePathFutures).thenApply( collection -> collection .stream() .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList())); }
preferredLocationsFuture = FutureUtils.combineAll(preferredLocationFutures); break; case ANY:
preferredLocationsFuture = FutureUtils.combineAll(preferredLocationFutures); break; case ANY:
preferredLocationsFuture = FutureUtils.combineAll(preferredLocationFutures); break; case ANY:
reconcileFuture = FutureUtils.combineAll(executionReconcileFutures).whenComplete( (ignore, throwable) -> { boolean allCreated = true;
Collection<JobException> exceptions = FutureUtils.combineAll(futures).get();
final ConjunctFuture<Collection<Void>> allAssignFutures = FutureUtils.combineAll(assignFutures);
@Override public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) { CompletableFuture<ResourceOverview> taskManagerOverviewFuture = resourceManagerGateway.requestResourceOverview(timeout); final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout)); CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation); CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection); final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview(); return allJobsFuture.thenCombine( taskManagerOverviewFuture, (Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> { final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview); return new ClusterOverview(resourceOverview, allJobsOverview); }); }
@Override public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) { CompletableFuture<ResourceOverview> taskManagerOverviewFuture = resourceManagerGateway.requestResourceOverview(timeout); final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout)); CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation); CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection); final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview(); return allJobsFuture.thenCombine( taskManagerOverviewFuture, (Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> { final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview); return new ClusterOverview(resourceOverview, allJobsOverview); }); }
@Override public CompletableFuture<ClusterOverview> requestClusterOverview(Time timeout) { CompletableFuture<ResourceOverview> taskManagerOverviewFuture = resourceManagerGateway.requestResourceOverview(timeout); final List<CompletableFuture<Optional<JobStatus>>> optionalJobInformation = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout)); CompletableFuture<Collection<Optional<JobStatus>>> allOptionalJobsFuture = FutureUtils.combineAll(optionalJobInformation); CompletableFuture<Collection<JobStatus>> allJobsFuture = allOptionalJobsFuture.thenApply(this::flattenOptionalCollection); final JobsOverview completedJobsOverview = archivedExecutionGraphStore.getStoredJobsOverview(); return allJobsFuture.thenCombine( taskManagerOverviewFuture, (Collection<JobStatus> runningJobsStatus, ResourceOverview resourceOverview) -> { final JobsOverview allJobsOverview = JobsOverview.create(runningJobsStatus).combine(completedJobsOverview); return new ClusterOverview(resourceOverview, allJobsOverview); }); }
final ConjunctFuture<Collection<Execution>> allAllocationsFuture = FutureUtils.combineAll(allAllocationFutures);
final ConjunctFuture<Collection<Execution>> allAllocationsFuture = FutureUtils.combineAll(allAllocationFutures);
locationFuture == null ? Collections.EMPTY_LIST : FutureUtils.combineAll(locationFuture).join(); return new Tuple2(toSchedule, new SlotProfile( computeResource(sharingGroup),