RocksDBStateDataTransfer(int threadNum) { if (threadNum > 1) { executorService = Executors.newFixedThreadPool(threadNum); } else { executorService = newDirectExecutorService(); } }
/** * Creates a instance that submits the programs to the JobManager defined in the * configuration. This method will try to resolve the JobManager hostname and throw an exception * if that is not possible. * * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer. * * @throws Exception we cannot create the high availability services */ public ClusterClient(Configuration flinkConfig) throws Exception { this( flinkConfig, HighAvailabilityServicesUtils.createHighAvailabilityServices( flinkConfig, Executors.directExecutor(), HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION), false); }
public YarnApplicationStatusMonitor( YarnClient yarnClient, ApplicationId yarnApplicationId, ScheduledExecutor scheduledExecutor) { this.yarnClient = Preconditions.checkNotNull(yarnClient); this.yarnApplicationId = Preconditions.checkNotNull(yarnApplicationId); applicationStatusUpdateFuture = scheduledExecutor.scheduleWithFixedDelay( this::updateApplicationStatus, 0L, UPDATE_INTERVAL, TimeUnit.MILLISECONDS); applicationStatus = ApplicationStatus.UNKNOWN; }
private static ClusterClient<String> createFailingClusterClient(Exception expectedException) throws Exception { final ClusterClient<String> clusterClient = mock(ClusterClient.class); when(clusterClient.triggerSavepoint(any(JobID.class), nullable(String.class))) .thenReturn(FutureUtils.completedExceptionally(expectedException)); return clusterClient; } }
private <C> CompletableFuture<C> retry( CheckedSupplier<CompletableFuture<C>> operation, Predicate<Throwable> retryPredicate) { return FutureUtils.retryWithDelay( CheckedSupplier.unchecked(operation), restClusterClientConfiguration.getRetryMaxAttempts(), Time.milliseconds(restClusterClientConfiguration.getRetryDelay()), retryPredicate, new ScheduledExecutorServiceAdapter(retryExecutorService)); }
private void waitForTaskManagers(int numberOfTaskManagers, DispatcherGateway dispatcherGateway, FiniteDuration timeLeft) throws ExecutionException, InterruptedException { FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(Time.milliseconds(timeLeft.toMillis())), Time.milliseconds(50L), org.apache.flink.api.common.time.Deadline.fromNow(Duration.ofMillis(timeLeft.toMillis())), clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= numberOfTaskManagers, new ScheduledExecutorServiceAdapter(Executors.newSingleThreadScheduledExecutor())) .get(); }
/** * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their * {@link StateHandleID}. */ private void downloadDataForAllStateHandles( Map<StateHandleID, StreamStateHandle> stateHandleMap, Path restoreInstancePath, CloseableRegistry closeableRegistry) throws Exception { try { List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry); List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size()); for (Runnable runnable : runnables) { futures.add(CompletableFuture.runAsync(runnable, executorService)); } FutureUtils.waitForAll(futures).get(); } catch (ExecutionException e) { Throwable throwable = ExceptionUtils.stripExecutionException(e); throwable = ExceptionUtils.stripException(throwable, RuntimeException.class); if (throwable instanceof IOException) { throw (IOException) throwable; } else { throw new FlinkRuntimeException("Failed to download data for state handles.", e); } } }
/** * Tries to shut down the given actor gracefully. * * @param actorRef specifying the actor to shut down * @param timeout for the graceful shut down * @return A future that finishes with {@code true} iff. the actor could be stopped gracefully * or {@code actorRef} was {@code null}. */ private CompletableFuture<Boolean> stopActor(@Nullable final ActorRef actorRef, FiniteDuration timeout) { if (actorRef == null) { return CompletableFuture.completedFuture(true); } return FutureUtils.toJava(Patterns.gracefulStop(actorRef, timeout)) .exceptionally( (Throwable throwable) -> { // The actor did not stop gracefully in time, try to directly stop it actorSystem.stop(actorRef); log.warn("Could not stop actor {} gracefully.", actorRef.path(), throwable); return false; } ); }
@Override protected CompletableFuture<Void> stopClusterServices(boolean cleanupHaData) { final CompletableFuture<Void> serviceShutDownFuture = super.stopClusterServices(cleanupHaData); return FutureUtils.runAfterwards( serviceShutDownFuture, () -> { if (mesosServices != null) { mesosServices.close(cleanupHaData); } }); }
@VisibleForTesting CompletableFuture<URL> getWebMonitorBaseUrl() { return FutureUtils.orTimeout( webMonitorLeaderRetriever.getLeaderFuture(), restClusterClientConfiguration.getAwaitLeaderTimeout(), TimeUnit.MILLISECONDS) .thenApplyAsync(leaderAddressSessionId -> { final String url = leaderAddressSessionId.f0; try { return new URL(url); } catch (MalformedURLException e) { throw new IllegalArgumentException("Could not parse URL from " + url, e); } }, executorService); } }
private Collection<JobID> waitForRunningJobs(ClusterClient<?> clusterClient, Time timeout) throws ExecutionException, InterruptedException { return FutureUtils.retrySuccessfulWithDelay( CheckedSupplier.unchecked(clusterClient::listJobs), Time.milliseconds(50L), Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())), jobs -> !jobs.isEmpty(), TestingUtils.defaultScheduledExecutor()) .get() .stream() .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); }
public OperatorSnapshotFinalizer( @Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException { SnapshotResult<KeyedStateHandle> keyedManaged = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture()); SnapshotResult<KeyedStateHandle> keyedRaw = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture()); SnapshotResult<OperatorStateHandle> operatorManaged = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture()); SnapshotResult<OperatorStateHandle> operatorRaw = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture()); jobManagerOwnedState = new OperatorSubtaskState( operatorManaged.getJobManagerOwnedSnapshot(), operatorRaw.getJobManagerOwnedSnapshot(), keyedManaged.getJobManagerOwnedSnapshot(), keyedRaw.getJobManagerOwnedSnapshot() ); taskLocalState = new OperatorSubtaskState( operatorManaged.getTaskLocalSnapshot(), operatorRaw.getTaskLocalSnapshot(), keyedManaged.getTaskLocalSnapshot(), keyedRaw.getTaskLocalSnapshot() ); }
@Override public <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P> sendRequest( final String targetAddress, final int targetPort, final M messageHeaders, final U messageParameters, final R request) throws IOException { if (failHttpRequest.test(messageHeaders, messageParameters, request)) { return FutureUtils.completedExceptionally(new IOException("expected")); } else { return super.sendRequest(targetAddress, targetPort, messageHeaders, messageParameters, request); } } };
@Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); jarDir = temporaryFolder.newFolder().toPath(); jarUploadHandler = new JarUploadHandler( () -> CompletableFuture.completedFuture(mockDispatcherGateway), Time.seconds(10), Collections.emptyMap(), JarUploadHeaders.getInstance(), jarDir, Executors.directExecutor()); }
FutureUtils.waitForAll(futures.values()).get();
@Override protected CompletableFuture<Void> stopClusterServices(boolean cleanupHaData) { final CompletableFuture<Void> serviceShutDownFuture = super.stopClusterServices(cleanupHaData); return FutureUtils.runAfterwards( serviceShutDownFuture, () -> { if (mesosServices != null) { mesosServices.close(cleanupHaData); } }); }
private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time timeout) throws ExecutionException, InterruptedException { FutureUtils.retrySuccessfulWithDelay( () -> dispatcherGateway.requestClusterOverview(timeout), Time.milliseconds(50L), Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())), clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= 1 && clusterOverview.getNumSlotsAvailable() == 0 && clusterOverview.getNumSlotsTotal() == 2, TestingUtils.defaultScheduledExecutor()) .get(); }
@Override protected CompletableFuture<JobExecutionResultResponseBody> handleRequest( @Nonnull HandlerRequest<EmptyRequestBody, JobMessageParameters> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException { if (jobExecutionResults.hasNext()) { lastJobExecutionResult = jobExecutionResults.next(); } checkState(lastJobExecutionResult != null); if (lastJobExecutionResult instanceof JobExecutionResultResponseBody) { return CompletableFuture.completedFuture((JobExecutionResultResponseBody) lastJobExecutionResult); } else if (lastJobExecutionResult instanceof RestHandlerException) { return FutureUtils.completedExceptionally((RestHandlerException) lastJobExecutionResult); } else { throw new AssertionError(); } } }
@Override public void close() throws Exception { // Free cluster resources clusterClient.cancel(jobId); // cancel() is non-blocking so do this to make sure the job finished CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay( () -> clusterClient.getJobStatus(jobId), Time.milliseconds(50), deadline, (jobStatus) -> jobStatus.equals(JobStatus.CANCELED), TestingUtils.defaultScheduledExecutor()); assertEquals( JobStatus.CANCELED, jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS)); } }
@Override public CompletableFuture<Void> postStop() { // shut down all components Throwable firstException = null; if (resourceManagerClient != null) { try { resourceManagerClient.stop(); } catch (Throwable t) { firstException = t; } } if (nodeManagerClient != null) { try { nodeManagerClient.stop(); } catch (Throwable t) { firstException = ExceptionUtils.firstOrSuppressed(t, firstException); } } final CompletableFuture<Void> terminationFuture = super.postStop(); if (firstException != null) { return FutureUtils.completedExceptionally(new FlinkException("Error while shutting down YARN resource manager", firstException)); } else { return terminationFuture; } }