/** * Submits a JobGraph detached. * @param jobGraph The JobGraph * @param classLoader User code class loader to deserialize the results and errors (may contain custom classes). * @return JobSubmissionResult * @throws ProgramInvocationException */ public JobSubmissionResult runDetached(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { waitForClusterToBeReady(); final ActorGateway jobManagerGateway; try { jobManagerGateway = getJobManagerGateway(); } catch (Exception e) { throw new ProgramInvocationException("Failed to retrieve the JobManager gateway.", jobGraph.getJobID(), e); } try { logAndSysout("Submitting Job with JobID: " + jobGraph.getJobID() + ". Returning after job submission."); JobClient.submitJobDetached( new AkkaJobManagerGateway(jobManagerGateway), flinkConfig, jobGraph, Time.milliseconds(timeout.toMillis()), classLoader); return new JobSubmissionResult(jobGraph.getJobID()); } catch (JobExecutionException e) { throw new ProgramInvocationException("The program execution failed: " + e.getMessage(), jobGraph.getJobID(), e); } }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { return requestJob(jobId, timeout).thenApply(JobResult::createFrom); }
@Override public CompletableFuture<Collection<String>> requestMetricQueryServicePaths(Time timeout) { final String jobManagerPath = getAddress(); final String jobManagerMetricQueryServicePath = jobManagerPath.substring(0, jobManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME; return CompletableFuture.completedFuture( Collections.singleton(jobManagerMetricQueryServicePath)); }
@Override public CompletableFuture<Collection<Tuple2<ResourceID, String>>> requestTaskManagerMetricQueryServicePaths(Time timeout) { return requestTaskManagerInstances(timeout) .thenApply( (Collection<Instance> instances) -> instances .stream() .map( (Instance instance) -> { final String taskManagerAddress = instance.getTaskManagerGateway().getAddress(); final String taskManagerMetricQuerServicePath = taskManagerAddress.substring(0, taskManagerAddress.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME + '_' + instance.getTaskManagerID().getResourceIdString(); return Tuple2.of(instance.getTaskManagerID(), taskManagerMetricQuerServicePath); }) .collect(Collectors.toList())); }
@Override public CompletableFuture<Collection<String>> requestMetricQueryServicePaths(Time timeout) { final String jobManagerPath = getAddress(); final String jobManagerMetricQueryServicePath = jobManagerPath.substring(0, jobManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME; return CompletableFuture.completedFuture( Collections.singleton(jobManagerMetricQueryServicePath)); }
@Override public CompletableFuture<Collection<Tuple2<ResourceID, String>>> requestTaskManagerMetricQueryServicePaths(Time timeout) { return requestTaskManagerInstances(timeout) .thenApply( (Collection<Instance> instances) -> instances .stream() .map( (Instance instance) -> { final String taskManagerAddress = instance.getTaskManagerGateway().getAddress(); final String taskManagerMetricQuerServicePath = taskManagerAddress.substring(0, taskManagerAddress.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME + '_' + instance.getTaskManagerID().getResourceIdString(); return Tuple2.of(instance.getTaskManagerID(), taskManagerMetricQuerServicePath); }) .collect(Collectors.toList())); }
/** * The class loader necessary to deserialize the result of a job execution, * i.e. JobExecutionResult or Exceptions * @return The class loader for the job id * @throws JobRetrievalException if anything goes wrong */ public ClassLoader getClassLoader() throws JobRetrievalException { if (classLoader == null) { // lazily initializes the class loader when it is needed classLoader = JobClient.retrieveClassLoader( jobID, new AkkaJobManagerGateway(getJobManager()), configuration, highAvailabilityServices, Time.milliseconds(timeout.toMillis())); LOG.info("Reconstructed class loader for Job {}", jobID); } return classLoader; }
@Override public CompletableFuture<Collection<String>> requestMetricQueryServicePaths(Time timeout) { final String jobManagerPath = getAddress(); final String jobManagerMetricQueryServicePath = jobManagerPath.substring(0, jobManagerPath.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME; return CompletableFuture.completedFuture( Collections.singleton(jobManagerMetricQueryServicePath)); }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { return requestJob(jobId, timeout).thenApply(JobResult::createFrom); }
@Override public CompletableFuture<Collection<Tuple2<ResourceID, String>>> requestTaskManagerMetricQueryServicePaths(Time timeout) { return requestTaskManagerInstances(timeout) .thenApply( (Collection<Instance> instances) -> instances .stream() .map( (Instance instance) -> { final String taskManagerAddress = instance.getTaskManagerGateway().getAddress(); final String taskManagerMetricQuerServicePath = taskManagerAddress.substring(0, taskManagerAddress.lastIndexOf('/') + 1) + MetricQueryService.METRIC_QUERY_SERVICE_NAME + '_' + instance.getTaskManagerID().getResourceIdString(); return Tuple2.of(instance.getTaskManagerID(), taskManagerMetricQuerServicePath); }) .collect(Collectors.toList())); }
/** * The class loader necessary to deserialize the result of a job execution, * i.e. JobExecutionResult or Exceptions * @return The class loader for the job id * @throws JobRetrievalException if anything goes wrong */ public ClassLoader getClassLoader() throws JobRetrievalException { if (classLoader == null) { // lazily initializes the class loader when it is needed classLoader = JobClient.retrieveClassLoader( jobID, new AkkaJobManagerGateway(getJobManager()), configuration, highAvailabilityServices, Time.milliseconds(timeout.toMillis())); LOG.info("Reconstructed class loader for Job {}", jobID); } return classLoader; }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { return requestJob(jobId, timeout).thenApply(JobResult::createFrom); }
/** * The class loader necessary to deserialize the result of a job execution, * i.e. JobExecutionResult or Exceptions * @return The class loader for the job id * @throws JobRetrievalException if anything goes wrong */ public ClassLoader getClassLoader() throws JobRetrievalException { if (classLoader == null) { // lazily initializes the class loader when it is needed classLoader = JobClient.retrieveClassLoader( jobID, new AkkaJobManagerGateway(getJobManager()), configuration, highAvailabilityServices, Time.milliseconds(timeout.toMillis())); LOG.info("Reconstructed class loader for Job {}", jobID); } return classLoader; }
@Override protected CompletableFuture<JobManagerGateway> createGateway(CompletableFuture<Tuple2<String, UUID>> leaderFuture) { return FutureUtils.retryWithDelay( () -> leaderFuture.thenCompose( (Tuple2<String, UUID> addressLeaderId) -> FutureUtils.toJava( AkkaUtils.getActorRefFuture( addressLeaderId.f0, actorSystem, FutureUtils.toFiniteDuration(timeout))) .thenApply( (ActorRef jobManagerRef) -> { ActorGateway leaderGateway = new AkkaActorGateway( jobManagerRef, addressLeaderId.f1); return new AkkaJobManagerGateway(leaderGateway); } )), retries, retryDelay, scheduledExecutor); } }
@Override protected CompletableFuture<JobManagerGateway> createGateway(CompletableFuture<Tuple2<String, UUID>> leaderFuture) { return FutureUtils.retryWithDelay( () -> leaderFuture.thenCompose( (Tuple2<String, UUID> addressLeaderId) -> FutureUtils.toJava( AkkaUtils.getActorRefFuture( addressLeaderId.f0, actorSystem, FutureUtils.toFiniteDuration(timeout))) .thenApply( (ActorRef jobManagerRef) -> { ActorGateway leaderGateway = new AkkaActorGateway( jobManagerRef, addressLeaderId.f1); return new AkkaJobManagerGateway(leaderGateway); } )), retries, retryDelay, scheduledExecutor); } }
@Override protected CompletableFuture<JobManagerGateway> createGateway(CompletableFuture<Tuple2<String, UUID>> leaderFuture) { return FutureUtils.retryWithDelay( () -> leaderFuture.thenCompose( (Tuple2<String, UUID> addressLeaderId) -> FutureUtils.toJava( AkkaUtils.getActorRefFuture( addressLeaderId.f0, actorSystem, FutureUtils.toFiniteDuration(timeout))) .thenApply( (ActorRef jobManagerRef) -> { ActorGateway leaderGateway = new AkkaActorGateway( jobManagerRef, addressLeaderId.f1); return new AkkaJobManagerGateway(leaderGateway); } )), retries, retryDelay, scheduledExecutor); } }
/** * Submits a JobGraph detached. * @param jobGraph The JobGraph * @param classLoader User code class loader to deserialize the results and errors (may contain custom classes). * @return JobSubmissionResult * @throws ProgramInvocationException */ public JobSubmissionResult runDetached(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { waitForClusterToBeReady(); final ActorGateway jobManagerGateway; try { jobManagerGateway = getJobManagerGateway(); } catch (Exception e) { throw new ProgramInvocationException("Failed to retrieve the JobManager gateway.", e); } try { logAndSysout("Submitting Job with JobID: " + jobGraph.getJobID() + ". Returning after job submission."); JobClient.submitJobDetached( new AkkaJobManagerGateway(jobManagerGateway), flinkConfig, jobGraph, Time.milliseconds(timeout.toMillis()), classLoader); return new JobSubmissionResult(jobGraph.getJobID()); } catch (JobExecutionException e) { throw new ProgramInvocationException("The program execution failed: " + e.getMessage(), e); } }
final AkkaJobManagerGateway akkaJobManagerGateway = new AkkaJobManagerGateway(jobManagerGateway);
final AkkaJobManagerGateway akkaJobManagerGateway = new AkkaJobManagerGateway(jobManagerGateway);
final AkkaJobManagerGateway akkaJobManagerGateway = new AkkaJobManagerGateway(jobManagerGateway);