@Override public JobExecutionResult execute(String jobName) throws Exception { final StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); final JobGraph jobGraph = streamGraph.getJobGraph(); for (Path jarFile : jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); return jobExecutor.executeJobBlocking(jobGraph); }
@Override public JobExecutionResult execute(String jobName) throws Exception { OptimizedPlan op = compileProgram(jobName); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jobGraph = jgg.compileJobGraph(op); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph); return this.lastJobExecutionResult; }
public static JobGraph getJobGraph(Configuration flinkConfig, FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) { JobGraph job; if (optPlan instanceof StreamingPlan) { job = ((StreamingPlan) optPlan).getJobGraph(); job.setSavepointRestoreSettings(savepointSettings); } else { JobGraphGenerator gen = new JobGraphGenerator(flinkConfig); job = gen.compileJobGraph((OptimizedPlan) optPlan); } for (URL jar : jarFiles) { try { job.addJar(new Path(jar.toURI())); } catch (URISyntaxException e) { throw new RuntimeException("URL is invalid. This should not happen.", e); } } job.setClasspaths(classpaths); return job; }
private void initJobGraph() throws IOException { stopJobSignal = YarnTestJob.StopJobSignal.usingMarkerFile(FOLDER.newFile().toPath()); job = YarnTestJob.stoppableJob(stopJobSignal); final File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests")); assertThat(testingJar, notNullValue()); job.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI())); }
jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
jobGraph.addJar(new Path(url.toURI())); } catch (URISyntaxException e) { throw new ProgramInvocationException("Invalid URL for jar file: " + url + '.', jobGraph.getJobID(), e);
@Override public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { final JobGraph jobGraph = streamGraph.getJobGraph(); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); return (JobExecutionResult) jobExecutor.executeJob(jobGraph, false); }
@Override public JobExecutionResult execute(String jobName) throws Exception { final StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); final JobGraph jobGraph = streamGraph.getJobGraph(); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); return miniCluster.submitJobAndWait(jobGraph, false); }
@Override public JobExecutionResult execute(String jobName) throws Exception { OptimizedPlan op = compileProgram(jobName); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jobGraph = jgg.compileJobGraph(op); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph); return this.lastJobExecutionResult; }
@Override public JobExecutionResult execute(String jobName) throws Exception { final StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); final JobGraph jobGraph = streamGraph.getJobGraph(); for (Path jarFile : jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); return jobExecutor.executeJobBlocking(jobGraph); }
@Override public JobExecutionResult execute(String jobName) throws Exception { final StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); final JobGraph jobGraph = streamGraph.getJobGraph(); for (Path jarFile : jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); return jobExecutor.executeJobBlocking(jobGraph); }
@Override public JobExecutionResult execute(String jobName) throws Exception { OptimizedPlan op = compileProgram(jobName); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jobGraph = jgg.compileJobGraph(op); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); this.lastJobExecutionResult = miniCluster.submitJobAndWait(jobGraph, false); return this.lastJobExecutionResult; }
@Override public JobExecutionResult execute(String jobName) throws Exception { OptimizedPlan op = compileProgram(jobName); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jobGraph = jgg.compileJobGraph(op); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph); return this.lastJobExecutionResult; }
public static JobGraph getJobGraph(Configuration flinkConfig, FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) { JobGraph job; if (optPlan instanceof StreamingPlan) { job = ((StreamingPlan) optPlan).getJobGraph(); job.setSavepointRestoreSettings(savepointSettings); } else { JobGraphGenerator gen = new JobGraphGenerator(flinkConfig); job = gen.compileJobGraph((OptimizedPlan) optPlan); } for (URL jar : jarFiles) { try { job.addJar(new Path(jar.toURI())); } catch (URISyntaxException e) { throw new RuntimeException("URL is invalid. This should not happen.", e); } } job.setClasspaths(classpaths); return job; }
public static JobGraph getJobGraph(Configuration flinkConfig, FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, List<URI> libjars, List<URI> files, SavepointRestoreSettings savepointSettings) { JobGraph job; if (optPlan instanceof StreamingPlan) { job = ((StreamingPlan) optPlan).getJobGraph(); job.setSavepointRestoreSettings(savepointSettings); } else { JobGraphGenerator gen = new JobGraphGenerator(flinkConfig); job = gen.compileJobGraph((OptimizedPlan) optPlan); } for (URL jar : jarFiles) { try { job.addJar(new Path(jar.toURI())); } catch (URISyntaxException e) { throw new RuntimeException("URL is invalid. This should not happen.", e); } } for (URI libjar : libjars) { job.addJar(new Path(libjar)); } for (URI file : files) { final String fileKey = file.getFragment() != null ? file.getFragment() : new Path(file).getName(); // Remove the part after '#' in file path since this part has been already set to file key. job.addUserArtifact(fileKey, new DistributedCacheEntry( org.apache.commons.lang3.StringUtils.substringBeforeLast(file.toString(), "#"), false, false)); } job.setClasspaths(classpaths); return job; }
@Override protected JobSubmissionResult executeInternal(String jobName, boolean detached, SavepointRestoreSettings savepointRestoreSettings) throws Exception { final StreamGraph streamGraph = getStreamGraph(); streamGraph.setJobName(jobName); final JobGraph jobGraph = streamGraph.getJobGraph(); for (Path jarFile : jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); for (Tuple2<String, DistributedCache.DistributedCacheEntry> file : cacheFile) { jobGraph.addUserArtifact(file.f0, file.f1); } return jobExecutor.executeJob(jobGraph, detached); }
private JobGraph getJobGraph(FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) { JobGraph job; if (optPlan instanceof StreamingPlan) { job = ((StreamingPlan) optPlan).getJobGraph(); job.setSavepointRestoreSettings(savepointSettings); } else { JobGraphGenerator gen = new JobGraphGenerator(this.flinkConfig); job = gen.compileJobGraph((OptimizedPlan) optPlan); } for (URL jar : jarFiles) { try { job.addJar(new Path(jar.toURI())); } catch (URISyntaxException e) { throw new RuntimeException("URL is invalid. This should not happen.", e); } } job.setClasspaths(classpaths); return job; }
jobGraph.addJar(new Path(uploadedJarUri));
@Override public JobSubmissionResult executeInternal(String jobName, boolean detached) throws Exception { OptimizedPlan op = compileProgram(jobName); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jobGraph = jgg.compileJobGraph(op); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); JobSubmissionResult submissionResult = jobExecutor.executeJob(jobGraph, detached); if (submissionResult.isJobExecutionResult()) { this.lastJobExecutionResult = submissionResult.getJobExecutionResult(); } return submissionResult; }
protected Tuple2<JobGraph, ClassLoader> getJobGraphAndClassLoader(JarActionHandlerConfig config) throws Exception { // generate the graph JobGraph graph = null; PackagedProgram program = new PackagedProgram( new File(jarDir, config.getJarFile()), config.getEntryClass(), config.getProgramArgs()); ClassLoader classLoader = program.getUserCodeClassLoader(); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), new Configuration()); FlinkPlan plan = ClusterClient.getOptimizedPlan(optimizer, program, config.getParallelism()); if (plan instanceof StreamingPlan) { graph = ((StreamingPlan) plan).getJobGraph(); } else if (plan instanceof OptimizedPlan) { graph = new JobGraphGenerator().compileJobGraph((OptimizedPlan) plan); } if (graph == null) { throw new CompilerException("A valid job graph couldn't be generated for the jar."); } // Set the savepoint settings graph.setSavepointRestoreSettings(config.getSavepointRestoreSettings()); for (URL jar : program.getAllLibraries()) { try { graph.addJar(new Path(jar.toURI())); } catch (URISyntaxException e) { throw new ProgramInvocationException("Invalid jar path. Unexpected error. :("); } } return Tuple2.of(graph, classLoader); }