@Override public JobExecutionResult execute(String jobName) throws Exception { OptimizedPlan op = compileProgram(jobName); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jobGraph = jgg.compileJobGraph(op); for (Path jarFile: jarFiles) { jobGraph.addJar(jarFile); } jobGraph.setClasspaths(new ArrayList<>(classPaths)); this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph); return this.lastJobExecutionResult; }
finalizeBulkIteration(iteration); } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) { finalizeWorksetIteration(iteration); } else { throw new CompilerException(); attachOperatorNamesAndDescriptions(); .map(entry -> Tuple2.of(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); addUserArtifactEntries(userArtifacts, graph);
DistributionPattern pattern = connectJobVertices( inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast); if (inConn.isOnDynamicPath()) { numChannelsDynamicPath++; numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern, sourceVertex.getParallelism(), targetVertex.getParallelism()); addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast); return 1;
/** * Translates a {@link org.apache.flink.optimizer.plan.OptimizedPlan} into a * {@link org.apache.flink.runtime.jobgraph.JobGraph}. * * @param program Optimized plan that is translated into a JobGraph. * @return JobGraph generated from the plan. */ public JobGraph compileJobGraph(OptimizedPlan program) { return compileJobGraph(program, null); }
finalizeBulkIteration(iteration); } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) { finalizeWorksetIteration(iteration); } else { throw new CompilerException(); attachOperatorNamesAndDescriptions();
assignDriverResources(node, config); return vertex;
private JobGraph createJobGraph() { // make sure that all vertices start immediately jobGraph.setScheduleMode(ScheduleMode.EAGER); // Generate deterministic hashes for the nodes in order to identify them across // submission iff they didn't change. Map<Integer, byte[]> hashes = defaultStreamGraphHasher.traverseStreamGraphAndGenerateHashes(streamGraph); // Generate legacy version hashes for backwards compatibility List<Map<Integer, byte[]>> legacyHashes = new ArrayList<>(legacyStreamGraphHashers.size()); for (StreamGraphHasher hasher : legacyStreamGraphHashers) { legacyHashes.add(hasher.traverseStreamGraphAndGenerateHashes(streamGraph)); } Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes = new HashMap<>(); setChaining(hashes, legacyHashes, chainedOperatorHashes); setPhysicalEdges(); setSlotSharingAndCoLocation(); configureCheckpointing(); JobGraphGenerator.addUserArtifactEntries(streamGraph.getEnvironment().getCachedFiles(), jobGraph); // set the ExecutionConfig last when it has been finalized try { jobGraph.setExecutionConfig(streamGraph.getExecutionConfig()); } catch (IOException e) { throw new IllegalConfigurationException("Could not serialize the ExecutionConfig." + "This indicates that non-serializable types (like custom serializers) were registered"); } return jobGraph; }
assignLocalStrategyResources(channel, config, inputNum);
/** * Translates a {@link org.apache.flink.optimizer.plan.OptimizedPlan} into a * {@link org.apache.flink.runtime.jobgraph.JobGraph}. * * @param program Optimized plan that is translated into a JobGraph. * @return JobGraph generated from the plan. */ public JobGraph compileJobGraph(OptimizedPlan program) { return compileJobGraph(program, null); }
finalizeBulkIteration(iteration); } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) { finalizeWorksetIteration(iteration); } else { throw new CompilerException(); attachOperatorNamesAndDescriptions();
private JobVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); final JobVertex vertex = new JobVertex(taskName); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); // set the driver strategy config.setDriver(ds.getDriverClass()); config.setDriverStrategy(ds); if (node.getComparator1() != null) { config.setDriverComparator(node.getComparator1(), 0); } if (node.getComparator2() != null) { config.setDriverComparator(node.getComparator2(), 1); } if (node.getPairComparator() != null) { config.setDriverPairComparator(node.getPairComparator()); } // assign memory, file-handles, etc. assignDriverResources(node, config); return vertex; }
@Test public void testArtifactCompression() throws IOException { Path plainFile1 = tmp.newFile("plainFile1").toPath(); Path plainFile2 = tmp.newFile("plainFile2").toPath(); Path directory1 = tmp.newFolder("directory1").toPath(); Files.createDirectory(directory1.resolve("containedFile1")); Path directory2 = tmp.newFolder("directory2").toPath(); Files.createDirectory(directory2.resolve("containedFile2")); JobGraph jb = new JobGraph(); final String executableFileName = "executableFile"; final String nonExecutableFileName = "nonExecutableFile"; final String executableDirName = "executableDir"; final String nonExecutableDirName = "nonExecutableDIr"; Collection<Tuple2<String, DistributedCache.DistributedCacheEntry>> originalArtifacts = Arrays.asList( Tuple2.of(executableFileName, new DistributedCache.DistributedCacheEntry(plainFile1.toString(), true)), Tuple2.of(nonExecutableFileName, new DistributedCache.DistributedCacheEntry(plainFile2.toString(), false)), Tuple2.of(executableDirName, new DistributedCache.DistributedCacheEntry(directory1.toString(), true)), Tuple2.of(nonExecutableDirName, new DistributedCache.DistributedCacheEntry(directory2.toString(), false)) ); JobGraphGenerator.addUserArtifactEntries(originalArtifacts, jb); Map<String, DistributedCache.DistributedCacheEntry> submittedArtifacts = jb.getUserArtifacts(); DistributedCache.DistributedCacheEntry executableFileEntry = submittedArtifacts.get(executableFileName); assertState(executableFileEntry, true, false); DistributedCache.DistributedCacheEntry nonExecutableFileEntry = submittedArtifacts.get(nonExecutableFileName); assertState(nonExecutableFileEntry, false, false); DistributedCache.DistributedCacheEntry executableDirEntry = submittedArtifacts.get(executableDirName); assertState(executableDirEntry, true, true); DistributedCache.DistributedCacheEntry nonExecutableDirEntry = submittedArtifacts.get(nonExecutableDirName); assertState(nonExecutableDirEntry, false, true); }
assignLocalStrategyResources(channel, config, inputNum);
public static JobGraph getJobGraph(Configuration flinkConfig, FlinkPlan optPlan, List<URL> jarFiles, List<URL> classpaths, SavepointRestoreSettings savepointSettings) { JobGraph job; if (optPlan instanceof StreamingPlan) { job = ((StreamingPlan) optPlan).getJobGraph(); job.setSavepointRestoreSettings(savepointSettings); } else { JobGraphGenerator gen = new JobGraphGenerator(flinkConfig); job = gen.compileJobGraph((OptimizedPlan) optPlan); } for (URL jar : jarFiles) { try { job.addJar(new Path(jar.toURI())); } catch (URISyntaxException e) { throw new RuntimeException("URL is invalid. This should not happen.", e); } } job.setClasspaths(classpaths); return job; }
/** * Translates a {@link org.apache.flink.optimizer.plan.OptimizedPlan} into a * {@link org.apache.flink.runtime.jobgraph.JobGraph}. * * @param program Optimized plan that is translated into a JobGraph. * @return JobGraph generated from the plan. */ public JobGraph compileJobGraph(OptimizedPlan program) { return compileJobGraph(program, null); }
finalizeBulkIteration(iteration); } else if (iteration.getIterationNode() instanceof WorksetIterationPlanNode) { finalizeWorksetIteration(iteration); } else { throw new CompilerException(); attachOperatorNamesAndDescriptions(); .map(entry -> Tuple2.of(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); addUserArtifactEntries(userArtifacts, graph);
DistributionPattern pattern = connectJobVertices( inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast); if (inConn.isOnDynamicPath()) { numChannelsDynamicPath++; numDynamicSenderTasksTotal += getNumberOfSendersPerReceiver(pattern, sourceVertex.getParallelism(), targetVertex.getParallelism()); addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast); return 1;
assignDriverResources(node, config); return vertex;
private JobGraph createJobGraph() { // make sure that all vertices start immediately jobGraph.setScheduleMode(ScheduleMode.EAGER); // Generate deterministic hashes for the nodes in order to identify them across // submission iff they didn't change. Map<Integer, byte[]> hashes = defaultStreamGraphHasher.traverseStreamGraphAndGenerateHashes(streamGraph); // Generate legacy version hashes for backwards compatibility List<Map<Integer, byte[]>> legacyHashes = new ArrayList<>(legacyStreamGraphHashers.size()); for (StreamGraphHasher hasher : legacyStreamGraphHashers) { legacyHashes.add(hasher.traverseStreamGraphAndGenerateHashes(streamGraph)); } Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes = new HashMap<>(); setChaining(hashes, legacyHashes, chainedOperatorHashes); setPhysicalEdges(); setSlotSharingAndCoLocation(); configureCheckpointing(); JobGraphGenerator.addUserArtifactEntries(streamGraph.getEnvironment().getCachedFiles(), jobGraph); // set the ExecutionConfig last when it has been finalized try { jobGraph.setExecutionConfig(streamGraph.getExecutionConfig()); } catch (IOException e) { throw new IllegalConfigurationException("Could not serialize the ExecutionConfig." + "This indicates that non-serializable types (like custom serializers) were registered"); } return jobGraph; }
assignLocalStrategyResources(channel, config, inputNum);