private PackagedProgram createPackagedProgram() throws FlinkException { try { final Class<?> mainClass = getClass().getClassLoader().loadClass(jobClassName); return new PackagedProgram(mainClass, programArguments); } catch (ClassNotFoundException | ProgramInvocationException e) { throw new FlinkException("Could not load the provided entrypoint class.", e); } } }
public JobGraph toJobGraph(Configuration configuration) { if (!Files.exists(jarFile)) { throw new CompletionException(new RestHandlerException( String.format("Jar file %s does not exist", jarFile), HttpResponseStatus.BAD_REQUEST)); } try { final PackagedProgram packagedProgram = new PackagedProgram( jarFile.toFile(), entryClass, programArgs.toArray(new String[0])); return PackagedProgramUtils.createJobGraph(packagedProgram, configuration, parallelism); } catch (final ProgramInvocationException e) { throw new CompletionException(e); } } }
/** * Creates a Packaged program from the given command line options. * * @return A PackagedProgram (upon success) */ PackagedProgram buildProgram(ProgramOptions options) throws FileNotFoundException, ProgramInvocationException { String[] programArgs = options.getProgramArgs(); String jarFilePath = options.getJarFilePath(); List<URL> classpaths = options.getClasspaths(); if (jarFilePath == null) { throw new IllegalArgumentException("The program JAR file was not specified."); } File jarFile = new File(jarFilePath); // Check if JAR file exists if (!jarFile.exists()) { throw new FileNotFoundException("JAR file does not exist: " + jarFile); } else if (!jarFile.isFile()) { throw new FileNotFoundException("JAR file is not a file: " + jarFile); } // Get assembler class String entryPointClass = options.getEntryPointClassName(); PackagedProgram program = entryPointClass == null ? new PackagedProgram(jarFile, classpaths, programArgs) : new PackagedProgram(jarFile, classpaths, entryPointClass, programArgs); program.setSavepointRestoreSettings(options.getSavepointRestoreSettings()); return program; }
program = new PackagedProgram(f, clazz, new String[0]); } catch (Exception ignored) {
@Test public void testCustomSplitJobWithCustomClassLoaderPath() throws IOException, ProgramInvocationException { URL classpath = new File(INPUT_SPLITS_PROG_JAR_FILE).toURI().toURL(); PackagedProgram inputSplitTestProg2 = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.<Path>emptyList(), Collections.singleton(classpath)); inputSplitTestProg2.invokeInteractiveModeForExecution(); }
@Test public void testExternalProgram() throws Exception { TestEnvironment.setAsContext( MINI_CLUSTER, PARALLELISM, Collections.singleton(new Path(JAR_FILE)), Collections.emptyList()); String testData = getClass().getResource(TEST_DATA_FILE).toString(); PackagedProgram program = new PackagedProgram(new File(JAR_FILE), new String[]{testData}); program.invokeInteractiveModeForExecution(); } }
@Test public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingInputSplitTestProg.invokeInteractiveModeForExecution(); }
@Test public void testUserCodeTypeJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(USERCODETYPE_JAR_PATH)), Collections.<URL>emptyList()); userCodeTypeProg.invokeInteractiveModeForExecution(); }
@Test public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException { // regular streaming job PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)), Collections.<URL>emptyList()); streamingProg.invokeInteractiveModeForExecution(); }
@Test public void testCustomSplitJobWithCustomClassLoaderJar() throws IOException, ProgramInvocationException { PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)), Collections.<URL>emptyList()); inputSplitTestProg.invokeInteractiveModeForExecution(); }
@Test public void testKMeansJobWithCustomClassLoader() throws IOException, ProgramInvocationException { PackagedProgram kMeansProg = new PackagedProgram( new File(KMEANS_JAR_PATH), new String[] { KMeansData.DATAPOINTS, KMeansData.INITIAL_CENTERS, "25" }); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(KMEANS_JAR_PATH)), Collections.<URL>emptyList()); kMeansProg.invokeInteractiveModeForExecution(); }
@Test public void testGetPreviewPlan() { try { PackagedProgram prog = new PackagedProgram(new File(CliFrontendTestUtils.getTestJarPath())); final PrintStream out = System.out; final PrintStream err = System.err; try { System.setOut(new PrintStream(new NullOutputStream())); System.setErr(new PrintStream(new NullOutputStream())); Assert.assertNotNull(prog.getPreviewPlan()); } finally { System.setOut(out); System.setErr(err); } } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail("Test is erroneous: " + e.getMessage()); } }
@Test public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException { File checkpointDir = FOLDER.newFolder(); File outputDir = FOLDER.newFolder(); final PackagedProgram program = new PackagedProgram( new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH), new String[] { checkpointDir.toURI().toString(), outputDir.toURI().toString() }); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)), Collections.<URL>emptyList()); expectedException.expectCause( Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class))); program.invokeInteractiveModeForExecution(); }
PackagedProgram streamingCheckpointedProg = new PackagedProgram(new File(STREAMING_CHECKPOINTED_PROG_JAR_FILE));
File outputDir = FOLDER.newFolder(); final PackagedProgram program = new PackagedProgram( new File(CUSTOM_KV_STATE_JAR_PATH), new String[] {
PackagedProgram prg = new PackagedProgram(TestExecuteTwice.class); out.run(prg, 1); fail(FAIL_MESSAGE); PackagedProgram prg = new PackagedProgram(TestEager.class); out.run(prg, 1); fail(FAIL_MESSAGE); PackagedProgram prg = new PackagedProgram(TestGetRuntime.class); out.run(prg, 1); fail(FAIL_MESSAGE); PackagedProgram prg = new PackagedProgram(TestGetJobID.class); out.run(prg, 1); fail(FAIL_MESSAGE); PackagedProgram prg = new PackagedProgram(TestGetAccumulator.class); out.run(prg, 1); fail(FAIL_MESSAGE); PackagedProgram prg = new PackagedProgram(TestGetAllAccumulator.class); out.run(prg, 1); fail(FAIL_MESSAGE);
@Test public void testGetExecutionPlan() { try { PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); InetAddress mockAddress = InetAddress.getLocalHost(); InetSocketAddress mockJmAddress = new InetSocketAddress(mockAddress, 12345); Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, mockJmAddress.getHostName()); config.setInteger(JobManagerOptions.PORT, mockJmAddress.getPort()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, -1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
@Test public void testGetExecutionPlan() { try { jobManagerSystem.actorOf( Props.create(FailureReturningActor.class), JobMaster.JOB_MANAGER_NAME); PackagedProgram prg = new PackagedProgram(TestOptimizerPlan.class, "/dev/random", "/tmp"); assertNotNull(prg.getPreviewPlan()); Optimizer optimizer = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), config); OptimizedPlan op = (OptimizedPlan) ClusterClient.getOptimizedPlan(optimizer, prg, 1); assertNotNull(op); PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator(); assertNotNull(dumper.getOptimizerPlanAsJSON(op)); // test HTML escaping PlanJSONDumpGenerator dumper2 = new PlanJSONDumpGenerator(); dumper2.setEncodeForHTML(true); String htmlEscaped = dumper2.getOptimizerPlanAsJSON(op); assertEquals(-1, htmlEscaped.indexOf('\\')); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile); final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1);
public JobGraph toJobGraph(Configuration configuration) { if (!Files.exists(jarFile)) { throw new CompletionException(new RestHandlerException( String.format("Jar file %s does not exist", jarFile), HttpResponseStatus.BAD_REQUEST)); } try { final PackagedProgram packagedProgram = new PackagedProgram( jarFile.toFile(), entryClass, programArgs.toArray(new String[0])); return PackagedProgramUtils.createJobGraph(packagedProgram, configuration, parallelism); } catch (final ProgramInvocationException e) { throw new CompletionException(e); } } }