private void constructEnvVarArgs( Map<String, String> env, String submitArgsEnvVariable) throws IOException { mergeEnvPathList(env, getLibPathEnvName(), getEffectiveConfig().get(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)); StringBuilder submitArgs = new StringBuilder(); for (String arg : buildSparkSubmitArgs()) { if (submitArgs.length() > 0) { submitArgs.append(" "); } submitArgs.append(quoteForCommandString(arg)); } env.put(submitArgsEnvVariable, submitArgs.toString()); }
private void constructEnvVarArgs( Map<String, String> env, String submitArgsEnvVariable) throws IOException { mergeEnvPathList(env, getLibPathEnvName(), getEffectiveConfig().get(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)); StringBuilder submitArgs = new StringBuilder(); for (String arg : buildSparkSubmitArgs()) { if (submitArgs.length() > 0) { submitArgs.append(" "); } submitArgs.append(quoteForCommandString(arg)); } env.put(submitArgsEnvVariable, submitArgs.toString()); }
private void constructEnvVarArgs( Map<String, String> env, String submitArgsEnvVariable) throws IOException { mergeEnvPathList(env, getLibPathEnvName(), getEffectiveConfig().get(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)); StringBuilder submitArgs = new StringBuilder(); for (String arg : buildSparkSubmitArgs()) { if (submitArgs.length() > 0) { submitArgs.append(" "); } submitArgs.append(quoteForCommandString(arg)); } env.put(submitArgsEnvVariable, submitArgs.toString()); }
private void constructEnvVarArgs( Map<String, String> env, String submitArgsEnvVariable) throws IOException { mergeEnvPathList(env, getLibPathEnvName(), getEffectiveConfig().get(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)); StringBuilder submitArgs = new StringBuilder(); for (String arg : buildSparkSubmitArgs()) { if (submitArgs.length() > 0) { submitArgs.append(" "); } submitArgs.append(quoteForCommandString(arg)); } env.put(submitArgsEnvVariable, submitArgs.toString()); }
private ProcessBuilder createBuilder() throws IOException { List<String> cmd = new ArrayList<>(); cmd.add(findSparkSubmit()); cmd.addAll(builder.buildSparkSubmitArgs());
String script = isWindows() ? "spark-submit.cmd" : "spark-submit"; cmd.add(join(File.separator, builder.getSparkHome(), "bin", script)); cmd.addAll(builder.buildSparkSubmitArgs());
private ProcessBuilder createBuilder() throws IOException { List<String> cmd = new ArrayList<>(); cmd.add(findSparkSubmit()); cmd.addAll(builder.buildSparkSubmitArgs());
String script = isWindows() ? "spark-submit.cmd" : "spark-submit"; cmd.add(join(File.separator, builder.getSparkHome(), "bin", script)); cmd.addAll(builder.buildSparkSubmitArgs());
cmd.addAll(buildSparkSubmitArgs()); return cmd;
cmd.addAll(buildSparkSubmitArgs()); return cmd;
cmd.addAll(buildSparkSubmitArgs()); return cmd;
cmd.addAll(buildSparkSubmitArgs()); return cmd;
@Test public void testShellCliParser() throws Exception { List<String> sparkSubmitArgs = Arrays.asList( parser.CLASS, "org.apache.spark.repl.Main", parser.MASTER, "foo", "--app-arg", "bar", "--app-switch", parser.FILES, "baz", parser.NAME, "appName"); List<String> args = newCommandBuilder(sparkSubmitArgs).buildSparkSubmitArgs(); List<String> expected = Arrays.asList("spark-shell", "--app-arg", "bar", "--app-switch"); assertEquals(expected, args.subList(args.size() - expected.size(), args.size())); }
@Test public void testShellCliParser() throws Exception { List<String> sparkSubmitArgs = Arrays.asList( parser.CLASS, "org.apache.spark.repl.Main", parser.MASTER, "foo", "--app-arg", "bar", "--app-switch", parser.FILES, "baz", parser.NAME, "appName"); List<String> args = newCommandBuilder(sparkSubmitArgs).buildSparkSubmitArgs(); List<String> expected = Arrays.asList("spark-shell", "--app-arg", "bar", "--app-switch"); assertEquals(expected, args.subList(args.size() - expected.size(), args.size())); }
@Test(expected = IllegalArgumentException.class) public void testMissingAppResource() { new SparkSubmitCommandBuilder().buildSparkSubmitArgs(); }
@Test(expected = IllegalArgumentException.class) public void testMissingAppResource() { new SparkSubmitCommandBuilder().buildSparkSubmitArgs(); }
/** * Starts a Spark application. * * @see AbstractLauncher#startApplication(SparkAppHandle.Listener...) * @param listeners Listeners to add to the handle before the app is launched. * @return A handle for the launched application. */ @Override public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { if (builder.isClientMode(builder.getEffectiveConfig())) { LOG.warning("It's not recommended to run client-mode applications using InProcessLauncher."); } Method main = findSparkSubmit(); LauncherServer server = LauncherServer.getOrCreateServer(); InProcessAppHandle handle = new InProcessAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); } String secret = server.registerHandle(handle); setConf(LauncherProtocol.CONF_LAUNCHER_PORT, String.valueOf(server.getPort())); setConf(LauncherProtocol.CONF_LAUNCHER_SECRET, secret); List<String> sparkArgs = builder.buildSparkSubmitArgs(); String[] argv = sparkArgs.toArray(new String[sparkArgs.size()]); String appName = CommandBuilderUtils.firstNonEmpty(builder.appName, builder.mainClass, "<unknown>"); handle.start(appName, main, argv); return handle; }
/** * Starts a Spark application. * * @see AbstractLauncher#startApplication(SparkAppHandle.Listener...) * @param listeners Listeners to add to the handle before the app is launched. * @return A handle for the launched application. */ @Override public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { if (builder.isClientMode(builder.getEffectiveConfig())) { LOG.warning("It's not recommended to run client-mode applications using InProcessLauncher."); } Method main = findSparkSubmit(); LauncherServer server = LauncherServer.getOrCreateServer(); InProcessAppHandle handle = new InProcessAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); } String secret = server.registerHandle(handle); setConf(LauncherProtocol.CONF_LAUNCHER_PORT, String.valueOf(server.getPort())); setConf(LauncherProtocol.CONF_LAUNCHER_SECRET, secret); List<String> sparkArgs = builder.buildSparkSubmitArgs(); String[] argv = sparkArgs.toArray(new String[sparkArgs.size()]); String appName = CommandBuilderUtils.firstNonEmpty(builder.appName, builder.mainClass, "<unknown>"); handle.start(appName, main, argv); return handle; }
@Test public void testAlternateSyntaxParsing() throws Exception { List<String> sparkSubmitArgs = Arrays.asList( parser.CLASS + "=org.my.Class", parser.MASTER + "=foo", parser.DEPLOY_MODE + "=bar", SparkLauncher.NO_RESOURCE); List<String> cmd = newCommandBuilder(sparkSubmitArgs).buildSparkSubmitArgs(); assertEquals("org.my.Class", findArgValue(cmd, parser.CLASS)); assertEquals("foo", findArgValue(cmd, parser.MASTER)); assertEquals("bar", findArgValue(cmd, parser.DEPLOY_MODE)); }
@Test public void testAlternateSyntaxParsing() throws Exception { List<String> sparkSubmitArgs = Arrays.asList( parser.CLASS + "=org.my.Class", parser.MASTER + "=foo", parser.DEPLOY_MODE + "=bar", SparkLauncher.NO_RESOURCE); List<String> cmd = newCommandBuilder(sparkSubmitArgs).buildSparkSubmitArgs(); assertEquals("org.my.Class", findArgValue(cmd, parser.CLASS)); assertEquals("foo", findArgValue(cmd, parser.MASTER)); assertEquals("bar", findArgValue(cmd, parser.DEPLOY_MODE)); }