launcher.addSparkArg(opts.HELP); try { launcher.addSparkArg(opts.PROXY_USER); fail("Expected IllegalArgumentException."); } catch (IllegalArgumentException e) { launcher.addSparkArg(opts.PROXY_USER, "someUser"); try { launcher.addSparkArg(opts.HELP, "someValue"); fail("Expected IllegalArgumentException."); } catch (IllegalArgumentException e) { launcher.addSparkArg("--future-argument"); launcher.addSparkArg("--future-argument", "someValue"); launcher.addSparkArg(opts.MASTER, "myMaster"); assertEquals("myMaster", launcher.builder.master); launcher.addSparkArg(opts.JARS, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.jars); launcher.addSparkArg(opts.FILES, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.files); launcher.addSparkArg(opts.PY_FILES, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.pyFiles); launcher.addSparkArg(opts.CONF, "spark.foo=bar"); assertEquals("bar", launcher.builder.conf.get("spark.foo"));
launcher.addSparkArg(opts.HELP); try { launcher.addSparkArg(opts.PROXY_USER); fail("Expected IllegalArgumentException."); } catch (IllegalArgumentException e) { launcher.addSparkArg(opts.PROXY_USER, "someUser"); try { launcher.addSparkArg(opts.HELP, "someValue"); fail("Expected IllegalArgumentException."); } catch (IllegalArgumentException e) { launcher.addSparkArg("--future-argument"); launcher.addSparkArg("--future-argument", "someValue"); launcher.addSparkArg(opts.MASTER, "myMaster"); assertEquals("myMaster", launcher.builder.master); launcher.addSparkArg(opts.JARS, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.jars); launcher.addSparkArg(opts.FILES, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.files); launcher.addSparkArg(opts.PY_FILES, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.pyFiles); launcher.addSparkArg(opts.CONF, "spark.foo=bar"); assertEquals("bar", launcher.builder.conf.get("spark.foo"));
launcher.addSparkArg(opts.HELP); try { launcher.addSparkArg(opts.PROXY_USER); fail("Expected IllegalArgumentException."); } catch (IllegalArgumentException e) { launcher.addSparkArg(opts.PROXY_USER, "someUser"); try { launcher.addSparkArg(opts.HELP, "someValue"); fail("Expected IllegalArgumentException."); } catch (IllegalArgumentException e) { launcher.addSparkArg("--future-argument"); launcher.addSparkArg("--future-argument", "someValue"); launcher.addSparkArg(opts.MASTER, "myMaster"); assertEquals("myMaster", launcher.builder.master); launcher.addSparkArg(opts.JARS, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.jars); launcher.addSparkArg(opts.FILES, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.files); launcher.addSparkArg(opts.PY_FILES, "bar"); assertEquals(Arrays.asList("bar"), launcher.builder.pyFiles); launcher.addSparkArg(opts.CONF, "spark.foo=bar"); assertEquals("bar", launcher.builder.conf.get("spark.foo"));
@Test public void testChildProcLauncher() throws Exception { // This test is failed on Windows due to the failure of initiating executors // by the path length limitation. See SPARK-18718. assumeTrue(!Utils.isWindows()); SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); Map<String, String> env = new HashMap<>(); env.put("SPARK_PRINT_LAUNCH_COMMAND", "1"); launcher .setMaster("local") .setAppResource(SparkLauncher.NO_RESOURCE) .addSparkArg(opts.CONF, String.format("%s=-Dfoo=ShouldBeOverriddenBelow", SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)) .setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Dfoo=bar -Dtest.appender=console") .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")) .addSparkArg(opts.CLASS, "ShouldBeOverriddenBelow") .setMainClass(SparkLauncherTestApp.class.getName()) .redirectError() .addAppArgs("proc"); final Process app = launcher.launch(); new OutputRedirector(app.getInputStream(), getClass().getName() + ".child", TF); assertEquals(0, app.waitFor()); }
@Test public void testChildProcLauncher() throws Exception { // This test is failed on Windows due to the failure of initiating executors // by the path length limitation. See SPARK-18718. assumeTrue(!Utils.isWindows()); SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); Map<String, String> env = new HashMap<>(); env.put("SPARK_PRINT_LAUNCH_COMMAND", "1"); launcher .setMaster("local") .setAppResource(SparkLauncher.NO_RESOURCE) .addSparkArg(opts.CONF, String.format("%s=-Dfoo=ShouldBeOverriddenBelow", SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)) .setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Dfoo=bar -Dtest.appender=console") .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")) .addSparkArg(opts.CLASS, "ShouldBeOverriddenBelow") .setMainClass(SparkLauncherTestApp.class.getName()) .redirectError() .addAppArgs("proc"); final Process app = launcher.launch(); new OutputRedirector(app.getInputStream(), getClass().getName() + ".child", TF); assertEquals(0, app.waitFor()); }
@Test public void testChildProcLauncher() throws Exception { // This test is failed on Windows due to the failure of initiating executors // by the path length limitation. See SPARK-18718. assumeTrue(!Utils.isWindows()); SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); Map<String, String> env = new HashMap<>(); env.put("SPARK_PRINT_LAUNCH_COMMAND", "1"); launcher .setMaster("local") .setAppResource(SparkLauncher.NO_RESOURCE) .addSparkArg(opts.CONF, String.format("%s=-Dfoo=ShouldBeOverriddenBelow", SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)) .setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Dfoo=bar -Dtest.appender=childproc") .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")) .addSparkArg(opts.CLASS, "ShouldBeOverriddenBelow") .setMainClass(SparkLauncherTestApp.class.getName()) .addAppArgs("proc"); final Process app = launcher.launch(); new OutputRedirector(app.getInputStream(), TF); new OutputRedirector(app.getErrorStream(), TF); assertEquals(0, app.waitFor()); }
launcher.addSparkArg("--proxy-user", conf.get(PROXY_USER));