sparkConf.setSparkHome(SPARK_HOME);
sparkConf.setSparkHome(SPARK_HOME);
@Bean public SparkConf sparkConf() { SparkConf sparkConf = new SparkConf() .setAppName(appName) .setSparkHome(sparkHome) .setMaster(masterUri); return sparkConf; }
config.setMaster(_master); config.setAppName(_jobName); config.setSparkHome(_sparkHome); config.setJars(_jars); config.set("spark.logConf", "true");
String sparkHome = System.getenv("SPARK_HOME"); if (StringUtils.isNotBlank(sparkHome)) { sparkConf.setSparkHome(sparkHome);
@ConditionalOnMissingBean(JavaSparkContext.class) @Bean(destroyMethod = "stop") JavaSparkContext sparkContext(SparkProperties sparkProperties) { SparkConf config = new SparkConf(). setAppName(sparkProperties.getApplicationName()). setMaster(sparkProperties.getMasterUrl()); if(sparkProperties.getSparkHome() != null) { config.setSparkHome(sparkProperties.getSparkHome()); } return new JavaSparkContext(config); }
sparkConf.setSparkHome(workingDirectory.getAbsolutePath());
System.setProperty("SPARK_YARN_MODE", "true"); SparkConf sparkConf = new SparkConf(); sparkConf.setSparkHome(SPARK_HOME); sparkConf.setMaster("yarn"); sparkConf.setAppName("spark-yarn");
sparkConf.setSparkHome(sparkHome); } else { LOG.warn("SPARK_HOME is not set");
@Override public void setup() { // Create a StreamingContext with a SparkConf configuration SparkConf sparkConf = new SparkConf(false) .setAppName("JaiSpark") .setSparkHome("target/sparkhome") .setMaster("local") .set("spark.executor.memory", "128m") .set("spark.local.dir", new File("target/sparkhome/tmp").getAbsolutePath()) .set("spark.cores.max", "2").set("spark.akka.threads", "2") .set("spark.akka.timeout", "60").set("spark.logConf", "true") .set("spark.cleaner.delay", "3700") .set("spark.cleaner.ttl", "86400") .set("spark.shuffle.spill", "false") .set("spark.driver.host", "localhost") .set("spark.driver.port", "43214"); jssc = new JavaStreamingContext(sparkConf, new Duration(5000)); String checkpointDir = hadoopClusterService.getHDFSUri() + "/sparkcheckpoint"; jssc.checkpoint(checkpointDir); startFlumeStream(); }