public static SparkSession createSparkSession(final StoreProperties storeProperties) { SparkSession.Builder builder = SparkSession.builder() .appName(storeProperties.get(SparkConstants.APP_NAME, SparkConstants.DEFAULT_APP_NAME)); if (Boolean.parseBoolean(storeProperties.get(SparkConstants.USE_SPARK_DEFAULT_CONF, "false"))) { final Properties properties = new Properties(); final String sparkDefaultConfPath = storeProperties.get(SparkConstants.SPARK_DEFAULT_CONF_PATH, SparkConstants.DEFAULT_SPARK_DEFAULT_CONF_PATH); try { properties.load(Files.newBufferedReader(Paths.get(sparkDefaultConfPath))); } catch (final IOException e) { throw new IllegalArgumentException("Failed to read spark-default conf from " + sparkDefaultConfPath, e); } for (final Map.Entry<Object, Object> entry : properties.entrySet()) { builder.config((String) entry.getKey(), (String) entry.getValue()); } final String sparkMaster = storeProperties.get(SparkConstants.MASTER); builder = (sparkMaster == null || sparkMaster.isEmpty()) ? builder : builder.master(sparkMaster); } else { builder.master(storeProperties.get(SparkConstants.MASTER, SparkConstants.MASTER_DEFAULT)); } builder.config(SparkConstants.DRIVER_ALLOW_MULTIPLE_CONTEXTS, "true") .config(SparkConstants.SERIALIZER, storeProperties.get(SparkConstants.SERIALIZER, SparkConstants.DEFAULT_SERIALIZER)) .config(SparkConstants.KRYO_REGISTRATOR, storeProperties.get(SparkConstants.KRYO_REGISTRATOR, SparkConstants.DEFAULT_KRYO_REGISTRATOR)); return builder.getOrCreate(); } }