static void checkSparkLibraryCompatibility(boolean throwOnIncompatible) { // check whether the correct es-hadoop is used with the correct Spark version boolean isSpark13Level = ObjectUtils.isClassPresent("org.apache.spark.sql.DataFrame", SparkConf.class.getClassLoader()); boolean isSpark20Level = ObjectUtils.isClassPresent("org.apache.spark.sql.streaming.StreamingQuery", SparkConf.class.getClassLoader()); CompatibilityLevel compatibilityLevel = ObjectUtils.instantiate("org.elasticsearch.spark.sql.SparkSQLCompatibilityLevel", CompatUtils.class.getClassLoader()); boolean isEshForSpark20 = "20".equals(compatibilityLevel.versionId()); String esSupportedSparkVersion = compatibilityLevel.versionDescription(); String errorMessage = null; if (!(isSpark13Level || isSpark20Level)) { String sparkVersion = getSparkVersionOr("1.0-1.2"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with unsupported Spark version %s", esSupportedSparkVersion, sparkVersion); } else if (isSpark20Level != isEshForSpark20) { // XOR can be applied as well but != increases readability String sparkVersion = getSparkVersionOr(isSpark13Level ? "1.3-1.6" : "2.0+"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with Spark %s", esSupportedSparkVersion, sparkVersion); } if (errorMessage != null) { if (throwOnIncompatible) { throw new EsHadoopIllegalStateException(errorMessage); } else { LogFactory.getLog("org.elasticsearch.spark.rdd.EsSpark").warn(errorMessage); } } }
static void checkSparkLibraryCompatibility(boolean throwOnIncompatible) { // check whether the correct es-hadoop is used with the correct Spark version boolean isSpark13Level = ObjectUtils.isClassPresent("org.apache.spark.sql.DataFrame", SparkConf.class.getClassLoader()); boolean isSpark20Level = ObjectUtils.isClassPresent("org.apache.spark.sql.streaming.StreamingQuery", SparkConf.class.getClassLoader()); CompatibilityLevel compatibilityLevel = ObjectUtils.instantiate("org.elasticsearch.spark.sql.SparkSQLCompatibilityLevel", CompatUtils.class.getClassLoader()); boolean isEshForSpark20 = "20".equals(compatibilityLevel.versionId()); String esSupportedSparkVersion = compatibilityLevel.versionDescription(); String errorMessage = null; if (!(isSpark13Level || isSpark20Level)) { String sparkVersion = getSparkVersionOr("1.0-1.2"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with unsupported Spark version %s", esSupportedSparkVersion, sparkVersion); } else if (isSpark20Level != isEshForSpark20) { // XOR can be applied as well but != increases readability String sparkVersion = getSparkVersionOr(isSpark13Level ? "1.3-1.6" : "2.0+"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with Spark %s", esSupportedSparkVersion, sparkVersion); } if (errorMessage != null) { if (throwOnIncompatible) { throw new EsHadoopIllegalStateException(errorMessage); } else { LogFactory.getLog("org.elasticsearch.spark.rdd.EsSpark").warn(errorMessage); } } }
static void checkSparkLibraryCompatibility(boolean throwOnIncompatible) { // check whether the correct es-hadoop is used with the correct Spark version boolean isSpark13Level = ObjectUtils.isClassPresent("org.apache.spark.sql.DataFrame", SparkConf.class.getClassLoader()); boolean isSpark20Level = ObjectUtils.isClassPresent("org.apache.spark.sql.streaming.StreamingQuery", SparkConf.class.getClassLoader()); CompatibilityLevel compatibilityLevel = ObjectUtils.instantiate("org.elasticsearch.spark.sql.SparkSQLCompatibilityLevel", CompatUtils.class.getClassLoader()); boolean isEshForSpark20 = "20".equals(compatibilityLevel.versionId()); String esSupportedSparkVersion = compatibilityLevel.versionDescription(); String errorMessage = null; if (!(isSpark13Level || isSpark20Level)) { String sparkVersion = getSparkVersionOr("1.0-1.2"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with unsupported Spark version %s", esSupportedSparkVersion, sparkVersion); } else if (isSpark20Level != isEshForSpark20) { // XOR can be applied as well but != increases readability String sparkVersion = getSparkVersionOr(isSpark13Level ? "1.3-1.6" : "2.0+"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with Spark %s", esSupportedSparkVersion, sparkVersion); } if (errorMessage != null) { if (throwOnIncompatible) { throw new EsHadoopIllegalStateException(errorMessage); } else { LogFactory.getLog("org.elasticsearch.spark.rdd.EsSpark").warn(errorMessage); } } }
static void checkSparkLibraryCompatibility(boolean throwOnIncompatible) { // check whether the correct es-hadoop is used with the correct Spark version boolean isSpark13Level = ObjectUtils.isClassPresent("org.apache.spark.sql.DataFrame", SparkConf.class.getClassLoader()); boolean isSpark20Level = ObjectUtils.isClassPresent("org.apache.spark.sql.streaming.StreamingQuery", SparkConf.class.getClassLoader()); CompatibilityLevel compatibilityLevel = ObjectUtils.instantiate("org.elasticsearch.spark.sql.SparkSQLCompatibilityLevel", CompatUtils.class.getClassLoader()); boolean isEshForSpark20 = "20".equals(compatibilityLevel.versionId()); String esSupportedSparkVersion = compatibilityLevel.versionDescription(); String errorMessage = null; if (!(isSpark13Level || isSpark20Level)) { String sparkVersion = getSparkVersionOr("1.0-1.2"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with unsupported Spark version %s", esSupportedSparkVersion, sparkVersion); } else if (isSpark20Level != isEshForSpark20) { // XOR can be applied as well but != increases readability String sparkVersion = getSparkVersionOr(isSpark13Level ? "1.3-1.6" : "2.0+"); errorMessage = String.format("Incorrect classpath detected; Elasticsearch Spark compiled for Spark %s but used with Spark %s", esSupportedSparkVersion, sparkVersion); } if (errorMessage != null) { if (throwOnIncompatible) { throw new EsHadoopIllegalStateException(errorMessage); } else { LogFactory.getLog("org.elasticsearch.spark.rdd.EsSpark").warn(errorMessage); } } }