@Override public Settings copy() { return new SparkSettings(cfg.clone()); }
@Override public Settings copy() { return new SparkSettings(cfg.clone()); }
private LinkedBlockingQueue<Kryo> initialize(final Configuration configuration) { // DCL is safe in this case due to volatility if (!INITIALIZED) { synchronized (UnshadedKryoShimService.class) { if (!INITIALIZED) { // so we don't get a WARN that a new configuration is being created within an active context final SparkConf sparkConf = null == Spark.getContext() ? new SparkConf() : Spark.getContext().getConf().clone(); configuration.getKeys().forEachRemaining(key -> sparkConf.set(key, configuration.getProperty(key).toString())); final KryoSerializer serializer = new KryoSerializer(sparkConf); // Setup a pool backed by our spark.serializer instance // Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf) KRYOS.clear(); final int poolSize = configuration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT); for (int i = 0; i < poolSize; i++) { KRYOS.add(serializer.newKryo()); } INITIALIZED = true; } } } return KRYOS; } }
@Override public Settings copy() { return new SparkSettings(cfg.clone()); }
@Override public Settings copy() { return new SparkSettings(cfg.clone()); }
@Override public Settings copy() { return new SparkSettings(cfg.clone()); }
public static SparkConf applyDefaultsToConfig(final SparkConf userConf) { SparkConf newConf = userConf.clone(); newConf = newConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); newConf = newConf.set( "spark.kryo.registrator", "org.locationtech.geowave.analytic.spark.GeoWaveRegistrator"); return newConf; }
private LinkedBlockingQueue<Kryo> initialize(final Configuration configuration) { // DCL is safe in this case due to volatility if (!INITIALIZED) { synchronized (UnshadedKryoShimService.class) { if (!INITIALIZED) { // so we don't get a WARN that a new configuration is being created within an active context final SparkConf sparkConf = null == Spark.getContext() ? new SparkConf() : Spark.getContext().getConf().clone(); configuration.getKeys().forEachRemaining(key -> sparkConf.set(key, configuration.getProperty(key).toString())); final KryoSerializer serializer = new KryoSerializer(sparkConf); // Setup a pool backed by our spark.serializer instance // Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf) KRYOS.clear(); final int poolSize = configuration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT); for (int i = 0; i < poolSize; i++) { KRYOS.add(serializer.newKryo()); } INITIALIZED = true; } } } return KRYOS; } }
try { SecurityManager securityManager; SparkConf conf = this.conf.clone(); conf.setAppName("Carbon Dictionary Server");