SparkConf conf = new SparkConf().set("spark.cassandra.connection.host", "127.0.0.1"); conf.setJars(SparkContext.jarOfObject(this))
SparkConf conf = new SparkConf().setAppName("LineCount"); conf.setMaster("spark://localhost:7077") conf.setJars(new String[] { "/home/path/MainTest.jar" });
MySQLDependenciesJob(Builder builder) { this.db = builder.db; this.day = builder.day; SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); df.setTimeZone(TimeZone.getTimeZone("UTC")); this.dateStamp = df.format(new Date(builder.day)); this.url = new StringBuilder("jdbc:mysql://") .append(builder.host).append(":").append(builder.port) .append("/").append(builder.db) .append("?autoReconnect=true") .append("&useSSL=").append(builder.useSsl).toString(); this.user = builder.user; this.password = builder.password; this.conf = new SparkConf(true) .setMaster(builder.sparkMaster) .setAppName(getClass().getName()); if (builder.jars != null) conf.setJars(builder.jars); for (Map.Entry<String, String> entry : builder.sparkProperties.entrySet()) { conf.set(entry.getKey(), entry.getValue()); } this.logInitializer = builder.logInitializer; }
CassandraDependenciesJob(Builder builder) { this.keyspace = builder.keyspace; this.day = builder.day; SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); df.setTimeZone(TimeZone.getTimeZone("UTC")); this.dateStamp = df.format(new Date(builder.day)); this.conf = new SparkConf(true).setMaster(builder.sparkMaster).setAppName(getClass().getName()); conf.set("spark.cassandra.connection.host", parseHosts(builder.contactPoints)); conf.set("spark.cassandra.connection.port", parsePort(builder.contactPoints)); if (builder.localDc != null) conf.set("connection.local_dc", builder.localDc); if (builder.jars != null) conf.setJars(builder.jars); for (Map.Entry<String, String> entry : builder.sparkProperties.entrySet()) { conf.set(entry.getKey(), entry.getValue()); } this.logInitializer = builder.logInitializer; }
CassandraDependenciesJob(Builder builder) { this.keyspace = builder.keyspace; this.day = builder.day; this.strictTraceId = builder.strictTraceId; this.inTest = builder.inTest; SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); df.setTimeZone(TimeZone.getTimeZone("UTC")); this.dateStamp = df.format(new Date(builder.day)); this.conf = new SparkConf(true).setMaster(builder.sparkMaster).setAppName(getClass().getName()); conf.set("spark.cassandra.connection.host", parseHosts(builder.contactPoints)); conf.set("spark.cassandra.connection.port", parsePort(builder.contactPoints)); if (builder.localDc != null) conf.set("connection.local_dc", builder.localDc); if (builder.jars != null) conf.setJars(builder.jars); for (Map.Entry<String, String> entry : builder.sparkProperties.entrySet()) { conf.set(entry.getKey(), entry.getValue()); } this.logInitializer = builder.logInitializer; }
config.setAppName(_jobName); config.setSparkHome(_sparkHome); config.setJars(_jars); config.set("spark.logConf", "true");
CassandraDependenciesJob(Builder builder) { this.keyspace = builder.keyspace; this.day = builder.day; SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); df.setTimeZone(TimeZone.getTimeZone("UTC")); this.conf = new SparkConf(true) .setMaster(builder.sparkMaster) .setAppName(getClass().getName()); conf.set("spark.cassandra.connection.host", parseHosts(builder.contactPoints)); conf.set("spark.cassandra.connection.port", parsePort(builder.contactPoints)); if (builder.localDc != null) { conf.set("connection.local_dc", builder.localDc); } if (builder.jars != null) { conf.setJars(builder.jars); } for (Map.Entry<String, String> entry : builder.sparkProperties.entrySet()) { conf.set(entry.getKey(), entry.getValue()); } }
@Memoized JavaStreamingContext jsc() { SparkConf conf = new SparkConf(true) .setMaster(master()) .setAppName(getClass().getName()); if (!jars().isEmpty()) conf.setJars(jars().toArray(new String[0])); for (Map.Entry<String, String> entry : conf().entrySet()) { conf.set(entry.getKey(), entry.getValue()); } return new JavaStreamingContext(conf, new Duration(batchDuration())); }
this.conf = new SparkConf(true).setMaster(builder.sparkMaster).setAppName(getClass().getName()); if (builder.jars != null) { conf.setJars(builder.jars);
private static JavaSparkContext createSparkContext(SparkContextOptions contextOptions) { if (usesProvidedSparkContext) { LOG.info("Using a provided Spark Context"); JavaSparkContext jsc = contextOptions.getProvidedSparkContext(); if (jsc == null || jsc.sc().isStopped()) { LOG.error("The provided Spark context " + jsc + " was not created or was stopped"); throw new RuntimeException("The provided Spark context was not created or was stopped"); } return jsc; } else { LOG.info("Creating a brand new Spark Context."); SparkConf conf = new SparkConf(); if (!conf.contains("spark.master")) { // set master if not set. conf.setMaster(contextOptions.getSparkMaster()); } if (contextOptions.getFilesToStage() != null && !contextOptions.getFilesToStage().isEmpty()) { conf.setJars(contextOptions.getFilesToStage().toArray(new String[0])); } conf.setAppName(contextOptions.getAppName()); // register immutable collections serializers because the SDK uses them. conf.set("spark.kryo.registrator", BeamSparkRunnerRegistrator.class.getName()); return new JavaSparkContext(conf); } } }
private JavaStreamingContext create(String streamingContextName, int port, long streamingBatchTime, String sparkHost) { SparkConf conf = new SparkConf(); conf.set("spark.ui.port", String.valueOf(port)); conf.setAppName(streamingContextName); conf.setJars(JavaStreamingContext.jarOfClass(StreamingEngine.class)); conf.setMaster(sparkHost); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); conf.registerKryoClasses(new Class[] { StratioStreamingMessage.class, InsertMessage.class, ColumnType.class, Action.class}); HashMap<String, String> tuningProperties = configurationContext.getSparkTunningProperties(); if (tuningProperties != null && tuningProperties.size() > 0) { tuningProperties.forEach( (key, value) -> conf.set(key, value)); } JavaStreamingContext streamingContext = new JavaStreamingContext(conf, new Duration(streamingBatchTime)); return streamingContext; }