/** * Create a new builder, the {@link SparkExecutor} can be constructed from * * @param appName the application name * @return builder */ public static Builder newBuilder(String appName) { return newBuilder(appName, new SparkConf()); }
@Override public CompletableFuture<Result> submit(Flow flow) { return CompletableFuture.supplyAsync(() -> execute(flow), submitExecutor); }
public SparkExecutor build() { conf.setAppName(appName); // make sure we use kryo conf.set("spark.serializer", KryoSerializer.class.getName()); // register euphoria-spark related classes if (kryoRequiredRegistrationDisabled) { LOG.warn("Required kryo registration is disabled. This is highly suboptimal!"); conf.set("spark.kryo.registrationRequired", "false"); } else { Preconditions.checkArgument(registrator != null, "You must provide a kryo registrator."); conf.set("spark.kryo.registrationRequired", "true"); conf.set("spark.kryo.registrator", registrator.getName()); } return new SparkExecutor( conf, storageLevel, desiredSplitSize, ImmutableMap.copyOf(comparators)); } }
private Result execute(Flow flow) { if (!isBoundedInput(flow)) { throw new UnsupportedOperationException("Spark executor doesn't support unbounded input");
@Override public Executor newExecutor(Config config, Collection<? extends Class<?>> dataClasses) throws IOException { SparkConf conf = new SparkConf(); conf.set("spark.serializer", KryoSerializer.class.getName()); conf.registerKryoClasses(dataClasses.toArray(new Class[dataClasses.size()])); return SparkExecutor .newBuilder(EuphoriaSparkTrends.class.getSimpleName(), conf) .build(); } }
@Override public Executor create(Class<?>... classes) { final SparkExecutor.Builder builder = SparkExecutor .newBuilder("euphoria-example") .registerKryoClasses(classes); if (test) { return builder.local().build(); } else { return builder.build(); } } }
@Override default ExecutorEnvironment newExecutorEnvironment() throws Exception { final Executor executor = SparkExecutor.newBuilder("test") .local() .allowMultipleContexts() .kryoRegistrator(TestRegistrator.class) .build(); return new ExecutorEnvironment() { @Override public Executor getExecutor() { return executor; } @Override public void shutdown() throws Exception { executor.shutdown(); } }; } }