/** * Creates a new instance. * * @param sparkContext the {@link JavaSparkContext} to be wrapped */ public SparkContextReference(CrossPlatformExecutor crossPlatformExecutor, JavaSparkContext sparkContext) { super(null); if (crossPlatformExecutor != null) { crossPlatformExecutor.registerGlobal(this); } this.sparkContext = sparkContext; }
private Executor getOrCreateExecutorFor(ExecutionStage stage) { return this.executors.computeIfAbsent( stage.getPlatformExecution().getPlatform(), platform -> { // It is important to register the Executor. This way, we ensure that it will also not be disposed // among disconnected PlatformExecutions. The downside is, that we only remove it, once the // execution is done. final Executor executor = platform.getExecutorFactory().create(this.job); this.registerGlobal(executor); return executor; } ); }