public long getValue() { if (accumulator != null) { return accumulator.value(); } else { return accumValue; } }
static void addOnCompletition(TaskContext taskContext, final Function0<?> function) { taskContext.addTaskCompletionListener(new TaskCompletionListener() { @Override public void onTaskCompletion(TaskContext context) { function.apply(); } }); }
@Override public final void onExecutorMetricsUpdate( SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { onEvent(executorMetricsUpdate); }
private SparkConf initializeSparkConf(String pluginNames) { return new SparkConf() .setMaster("local") .setAppName("test") .set(EXECUTOR_PLUGIN_CONF_NAME, pluginNames); }
@Override public SparkConf getSparkConf() { SparkConf sparkConf = new SparkConf(); sparkConf.setMaster("local[1]"); sparkConf.setAppName("appName1"); return sparkConf; }
@Override public SparkConf getSparkConf() { return new SparkConf(); }
@Override public void onTaskCompletion(TaskContext context) { context.isCompleted(); context.isInterrupted(); context.stageId(); context.stageAttemptNumber(); context.partitionId(); context.addTaskCompletionListener(this); } }
/** * Get the root directory that contains files added through `SparkContext.addFile()`. */ public static String getRootDirectory() { return SparkEnv.get().sparkFilesDir(); } }
public void increment(long incr) { accumulator.add(incr); }
private SparkConf initializeSparkConf(String pluginNames) { return new SparkConf() .setMaster("local") .setAppName("test") .set(EXECUTOR_PLUGIN_CONF_NAME, pluginNames); }
@Override public void onTaskCompletion(TaskContext context) { context.isCompleted(); context.isInterrupted(); context.stageId(); context.stageAttemptNumber(); context.partitionId(); context.addTaskCompletionListener(this); } }
static void addOnCompletition(TaskContext taskContext, final Function0<?> function) { taskContext.addTaskCompletionListener(new TaskCompletionListener() { @Override public void onTaskCompletion(TaskContext context) { function.apply(); } }); }
@Override public final void onExecutorMetricsUpdate( SparkListenerExecutorMetricsUpdate executorMetricsUpdate) { onEvent(executorMetricsUpdate); }
@Override public final void onExecutorUnblacklisted( SparkListenerExecutorUnblacklisted executorUnblacklisted) { onEvent(executorUnblacklisted); }
@Override public void onExecutorBlacklistedForStage( SparkListenerExecutorBlacklistedForStage executorBlacklistedForStage) { onEvent(executorBlacklistedForStage); }
@Override public final void onExecutorUnblacklisted( SparkListenerExecutorUnblacklisted executorUnblacklisted) { onEvent(executorUnblacklisted); }
@Override public void onNodeBlacklistedForStage( SparkListenerNodeBlacklistedForStage nodeBlacklistedForStage) { onEvent(nodeBlacklistedForStage); }
@Override public void onOtherEvent(SparkListenerEvent event) { onEvent(event); } }
@Override public void onExecutorBlacklistedForStage( SparkListenerExecutorBlacklistedForStage executorBlacklistedForStage) { onEvent(executorBlacklistedForStage); }
@Override public void onOtherEvent(SparkListenerEvent event) { onEvent(event); } }