@Override public boolean cancelJob() { int id = Integer.parseInt(jobId); javaSparkContext.sc().cancelJob(id); return true; }
@Override public SparkConf getSparkConf() { return sc.sc().conf(); }
@Override public int getDefaultParallelism() throws Exception { return sc.sc().defaultParallelism(); }
@Override public String getAppID() { return sparkContext.sc().applicationId(); }
@Override public Integer call(JobContext jc) throws Exception { return jc.sc().sc().defaultParallelism(); } }
@Override public String call(JobContext jc) throws Exception { return jc.sc().sc().applicationId(); } }
@Override public void cleanup() { jobMetricsListener.cleanup(jobId); if (cachedRDDIds != null) { for (Integer cachedRDDId: cachedRDDIds) { sparkContext.sc().unpersistRDD(cachedRDDId, false); } } }
@Override public int getExecutorCount() { return sc.sc().getExecutorMemoryStatus().size(); }
@Override public String call(JobContext jc) throws Exception { return jc.sc().sc().applicationId(); } }
@Override public void cleanup() { jobMetricsListener.cleanup(jobId); if (cachedRDDIds != null) { for (Integer cachedRDDId: cachedRDDIds) { sparkContext.sc().unpersistRDD(cachedRDDId, false); } } }
/** * Release cached RDDs as soon as the job is done. * This is different from local Spark client so as * to save a RPC call/trip, avoid passing cached RDD * id information around. Otherwise, we can follow * the local Spark client way to be consistent. */ void releaseCache() { if (cachedRDDIds != null) { for (Integer cachedRDDId: cachedRDDIds) { jc.sc().sc().unpersistRDD(cachedRDDId, false); } } }
@Override public Integer call(JobContext jc) throws Exception { // minus 1 here otherwise driver is also counted as an executor int count = jc.sc().sc().getExecutorMemoryStatus().size() - 1; return Integer.valueOf(count); }
private LocalHiveSparkClient(SparkConf sparkConf) { sc = new JavaSparkContext(sparkConf); jobMetricsListener = new JobMetricsListener(); sc.sc().listenerBus().addListener(jobMetricsListener); }
private SparkContext getSparkContext(SparkSession sparkSession) throws ReflectiveOperationException { HiveSparkClient sparkClient = getSparkClient(sparkSession); Assert.assertNotNull(sparkClient); return getSparkContext(sparkClient).sc(); }
@Test public void foreachPartition() { LongAccumulator accum = sc.sc().longAccumulator(); JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World")); rdd.foreachPartition(iter -> { while (iter.hasNext()) { iter.next(); accum.add(1); } }); assertEquals(2, accum.value().intValue()); }
@Test public void foreachPartition() { LongAccumulator accum = sc.sc().longAccumulator(); JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World")); rdd.foreachPartition(iter -> { while (iter.hasNext()) { iter.next(); accum.add(1); } }); assertEquals(2, accum.value().intValue()); }
@Test public void foreach() { LongAccumulator accum = sc.sc().longAccumulator(); JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World")); rdd.foreach(s -> accum.add(1)); assertEquals(2, accum.value().intValue()); }
@Test public void foreach() { LongAccumulator accum = sc.sc().longAccumulator(); JavaRDD<String> rdd = sc.parallelize(Arrays.asList("Hello", "World")); rdd.foreach(s -> accum.add(1)); assertEquals(2, accum.value().intValue()); }