@Override public Throwable getSparkJobException() { if (future.isDone()) { try { future.get(); } catch (Throwable e) { return e; } } return null; }
@Override public Throwable getError() { if (error != null) { return error; } if (future.isDone()) { try { future.get(); } catch (Throwable e) { return e; } } return null; }
@Override public JobExecutionStatus getState() { SparkJobInfo sparkJobInfo = getJobInfo(); // For spark job with empty source data, it's not submitted actually, so we would never // receive JobStart/JobEnd event in JobStateListener, use JavaFutureAction to get current // job state. if (sparkJobInfo == null && future.isDone()) { try { future.get(); } catch (Exception e) { LOG.error("Failed to run job " + jobId, e); return JobExecutionStatus.FAILED; } return JobExecutionStatus.SUCCEEDED; } return sparkJobInfo == null ? null : sparkJobInfo.status(); }
@Override public JobExecutionStatus getState() { SparkJobInfo sparkJobInfo = getJobInfo(); // For spark job with empty source data, it's not submitted actually, so we would never // receive JobStart/JobEnd event in JobStateListener, use JavaFutureAction to get current // job state. if (sparkJobInfo == null && future.isDone()) { try { future.get(); } catch (Exception e) { LOG.error("Failed to run job " + jobId, e); return JobExecutionStatus.FAILED; } return JobExecutionStatus.SUCCEEDED; } return sparkJobInfo == null ? null : sparkJobInfo.status(); }
@Test public void takeAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.takeAsync(1); List<Integer> result = future.get(); assertEquals(1, result.size()); assertEquals((Integer) 1, result.get(0)); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void takeAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.takeAsync(1); List<Integer> result = future.get(); assertEquals(1, result.size()); assertEquals((Integer) 1, result.get(0)); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void testAsyncActionErrorWrapping() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.map(new BuggyMapFunction<>()).countAsync(); try { future.get(2, TimeUnit.SECONDS); fail("Expected future.get() for failed job to throw ExcecutionException"); } catch (ExecutionException ee) { assertTrue(Throwables.getStackTraceAsString(ee).contains("Custom exception!")); } assertTrue(future.isDone()); }
@Test public void takeAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.takeAsync(1); List<Integer> result = future.get(); assertEquals(1, result.size()); assertEquals((Integer) 1, result.get(0)); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void testAsyncActionErrorWrapping() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.map(new BuggyMapFunction<>()).countAsync(); try { future.get(2, TimeUnit.SECONDS); fail("Expected future.get() for failed job to throw ExcecutionException"); } catch (ExecutionException ee) { assertTrue(Throwables.getStackTraceAsString(ee).contains("Custom exception!")); } assertTrue(future.isDone()); }
@Test public void testAsyncActionErrorWrapping() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.map(new BuggyMapFunction<>()).countAsync(); try { future.get(2, TimeUnit.SECONDS); fail("Expected future.get() for failed job to throw ExcecutionException"); } catch (ExecutionException ee) { assertTrue(Throwables.getStackTraceAsString(ee).contains("Custom exception!")); } assertTrue(future.isDone()); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void countAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.countAsync(); long count = future.get(); assertEquals(data.size(), count); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void countAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.countAsync(); long count = future.get(); assertEquals(data.size(), count); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void countAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Long> future = rdd.countAsync(); long count = future.get(); assertEquals(data.size(), count); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void foreachAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> {}); future.get(); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void collectAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.collectAsync(); List<Integer> result = future.get(); assertEquals(data, result); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void collectAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.collectAsync(); List<Integer> result = future.get(); assertEquals(data, result); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void collectAsync() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<List<Integer>> future = rdd.collectAsync(); List<Integer> result = future.get(); assertEquals(data, result); assertFalse(future.isCancelled()); assertTrue(future.isDone()); assertEquals(1, future.jobIds().size()); }
@Test public void testAsyncActionCancellation() throws Exception { List<Integer> data = Arrays.asList(1, 2, 3, 4, 5); JavaRDD<Integer> rdd = sc.parallelize(data, 1); JavaFutureAction<Void> future = rdd.foreachAsync(integer -> { Thread.sleep(10000); // To ensure that the job won't finish before it's cancelled. }); future.cancel(true); assertTrue(future.isCancelled()); assertTrue(future.isDone()); try { future.get(2000, TimeUnit.MILLISECONDS); fail("Expected future.get() for cancelled job to throw CancellationException"); } catch (CancellationException ignored) { // pass } }