@Override public void onJobStart(SparkListenerJobStart jobStart) { super.onJobStart(jobStart); }
@Override public void onJobEnd(SparkListenerJobEnd jobEnd) { super.onJobEnd(jobEnd); }
@Override public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { super.onStageSubmitted(stageSubmitted); sparkUIManager.startStage(stageSubmitted.stageInfo().stageId(), stageSubmitted.stageInfo().numTasks()); }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd.reason() instanceof org.apache.spark.Success$ && !taskEnd.taskInfo().speculative()) { Metrics metrics = new Metrics(taskEnd.taskMetrics(), taskEnd.taskInfo()); Integer jobId; synchronized (stageToJobId) { jobId = stageToJobId.get(taskEnd.stageId()); } // TODO: implement implicit AsyncRDDActions conversion instead of jc.monitor()? // TODO: how to handle stage failures? String clientId = getClientId(jobId); if (clientId != null) { protocol.sendMetrics(clientId, jobId, taskEnd.stageId(), taskEnd.taskInfo().taskId(), metrics); } } }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { super.onTaskEnd(taskEnd); String reason = taskEnd.reason().toString(); if (reason.equals("Success")) { sparkUIManager.taskEnd(taskEnd.stageId(), taskEnd.taskInfo().taskId()); } else if (reason.contains("stage cancelled")) { sparkUIManager.taskCancelled(taskEnd.stageId(), taskEnd.taskInfo().taskId()); } } });
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<Integer, List<Map.Entry<TaskMetrics, TaskInfo>>> jobMetrics = allJobMetrics.computeIfAbsent( jobId, k -> Maps.newHashMap()); List<Map.Entry<TaskMetrics, TaskInfo>> stageMetrics = jobMetrics.computeIfAbsent(stageId, k -> Lists.newLinkedList()); stageMetrics.add(new AbstractMap.SimpleEntry<>(taskEnd.taskMetrics(), taskEnd.taskInfo())); } }
@Override public synchronized void onTaskEnd(SparkListenerTaskEnd taskEnd) { int stageId = taskEnd.stageId(); Integer jobId = stageIdToJobId.get(stageId); if (jobId == null) { LOG.warn("Can not find job id for stage[" + stageId + "]."); } else { Map<Integer, List<TaskMetrics>> jobMetrics = allJobMetrics.get(jobId); if (jobMetrics == null) { jobMetrics = Maps.newHashMap(); allJobMetrics.put(jobId, jobMetrics); } List<TaskMetrics> stageMetrics = jobMetrics.get(stageId); if (stageMetrics == null) { stageMetrics = Lists.newLinkedList(); jobMetrics.put(stageId, stageMetrics); } stageMetrics.add(taskEnd.taskMetrics()); } }
@Override public void onTaskStart(SparkListenerTaskStart taskStart) { super.onTaskStart(taskStart); sparkUIManager.taskStart(taskStart.stageId(), taskStart.taskInfo().taskId()); }
@Override public void onTaskEnd(SparkListenerTaskEnd taskEnd) { if (taskEnd != null && taskEnd.taskMetrics() != null && taskEnd.taskMetrics().outputMetrics() != null) { metrics.bytesWritten += taskEnd.taskMetrics().outputMetrics().bytesWritten(); metrics.recordsWritten += taskEnd.taskMetrics().outputMetrics().recordsWritten(); } }
@Override public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { super.onStageCompleted(stageCompleted); sparkUIManager.endStage(stageCompleted.stageInfo().stageId()); }
@Override public void onJobEnd(SparkListenerJobEnd jobEnd) { synchronized (stageToJobId) { for (Iterator<Map.Entry<Integer, Integer>> it = stageToJobId.entrySet().iterator(); it.hasNext();) { Map.Entry<Integer, Integer> e = it.next(); if (e.getValue() == jobEnd.jobId()) { it.remove(); } } } String clientId = getClientId(jobEnd.jobId()); if (clientId != null) { activeJobs.get(clientId).jobDone(); } }
@VisibleForTesting void closeAndWriteOutput() throws IOException { assert(sorter != null); updatePeakMemoryUsed(); serBuffer = null; serOutputStream = null; final SpillInfo[] spills = sorter.closeAndGetSpills(); sorter = null; final long[] partitionLengths; final File output = shuffleBlockResolver.getDataFile(shuffleId, mapId); final File tmp = Utils.tempFileWith(output); try { try { partitionLengths = mergeSpills(spills, tmp); } finally { for (SpillInfo spill : spills) { if (spill.file.exists() && ! spill.file.delete()) { logger.error("Error while deleting spill file {}", spill.file.getPath()); } } } shuffleBlockResolver.writeIndexFileAndCommit(shuffleId, mapId, partitionLengths, tmp); } finally { if (tmp.exists() && !tmp.delete()) { logger.error("Error while deleting temp file {}", tmp.getAbsolutePath()); } } mapStatus = MapStatus$.MODULE$.apply(blockManager.shuffleServerId(), partitionLengths); }
private LocalHiveSparkClient(SparkConf sparkConf) { sc = new JavaSparkContext(sparkConf); jobMetricsListener = new JobMetricsListener(); sc.sc().listenerBus().addListener(jobMetricsListener); }
@Override public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) { super.onApplicationEnd(applicationEnd); SparkUIApi sparkUI = SparkVariable.getSparkUI(); sparkUI.applicationEnd(); } }
@Test public void onApplicationEnd() { //given StartStopSparkListener listener = new StartStopSparkListener(); //when listener.onApplicationEnd(new SparkListenerApplicationEnd(123)); //then Assertions.assertThat(sparkUIMock.ended).isTrue(); }
public Metrics(TaskMetrics metrics, TaskInfo taskInfo) { this( metrics.executorDeserializeTime(), TimeUnit.NANOSECONDS.toMillis(metrics.executorDeserializeCpuTime()), metrics.executorRunTime(), TimeUnit.NANOSECONDS.toMillis(metrics.executorCpuTime()), metrics.resultSize(), metrics.jvmGCTime(), metrics.resultSerializationTime(), metrics.memoryBytesSpilled(), metrics.diskBytesSpilled(), taskInfo.duration(), optionalInputMetric(metrics), optionalShuffleReadMetric(metrics), optionalShuffleWriteMetrics(metrics), optionalOutputMetrics(metrics)); }
@Override public synchronized void onJobStart(SparkListenerJobStart jobStart) { int jobId = jobStart.jobId(); int size = jobStart.stageIds().size(); int[] intStageIds = new int[size]; for (int i = 0; i < size; i++) { Integer stageId = (Integer) jobStart.stageIds().apply(i); intStageIds[i] = stageId; stageIdToJobId.put(stageId, jobId); } jobIdToStageId.put(jobId, intStageIds); }
@VisibleForTesting void closeAndWriteOutput() throws IOException { assert(sorter != null); updatePeakMemoryUsed(); serBuffer = null; serOutputStream = null; final SpillInfo[] spills = sorter.closeAndGetSpills(); sorter = null; final long[] partitionLengths; final File output = shuffleBlockResolver.getDataFile(shuffleId, mapId); final File tmp = Utils.tempFileWith(output); try { try { partitionLengths = mergeSpills(spills, tmp); } finally { for (SpillInfo spill : spills) { if (spill.file.exists() && ! spill.file.delete()) { logger.error("Error while deleting spill file {}", spill.file.getPath()); } } } shuffleBlockResolver.writeIndexFileAndCommit(shuffleId, mapId, partitionLengths, tmp); } finally { if (tmp.exists() && !tmp.delete()) { logger.error("Error while deleting temp file {}", tmp.getAbsolutePath()); } } mapStatus = MapStatus$.MODULE$.apply(blockManager.shuffleServerId(), partitionLengths); }
@Override public synchronized void onJobStart(SparkListenerJobStart jobStart) { int jobId = jobStart.jobId(); int size = jobStart.stageIds().size(); int[] intStageIds = new int[size]; for (int i = 0; i < size; i++) { Integer stageId = (Integer) jobStart.stageIds().apply(i); intStageIds[i] = stageId; stageIdToJobId.put(stageId, jobId); } jobIdToStageId.put(jobId, intStageIds); }