@Override public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { super.onStageCompleted(stageCompleted); sparkUIManager.endStage(stageCompleted.stageInfo().stageId()); }
@Override public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { super.onStageSubmitted(stageSubmitted); sparkUIManager.startStage(stageSubmitted.stageInfo().stageId(), stageSubmitted.stageInfo().numTasks()); }
public static void removeStageInfo(@NonNull final StageInfo stageInfo) { final int stageId = stageInfo.stageId(); if (!stageIdRelatedMethodMap.containsKey(stageId)) { log.warn("Cannot find StageId {} for removeStageInfo.", stageId); } else { stageIdRelatedMethodMap.remove(stageId); } }
public static void recordStageInfo(@NonNull final StageInfo stageInfo, @NonNull final Properties properties) { // record all registered methods related to the stage MethodManager.recordStageInfo(stageInfo); // add the stageId-jobName mapping final String jobName = properties.getProperty(JOB_NAME_PROP, UNKNOWN_JOB_NAME); stageIdNameMap.put(stageInfo.stageId(), jobName); if (jobName.equals(UNKNOWN_JOB_NAME)) { log.warn("Stage Id #{} has an unknown Job Name.", stageInfo.stageId()); } else { log.debug("Stage #{} has a JobName {}.", stageInfo.stageId(), jobName); } }
public static void recordStageInfo(@NonNull final StageInfo stageInfo) { final int stageId = stageInfo.stageId(); if (!stageIdRelatedMethodMap.containsKey(stageId)) { stageIdRelatedMethodMap.putIfAbsent(stageId, getRelatedMethods(stageInfo)); } }
public static void recordStageTime(@NonNull final StageInfo stageInfo, final long timeInMs) { final String jobName = getJobName(stageInfo.stageId()); final JobRunTimeManager jobRunTimeManager = getJobRunTimeManager(jobName); final List<String> methods = MethodManager.getRelatedMethods(stageInfo.stageId()); methods.forEach(method -> jobRunTimeManager.reportStageTime(method, timeInMs)); }
@Override public void onStageCompleted(final SparkListenerStageCompleted stageCompleted) { TimeoutManager.getInstance().setLastEventTime(stageCompleted.stageInfo().stageId()); final StageInfo stageInfo = stageCompleted.stageInfo(); if (stageInfo.completionTime().isDefined() && stageInfo.submissionTime().isDefined()) { SparkJobTracker.recordStageTime(stageInfo, (long) stageInfo.completionTime().get() - (long) stageInfo.submissionTime().get()); } else { log.error("Stage completed without submission or completion time. Stage {}: {}", stageInfo.stageId(), stageInfo.name()); } SparkJobTracker.removeStageInfo(stageInfo); }
@Override public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { try { long submissionTime = stageSubmitted.stageInfo().submissionTime().getOrElse(currentTimeLongScala); String name = stageSubmitted.stageInfo().name(); String stageId = String.valueOf(stageSubmitted.stageInfo().stageId()); String attemptId = String.valueOf(stageSubmitted.stageInfo().attemptId()); int numTasks = stageSubmitted.stageInfo().numTasks(); sendStageStateEvent(submissionTime, State.BEGIN, name, stageId, attemptId, numTasks); } catch (Throwable t) { LOGGER.warn("Failed to send event for onStageSubmitted", t); } }
long completionTime = stageCompleted.stageInfo().completionTime().getOrElse(currentTimeLongScala); String name = stageCompleted.stageInfo().name(); String stageId = String.valueOf(stageCompleted.stageInfo().stageId()); String attemptId = String.valueOf(stageCompleted.stageInfo().attemptId()); int numTasks = stageCompleted.stageInfo().numTasks();