@Override public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { try { long submissionTime = stageCompleted.stageInfo().submissionTime().getOrElse(zeroLongScala); long completionTime = stageCompleted.stageInfo().completionTime().getOrElse(currentTimeLongScala); String name = stageCompleted.stageInfo().name(); String stageId = String.valueOf(stageCompleted.stageInfo().stageId()); String attemptId = String.valueOf(stageCompleted.stageInfo().attemptId()); int numTasks = stageCompleted.stageInfo().numTasks(); String status = getValOrNull(() -> stageCompleted.stageInfo().getStatusString()); tryToSet(() -> stageEventBuilder.setExecutorCpuTime(stageCompleted.stageInfo().taskMetrics().executorCpuTime())); tryToSet(() -> stageEventBuilder.setExecutorDeserializeCpuTime(stageCompleted.stageInfo().taskMetrics().executorDeserializeCpuTime())); tryToSet(() -> stageEventBuilder.setExecutorRunTime(stageCompleted.stageInfo().taskMetrics().executorRunTime())); tryToSet(() -> stageEventBuilder.setJvmGcTime(stageCompleted.stageInfo().taskMetrics().jvmGCTime())); tryToSet(() -> stageEventBuilder.setExecutorDeserializeTime(stageCompleted.stageInfo().taskMetrics().executorDeserializeTime())); tryToSet(() -> stageEventBuilder.setResultSerializationTime(stageCompleted.stageInfo().taskMetrics().resultSerializationTime())); tryToSet(() -> stageEventBuilder.setResultSize(stageCompleted.stageInfo().taskMetrics().resultSize())); tryToSet(() -> stageEventBuilder.setPeakExecutionMemory(stageCompleted.stageInfo().taskMetrics().peakExecutionMemory())); tryToSet(() -> stageEventBuilder.setDiskBytesSpilled(stageCompleted.stageInfo().taskMetrics().diskBytesSpilled())); tryToSet(() -> stageEventBuilder.setMemoryBytesSpilled(stageCompleted.stageInfo().taskMetrics().memoryBytesSpilled())); tryToSet(() -> stageEventBuilder.setShuffleReadRecords(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().recordsRead())); tryToSet(() -> stageEventBuilder.setShuffleReadFetchWaitTime(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().fetchWaitTime())); tryToSet(() -> stageEventBuilder.setShuffleReadLocalBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().localBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadRemoteBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().remoteBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadTotalBytes(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics().totalBytesRead())); tryToSet(() -> stageEventBuilder.setShuffleReadLocalBlocksFetched(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics() .localBlocksFetched())); tryToSet(() -> stageEventBuilder.setShuffleReadRemoteBlocksFetched(stageCompleted.stageInfo().taskMetrics().shuffleReadMetrics()
@Override public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { try { long submissionTime = stageSubmitted.stageInfo().submissionTime().getOrElse(currentTimeLongScala); String name = stageSubmitted.stageInfo().name(); String stageId = String.valueOf(stageSubmitted.stageInfo().stageId()); String attemptId = String.valueOf(stageSubmitted.stageInfo().attemptId()); int numTasks = stageSubmitted.stageInfo().numTasks(); sendStageStateEvent(submissionTime, State.BEGIN, name, stageId, attemptId, numTasks); } catch (Throwable t) { LOGGER.warn("Failed to send event for onStageSubmitted", t); } }
@Override public void onStageCompleted(final SparkListenerStageCompleted stageCompleted) { TimeoutManager.getInstance().setLastEventTime(stageCompleted.stageInfo().stageId()); final StageInfo stageInfo = stageCompleted.stageInfo(); if (stageInfo.completionTime().isDefined() && stageInfo.submissionTime().isDefined()) { SparkJobTracker.recordStageTime(stageInfo, (long) stageInfo.completionTime().get() - (long) stageInfo.submissionTime().get()); } else { log.error("Stage completed without submission or completion time. Stage {}: {}", stageInfo.stageId(), stageInfo.name()); } SparkJobTracker.removeStageInfo(stageInfo); }
@Override public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { super.onStageCompleted(stageCompleted); sparkUIManager.endStage(stageCompleted.stageInfo().stageId()); }
@Override public void onStageCompleted(SparkListenerStageCompleted sparkListenerStageCompleted) { logDebug("Spark scheduler marked '" + sparkListenerStageCompleted.stageInfo().name() + "' as " + sparkListenerStageCompleted.stageInfo().getStatusString()); }
private static List<String> getRelatedMethods(@NonNull final StageInfo stageInfo) { if (registeredMethods.isEmpty()) { return Collections.singletonList(stageInfo.name()); } else { final List<String> stack = parse(stageInfo.details()); stack.removeIf(s -> !registeredMethods.containsKey(s)); return stack; } } }
@Override public void onStageSubmitted(SparkListenerStageSubmitted sparkListenerStageSubmitted) { logDebug("Spark scheduler submitted the missing tasks of a stage '" + sparkListenerStageSubmitted.stageInfo().name() + "'."); }
public static void removeStageInfo(@NonNull final StageInfo stageInfo) { final int stageId = stageInfo.stageId(); if (!stageIdRelatedMethodMap.containsKey(stageId)) { log.warn("Cannot find StageId {} for removeStageInfo.", stageId); } else { stageIdRelatedMethodMap.remove(stageId); } }
public static void recordStageInfo(@NonNull final StageInfo stageInfo, @NonNull final Properties properties) { // record all registered methods related to the stage MethodManager.recordStageInfo(stageInfo); // add the stageId-jobName mapping final String jobName = properties.getProperty(JOB_NAME_PROP, UNKNOWN_JOB_NAME); stageIdNameMap.put(stageInfo.stageId(), jobName); if (jobName.equals(UNKNOWN_JOB_NAME)) { log.warn("Stage Id #{} has an unknown Job Name.", stageInfo.stageId()); } else { log.debug("Stage #{} has a JobName {}.", stageInfo.stageId(), jobName); } }
public static void recordStageInfo(@NonNull final StageInfo stageInfo) { final int stageId = stageInfo.stageId(); if (!stageIdRelatedMethodMap.containsKey(stageId)) { stageIdRelatedMethodMap.putIfAbsent(stageId, getRelatedMethods(stageInfo)); } }
public static void recordStageTime(@NonNull final StageInfo stageInfo, final long timeInMs) { final String jobName = getJobName(stageInfo.stageId()); final JobRunTimeManager jobRunTimeManager = getJobRunTimeManager(jobName); final List<String> methods = MethodManager.getRelatedMethods(stageInfo.stageId()); methods.forEach(method -> jobRunTimeManager.reportStageTime(method, timeInMs)); }