@Override public void onStageSubmitted(SparkListenerStageSubmitted sparkListenerStageSubmitted) { logDebug("Spark scheduler submitted the missing tasks of a stage '" + sparkListenerStageSubmitted.stageInfo().name() + "'."); }
private static List<String> getRelatedMethods(@NonNull final StageInfo stageInfo) { if (registeredMethods.isEmpty()) { return Collections.singletonList(stageInfo.name()); } else { final List<String> stack = parse(stageInfo.details()); stack.removeIf(s -> !registeredMethods.containsKey(s)); return stack; } } }
@Override public void onStageCompleted(SparkListenerStageCompleted sparkListenerStageCompleted) { logDebug("Spark scheduler marked '" + sparkListenerStageCompleted.stageInfo().name() + "' as " + sparkListenerStageCompleted.stageInfo().getStatusString()); }
@Override public void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) { try { long submissionTime = stageSubmitted.stageInfo().submissionTime().getOrElse(currentTimeLongScala); String name = stageSubmitted.stageInfo().name(); String stageId = String.valueOf(stageSubmitted.stageInfo().stageId()); String attemptId = String.valueOf(stageSubmitted.stageInfo().attemptId()); int numTasks = stageSubmitted.stageInfo().numTasks(); sendStageStateEvent(submissionTime, State.BEGIN, name, stageId, attemptId, numTasks); } catch (Throwable t) { LOGGER.warn("Failed to send event for onStageSubmitted", t); } }
@Override public void onStageCompleted(final SparkListenerStageCompleted stageCompleted) { TimeoutManager.getInstance().setLastEventTime(stageCompleted.stageInfo().stageId()); final StageInfo stageInfo = stageCompleted.stageInfo(); if (stageInfo.completionTime().isDefined() && stageInfo.submissionTime().isDefined()) { SparkJobTracker.recordStageTime(stageInfo, (long) stageInfo.completionTime().get() - (long) stageInfo.submissionTime().get()); } else { log.error("Stage completed without submission or completion time. Stage {}: {}", stageInfo.stageId(), stageInfo.name()); } SparkJobTracker.removeStageInfo(stageInfo); }
long submissionTime = stageCompleted.stageInfo().submissionTime().getOrElse(zeroLongScala); long completionTime = stageCompleted.stageInfo().completionTime().getOrElse(currentTimeLongScala); String name = stageCompleted.stageInfo().name(); String stageId = String.valueOf(stageCompleted.stageInfo().stageId()); String attemptId = String.valueOf(stageCompleted.stageInfo().attemptId());