@Override public int monitorJob() { RemoteSparkJobMonitor remoteSparkJobMonitor = new RemoteSparkJobMonitor(hiveConf, sparkJobStatus); return remoteSparkJobMonitor.startMonitor(); } }
@Override public int monitorJob() { LocalSparkJobMonitor localSparkJobMonitor = new LocalSparkJobMonitor(hiveConf, sparkJobStatus); return localSparkJobMonitor.startMonitor(); } }
@Override public boolean equals(Object obj) { if (obj instanceof SparkStageProgress) { SparkStageProgress other = (SparkStageProgress) obj; return getTotalTaskCount() == other.getTotalTaskCount() && getSucceededTaskCount() == other.getSucceededTaskCount() && getRunningTaskCount() == other.getRunningTaskCount() && getFailedTaskCount() == other.getFailedTaskCount(); } return false; }
private Map<SparkStage, SparkStageProgress> progressMap() { return new HashMap<SparkStage, SparkStageProgress>() {{ put(new SparkStage(1, 0), new SparkStageProgress(4, 3, 1, 0)); put(new SparkStage(3, 1), new SparkStageProgress(6, 4, 1, 1)); put(new SparkStage(9, 0), new SparkStageProgress(5, 5, 0, 0)); put(new SparkStage(10, 2), new SparkStageProgress(5, 3, 2, 0)); put(new SparkStage(15, 1), new SparkStageProgress(4, 3, 1, 0)); put(new SparkStage(15, 2), new SparkStageProgress(4, 4, 0, 0)); put(new SparkStage(20, 3), new SparkStageProgress(3, 1, 1, 1)); put(new SparkStage(21, 1), new SparkStageProgress(2, 2, 0, 0)); }}; }
private int getCompletedStages() { int completed = 0; SortedSet<SparkStage> keys = new TreeSet<SparkStage>(progressMap.keySet()); for (SparkStage stage : keys) { SparkStageProgress progress = progressMap.get(stage); final int complete = progress.getSucceededTaskCount(); final int total = progress.getTotalTaskCount(); if (total > 0 && complete == total) { completed++; } } return completed; }
@Override public void printStatus(Map<SparkStage, SparkStageProgress> progressMap, Map<SparkStage, SparkStageProgress> lastProgressMap) { // do not print duplicate status while still in middle of print interval. boolean isDuplicateState = isSameAsPreviousProgress(progressMap, lastProgressMap); boolean withinInterval = System.currentTimeMillis() <= lastPrintTime + PRINT_INTERVAL; if (isDuplicateState && withinInterval) { return; } String report = getReport(progressMap); renderProgress(monitor.getProgressMonitor(progressMap)); if (showReport(report)) { renderReport(report); lastReport = report; lastPrintTime = System.currentTimeMillis(); } }
private void killJob() { LOG.debug("Killing Spark job with job handle id " + sparkJobHandleId); boolean needToKillJob = false; if (jobRef != null && !jobKilled) { synchronized (this) { if (!jobKilled) { jobKilled = true; needToKillJob = true; } } } if (needToKillJob) { try { jobRef.cancelJob(); } catch (Exception e) { LOG.warn("Failed to kill Spark job", e); } } }
protected int getStageMaxTaskCount(Map<SparkStage, SparkStageProgress> progressMap) { int stageMaxTasks = 0; for (SparkStageProgress progress : progressMap.values()) { int tasks = progress.getTotalTaskCount(); if (tasks > stageMaxTasks) { stageMaxTasks = tasks; } } return stageMaxTasks; }
private RenderStrategy.UpdateFunction updateFunction() { return inPlaceUpdate && !SessionState.get().isHiveServerQuery() ? new RenderStrategy.InPlaceUpdateFunction( this) : new RenderStrategy.LogToFileFunction(this); } }
@Override public String executionStatus() { if (getCompletedStages() == progressMap.keySet().size()) { return SparkJobMonitor.StageState.FINISHED.toString(); } else { return SparkJobMonitor.StageState.RUNNING.toString(); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SparkStage that = (SparkStage) o; return getStageId() == that.getStageId() && getAttemptId() == that.getAttemptId(); }
ProgressMonitor getProgressMonitor(Map<SparkStage, SparkStageProgress> progressMap) { return new SparkProgressMonitor(progressMap, startTime); }
@Override public String footerSummary() { return String.format("STAGES: %02d/%02d", getCompletedStages(), progressMap.keySet().size()); }
@Override public boolean equals(Object obj) { if (obj instanceof SparkStageProgress) { SparkStageProgress other = (SparkStageProgress) obj; return getTotalTaskCount() == other.getTotalTaskCount() && getSucceededTaskCount() == other.getSucceededTaskCount() && getRunningTaskCount() == other.getRunningTaskCount() && getFailedTaskCount() == other.getFailedTaskCount(); } return false; }
@Override public int monitorJob() { RemoteSparkJobMonitor remoteSparkJobMonitor = new RemoteSparkJobMonitor(hiveConf, sparkJobStatus); return remoteSparkJobMonitor.startMonitor(); } }
@Override public int monitorJob() { LocalSparkJobMonitor localSparkJobMonitor = new LocalSparkJobMonitor(hiveConf, sparkJobStatus); return localSparkJobMonitor.startMonitor(); } }
@Override public double progressedPercentage() { SortedSet<SparkStage> keys = new TreeSet<SparkStage>(progressMap.keySet()); int sumTotal = 0; int sumComplete = 0; for (SparkStage stage : keys) { SparkStageProgress progress = progressMap.get(stage); final int complete = progress.getSucceededTaskCount(); final int total = progress.getTotalTaskCount(); sumTotal += total; sumComplete += complete; } double progress = (sumTotal == 0) ? 1.0f : (float) sumComplete / (float) sumTotal; return progress; }
protected int getTotalTaskCount(Map<SparkStage, SparkStageProgress> progressMap) { int totalTasks = 0; for (SparkStageProgress progress : progressMap.values()) { totalTasks += progress.getTotalTaskCount(); } return totalTasks; }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("TotalTasks: "); sb.append(getTotalTaskCount()); sb.append(" Succeeded: "); sb.append(getSucceededTaskCount()); sb.append(" Running: "); sb.append(getRunningTaskCount()); sb.append(" Failed: "); sb.append(getFailedTaskCount()); return sb.toString(); } }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("TotalTasks: "); sb.append(getTotalTaskCount()); sb.append(" Succeeded: "); sb.append(getSucceededTaskCount()); sb.append(" Running: "); sb.append(getRunningTaskCount()); sb.append(" Failed: "); sb.append(getFailedTaskCount()); return sb.toString(); } }