/** * Log a raw record type with keys and values. This is method is generally not used directly. * @param recordType type of log event * @param key key * @param value value */ public static void log(PrintWriter out, RecordTypes recordType, Keys key, String value){ value = escapeString(value); out.println(recordType.name() + DELIMITER + key + "=\"" + value + "\"" + DELIMITER + LINE_DELIMITER_CHAR); }
/** * Logs history meta-info to the history file. This needs to be called once * per history file. * @param jobId job id, assigned by jobtracker. */ static void logMetaInfo(ArrayList<PrintWriter> writers){ if (!disableHistory){ if (null != writers){ JobHistory.log(writers, RecordTypes.Meta, new Keys[] {Keys.VERSION}, new String[] {String.valueOf(VERSION)}); } } } }
/** * Populates a JobInfo object from the job's history log file. * @param jobHistoryFile history file for this job. * @param job a precreated JobInfo object, should be non-null. * @param fs FileSystem where historyFile is present. * @throws IOException */ public static void parseJobTasks(String jobHistoryFile, JobHistory.JobInfo job, FileSystem fs) throws IOException { JobHistory.parseHistoryFromFS(jobHistoryFile, new JobTasksParseListener(job), fs); }
infoServer.setAttribute("job.tracker", this); boolean historyInitialized = JobHistory.init(conf, this.localMachine, this.startTime); String historyLogDir = null; && !JobHistory.isDisableHistory() && systemDirData != null) { for (FileStatus status : systemDirData) { JobHistory.init(conf, this.localMachine, this.startTime); historyLogDir = conf.get("hadoop.job.history.location"); infoServer.setAttribute("historyLogDir", historyLogDir);
infoServer.setAttribute("job.tracker", this); boolean historyInitialized = JobHistory.init(this, conf, this.localMachine, this.startTime); JobHistory.initDone(conf, fs); String historyLogDir = JobHistory.getCompletedJobHistoryLocation().toString(); infoServer.setAttribute("historyLogDir", historyLogDir); FileSystem historyFS = new Path(historyLogDir).getFileSystem(conf);
taskList.append(attempt.get(Keys.HOSTNAME)).append("\t"); taskList.append(attempt.get(Keys.ERROR)); String taskLogsUrl = JobHistory.getTaskLogsUrl(attempt); taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a"); System.out.println(taskList.toString());
continue; parseLine(buf.toString(), l, isEscaped); buf = new StringBuffer(); } while ((line = reader.readLine())!= null);
private boolean minConditionToRetire(JobInProgress job, long now) { return job.getStatus().getRunState() != JobStatus.RUNNING && job.getStatus().getRunState() != JobStatus.PREP && (job.getFinishTime() + MIN_TIME_BEFORE_RETIRE < now) && (job.isHistoryFileCopied() || JobHistory.isDisableHistory()); } /**
taskList.append(attempt.get(Keys.HOSTNAME)).append("\t"); taskList.append(attempt.get(Keys.ERROR)); String taskLogsUrl = JobHistory.getTaskLogsUrl(attempt); taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a"); System.out.println(taskList.toString());
continue; parseLine(buf.toString(), l, isEscaped); buf = new StringBuffer(); } while ((line = reader.readLine())!= null);
/** * Logs history meta-info to the history file. This needs to be called once * per history file. * @param jobId job id, assigned by jobtracker. */ static void logMetaInfo(ArrayList<PrintWriter> writers){ if (!disableHistory){ if (null != writers){ JobHistory.log(writers, RecordTypes.Meta, new Keys[] {Keys.VERSION}, new String[] {String.valueOf(VERSION)}); } } } }
/** * Populates a JobInfo object from the job's history log file. * @param jobHistoryFile history file for this job. * @param job a precreated JobInfo object, should be non-null. * @param fs FileSystem where historyFile is present. * @throws IOException */ public static void parseJobTasks(String jobHistoryFile, JobHistory.JobInfo job, FileSystem fs) throws IOException { JobHistory.parseHistoryFromFS(jobHistoryFile, new JobTasksParseListener(job), fs); }
/** * Log a raw record type with keys and values. This is method is generally not used directly. * @param recordType type of log event * @param key key * @param value value */ static void log(PrintWriter out, RecordTypes recordType, Keys key, String value){ value = escapeString(value); out.println(recordType.name() + DELIMITER + key + "=\"" + value + "\"" + DELIMITER + LINE_DELIMITER_CHAR); }
/** * Log job's submit-time/launch-time * @param jobid job id * @param submitTime job's submit time * @param launchTime job's launch time * @param restartCount number of times the job got restarted */ public static void logJobInfo(JobID jobid, long submitTime, long launchTime, int restartCount){ if (!disableHistory){ String logFileKey = JOBTRACKER_UNIQUE_STRING + jobid; ArrayList<PrintWriter> writer = openJobs.get(logFileKey); if (null != writer){ JobHistory.log(writer, RecordTypes.Job, new Keys[] {Keys.JOBID, Keys.SUBMIT_TIME, Keys.LAUNCH_TIME, Keys.RESTART_COUNT}, new String[] {jobid.toString(), String.valueOf(submitTime), String.valueOf(launchTime), String.valueOf(restartCount)}); } } } }
private void printFailedAttempts(NodesFilter filter) throws IOException { JobHistory.parseHistoryFromFS(jobLogFile, filter, fs); Map<String, Set<String>> badNodes = filter.getValues(); StringBuffer attempts = new StringBuffer(); if (badNodes.size() > 0) { attempts.append("\n").append(filter.getFailureType()); attempts.append(" task attempts by nodes"); attempts.append("\nHostname\tFailedTasks"); attempts.append("\n==============================="); System.out.println(attempts.toString()); for (Map.Entry<String, Set<String>> entry : badNodes.entrySet()) { String node = entry.getKey(); Set<String> failedTasks = entry.getValue(); attempts.setLength(0); attempts.append(node).append("\t"); for (String t : failedTasks) { attempts.append(t).append(", "); } System.out.println(attempts.toString()); } } }
/** * Log a number of keys and values with record. the array length of keys and values * should be same. * @param recordType type of log event * @param keys type of log event * @param values type of log event */ static void log(ArrayList<PrintWriter> writers, RecordTypes recordType, Keys[] keys, String[] values) { StringBuffer buf = new StringBuffer(recordType.name()); buf.append(DELIMITER); for(int i =0; i< keys.length; i++){ buf.append(keys[i]); buf.append("=\""); values[i] = escapeString(values[i]); buf.append(values[i]); buf.append("\""); buf.append(DELIMITER); } buf.append(LINE_DELIMITER_CHAR); for (PrintWriter out : writers) { out.println(buf.toString()); } }
public static void logJobInfo(JobID jobid, long submitTime, long launchTime) { if (!disableHistory){ ArrayList<PrintWriter> writer = fileManager.getWriters(jobid); if (null != writer){ JobHistory.log(writer, RecordTypes.Job, new Keys[] {Keys.JOBID, Keys.SUBMIT_TIME, Keys.LAUNCH_TIME}, new String[] {jobid.toString(), String.valueOf(submitTime), String.valueOf(launchTime)}); } } } }
private void printFailedAttempts(NodesFilter filter) throws IOException { JobHistory.parseHistoryFromFS(jobLogFile, filter, fs); Map<String, Set<String>> badNodes = filter.getValues(); StringBuffer attempts = new StringBuffer(); if (badNodes.size() > 0) { attempts.append("\n").append(filter.getFailureType()); attempts.append(" task attempts by nodes"); attempts.append("\nHostname\tFailedTasks"); attempts.append("\n==============================="); System.out.println(attempts.toString()); for (Map.Entry<String, Set<String>> entry : badNodes.entrySet()) { String node = entry.getKey(); Set<String> failedTasks = entry.getValue(); attempts.setLength(0); attempts.append(node).append("\t"); for (String t : failedTasks) { attempts.append(t).append(", "); } System.out.println(attempts.toString()); } } }
/** * Log a number of keys and values with record. the array length of keys and values * should be same. * @param recordType type of log event * @param keys type of log event * @param values type of log event */ static void log(ArrayList<PrintWriter> writers, RecordTypes recordType, Keys[] keys, String[] values) { StringBuffer buf = new StringBuffer(recordType.name()); buf.append(DELIMITER); for(int i =0; i< keys.length; i++){ buf.append(keys[i]); buf.append("=\""); values[i] = escapeString(values[i]); buf.append(values[i]); buf.append("\""); buf.append(DELIMITER); } buf.append(LINE_DELIMITER_CHAR); for (PrintWriter out : writers) { LogTask task = new LogTask(out, buf.toString()); fileManager.addWriteTask(task); } }
/** * Logs job as running * @param jobId job id, assigned by jobtracker. */ public static void logStarted(JobID jobId){ if (!disableHistory){ String logFileKey = JOBTRACKER_UNIQUE_STRING + jobId; ArrayList<PrintWriter> writer = openJobs.get(logFileKey); if (null != writer){ JobHistory.log(writer, RecordTypes.Job, new Keys[] {Keys.JOBID, Keys.JOB_STATUS}, new String[] {jobId.toString(), Values.RUNNING.name()}); } } }