protected void closeEventWriter(JobId jobId) throws IOException { final MetaInfo mi = fileMap.get(jobId); if (mi == null) { throw new IOException("No MetaInfo found for JobId: [" + jobId + "]"); } if (!mi.isWriterActive()) { throw new IOException( "Inactive Writer: Likely received multiple JobFinished / " + "JobUnsuccessful events for JobId: [" + jobId + "]"); } // Close the Writer try { mi.closeWriter(); } catch (IOException e) { LOG.error("Error closing writer for JobID: " + jobId); throw e; } }
void maybeFlush(HistoryEvent historyEvent) throws IOException { if ((eventQueue.size() < minQueueSizeForBatchingFlushes && numUnflushedCompletionEvents > 0) || numUnflushedCompletionEvents >= maxUnflushedCompletionEvents || isJobCompletionEvent(historyEvent)) { this.flush(); } }
HistoryEvent historyEvent = event.getHistoryEvent(); if (! (historyEvent instanceof NormalizedResourceEvent)) { mi.writeEvent(historyEvent); processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID()); if (LOG.isDebugEnabled()) { JobSubmittedEvent jobSubmittedEvent = (JobSubmittedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setSubmitTime(jobSubmittedEvent.getSubmitTime()); mi.getJobIndexInfo().setQueueName(jobSubmittedEvent.getJobQueueName()); mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime()); JobQueueChangeEvent jQueueEvent = (JobQueueChangeEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setQueueName(jQueueEvent.getJobQueueName()); JobFinishedEvent jFinishedEvent = (JobFinishedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime()); mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getSucceededMaps()); mi.getJobIndexInfo().setNumReduces( jFinishedEvent.getSucceededReduces()); mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString()); closeEventWriter(event.getJobID()); processDoneFiles(event.getJobID());
HistoryEvent historyEvent = event.getHistoryEvent(); if (! (historyEvent instanceof NormalizedResourceEvent)) { mi.writeEvent(historyEvent); processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID()); if (timelineClient != null) { JobSubmittedEvent jobSubmittedEvent = (JobSubmittedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setSubmitTime(jobSubmittedEvent.getSubmitTime()); mi.getJobIndexInfo().setQueueName(jobSubmittedEvent.getJobQueueName()); mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime()); JobQueueChangeEvent jQueueEvent = (JobQueueChangeEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setQueueName(jQueueEvent.getJobQueueName()); JobFinishedEvent jFinishedEvent = (JobFinishedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime()); mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getFinishedMaps()); mi.getJobIndexInfo().setNumReduces( jFinishedEvent.getFinishedReduces()); mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString()); closeEventWriter(event.getJobID()); processDoneFiles(event.getJobID());
HistoryEvent historyEvent = event.getHistoryEvent(); if (! (historyEvent instanceof NormalizedResourceEvent)) { mi.writeEvent(historyEvent); processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID()); if (timelineClient != null) { JobSubmittedEvent jobSubmittedEvent = (JobSubmittedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setSubmitTime(jobSubmittedEvent.getSubmitTime()); mi.getJobIndexInfo().setQueueName(jobSubmittedEvent.getJobQueueName()); mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime()); JobQueueChangeEvent jQueueEvent = (JobQueueChangeEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setQueueName(jQueueEvent.getJobQueueName()); JobFinishedEvent jFinishedEvent = (JobFinishedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setFinishTime(jFinishedEvent.getFinishTime()); mi.getJobIndexInfo().setNumMaps(jFinishedEvent.getFinishedMaps()); mi.getJobIndexInfo().setNumReduces( jFinishedEvent.getFinishedReduces()); mi.getJobIndexInfo().setJobStatus(JobState.SUCCEEDED.toString()); closeEventWriter(event.getJobID()); processDoneFiles(event.getJobID());
LOG.debug("Shutting down timer for " + mi); mi.shutDownTimer(); } catch (IOException e) { LOG.info("Exception while canceling delayed flush timer. " JobId toClose = jobIt.getKey(); MetaInfo mi = jobIt.getValue(); if(mi != null && mi.isWriterActive()) { LOG.warn("Found jobId " + toClose + " to have not been closed. Will close"); job.getKilledMaps(), job.getKilledReduces(), createJobStateForJobUnsuccessfulCompletionEvent( mi.getForcedJobStateOnShutDown()), job.getDiagnostics()); JobHistoryEvent jfEvent = new JobHistoryEvent(toClose, jucEvent); mi.closeWriter(); } catch (IOException e) { LOG.info("Exception while closing file " + e.getMessage());
MetaInfo fi = new MetaInfo(historyFile, logDirConfPath, writer, user, jobName, jobId, amStartedEvent.getForcedJobStateOnShutDown(), queueName); fi.getJobSummary().setJobId(jobId); fi.getJobSummary().setJobLaunchTime(amStartedEvent.getStartTime()); fi.getJobSummary().setJobSubmitTime(amStartedEvent.getSubmitTime()); fi.getJobIndexInfo().setJobStartTime(amStartedEvent.getStartTime()); fi.getJobIndexInfo().setSubmitTime(amStartedEvent.getSubmitTime()); fileMap.put(jobId, fi);
MetaInfo fi = new MetaInfo(historyFile, logDirConfPath, writer, user, jobName, jobId, amStartedEvent.getForcedJobStateOnShutDown(), queueName); fi.getJobSummary().setJobId(jobId); fi.getJobSummary().setJobLaunchTime(amStartedEvent.getStartTime()); fi.getJobSummary().setJobSubmitTime(amStartedEvent.getSubmitTime()); fi.getJobIndexInfo().setJobStartTime(amStartedEvent.getStartTime()); fi.getJobIndexInfo().setSubmitTime(amStartedEvent.getSubmitTime()); fileMap.put(jobId, fi);
LOG.debug("Shutting down timer for " + mi); mi.shutDownTimer(); } catch (IOException e) { LOG.info("Exception while cancelling delayed flush timer. " JobId toClose = jobIt.getKey(); MetaInfo mi = jobIt.getValue(); if(mi != null && mi.isWriterActive()) { LOG.warn("Found jobId " + toClose + " to have not been closed. Will close"); job.getCompletedReduces(), createJobStateForJobUnsuccessfulCompletionEvent( mi.getForcedJobStateOnShutDown()), job.getDiagnostics()); JobHistoryEvent jfEvent = new JobHistoryEvent(toClose, jucEvent); mi.closeWriter(); } catch (IOException e) { LOG.info("Exception while closing file " + e.getMessage());
LOG.debug("Shutting down timer for " + mi); mi.shutDownTimer(); } catch (IOException e) { LOG.info("Exception while cancelling delayed flush timer. " JobId toClose = jobIt.getKey(); MetaInfo mi = jobIt.getValue(); if(mi != null && mi.isWriterActive()) { LOG.warn("Found jobId " + toClose + " to have not been closed. Will close"); job.getCompletedReduces(), createJobStateForJobUnsuccessfulCompletionEvent( mi.getForcedJobStateOnShutDown()), job.getDiagnostics()); JobHistoryEvent jfEvent = new JobHistoryEvent(toClose, jucEvent); mi.closeWriter(); } catch (IOException e) { LOG.info("Exception while closing file " + e.getMessage());
if (mi.getHistoryFile() == null) { LOG.warn("No file for job-history with " + jobId + " found in cache!"); if (mi.getConfFile() == null) { LOG.warn("No file for jobconf with " + jobId + " found in cache!"); doneDirPrefixPath, doneSummaryFileName)); summaryFileOut = doneDirFS.create(qualifiedSummaryDoneFile, true); summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString()); summaryFileOut.close(); doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission( if (mi.getHistoryFile() != null) { Path historyFile = mi.getHistoryFile(); Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile); String doneJobHistoryFileName = getTempFileName(FileNameIndexUtils.getDoneFileName(mi .getJobIndexInfo())); qualifiedDoneFile = doneDirFS.makeQualified(new Path(doneDirPrefixPath, if (mi.getConfFile() != null) { Path confFile = mi.getConfFile(); Path qualifiedConfFile = stagingDirFS.makeQualified(confFile); String doneConfFileName =
MetaInfo fi = new MetaInfo(historyFile, logDirConfPath, writer, user, jobName, jobId, amStartedEvent.getForcedJobStateOnShutDown(), queueName); fi.getJobSummary().setJobId(jobId); fi.getJobSummary().setJobLaunchTime(amStartedEvent.getStartTime()); fi.getJobSummary().setJobSubmitTime(amStartedEvent.getSubmitTime()); fi.getJobIndexInfo().setJobStartTime(amStartedEvent.getStartTime()); fi.getJobIndexInfo().setSubmitTime(amStartedEvent.getSubmitTime()); fileMap.put(jobId, fi);
if (mi.getHistoryFile() == null) { LOG.warn("No file for job-history with " + jobId + " found in cache!"); if (mi.getConfFile() == null) { LOG.warn("No file for jobconf with " + jobId + " found in cache!"); doneDirPrefixPath, doneSummaryFileName)); summaryFileOut = doneDirFS.create(qualifiedSummaryDoneFile, true); summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString()); summaryFileOut.close(); doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission( if (mi.getHistoryFile() != null) { Path historyFile = mi.getHistoryFile(); Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile); int jobNameLimit = String doneJobHistoryFileName = getTempFileName(FileNameIndexUtils.getDoneFileName(mi .getJobIndexInfo(), jobNameLimit)); qualifiedDoneFile = doneDirFS.makeQualified(new Path(doneDirPrefixPath, if (mi.getConfFile() != null) { Path confFile = mi.getConfFile(); Path qualifiedConfFile = stagingDirFS.makeQualified(confFile); String doneConfFileName =
if (mi.getHistoryFile() == null) { LOG.warn("No file for job-history with " + jobId + " found in cache!"); if (mi.getConfFile() == null) { LOG.warn("No file for jobconf with " + jobId + " found in cache!"); doneDirPrefixPath, doneSummaryFileName)); summaryFileOut = doneDirFS.create(qualifiedSummaryDoneFile, true); summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString()); summaryFileOut.close(); doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission( if (mi.getHistoryFile() != null) { Path historyFile = mi.getHistoryFile(); Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile); int jobNameLimit = String doneJobHistoryFileName = getTempFileName(FileNameIndexUtils.getDoneFileName(mi .getJobIndexInfo(), jobNameLimit)); qualifiedDoneFile = doneDirFS.makeQualified(new Path(doneDirPrefixPath, if (mi.getConfFile() != null) { Path confFile = mi.getConfFile(); Path qualifiedConfFile = stagingDirFS.makeQualified(confFile); String doneConfFileName =
Assert.assertEquals(mi.getJobIndexInfo().getSubmitTime(), 100); Assert.assertEquals(mi.getJobIndexInfo().getJobStartTime(), 200); Assert.assertEquals(mi.getJobSummary().getJobSubmitTime(), 100); Assert.assertEquals(mi.getJobSummary().getJobLaunchTime(), 200); 0, 0, 0, 0, 0, 0, JobStateInternal.FAILED.toString()))); Assert.assertEquals(mi.getJobIndexInfo().getSubmitTime(), 100); Assert.assertEquals(mi.getJobIndexInfo().getJobStartTime(), 200); Assert.assertEquals(mi.getJobSummary().getJobSubmitTime(), 100); Assert.assertEquals(mi.getJobSummary().getJobLaunchTime(), 200); verify(jheh, times(1)).processDoneFiles(t.jobId);
protected void closeEventWriter(JobId jobId) throws IOException { final MetaInfo mi = fileMap.get(jobId); if (mi == null) { throw new IOException("No MetaInfo found for JobId: [" + jobId + "]"); } if (!mi.isWriterActive()) { throw new IOException( "Inactive Writer: Likely received multiple JobFinished / " + "JobUnsuccessful events for JobId: [" + jobId + "]"); } // Close the Writer try { mi.closeWriter(); } catch (IOException e) { LOG.error("Error closing writer for JobID: " + jobId); throw e; } }
protected void closeEventWriter(JobId jobId) throws IOException { final MetaInfo mi = fileMap.get(jobId); if (mi == null) { throw new IOException("No MetaInfo found for JobId: [" + jobId + "]"); } if (!mi.isWriterActive()) { throw new IOException( "Inactive Writer: Likely received multiple JobFinished / " + "JobUnsuccessful events for JobId: [" + jobId + "]"); } // Close the Writer try { mi.closeWriter(); } catch (IOException e) { LOG.error("Error closing writer for JobID: " + jobId); throw e; } }
/** Close the event writer for this id * @throws IOException */ public void closeWriter(JobId id) throws IOException { try { final MetaInfo mi = fileMap.get(id); if (mi != null) { mi.closeWriter(); } } catch (IOException e) { LOG.error("Error closing writer for JobID: " + id); throw e; } }
/** Close the event writer for this id * @throws IOException */ public void closeWriter(JobId id) throws IOException { try { final MetaInfo mi = fileMap.get(id); if (mi != null) { mi.closeWriter(); } } catch (IOException e) { LOG.error("Error closing writer for JobID: " + id); throw e; } }
@Override public void run() { LOG.debug("In flush timer task"); synchronized (lock) { try { if (!metaInfo.isTimerShutDown() && shouldRun) metaInfo.flush(); } catch (IOException e) { ioe = e; } } }