@Override public RMAppMetrics getRMAppMetrics() { return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0); } @Override
info("Application Metrics") ._("Total Resource Preempted:", appMetrics == null ? "N/A" : appMetrics.getResourcePreempted()) ._("Total Number of Non-AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumNonAMContainersPreempted()) ._("Total Number of AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumAMContainersPreempted()) ._("Resource Preempted from Current Attempt:", attemptResourcePreempted) ._("Aggregate Resource Allocation:", String.format("%d MB-seconds, %d vcore-seconds", appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())); pdiv._();
.add("finishTime", app.getFinishTime()) .add("finalStatus", app.getFinalApplicationStatus()) .add("memorySeconds", metrics.getMemorySeconds()) .add("vcoreSeconds", metrics.getVcoreSeconds()) .add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds()) .add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds()) .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) .add("applicationType", app.getApplicationType()) .add("resourceSeconds", StringHelper .getResourceSecondsString(metrics.getResourceSecondsMap())) .add("preemptedResourceSeconds", StringHelper .getResourceSecondsString( metrics.getPreemptedResourceSecondsMap())); return summary;
private Set<TimelineMetric> getTimelinelineAppMetrics( RMAppMetrics appMetrics, long timestamp) { Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>(); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_CPU_METRICS, timestamp, appMetrics.getVcoreSeconds())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_MEM_METRICS, timestamp, appMetrics.getMemorySeconds())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp, appMetrics.getPreemptedMemorySeconds())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, timestamp, appMetrics.getPreemptedVcoreSeconds())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_RESOURCE_PREEMPTED_CPU, timestamp, appMetrics.getResourcePreempted().getVirtualCores())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_RESOURCE_PREEMPTED_MEM, timestamp, appMetrics.getResourcePreempted().getMemorySize())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_NON_AM_CONTAINER_PREEMPTED, timestamp, appMetrics.getNumNonAMContainersPreempted())); entityMetrics.add(getTimelineMetric( ApplicationMetricsConstants.APP_AM_CONTAINER_PREEMPTED, timestamp, appMetrics.getNumAMContainersPreempted())); return entityMetrics; }
appMetrics.getVcoreSeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, appMetrics.getMemorySeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, appMetrics.getPreemptedMemorySeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, appMetrics.getPreemptedVcoreSeconds()); tEvent.setEventInfo(eventInfo);
private void waitForAppPreemptionInfo(RMApp app, Resource preempted, int numAMPreempted, int numTaskPreempted, Resource currentAttemptPreempted, boolean currentAttemptAMPreempted, int numLatestAttemptTaskPreempted) throws InterruptedException { while (true) { RMAppMetrics appPM = app.getRMAppMetrics(); RMAppAttemptMetrics attemptPM = app.getCurrentAppAttempt().getRMAppAttemptMetrics(); if (appPM.getResourcePreempted().equals(preempted) && appPM.getNumAMContainersPreempted() == numAMPreempted && appPM.getNumNonAMContainersPreempted() == numTaskPreempted && attemptPM.getResourcePreempted().equals(currentAttemptPreempted) && app.getCurrentAppAttempt().getRMAppAttemptMetrics() .getIsPreempted() == currentAttemptAMPreempted && attemptPM.getNumNonAMContainersPreempted() == numLatestAttemptTaskPreempted) { return; } Thread.sleep(500); } }
private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) { TimelineEntity entity = createApplicationEntity(event.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( ApplicationMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(event.getTimestamp()); Map<String, Object> eventInfo = new HashMap<String, Object>(); eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, event.getDiagnosticsInfo()); eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, event.getFinalApplicationStatus().toString()); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, event.getYarnApplicationState().toString()); if (event.getLatestApplicationAttemptId() != null) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, event.getLatestApplicationAttemptId().toString()); } RMAppMetrics appMetrics = event.getAppMetrics(); entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS, appMetrics.getVcoreSeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, appMetrics.getMemorySeconds()); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); }
info("Application Metrics") .__("Total Resource Preempted:", appMetrics == null ? "N/A" : appMetrics.getResourcePreempted()) .__("Total Number of Non-AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumNonAMContainersPreempted()) .__("Total Number of AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumAMContainersPreempted()) .__("Resource Preempted from Current Attempt:", attemptResourcePreempted) .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" : StringHelper .getResourceSecondsString(appMetrics.getResourceSecondsMap())) .__("Aggregate Preempted Resource Allocation:", appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString( appMetrics.getPreemptedResourceSecondsMap()));
private void waitForAppPreemptionInfo(RMApp app, Resource preempted, int numAMPreempted, int numTaskPreempted, Resource currentAttemptPreempted, boolean currentAttemptAMPreempted, int numLatestAttemptTaskPreempted) throws InterruptedException { while (true) { RMAppMetrics appPM = app.getRMAppMetrics(); RMAppAttemptMetrics attemptPM = app.getCurrentAppAttempt().getRMAppAttemptMetrics(); if (appPM.getResourcePreempted().equals(preempted) && appPM.getNumAMContainersPreempted() == numAMPreempted && appPM.getNumNonAMContainersPreempted() == numTaskPreempted && attemptPM.getResourcePreempted().equals(currentAttemptPreempted) && app.getCurrentAppAttempt().getRMAppAttemptMetrics() .getIsPreempted() == currentAttemptAMPreempted && attemptPM.getNumNonAMContainersPreempted() == numLatestAttemptTaskPreempted) { return; } Thread.sleep(500); } }
private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) { TimelineEntity entity = createApplicationEntity(event.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( ApplicationMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setTimestamp(event.getTimestamp()); Map<String, Object> eventInfo = new HashMap<String, Object>(); eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, event.getDiagnosticsInfo()); eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, event.getFinalApplicationStatus().toString()); eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, event.getYarnApplicationState().toString()); if (event.getLatestApplicationAttemptId() != null) { eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, event.getLatestApplicationAttemptId().toString()); } RMAppMetrics appMetrics = event.getAppMetrics(); entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS, appMetrics.getVcoreSeconds()); entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS, appMetrics.getMemorySeconds()); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); putEntity(entity); }
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); Assert.assertEquals( app.getRMAppMetrics().getMemorySeconds(), Long.parseLong(entity.getOtherInfo() .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString())); Assert.assertEquals( app.getRMAppMetrics().getVcoreSeconds(), Long.parseLong(entity.getOtherInfo() .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString())); Assert.assertEquals( app.getRMAppMetrics().getPreemptedMemorySeconds(), Long.parseLong(entity.getOtherInfo() .get(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS) .toString())); Assert.assertEquals( app.getRMAppMetrics().getPreemptedVcoreSeconds(), Long.parseLong(entity.getOtherInfo() .get(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS)
info("Application Metrics") ._("Total Resource Preempted:", appMetrics == null ? "N/A" : appMetrics.getResourcePreempted()) ._("Total Number of Non-AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumNonAMContainersPreempted()) ._("Total Number of AM Containers Preempted:", appMetrics == null ? "N/A" : appMetrics.getNumAMContainersPreempted()) ._("Resource Preempted from Current Attempt:", attemptResourcePreempted) ._("Aggregate Resource Allocation:", String.format("%d MB-seconds, %d vcore-seconds", appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(), appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds())); pdiv._();
numAMContainerPreempted = appMetrics.getNumAMContainersPreempted(); preemptedResourceMB = appMetrics.getResourcePreempted().getMemorySize(); numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds(); resourceSecondsMap = appMetrics.getResourceSecondsMap(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap(); ApplicationSubmissionContext appSubmissionContext = app.getApplicationSubmissionContext();
assertEquals("numContainers doesn't match", 1, numContainers); assertEquals("preemptedResourceMB doesn't match", app .getRMAppMetrics().getResourcePreempted().getMemory(), preemptedResourceMB); assertEquals("preemptedResourceVCores doesn't match", app .getRMAppMetrics().getResourcePreempted().getVirtualCores(), preemptedResourceVCores); assertEquals("numNonAMContainerPreempted doesn't match", app .getRMAppMetrics().getNumNonAMContainersPreempted(), numNonAMContainerPreempted); assertEquals("numAMContainerPreempted doesn't match", app .getRMAppMetrics().getNumAMContainersPreempted(), numAMContainerPreempted);
Assert.assertTrue( "Before app submittion, memory seconds should have been 0 but was " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() == 0); Assert.assertTrue( "Before app submission, vcore seconds should have been 0 but was " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() == 0); while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; Assert.assertTrue( "While app is running, memory seconds should be >0 but is " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() > 0); Assert.assertTrue( "While app is running, vcore seconds should be >0 but is " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() > 0); ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds()); Assert.assertEquals("Unexpected VcoreSeconds value", ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
@Override public RMAppMetrics getRMAppMetrics() { return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), new HashMap<>()); }
appMetrics.getNumAMContainersPreempted(); preemptedResourceMB = appMetrics.getResourcePreempted().getMemory(); numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted(); preemptedResourceVCores = appMetrics.getResourcePreempted().getVirtualCores(); memorySeconds = appMetrics.getMemorySeconds(); vcoreSeconds = appMetrics.getVcoreSeconds();
assertEquals("numContainers doesn't match", 1, numContainers); assertEquals("preemptedResourceMB doesn't match", app .getRMAppMetrics().getResourcePreempted().getMemorySize(), preemptedResourceMB); assertEquals("preemptedResourceVCores doesn't match", app .getRMAppMetrics().getResourcePreempted().getVirtualCores(), preemptedResourceVCores); assertEquals("numNonAMContainerPreempted doesn't match", app .getRMAppMetrics().getNumNonAMContainersPreempted(), numNonAMContainerPreempted); assertEquals("numAMContainerPreempted doesn't match", app .getRMAppMetrics().getNumAMContainersPreempted(), numAMContainerPreempted); assertEquals("Log aggregation Status doesn't match", app
Assert.assertTrue( "Before app submittion, memory seconds should have been 0 but was " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() == 0); Assert.assertTrue( "Before app submission, vcore seconds should have been 0 but was " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() == 0); while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; Assert.assertTrue( "While app is running, memory seconds should be >0 but is " + rmAppMetrics.getMemorySeconds(), rmAppMetrics.getMemorySeconds() > 0); Assert.assertTrue( "While app is running, vcore seconds should be >0 but is " + rmAppMetrics.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds() > 0); ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds()); Assert.assertEquals("Unexpected VcoreSeconds value", ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
@Override public RMAppMetrics getRMAppMetrics() { return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), new HashMap<>()); } @Override