this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); this.state = app.createApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() || YarnApplicationState.NEW == this.state || YarnApplicationState.ACCEPTED == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app .getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = this.applicationId = app.getApplicationId(); this.applicationType = app.getApplicationType(); this.appIdNum = String.valueOf(app.getApplicationId().getId()); this.id = app.getApplicationId().toString(); this.user = app.getUser().toString(); this.name = app.getName().toString(); this.queue = app.getQueue().toString(); this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) { this.diagnostics = ""; if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) { this.applicationTags = Joiner.on(',').join(app.getApplicationTags()); this.finalStatus = app.getFinalApplicationStatus();
@SuppressWarnings("unchecked") @Override public void appCreated(RMApp app, long createdTime) { TimelineEntity entity = createApplicationEntity(app.getApplicationId()); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, app.getName()); entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, app.getApplicationType()); entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, app.getUser()); entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, app.getQueue()); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, app.getSubmitTime()); entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, app.getApplicationTags()); entityInfo.put( ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, app.getApplicationSubmissionContext().getUnmanagedAM()); entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, app.getApplicationPriority().getPriority()); entityInfo.put(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION, app.getAmNodeLabelExpression()); entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, app.getAppNodeLabelExpression()); if (app.getCallerContext() != null) { if (app.getCallerContext().getContext() != null) { entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT, app.getCallerContext().getContext()); if (app.getCallerContext().getSignature() != null) {
/** * Non-blocking API * ResourceManager services call this to remove an application from the state * store * This does not block the dispatcher threads * There is no notification of completion for this operation. */ @SuppressWarnings("unchecked") public void removeApplication(RMApp app) { ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getApplicationSubmissionContext(), app.getUser(), app.getCallerContext()); for(RMAppAttempt appAttempt : app.getAppAttempts().values()) { appState.attempts.put(appAttempt.getAppAttemptId(), null); } getRMStateStoreEventHandler().handle( new RMStateStoreRemoveAppEvent(appState)); }
@SuppressWarnings("unchecked") public void applicationStarted(RMApp app) { if (historyServiceEnabled) { dispatcher.getEventHandler().handle( new WritingApplicationStartEvent(app.getApplicationId(), ApplicationStartData.newInstance(app.getApplicationId(), app.getName(), app.getApplicationType(), app.getQueue(), app.getUser(), app.getSubmitTime(), app.getStartTime()))); } }
/** * Non-Blocking API * ResourceManager services use this to store the application's state * This does not block the dispatcher threads * RMAppStoredEvent will be sent on completion to notify the RMApp */ @SuppressWarnings("unchecked") public void storeNewApplication(RMApp app) { ApplicationSubmissionContext context = app .getApplicationSubmissionContext(); assert context instanceof ApplicationSubmissionContextPBImpl; ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), context, app.getUser(), app.getCallerContext()); appState.setApplicationTimeouts(app.getApplicationTimeouts()); getRMStateStoreEventHandler().handle(new RMStateStoreAppEvent(appState)); }
private String getTimelineEntityDir(RMApp app) { String outputDirApp = testRootDir.getAbsolutePath() + "/" + FileSystemTimelineWriterImpl.ENTITIES_DIR + "/" + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" + app.getUser() + "/" + app.getName() + "/" + TimelineUtils.DEFAULT_FLOW_VERSION + "/" + app.getStartTime() + "/" + app.getApplicationId(); return outputDirApp; }
private static RMApp createRMApp(ApplicationId appId) { RMApp app = mock(RMAppImpl.class); when(app.getApplicationId()).thenReturn(appId); when(app.getName()).thenReturn("test app"); when(app.getApplicationType()).thenReturn("test app type"); when(app.getUser()).thenReturn("testUser"); when(app.getQueue()).thenReturn("test queue"); when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L); when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L); when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L); when(app.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info")); RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()) .thenReturn(ApplicationAttemptId.newInstance(appId, 1)); when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()) .thenReturn(FinalApplicationStatus.UNDEFINED); Map<String, Long> resourceSecondsMap = new HashMap<>(); resourceSecondsMap .put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap, new HashMap<>())); when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet()); ApplicationSubmissionContext appSubmissionContext = mock(ApplicationSubmissionContext.class); when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10));
this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); this.state = app.createApplicationState(); this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty() || YarnApplicationState.NEW == this.state || YarnApplicationState.ACCEPTED == this.state; this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app.getFinishTime() == 0 ? "ApplicationMaster" : "History"); if (!trackingUrlIsNotReady) { this.trackingUrl = this.applicationId = app.getApplicationId(); this.applicationType = app.getApplicationType(); this.appIdNum = String.valueOf(app.getApplicationId().getId()); this.id = app.getApplicationId().toString(); this.user = app.getUser().toString(); this.name = app.getName().toString(); this.queue = app.getQueue().toString(); this.priority = 0; this.masterNodeId = ""; if (app.getApplicationPriority() != null) { this.priority = app.getApplicationPriority().getPriority(); this.progress = app.getProgress() * 100; this.diagnostics = app.getDiagnostics().toString(); if (diagnostics == null || diagnostics.isEmpty()) {
when(app.getTrackingUrl()).thenReturn("http://host:123"); when(app.getState()).thenReturn(RMAppState.FAILED); when(app.getApplicationId()).thenReturn(APP_ID); when(app.getApplicationType()).thenReturn("Type"); when(app.getUser()).thenReturn("user"); when(app.getName()).thenReturn("Name"); when(app.getQueue()).thenReturn("queue"); when(app.getDiagnostics()).thenReturn(new StringBuilder()); when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED); when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED); when(app.getStartTime()).thenReturn(0L); when(app.getFinishTime()).thenReturn(0L); when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED); when(app.getRMAppMetrics()).thenReturn(appMetrics);
/** * Non-Blocking API * ResourceManager services use this to store the application's state * This does not block the dispatcher threads * RMAppStoredEvent will be sent on completion to notify the RMApp */ @SuppressWarnings("unchecked") public void storeNewApplication(RMApp app) { ApplicationSubmissionContext context = app .getApplicationSubmissionContext(); assert context instanceof ApplicationSubmissionContextPBImpl; ApplicationStateData appState = ApplicationStateData.newInstance( app.getSubmitTime(), app.getStartTime(), context, app.getUser()); dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState)); }
private static void testAppStartState(ApplicationId applicationId, String user, String name, String queue, RMApp application) { Assert.assertTrue("application start time is not greater than 0", application.getStartTime() > 0); Assert.assertTrue("application start time is before currentTime", application.getStartTime() <= System.currentTimeMillis()); Assert.assertEquals("application user is not correct", user, application.getUser()); Assert.assertEquals("application id is not correct", applicationId, application.getApplicationId()); Assert.assertEquals("application progress is not correct", (float)0.0, application.getProgress(), (float)0.0); Assert.assertEquals("application queue is not correct", queue, application.getQueue()); Assert.assertEquals("application name is not correct", name, application.getName()); Assert.assertEquals("application finish time is not 0 and should be", 0, application.getFinishTime()); Assert.assertEquals("application tracking url is not correct", null, application.getTrackingUrl()); StringBuilder diag = application.getDiagnostics(); Assert.assertEquals("application diagnostics is not correct", 0, diag.length()); }
!callerUGI.getUserName().equals(application.getUser())) { continue; ? application.getApplicationType() : StringUtils.toLowerCase(application.getApplicationType()); if (!applicationTypes.contains(appTypeToMatch)) { continue; .createApplicationState())) { continue; !users.contains(application.getUser())) { continue; if (start != null && !start.containsLong(application.getStartTime())) { continue; if (finish != null && !finish.containsLong(application.getFinishTime())) { continue; Set<String> appTags = application.getApplicationTags(); if (appTags == null || appTags.isEmpty()) { continue; boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); if (scope == ApplicationsRequestScope.VIEWABLE && !allowAccess) { reports.add(application.createAndGetApplicationReport(
sb.append(String.format( " %s %9s %5.2f%% %5ds", rmApp.getState(), (rmApp.getApplicationType() == null)? "" : rmApp.getApplicationType(), 100.0 * rmApp.getProgress(), (mclock.getTime() - rmApp.getStartTime()) / 1000));
private ApplicationReport verifyAppReportAfterRMRestart(RMApp app, MockRM rm) throws Exception { GetApplicationReportRequest reportRequest = GetApplicationReportRequest.newInstance(app.getApplicationId()); GetApplicationReportResponse response = rm.getClientRMService().getApplicationReport(reportRequest); ApplicationReport report = response.getApplicationReport(); Assert.assertEquals(app.getStartTime(), report.getStartTime()); Assert.assertEquals(app.getFinishTime(), report.getFinishTime()); Assert.assertEquals(app.createApplicationState(), report.getYarnApplicationState()); Assert.assertTrue(1 == report.getProgress()); return response.getApplicationReport(); }
protected MockAM launchAM(RMApp app, MockRM rm, MockNM nm) throws Exception { RMAppAttempt attempt = app.getCurrentAppAttempt(); nm.nodeHeartbeat(true); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); am.registerAppAttempt(); rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); rm.waitForState(app.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.RUNNING); return am; }
boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); ApplicationReport report = application.createAndGetApplicationReport(callerUGI.getUserName(), allowAccess);
"non-existing app " + appId); String userId = app.getUser(); TimelineCollectorContext context = collector.getTimelineEntityContext(); if (userId != null && !userId.isEmpty()) { app.getName(), appId)); context.setFlowVersion(TimelineUtils.DEFAULT_FLOW_VERSION); context.setFlowRunId(app.getStartTime()); for (String tag : app.getApplicationTags()) { String[] parts = tag.split(":", 2); if (parts.length != 2 || parts[1].isEmpty()) {
/** * Wait until an attempt has reached a specified state. * The timeout is 40 seconds. * @param finalState the attempt state waited * @throws InterruptedException * if interrupted while waiting for the state transition */ private void waitForState(RMAppAttemptState finalState) throws InterruptedException { RMApp app = context.getRMApps().get(attemptId.getApplicationId()); RMAppAttempt attempt = app.getRMAppAttempt(attemptId); MockRM.waitForState(attempt, finalState); }
protected void addToCompletedApps(TestRMAppManager appMonitor, RMContext rmContext) { for (RMApp app : rmContext.getRMApps().values()) { if (app.getState() == RMAppState.FINISHED || app.getState() == RMAppState.KILLED || app.getState() == RMAppState.FAILED) { appMonitor.finishApplication(app.getApplicationId()); } } }
protected RMApp createMockRMApp(ApplicationAttemptId attemptId) { RMApp app = mock(RMAppImpl.class); when(app.getApplicationId()).thenReturn(attemptId.getApplicationId()); RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class); when(attempt.getAppAttemptId()).thenReturn(attemptId); RMAppAttemptMetrics attemptMetric = mock(RMAppAttemptMetrics.class); when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); when(app.getCurrentAppAttempt()).thenReturn(attempt); ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getUnmanagedAM()).thenReturn(false); when(attempt.getSubmissionContext()).thenReturn(submissionContext); when(app.getApplicationSubmissionContext()).thenReturn(submissionContext); resourceManager.getRMContext().getRMApps() .put(attemptId.getApplicationId(), app); return app; }