private void verifyRMAppFieldsForFinalTransitions(RMApp application) { assertEquals(null, application.getApplicationSubmissionContext(). getAMContainerSpec()); assertEquals(null, application.getApplicationSubmissionContext(). getLogAggregationContext()); } }
private static RMApp createMockAppForRemove(ApplicationId appId, ApplicationAttemptId... attemptIds) { RMApp app = mock(RMApp.class); ApplicationSubmissionContextPBImpl context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); when(app.getApplicationSubmissionContext()).thenReturn(context); when(app.getUser()).thenReturn("test"); if (attemptIds.length > 0) { HashMap<ApplicationAttemptId, RMAppAttempt> attempts = new HashMap<>(); for (ApplicationAttemptId attemptId : attemptIds) { RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()).thenReturn(attemptId); attempts.put(attemptId, appAttempt); } when(app.getAppAttempts()).thenReturn(attempts); } return app; }
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); store.storeNewApplication(mockApp); return mockApp; }
protected RMApp createMockRMApp(ApplicationAttemptId attemptId) { RMApp app = mock(RMAppImpl.class); when(app.getApplicationId()).thenReturn(attemptId.getApplicationId()); RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class); when(attempt.getAppAttemptId()).thenReturn(attemptId); RMAppAttemptMetrics attemptMetric = mock(RMAppAttemptMetrics.class); when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); when(app.getCurrentAppAttempt()).thenReturn(attempt); ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getUnmanagedAM()).thenReturn(false); when(attempt.getSubmissionContext()).thenReturn(submissionContext); when(app.getApplicationSubmissionContext()).thenReturn(submissionContext); resourceManager.getRMContext().getRMApps() .put(attemptId.getApplicationId(), app); return app; }
public List<Container> getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); SchedulerApplication<T> app = applications.get(appId); List<Container> containerList = new ArrayList<Container>(); RMApp appImpl = this.rmContext.getRMApps().get(appId); if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) { return containerList; } if (app == null) { return containerList; } Collection<RMContainer> liveContainers = app.getCurrentAppAttempt().getLiveContainers(); ContainerId amContainerId = rmContext.getRMApps().get(appId).getCurrentAppAttempt() .getMasterContainer().getId(); for (RMContainer rmContainer : liveContainers) { if (!rmContainer.getContainerId().equals(amContainerId)) { containerList.add(rmContainer.getContainer()); } } return containerList; }
public List<Container> getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); SchedulerApplication<T> app = applications.get(appId); List<Container> containerList = new ArrayList<Container>(); RMApp appImpl = this.rmContext.getRMApps().get(appId); if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) { return containerList; } if (app == null) { return containerList; } Collection<RMContainer> liveContainers = app.getCurrentAppAttempt().getLiveContainers(); ContainerId amContainerId = rmContext.getRMApps().get(appId).getCurrentAppAttempt() .getMasterContainer().getId(); for (RMContainer rmContainer : liveContainers) { if (!rmContainer.getContainerId().equals(amContainerId)) { containerList.add(rmContainer.getContainer()); } } return containerList; }
/** * Non-blocking API * ResourceManager services call this to remove an application from the state * store * This does not block the dispatcher threads * There is no notification of completion for this operation. */ @SuppressWarnings("unchecked") public void removeApplication(RMApp app) { ApplicationStateData appState = ApplicationStateData.newInstance( app.getSubmitTime(), app.getStartTime(), app.getApplicationSubmissionContext(), app.getUser()); for(RMAppAttempt appAttempt : app.getAppAttempts().values()) { appState.attempts.put(appAttempt.getAppAttemptId(), null); } dispatcher.getEventHandler().handle(new RMStateStoreRemoveAppEvent(appState)); }
/** * Non-blocking API * ResourceManager services call this to remove an application from the state * store * This does not block the dispatcher threads * There is no notification of completion for this operation. */ @SuppressWarnings("unchecked") public void removeApplication(RMApp app) { ApplicationStateData appState = ApplicationStateData.newInstance( app.getSubmitTime(), app.getStartTime(), app.getApplicationSubmissionContext(), app.getUser()); for(RMAppAttempt appAttempt : app.getAppAttempts().values()) { appState.attempts.put(appAttempt.getAppAttemptId(), null); } dispatcher.getEventHandler().handle(new RMStateStoreRemoveAppEvent(appState)); }
/** * Non-blocking API * ResourceManager services call this to remove an application from the state * store * This does not block the dispatcher threads * There is no notification of completion for this operation. */ @SuppressWarnings("unchecked") public void removeApplication(RMApp app) { ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getApplicationSubmissionContext(), app.getUser(), app.getCallerContext()); for(RMAppAttempt appAttempt : app.getAppAttempts().values()) { appState.attempts.put(appAttempt.getAppAttemptId(), null); } getRMStateStoreEventHandler().handle( new RMStateStoreRemoveAppEvent(appState)); }
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); context.setAMContainerSpec(new ContainerLaunchContextPBImpl()); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); when(mockApp.getCallerContext()) .thenReturn(new CallerContext.Builder("context").build()); store.storeNewApplication(mockApp); return mockApp; }
/** * Non-Blocking API * ResourceManager services use this to store the application's state * This does not block the dispatcher threads * RMAppStoredEvent will be sent on completion to notify the RMApp */ @SuppressWarnings("unchecked") public void storeNewApplication(RMApp app) { ApplicationSubmissionContext context = app .getApplicationSubmissionContext(); assert context instanceof ApplicationSubmissionContextPBImpl; ApplicationStateData appState = ApplicationStateData.newInstance( app.getSubmitTime(), app.getStartTime(), context, app.getUser()); dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState)); }
/** * Non-Blocking API * ResourceManager services use this to store the application's state * This does not block the dispatcher threads * RMAppStoredEvent will be sent on completion to notify the RMApp */ @SuppressWarnings("unchecked") public void storeNewApplication(RMApp app) { ApplicationSubmissionContext context = app .getApplicationSubmissionContext(); assert context instanceof ApplicationSubmissionContextPBImpl; ApplicationStateData appState = ApplicationStateData.newInstance( app.getSubmitTime(), app.getStartTime(), context, app.getUser()); dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState)); }
protected ApplicationAttemptId createSchedulingRequest(String queueId, String userId, List<ResourceRequest> ask) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id.getApplicationId(), queueId, userId, false); // This conditional is for testAclSubmitApplication where app is rejected // and no app is added. if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) { scheduler.addApplicationAttempt(id, false, false); } RMApp rmApp = mock(RMApp.class); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt); when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn( new RMAppAttemptMetrics(id,resourceManager.getRMContext())); ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getUnmanagedAM()).thenReturn(false); when(rmAppAttempt.getSubmissionContext()).thenReturn(submissionContext); when(rmApp.getApplicationSubmissionContext()).thenReturn(submissionContext); resourceManager.getRMContext().getRMApps() .put(id.getApplicationId(), rmApp); scheduler.allocate(id, ask, null, new ArrayList<ContainerId>(), null, null, NULL_UPDATE_REQUESTS); return id; }
static int getExpectedNumberOfElements(RMApp app) { int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests if (app.getApplicationSubmissionContext() .getNodeLabelExpression() != null) { expectedNumberOfElements++; } if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { expectedNumberOfElements++; } if (AppInfo .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) { expectedNumberOfElements++; } return expectedNumberOfElements; } }
@Override public void finishApplicationMaster( ApplicationAttemptId applicationAttemptId, FinishApplicationMasterRequest request, FinishApplicationMasterResponse response) { RMApp app = getRmContext().getRMApps().get(applicationAttemptId.getApplicationId()); // For UnmanagedAMs, return true so they don't retry response.setIsUnregistered( app.getApplicationSubmissionContext().getUnmanagedAM()); getRmContext().getDispatcher().getEventHandler().handle( new RMAppAttemptUnregistrationEvent(applicationAttemptId, request .getTrackingUrl(), request.getFinalApplicationStatus(), request .getDiagnostics())); }
public void createRMStateForApplications( Map<ApplicationId, ApplicationStateData> applicationState, RMAppState rmAppState) { RMApp app = createNewTestApp(null); ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getUser(), app.getApplicationSubmissionContext(), rmAppState, null, app.getFinishTime()); applicationState.put(app.getApplicationId(), appState); }
/** * Non-Blocking API * ResourceManager services use this to store the application's state * This does not block the dispatcher threads * RMAppStoredEvent will be sent on completion to notify the RMApp */ @SuppressWarnings("unchecked") public void storeNewApplication(RMApp app) { ApplicationSubmissionContext context = app .getApplicationSubmissionContext(); assert context instanceof ApplicationSubmissionContextPBImpl; ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), context, app.getUser(), app.getCallerContext()); appState.setApplicationTimeouts(app.getApplicationTimeouts()); getRMStateStoreEventHandler().handle(new RMStateStoreAppEvent(appState)); }
public void createRMStateForApplications( Map<ApplicationId, ApplicationStateData> applicationState, RMAppState rmAppState) throws IOException { RMApp app = createNewTestApp(null); ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getUser(), app.getApplicationSubmissionContext(), rmAppState, null, app.getLaunchTime(), app.getFinishTime(), null); applicationState.put(app.getApplicationId(), appState); }
@Test public void testMaxPriorityValidation() throws Exception { Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); // Set Max Application Priority as 10 conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10); Priority maxPriority = Priority.newInstance(10); MockRM rm = new MockRM(conf); rm.start(); Priority appPriority1 = Priority.newInstance(15); rm.registerNode("127.0.0.1:1234", 8 * GB); RMApp app1 = rm.submitApp(1 * GB, appPriority1); // Application submission should be successful and verify priority Assert.assertEquals(app1.getApplicationSubmissionContext().getPriority(), maxPriority); rm.stop(); }
private void testApplicationPriorityUpdation(ClientRMService rmService, RMApp app1, int tobeUpdatedPriority, int expected) throws YarnException, IOException { UpdateApplicationPriorityRequest updateRequest = UpdateApplicationPriorityRequest.newInstance(app1.getApplicationId(), Priority.newInstance(tobeUpdatedPriority)); UpdateApplicationPriorityResponse updateApplicationPriority = rmService.updateApplicationPriority(updateRequest); Assert.assertEquals("Incorrect priority has been set to application", expected, app1.getApplicationSubmissionContext().getPriority() .getPriority()); Assert.assertEquals("Incorrect priority has been returned", expected, updateApplicationPriority.getApplicationPriority().getPriority()); }