private ApplicationSubmissionContextPBImpl convertFromProtoFormat(ApplicationSubmissionContextProto p) { return new ApplicationSubmissionContextPBImpl(p); }
private ApplicationSubmissionContextPBImpl convertFromProtoFormat(ApplicationSubmissionContextProto p) { return new ApplicationSubmissionContextPBImpl(p); }
private ApplicationSubmissionContextPBImpl convertFromProtoFormat(ApplicationSubmissionContextProto p) { return new ApplicationSubmissionContextPBImpl(p); }
@Override public ApplicationSubmissionContext getApplicationSubmissionContext() { return new ApplicationSubmissionContextPBImpl(); }
@Override public ApplicationSubmissionContext getApplicationSubmissionContext() { return new ApplicationSubmissionContextPBImpl(); }
@Override public ApplicationSubmissionContext getApplicationSubmissionContext() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; if(applicationSubmissionContext != null) { return applicationSubmissionContext; } if (!p.hasApplicationSubmissionContext()) { return null; } applicationSubmissionContext = new ApplicationSubmissionContextPBImpl( p.getApplicationSubmissionContext()); return applicationSubmissionContext; }
@Override public ApplicationSubmissionContext getApplicationSubmissionContext() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; if(applicationSubmissionContext != null) { return applicationSubmissionContext; } if (!p.hasApplicationSubmissionContext()) { return null; } applicationSubmissionContext = new ApplicationSubmissionContextPBImpl( p.getApplicationSubmissionContext()); return applicationSubmissionContext; }
@Override public ApplicationSubmissionContext getApplicationSubmissionContext() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; if(applicationSubmissionContext != null) { return applicationSubmissionContext; } if (!p.hasApplicationSubmissionContext()) { return null; } applicationSubmissionContext = new ApplicationSubmissionContextPBImpl( p.getApplicationSubmissionContext()); return applicationSubmissionContext; }
private static RMApp createMockAppForRemove(ApplicationId appId, ApplicationAttemptId... attemptIds) { RMApp app = mock(RMApp.class); ApplicationSubmissionContextPBImpl context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); when(app.getApplicationSubmissionContext()).thenReturn(context); when(app.getUser()).thenReturn("test"); if (attemptIds.length > 0) { HashMap<ApplicationAttemptId, RMAppAttempt> attempts = new HashMap<>(); for (ApplicationAttemptId attemptId : attemptIds) { RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()).thenReturn(attemptId); attempts.put(attemptId, appAttempt); } when(app.getAppAttempts()).thenReturn(attempts); } return app; }
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); store.storeNewApplication(mockApp); return mockApp; }
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); context.setAMContainerSpec(new ContainerLaunchContextPBImpl()); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); when(mockApp.getCallerContext()) .thenReturn(new CallerContext.Builder("context").build()); store.storeNewApplication(mockApp); return mockApp; }
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) { ApplicationId applicationId = MockApps.newAppID(appId++); String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); // ensure max application attempts set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); if(submissionContext == null) { submissionContext = new ApplicationSubmissionContextPBImpl(); } // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, System.currentTimeMillis(), "YARN", null, null); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), application); return application; }
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) throws IOException { ApplicationId applicationId = MockApps.newAppID(appId++); String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); // ensure max application attempts set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); if(submissionContext == null) { submissionContext = new ApplicationSubmissionContextPBImpl(); } // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); submissionContext.setPriority(Priority.newInstance(0)); submissionContext.setAMContainerSpec(prepareContainerLaunchContext()); submissionContext.setLogAggregationContext(getLogAggregationContext()); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, System.currentTimeMillis(), "YARN", null, new ArrayList<ResourceRequest>()); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), application); return application; }
@Test public void testUnmanagedApp() throws IOException { ApplicationSubmissionContext subContext = new ApplicationSubmissionContextPBImpl(); subContext.setUnmanagedAM(true); // test success path LOG.info("--- START: testUnmanagedAppSuccessPath ---"); final String diagMsg = "some diagnostics"; RMApp application = testCreateAppFinished(subContext, diagMsg); Assert.assertTrue("Finished app missing diagnostics", application.getDiagnostics().indexOf(diagMsg) != -1); // reset the counter of Mockito.verify reset(writer); reset(publisher); // test app fails after 1 app attempt failure LOG.info("--- START: testUnmanagedAppFailPath ---"); application = testCreateAppRunning(subContext); RMAppEvent event = new RMAppFailedAttemptEvent( application.getApplicationId(), RMAppEventType.ATTEMPT_FAILED, "", false); application.handle(event); rmDispatcher.await(); RMAppAttempt appAttempt = application.getCurrentAppAttempt(); Assert.assertEquals(1, appAttempt.getAppAttemptId().getAttemptId()); sendAppUpdateSavedEvent(application); assertFailed(application, ".*Unmanaged application.*Failing the application.*"); assertAppFinalStateSaved(application); }
null, null, new ApplicationSubmissionContextPBImpl(), null, null, System.currentTimeMillis(), "YARN", null, null) { @Override
@Test public void testUnmanagedApp() throws IOException { ApplicationSubmissionContext subContext = new ApplicationSubmissionContextPBImpl(); subContext.setUnmanagedAM(true); // test success path LOG.info("--- START: testUnmanagedAppSuccessPath ---"); final String diagMsg = "some diagnostics"; RMApp application = testCreateAppFinished(subContext, diagMsg); Assert.assertTrue("Finished app missing diagnostics", application.getDiagnostics().indexOf(diagMsg) != -1); // reset the counter of Mockito.verify reset(writer); reset(publisher); // test app fails after 1 app attempt failure LOG.info("--- START: testUnmanagedAppFailPath ---"); application = testCreateAppRunning(subContext); RMAppEvent event = new RMAppFailedAttemptEvent( application.getApplicationId(), RMAppEventType.ATTEMPT_FAILED, "", false); application.handle(event); rmDispatcher.await(); RMAppAttempt appAttempt = application.getCurrentAppAttempt(); Assert.assertEquals(1, appAttempt.getAppAttemptId().getAttemptId()); sendAppUpdateSavedEvent(application); assertFailed(application, ".*Unmanaged application.*Failing the application.*"); assertAppFinalStateSaved(application); verifyRMAppFieldsForFinalTransitions(application); }
@Test public void testDuplicateRMAppDeletion() throws Exception { TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester(); long submitTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis() + 1234; RMStateStore store = zkTester.getRMStateStore(); TestDispatcher dispatcher = new TestDispatcher(); store.setRMDispatcher(dispatcher); ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString( "appattempt_1352994193343_0002_000001"); ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId(); storeApp(store, appIdRemoved, submitTime, startTime); storeAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null, null, dispatcher); ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appIdRemoved); ApplicationStateData appStateRemoved = ApplicationStateData.newInstance( submitTime, startTime, context, "user1"); appStateRemoved.attempts.put(attemptIdRemoved, null); store.removeApplicationStateInternal(appStateRemoved); try { store.removeApplicationStateInternal(appStateRemoved); } catch (KeeperException.NoNodeException nne) { fail("NoNodeException should not happen."); } store.close(); }
new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appIdRemoved); ApplicationStateData appStateRemoved =
private void finishAppWithAttempts(RMState state, RMStateStore store, TestDispatcher dispatcher, ApplicationAttemptId attemptId, long submitTime, long startTime, int amExitStatus, long finishTime, boolean createNewApp) throws Exception { ApplicationId appId = attemptId.getApplicationId(); ApplicationStateData appStateNew = null; if (createNewApp) { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); context.setAMContainerSpec(new ContainerLaunchContextPBImpl()); appStateNew = createAppState(context, submitTime, startTime, finishTime, true); } else { ApplicationStateData appState = state.getApplicationState().get(appId); appStateNew = createAppState(appState.getApplicationSubmissionContext(), submitTime, startTime, finishTime, true); appStateNew.attempts.putAll(appState.attempts); } store.updateApplicationState(appStateNew); waitNotify(dispatcher); Container container = new ContainerPBImpl(); container.setId(ContainerId.newContainerId(attemptId, 1)); ApplicationAttemptStateData newAttemptState = createFinishedAttempt(attemptId, container, startTime, amExitStatus); updateAttempt(store, dispatcher, newAttemptState); }
private void pruneAppState(ApplicationStateData appState) { ApplicationSubmissionContext srcCtx = appState.getApplicationSubmissionContext(); ApplicationSubmissionContextPBImpl context = new ApplicationSubmissionContextPBImpl(); // most fields in the ApplicationSubmissionContext are not needed, // but the following few need to be present for recovery to succeed context.setApplicationId(srcCtx.getApplicationId()); context.setResource(srcCtx.getResource()); context.setQueue(srcCtx.getQueue()); context.setAMContainerResourceRequests( srcCtx.getAMContainerResourceRequests()); context.setApplicationName(srcCtx.getApplicationName()); context.setPriority(srcCtx.getPriority()); context.setApplicationTags(srcCtx.getApplicationTags()); context.setApplicationType(srcCtx.getApplicationType()); context.setUnmanagedAM(srcCtx.getUnmanagedAM()); context.setNodeLabelExpression(srcCtx.getNodeLabelExpression()); ContainerLaunchContextPBImpl amContainerSpec = new ContainerLaunchContextPBImpl(); amContainerSpec.setApplicationACLs( srcCtx.getAMContainerSpec().getApplicationACLs()); context.setAMContainerSpec(amContainerSpec); appState.setApplicationSubmissionContext(context); } }