public void submit() throws IOException, YarnException { // SET EXEC COMMAND final List<String> launchCommand = new JavaLaunchCommandBuilder(launcherClazz, commandPrefixList) .setConfigurationFilePaths(configurationFilePaths) .setClassPath(this.classpath.getDriverClasspath()) .setMemory(this.applicationSubmissionContext.getResource().getMemory()) .setStandardOut(driverStdoutFilePath) .setStandardErr(driverStderrFilePath) .build(); if (this.applicationSubmissionContext.getKeepContainersAcrossApplicationAttempts() && this.applicationSubmissionContext.getMaxAppAttempts() == 1) { LOG.log(Level.WARNING, "Application will not be restarted even though preserve evaluators is set to true" + " since the max application submissions is 1. Proceeding to submit application..."); } final ContainerLaunchContext containerLaunchContext = YarnTypes.getContainerLaunchContext( launchCommand, this.resources, tokenProvider.getTokens()); this.applicationSubmissionContext.setAMContainerSpec(containerLaunchContext); LOG.log(Level.INFO, "Submitting REEF Application to YARN. ID: {0}", this.applicationId); if (LOG.isLoggable(Level.INFO)) { LOG.log(Level.INFO, "REEF app command: {0}", StringUtils.join(launchCommand, ' ')); } this.yarnClient.submitApplication(applicationSubmissionContext); if (this.isUnmanaged) { // For Unmanaged AM mode, add a new app token to the // current process so it can talk to the RM as an AM. final Token<AMRMTokenIdentifier> token = this.yarnClient.getAMRMToken(this.applicationId); this.yarnProxyUser.set("reef-proxy", UserGroupInformation.getCurrentUser(), token); this.tokenProvider.addTokens(UserCredentialSecurityTokenProvider.serializeToken(token)); } }
.add("queue", appContext.getQueue()) .add("user", UserGroupInformation.getLoginUser()) .add("resource", appContext.getResource()) .toString(); LOG.info(specStr);
@Test public void testApplicationSubmissionContextPBImpl() throws Exception { validatePBImplRecord(ApplicationSubmissionContextPBImpl.class, ApplicationSubmissionContextProto.class); ApplicationSubmissionContext ctx = ApplicationSubmissionContext.newInstance(null, null, null, null, null, false, false, 0, Resources.none(), null, false, null, null); Assert.assertNotNull(ctx.getResource()); }
submissionContext.getAMContainerResourceRequests(); if (amReqs == null || amReqs.isEmpty()) { if (submissionContext.getResource() != null) { amReqs = Collections.singletonList(BuilderUtils .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); } else { throw new InvalidResourceRequestException("Invalid resource request, "
@Test public void testApplicationSubmissionContextPBImpl() throws Exception { validatePBImplRecord(ApplicationSubmissionContextPBImpl.class, ApplicationSubmissionContextProto.class); ApplicationSubmissionContext ctx = ApplicationSubmissionContext.newInstance(null, null, null, null, null, false, false, 0, Resources.none(), null, false, null, null); Assert.assertNotNull(ctx.getResource()); }
@Test public void testApplicationSubmissionContextPBImpl() throws Exception { validatePBImplRecord(ApplicationSubmissionContextPBImpl.class, ApplicationSubmissionContextProto.class); ApplicationSubmissionContext ctx = ApplicationSubmissionContext.newInstance(null, null, null, null, null, false, false, 0, Resources.none(), null, false, null, null); Assert.assertNotNull(ctx.getResource()); }
null, null, null, null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)), application) { @Override protected void onInvalidTranstion(
null, null, null, null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1));
amReq = BuilderUtils .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1);
amReq = BuilderUtils .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1);
Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)), application); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true);
true, BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true);
Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)), application);
BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1));
private void pruneAppState(ApplicationStateData appState) { ApplicationSubmissionContext srcCtx = appState.getApplicationSubmissionContext(); ApplicationSubmissionContextPBImpl context = new ApplicationSubmissionContextPBImpl(); // most fields in the ApplicationSubmissionContext are not needed, // but the following few need to be present for recovery to succeed context.setApplicationId(srcCtx.getApplicationId()); context.setResource(srcCtx.getResource()); context.setQueue(srcCtx.getQueue()); context.setAMContainerResourceRequests( srcCtx.getAMContainerResourceRequests()); context.setApplicationName(srcCtx.getApplicationName()); context.setPriority(srcCtx.getPriority()); context.setApplicationTags(srcCtx.getApplicationTags()); context.setApplicationType(srcCtx.getApplicationType()); context.setUnmanagedAM(srcCtx.getUnmanagedAM()); context.setNodeLabelExpression(srcCtx.getNodeLabelExpression()); ContainerLaunchContextPBImpl amContainerSpec = new ContainerLaunchContextPBImpl(); amContainerSpec.setApplicationACLs( srcCtx.getAMContainerSpec().getApplicationACLs()); context.setAMContainerSpec(amContainerSpec); appState.setApplicationSubmissionContext(context); } }
Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1))); Assert.assertEquals(RMAppState.NEW, application.getState());
assertEquals("a_queue", ctx.getQueue()); assertNotNull(ctx.getAMContainerSpec()); assertEquals(17L, ctx.getResource().getMemorySize()); assertEquals("ApplicationSchedulingPropertiesMap should have been pruned" + " from the application submission context when in FINISHED STATE",
oldAppSubCtxt.getCancelTokensWhenComplete(), oldAppSubCtxt.getMaxAppAttempts(), oldAppSubCtxt.getResource() ); context.setAttemptFailuresValidityInterval(
public void testRecoverApplication(ApplicationStateData appState, RMState rmState) throws Exception { ApplicationSubmissionContext submissionContext = appState.getApplicationSubmissionContext(); RMAppImpl application = new RMAppImpl( appState.getApplicationSubmissionContext().getApplicationId(), rmContext, conf, submissionContext.getApplicationName(), null, submissionContext.getQueue(), submissionContext, null, null, appState.getSubmitTime(), submissionContext.getApplicationType(), submissionContext.getApplicationTags(), BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); Assert.assertEquals(RMAppState.NEW, application.getState()); RMAppEvent recoverEvent = new RMAppRecoverEvent(application.getApplicationId(), rmState); // Trigger RECOVER event. application.handle(recoverEvent); // Application final status looked from recoveredFinalStatus Assert.assertTrue("Application is not in recoveredFinalStatus.", RMAppImpl.isAppInFinalState(application)); rmDispatcher.await(); RMAppState finalState = appState.getState(); Assert.assertEquals("Application is not in finalState.", finalState, application.getState()); }