private void sendAMContainerToNM(RMAppAttemptImpl appAttempt, RMAppAttemptContainerFinishedEvent containerFinishedEvent) { NodeId nodeId = containerFinishedEvent.getNodeId(); finishedContainersSentToAM.putIfAbsent(nodeId, new ArrayList<ContainerStatus>()); appAttempt.finishedContainersSentToAM.get(nodeId).add( containerFinishedEvent.getContainerStatus()); if (!appAttempt.getSubmissionContext() .getKeepContainersAcrossApplicationAttempts()) { appAttempt.sendFinishedContainersToNM(); } }
private void sendAMContainerToNM(RMAppAttemptImpl appAttempt, RMAppAttemptContainerFinishedEvent containerFinishedEvent) { NodeId nodeId = containerFinishedEvent.getNodeId(); finishedContainersSentToAM.putIfAbsent(nodeId, new ArrayList<ContainerStatus>()); appAttempt.finishedContainersSentToAM.get(nodeId).add( containerFinishedEvent.getContainerStatus()); if (!appAttempt.getSubmissionContext() .getKeepContainersAcrossApplicationAttempts()) { appAttempt.sendFinishedContainersToNM(); } }
.getKeepContainersAcrossApplicationAttempts(); for (Map.Entry<NodeId, List<ContainerStatus>> entry : justFinishedContainers.entrySet()) {
.getKeepContainersAcrossApplicationAttempts(); for (NodeId nodeId:justFinishedContainers.keySet()) {
.getKeepContainersAcrossApplicationAttempts(); for (Map.Entry<NodeId, List<ContainerStatus>> entry : justFinishedContainers.entrySet()) {
private static void amContainerFinished(RMAppAttemptImpl appAttempt, RMAppAttemptContainerFinishedEvent containerFinishedEvent) { NodeId nodeId = containerFinishedEvent.getNodeId(); ContainerStatus containerStatus = containerFinishedEvent.getContainerStatus(); if (containerStatus != null) { int exitStatus = containerStatus.getExitStatus(); if (Apps.shouldCountTowardsNodeBlacklisting(exitStatus)) { appAttempt.addAMNodeToBlackList(nodeId); } } else { LOG.warn("No ContainerStatus in containerFinishedEvent"); } if (!appAttempt.getSubmissionContext() .getKeepContainersAcrossApplicationAttempts()) { appAttempt.finishedContainersSentToAM.putIfAbsent(nodeId, new ArrayList<>()); appAttempt.finishedContainersSentToAM.get(nodeId).add(containerStatus); appAttempt.sendFinishedContainersToNM(); } else { appAttempt.sendFinishedAMContainerToNM(nodeId, containerStatus.getContainerId()); } }
.getKeepContainersAcrossApplicationAttempts() && event.getType() == RMAppAttemptEventType.CONTAINER_FINISHED) {
public void submit() throws IOException, YarnException { // SET EXEC COMMAND final List<String> launchCommand = new JavaLaunchCommandBuilder(launcherClazz, commandPrefixList) .setConfigurationFilePaths(configurationFilePaths) .setClassPath(this.classpath.getDriverClasspath()) .setMemory(this.applicationSubmissionContext.getResource().getMemory()) .setStandardOut(driverStdoutFilePath) .setStandardErr(driverStderrFilePath) .build(); if (this.applicationSubmissionContext.getKeepContainersAcrossApplicationAttempts() && this.applicationSubmissionContext.getMaxAppAttempts() == 1) { LOG.log(Level.WARNING, "Application will not be restarted even though preserve evaluators is set to true" + " since the max application submissions is 1. Proceeding to submit application..."); } final ContainerLaunchContext containerLaunchContext = YarnTypes.getContainerLaunchContext( launchCommand, this.resources, tokenProvider.getTokens()); this.applicationSubmissionContext.setAMContainerSpec(containerLaunchContext); LOG.log(Level.INFO, "Submitting REEF Application to YARN. ID: {0}", this.applicationId); if (LOG.isLoggable(Level.INFO)) { LOG.log(Level.INFO, "REEF app command: {0}", StringUtils.join(launchCommand, ' ')); } this.yarnClient.submitApplication(applicationSubmissionContext); if (this.isUnmanaged) { // For Unmanaged AM mode, add a new app token to the // current process so it can talk to the RM as an AM. final Token<AMRMTokenIdentifier> token = this.yarnClient.getAMRMToken(this.applicationId); this.yarnProxyUser.set("reef-proxy", UserGroupInformation.getCurrentUser(), token); this.tokenProvider.addTokens(UserCredentialSecurityTokenProvider.serializeToken(token)); } }
@Override public SubmitApplicationResponse submitApplication( SubmitApplicationRequest request) throws YarnException, IOException { validateRunning(); ApplicationId appId = null; if (request.getApplicationSubmissionContext() != null) { appId = request.getApplicationSubmissionContext().getApplicationId(); } LOG.info("Application submitted: " + appId); applicationMap.add(appId); if (request.getApplicationSubmissionContext().getUnmanagedAM() || request.getApplicationSubmissionContext() .getKeepContainersAcrossApplicationAttempts()) { keepContainerOnUams.add(appId); } return SubmitApplicationResponse.newInstance(); }
@Test public void testUnmanagedAMContainersCleanup() { unmanagedAM = true; when(submissionContext.getUnmanagedAM()).thenReturn(true); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); // submit AM and check it goes to SUBMITTED state submitApplicationAttempt(); // launch AM and verify attempt failed applicationAttempt.handle(new RMAppAttemptRegistrationEvent( applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl")); assertEquals(YarnApplicationAttemptState.SUBMITTED, applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertFalse(transferStateFromPreviousAttempt); }
@Test public void testUnmanagedAMContainersCleanup() { unmanagedAM = true; when(submissionContext.getUnmanagedAM()).thenReturn(true); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); // submit AM and check it goes to SUBMITTED state submitApplicationAttempt(); // launch AM and verify attempt failed applicationAttempt.handle(new RMAppAttemptRegistrationEvent( applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl")); assertEquals(YarnApplicationAttemptState.SUBMITTED, applicationAttempt.createApplicationAttemptState()); sendAttemptUpdateSavedEvent(applicationAttempt); assertFalse(transferStateFromPreviousAttempt); }
rmContext.getRMApps().get(appID).getApplicationSubmissionContext(); if (!(appContext.getUnmanagedAM() && appContext.getKeepContainersAcrossApplicationAttempts())) { String message = AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE + appID;
.getKeepContainersAcrossApplicationAttempts() && !appAttempt.submissionContext.getUnmanagedAM()) {
.getKeepContainersAcrossApplicationAttempts() && !appAttempt.submissionContext.getUnmanagedAM()) {
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)), application); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); when(submissionContext.getMaxAppAttempts()).thenReturn(1);
.getKeepContainersAcrossApplicationAttempts() && !appAttempt.submissionContext.getUnmanagedAM()) { int numberOfFailure = ((RMAppImpl)appAttempt.rmApp)
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); when(submissionContext.getMaxAppAttempts()).thenReturn(1);
@Test public void testFailedToFailed() { when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); when(application.getMaxAppAttempts()).thenReturn(2);
@Test public void testFailedToFailed() { when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); Container amContainer = allocateApplicationAttempt();
oldAppSubCtxt.getAttemptFailuresValidityInterval()); context.setKeepContainersAcrossApplicationAttempts( oldAppSubCtxt.getKeepContainersAcrossApplicationAttempts()); context.setAMContainerResourceRequests( oldAppSubCtxt.getAMContainerResourceRequests());