ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(jstormClientContext.keepContainers); if (!StringUtils.isBlank(jstormClientContext.instanceName) && JOYConstants.CLIIENT_CLASS.equals(jstormClientContext.appName)) { appContext.setApplicationName(jstormClientContext.appName + JOYConstants.DOT + jstormClientContext.instanceName); } else { appContext.setApplicationName(jstormClientContext.appName); appContext.setAttemptFailuresValidityInterval(jstormClientContext.attemptFailuresValidityInterval); vargs.add(JOYConstants.CLI_PREFIX + JOYConstants.NUM_CONTAINERS + JOYConstants.BLANK + String.valueOf(jstormClientContext.numContainers)); if (null != jstormClientContext.nodeLabelExpression) { appContext.setNodeLabelExpression(jstormClientContext.nodeLabelExpression); appContext.setResource(capability); appContext.setAMContainerSpec(amContainer); appContext.setPriority(pri); appContext.setQueue(jstormClientContext.amQueue);
applicationContext.setApplicationId(applicationId); applicationContext.setApplicationName(clientArguments.appName); applicationContext.setApplicationType(clientArguments.appType); Path appJarSrc = new Path(clientArguments.appMasterJar); Path appJarDst = Utilities capability.setMemory(conf.getInt(XLearningConfiguration.XLEARNING_AM_MEMORY, XLearningConfiguration.DEFAULT_XLEARNING_AM_MEMORY)); capability.setVirtualCores(conf.getInt(XLearningConfiguration.XLEARNING_AM_CORES, XLearningConfiguration.DEFAULT_XLEARNING_AM_CORES)); applicationContext.setResource(capability); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( localResources, appMasterEnv, appMasterLaunchcommands, null, null, null); applicationContext.setAMContainerSpec(amContainer); applicationContext.setPriority(priority); applicationContext.setQueue(conf.get(XLearningConfiguration.XLEARNING_APP_QUEUE, XLearningConfiguration.DEFAULT_XLEARNING_APP_QUEUE));
final ApplicationId appId = appContext.getApplicationId(); appContext.setMaxAppAttempts( configuration.getInteger( YarnConfigOptions.APPLICATION_ATTEMPTS.key(), } else { appContext.setMaxAppAttempts( configuration.getInteger( YarnConfigOptions.APPLICATION_ATTEMPTS.key(), appContext.setApplicationName(customApplicationName); appContext.setApplicationType("Apache Flink"); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue(yarnQueue);
appContext.setApplicationName(appName); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(getCapability()); appContext.setQueue(queueName); // queue appContext.setPriority(Priority.newInstance(priority)); if (!DoYUtil.isBlank(nodeLabelExpr)) { LOG.info( "Requesting to run the AM using node expression: " + nodeLabelExpr); appContext.setNodeLabelExpression(nodeLabelExpr); appContext.setUnmanagedAM(unmanaged); appContext.setMaxAppAttempts(1);
YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); ApplicationId applicationId = appSubmissionContext.getApplicationId(); appSubmissionContext.setApplicationName(this.applicationName); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
@Public @Stable public static ApplicationSubmissionContext newInstance( ApplicationId applicationId, String applicationName, String queue, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, String applicationType, boolean keepContainers, String appLabelExpression, ResourceRequest resourceRequest) { ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setApplicationType(applicationType); context.setKeepContainersAcrossApplicationAttempts(keepContainers); context.setNodeLabelExpression(appLabelExpression); context.setAMContainerResourceRequest(resourceRequest); return context; }
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
@SuppressWarnings("deprecation") private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
@SuppressWarnings("deprecation") public synchronized void submit() throws IOException, YarnException { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(this.applicationId); context.setQueue(this.queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); context.setAMContainerSpec(amContainer); context.setResource(Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); SubmitApplicationRequest request = recordFactory .newRecordInstance(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(context); final ResourceScheduler scheduler = resourceManager.getResourceScheduler(); resourceManager.getClientRMService().submitApplication(request); // Notify scheduler AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId, this.queue, "user"); scheduler.handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false); scheduler.handle(addAttemptEvent); }
appContext.setApplicationName("ignition"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue ApplicationId appId = appContext.getApplicationId();
appState.getApplicationSubmissionContext(); ApplicationSubmissionContext context = ApplicationSubmissionContext.newInstance( oldAppSubCtxt.getApplicationId(), oldAppSubCtxt.getApplicationName(), oldAppSubCtxt.getQueue(), oldAppSubCtxt.getPriority(), oldAppSubCtxt.getAMContainerSpec(), oldAppSubCtxt.getUnmanagedAM(), oldAppSubCtxt.getCancelTokensWhenComplete(), oldAppSubCtxt.getMaxAppAttempts(), oldAppSubCtxt.getResource() ); context.setAttemptFailuresValidityInterval( oldAppSubCtxt.getAttemptFailuresValidityInterval()); context.setKeepContainersAcrossApplicationAttempts( oldAppSubCtxt.getKeepContainersAcrossApplicationAttempts()); context.setAMContainerResourceRequests( oldAppSubCtxt.getAMContainerResourceRequests()); context.setLogAggregationContext(oldAppSubCtxt.getLogAggregationContext()); context.setApplicationType(oldAppSubCtxt.getApplicationType()); context.setNodeLabelExpression(oldAppSubCtxt.getNodeLabelExpression()); this.appSubCtxtCopy.put(oldAppSubCtxt.getApplicationId(), context);
submissionContext.setPriority(pri); submissionContext.setQueue(queue); submissionContext.setResource(resource); submissionContext.setLogAggregationContext(logAggregationContext); submissionContext.setKeepContainersAcrossApplicationAttempts(true); submissionContext.setMaxAppAttempts(maxAppAttempts); submissionContext.setAMContainerSpec(containerLaunchContext); return submissionContext;
appId = submissionContext.getApplicationId(); submissionContext.setApplicationName(name); submissionContext.setApplicationType(type); if (!applicationTags.isEmpty()) { submissionContext.setApplicationTags(applicationTags); submissionContext.setNodeLabelExpression(extractLabelExpression(options));
ApplicationId createApplication() throws IOException, YarnException { YarnClientApplication app = yarnClient.createApplication(); return app.getApplicationSubmissionContext().getApplicationId(); }
private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs) throws Exception { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance(); ApplicationClientProtocol submitterClient = getRMClientForUser(submitter); ApplicationId applicationId = submitterClient.getNewApplication(newAppRequest).getApplicationId(); Resource resource = BuilderUtils.newResource(1024, 1); Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.newInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.newInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.setApplicationId(applicationId); appSubmissionContext.setQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.newInstance(appSubmissionContext); submitterClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
@Override public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context) { ContainerLaunchContext launchContext = context.getLaunchContext(); YarnClient yarnClient = createYarnClient(); try { addRMToken(launchContext, yarnClient, appId); appSubmissionContext.setAMContainerSpec(launchContext); appSubmissionContext.setResource(capability); configureAppSubmissionContext(appSubmissionContext); yarnClient.submitApplication(appSubmissionContext); return new ProcessControllerImpl(appId); } catch (YarnException | IOException e) { throw new RuntimeException("Failed to submit application " + appId, e); } finally { yarnClient.stop(); } } };
private ApplicationSubmissionContext prepareForAppSubmission(String submitter, String queueName, Priority priority) throws Exception { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest .newInstance(); ApplicationClientProtocol submitterClient = getRMClientForUser(submitter); ApplicationId applicationId = submitterClient .getNewApplication(newAppRequest).getApplicationId(); Resource resource = BuilderUtils.newResource(1024, 1); ContainerLaunchContext amContainerSpec = ContainerLaunchContext .newInstance(null, null, null, null, null, null); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext .newInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.setApplicationId(applicationId); appSubmissionContext.setQueue(queueName); if (null != priority) { appSubmissionContext.setPriority(priority); } return appSubmissionContext; }
protected RMApp createNewTestApp(ApplicationSubmissionContext submissionContext) throws IOException { ApplicationId applicationId = MockApps.newAppID(appId++); String user = MockApps.newUserName(); String name = MockApps.newAppName(); String queue = MockApps.newQueue(); // ensure max application attempts set to known value conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); if(submissionContext == null) { submissionContext = new ApplicationSubmissionContextPBImpl(); } // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); submissionContext.setPriority(Priority.newInstance(0)); submissionContext.setAMContainerSpec(prepareContainerLaunchContext()); submissionContext.setLogAggregationContext(getLogAggregationContext()); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, System.currentTimeMillis(), "YARN", null, new ArrayList<ResourceRequest>()); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), application); return application; }
/** * Assign this job submission to a queue. * @param queueName YARN queue name. * @return reference to self for chain calls. */ UnmanagedAmYarnSubmissionHelper setQueue(final String queueName) { this.applicationSubmissionContext.setQueue(queueName); return this; }
UnmanagedAmYarnSubmissionHelper( final YarnConfiguration yarnConfiguration, final YarnProxyUser yarnProxyUser, final SecurityTokenProvider tokenProvider) throws IOException, YarnException { this.tokenProvider = tokenProvider; this.yarnProxyUser = yarnProxyUser; LOG.log(Level.FINE, "Initializing YARN Client"); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(yarnConfiguration); this.yarnClient.start(); LOG.log(Level.FINE, "Initialized YARN Client"); LOG.log(Level.FINE, "Requesting UNMANAGED Application ID from YARN."); final ContainerLaunchContext launchContext = YarnTypes.getContainerLaunchContext( Collections.<String>emptyList(), Collections.<String, LocalResource>emptyMap(), tokenProvider.getTokens()); final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication(); this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext(); this.applicationSubmissionContext.setAMContainerSpec(launchContext); this.applicationSubmissionContext.setUnmanagedAM(true); this.applicationId = this.applicationSubmissionContext.getApplicationId(); LOG.log(Level.INFO, "YARN UNMANAGED Application ID: {0}", this.applicationId); }