private ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId); Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME); Map<String, LocalResource> resourceMap = Maps.newHashMap(); addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap); addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap); if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) { addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap); } ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(resourceMap); containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName))); if (UserGroupInformation.isSecurityEnabled()) { containerLaunchContext.setTokens(this.tokens.duplicate()); } return containerLaunchContext; }
taskExecutorLaunchContext.getEnvironment() .put(ENV_FLINK_CONTAINER_ID, containerId); taskExecutorLaunchContext.getEnvironment() .put(ENV_FLINK_NODE_ID, host); return taskExecutorLaunchContext;
throw new RuntimeException("Error cacheFile path format " + path); pathRemote = new Path(paths[0]); } else { pathRemote = new Path(path); if (!pathRemote.getFileSystem(conf).exists(pathRemote)) { throw new IOException("cacheFile path " + pathRemote + " not existed!"); appMasterLaunchcommands.add(command.toString()); Resource capability = Records.newRecord(Resource.class); capability.setMemory(conf.getInt(XLearningConfiguration.XLEARNING_AM_MEMORY, XLearningConfiguration.DEFAULT_XLEARNING_AM_MEMORY)); capability.setVirtualCores(conf.getInt(XLearningConfiguration.XLEARNING_AM_CORES, XLearningConfiguration.DEFAULT_XLEARNING_AM_CORES)); applicationContext.setResource(capability); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( localResources, appMasterEnv, appMasterLaunchcommands, null, null, null); Priority priority = Records.newRecord(Priority.class); priority.setPriority(conf.getInt(XLearningConfiguration.XLEARNING_APP_PRIORITY, XLearningConfiguration.DEFAULT_XLEARNING_APP_PRIORITY)); applicationContext.setPriority(priority);
/** * Given this generic description of an application, create the detailed YARN * application submission context required to launch the application. * * @param conf * the YARN configuration obtained by reading the Hadoop * configuration files * @return the completed application launch context for the given application * @throws IOException * if localized resources are not found in the distributed file * system (such as HDFS) */ public ContainerLaunchContext createLaunchContext(YarnConfiguration conf) throws IOException { // Set up the container launch context ContainerLaunchContext container = Records .newRecord(ContainerLaunchContext.class); // Set up the list of commands to run. Here, we assume that we run only // one command. container.setCommands(Collections.singletonList(getCommand())); // Add localized resources container.setLocalResources(resources); // Environment. container.setEnvironment(env); return container; }
fs, appId, new Path(tmpConfigurationFile.getAbsolutePath()), localResources, homeDir, fs, appId, new Path(fp.toURI()), localResources, homeDir, File krb5 = new File(krb5Config); LOG.info("Adding KRB5 configuration {} to the AM container local resource bucket", krb5.getAbsolutePath()); Path krb5ConfPath = new Path(krb5.getAbsolutePath()); remoteKrb5Path = setupSingleLocalResource( Utils.KRB5_FILE_NAME, amContainer.setLocalResources(localResources); fs.close(); amContainer.setEnvironment(appMasterEnv); Resource capability = Records.newRecord(Resource.class); capability.setMemory(clusterSpecification.getMasterMemoryMB()); capability.setVirtualCores(1);
Path renamedScriptPath; if (Shell.WINDOWS) { renamedScriptPath = new Path(jstormMasterContext.scriptPath + ".bat"); } else { renamedScriptPath = new Path(jstormMasterContext.scriptPath + ".sh"); try { yarnUrl = ConverterUtils.getYarnUrlFromURI( new URI(renamedScriptPath.toString())); } catch (URISyntaxException e) { LOG.error("Error when trying to use shell script path specified" ContainerLaunchContext ctx = ContainerLaunchContext.newInstance( localResources, jstormMasterContext.shellEnv, commands, null, jstormMasterContext.allTokens.duplicate(), null); containerListener.addContainer(container.getId(), container);
long hdfsShellScriptTimestamp = 0; if (!jstormClientContext.shellScriptPath.isEmpty()) { Path shellSrc = new Path(jstormClientContext.shellScriptPath); String shellPathSuffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId.toString() + JOYConstants.BACKLASH + JOYConstants.SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( localResources, env, commands, null, null, null); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens);
@Test public void testCLCPBImplNullResourceType() throws IOException { RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); try { LocalResource resource = recordFactory.newRecordInstance(LocalResource.class); resource.setResource(URL.fromPath(new Path("."))); resource.setSize(-1); resource.setVisibility(LocalResourceVisibility.APPLICATION); resource.setType(null); resource.setTimestamp(System.currentTimeMillis()); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put("null_type_resource", resource); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(localResources); Assert.fail("Setting an invalid local resource should be an error!"); } catch (NullPointerException e) { Assert.assertTrue(e.getMessage().contains("Null resource type for local resource")); } }
false, (int) job.taskManagerMemoryMb()); amContainer.setLocalResources(localResources); final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString()) .collect(Collectors.joining(",")); ); amContainer.setEnvironment(appMasterEnv); Resource capability = Records.newRecord(Resource.class); capability.setMemory(getFlinkConfiguration() .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY));
String modelScript = localizeResources(localResources, new Path(request.getPath()), appJarLocation); for(Map.Entry<String, LocalResource> entry : localResources.entrySet()) { LOG.info(entry.getKey() + " localized: " + entry.getValue().getResource() ); ContainerLaunchContext ctx = ContainerLaunchContext.newInstance( localResources, env, commands, null, allTokens.duplicate(), null);
,ApplicationMaster.AMOptions.APP_JAR_PATH.of(ajPath.toString()) ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( localResources, env, commands, null, null, null); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens);
super.launchContainer(containerStartContext); int exitCode = 0; if (container.getLaunchContext().getCommands().toString().contains("bin/java")) { ExecJavaCliParser result = this.createExecCommandParser(containerWorkDir.toString()); try { exitCode = this.doLaunch(container, containerWorkDir); String cmd = container.getLaunchContext().getCommands().get(0); if (logger.isInfoEnabled()) { logger.info("Running Command: " + cmd);
public static ContainerLaunchContext newContainerLaunchContext( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
private void addDockerClientConfigToRunCommand(ContainerRuntimeContext ctx, DockerRunCommand dockerRunCommand) throws ContainerExecutionException { ByteBuffer tokens = ctx.getContainer().getLaunchContext().getTokens(); Credentials credentials; if (tokens != null) { Path nmPrivateDir = ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH) .getParent(); File dockerConfigPath = new File(nmPrivateDir + "/config.json"); try {
"Cannot start a continer before connecting to the container manager!"); ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setContainerId(container.getId()); ctx.setResource(container.getResource()); ctx.setLocalResources(localResources); ctx.setCommands(commands); ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); ctx.setEnvironment(env); + ", containerId=" + ctx.getContainerId() + ", memory=" + ctx.getResource().getMemory() + ", localResources=" + ctx.getLocalResources().toString() + ", commands=" + ctx.getCommands().toString() + ", env=" + ctx.getEnvironment().toString()); StartContainerRequest request = Records.newRecord(StartContainerRequest.class); request.setContainerLaunchContext(ctx);
private ContainerLaunchContext prepareContainerLaunchContext() throws IOException { ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class); clc.setCommands(Arrays.asList("/bin/sleep 5")); if (UserGroupInformation.isSecurityEnabled()) { clc.setTokens(getTokens()); clc.setTokensConf(getTokensConf()); } clc.setLocalResources(getLocalResources()); clc.setEnvironment(getEnvironment()); clc.setContainerRetryContext(getContainerRetryContext()); clc.setServiceData(getServiceData()); return clc; }
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); BootstrapTools.getStartCommand(commandTemplate, startCommandValues); amContainer.setCommands(Collections.singletonList(amCommand));
/** * @return a ContainerLaunchContext with the given commands and LocalResources. */ public static final ContainerLaunchContext getContainerLaunchContext( final List<String> commands, final Map<String, LocalResource> localResources) { final ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setLocalResources(localResources); context.setCommands(commands); return context; }
@Test (timeout = 30000) public void testAppRecoverPath() throws IOException { LOG.info("--- START: testAppRecoverPath ---"); ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class); ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class); Credentials credentials = new Credentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); clc.setTokens(securityTokens); sub.setAMContainerSpec(clc); testCreateAppSubmittedRecovery(sub); }
/** * Adds RM delegation token to the given {@link ContainerLaunchContext} so that the AM can authenticate itself * with RM using the delegation token. */ protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) { if (!UserGroupInformation.isSecurityEnabled()) { return; } try { Credentials credentials = YarnUtils.decodeCredentials(context.getTokens()); Configuration config = yarnClient.getConfig(); Token<TokenIdentifier> token = ConverterUtils.convertFromYarn( yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))), YarnUtils.getRMAddress(config)); LOG.debug("Added RM delegation token {} for application {}", token, appId); credentials.addToken(token.getService(), token); context.setTokens(YarnUtils.encodeCredentials(credentials)); } catch (YarnException | IOException e) { throw new RuntimeException("Failed to acquire RM delegation token", e); } }