public static ContainerSpecification createContainerSpec(Configuration configuration, Configuration dynamicProperties) throws Exception { // generate a container spec which conveys the artifacts/vars needed to launch a TM ContainerSpecification spec = new ContainerSpecification(); // propagate the AM dynamic configuration to the TM spec.getDynamicConfiguration().addAll(dynamicProperties); applyOverlays(configuration, spec); return spec; }
dynamicProperties.addAll(containerSpec.getDynamicConfiguration()); Set<String> tmPortKeys = extractPortKeys(containerSpec.getDynamicConfiguration()); List<Protos.Resource> portResources = allocation.takeRanges("ports", tmPortKeys.size(), roles); taskInfo.addAllResources(portResources); for (ContainerSpecification.Artifact artifact : containerSpec.getArtifacts()) { cmd.addUris(Utils.uri(resolver, artifact)); env.addVariables(variable(entry.getKey(), entry.getValue())); for (Map.Entry<String, String> entry : containerSpec.getEnvironmentVariables().entrySet()) { env.addVariables(variable(entry.getKey(), entry.getValue())); ContainerSpecification.formatSystemProperties(containerSpec.getSystemProperties())); .append(params.command()) .append(" ") .append(ContainerSpecification.formatSystemProperties(dynamicProperties)); cmd.setValue(launchCommand.toString());
@Override public void configure(ContainerSpecification container) throws IOException { if(hadoopConfDir == null) { return; } File coreSitePath = new File(hadoopConfDir, "core-site.xml"); File hdfsSitePath = new File(hadoopConfDir, "hdfs-site.xml"); container.getEnvironmentVariables().put("HADOOP_CONF_DIR", TARGET_CONF_DIR.toString()); container.getDynamicConfiguration().setString(ConfigConstants.PATH_HADOOP_CONFIG, TARGET_CONF_DIR.toString()); container.getArtifacts().add(ContainerSpecification.Artifact .newBuilder() .setSource(new Path(coreSitePath.toURI())) .setDest(new Path(TARGET_CONF_DIR, coreSitePath.getName())) .setCachable(true) .build()); container.getArtifacts().add(ContainerSpecification.Artifact .newBuilder() .setSource(new Path(hdfsSitePath.toURI())) .setDest(new Path(TARGET_CONF_DIR, hdfsSitePath.getName())) .setCachable(true) .build()); }
@Override public void configure(ContainerSpecification container) throws IOException { if(keytab != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(keytab) .setDest(TARGET_PATH) .setCachable(false) .build()); container.getDynamicConfiguration().setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(krb5Conf != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(krb5Conf) .setDest(TARGET_PATH) .setCachable(true) .build()); container.getSystemProperties().setString(JAVA_SECURITY_KRB5_CONF, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(ugi != null) { // overlay the Hadoop user identity (w/ tokens) container.getEnvironmentVariables().put("HADOOP_USER_NAME", ugi.getUserName()); } }
/** * Configures an artifact server to serve the artifacts associated with a container specification. * @param server the server to configure. * @param container the container with artifacts to serve. * @throws IOException if the artifacts cannot be accessed. */ static void configureArtifactServer(MesosArtifactServer server, ContainerSpecification container) throws IOException { // serve the artifacts associated with the container environment for (ContainerSpecification.Artifact artifact : container.getArtifacts()) { server.addPath(artifact.source, artifact.dest); } } }
@Override protected Object clone() throws CloneNotSupportedException { ContainerSpecification clone = new ContainerSpecification(); clone.artifacts.addAll(this.artifacts); clone.environmentVariables.putAll(this.environmentVariables); clone.systemProperties.addAll(this.systemProperties); clone.dynamicConfiguration.addAll(this.dynamicConfiguration); return clone; }
@Override public int getPorts() { return extractPortKeys(containerSpec.getDynamicConfiguration()).size(); }
@Override public void configure(ContainerSpecification container) throws IOException { if(hadoopConfDir == null) { return; } File coreSitePath = new File(hadoopConfDir, "core-site.xml"); File hdfsSitePath = new File(hadoopConfDir, "hdfs-site.xml"); container.getEnvironmentVariables().put("HADOOP_CONF_DIR", TARGET_CONF_DIR.toString()); container.getDynamicConfiguration().setString(ConfigConstants.PATH_HADOOP_CONFIG, TARGET_CONF_DIR.toString()); container.getArtifacts().add(ContainerSpecification.Artifact .newBuilder() .setSource(new Path(coreSitePath.toURI())) .setDest(new Path(TARGET_CONF_DIR, coreSitePath.getName())) .setCachable(true) .build()); container.getArtifacts().add(ContainerSpecification.Artifact .newBuilder() .setSource(new Path(hdfsSitePath.toURI())) .setDest(new Path(TARGET_CONF_DIR, hdfsSitePath.getName())) .setCachable(true) .build()); }
@Override public void configure(ContainerSpecification container) throws IOException { if(keytab != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(keytab) .setDest(TARGET_PATH) .setCachable(false) .build()); container.getDynamicConfiguration().setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(krb5Conf != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(krb5Conf) .setDest(TARGET_PATH) .setCachable(true) .build()); container.getSystemProperties().setString(JAVA_SECURITY_KRB5_CONF, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(ugi != null) { // overlay the Hadoop user identity (w/ tokens) container.getEnvironmentVariables().put("HADOOP_USER_NAME", ugi.getUserName()); } }
@Override public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs) throws IOException { java.nio.file.Path relativePath = sourceRoot.relativize(file); ContainerSpecification.Artifact.Builder artifact = ContainerSpecification.Artifact.newBuilder() .setSource(new Path(file.toUri())) .setDest(new Path(targetPath, relativePath.toString())) .setExecutable(Files.isExecutable(file)) .setCachable(true) .setExtract(false); env.getArtifacts().add(artifact.build()); return super.visitFile(file, attrs); } });
@Override protected Object clone() throws CloneNotSupportedException { ContainerSpecification clone = new ContainerSpecification(); clone.artifacts.addAll(this.artifacts); clone.environmentVariables.putAll(this.environmentVariables); clone.systemProperties.addAll(this.systemProperties); clone.dynamicConfiguration.addAll(this.dynamicConfiguration); return clone; }
@Override public void configure(ContainerSpecification container) throws IOException { if(hadoopConfDir == null) { return; } File coreSitePath = new File(hadoopConfDir, "core-site.xml"); File hdfsSitePath = new File(hadoopConfDir, "hdfs-site.xml"); container.getEnvironmentVariables().put("HADOOP_CONF_DIR", TARGET_CONF_DIR.toString()); container.getDynamicConfiguration().setString(ConfigConstants.PATH_HADOOP_CONFIG, TARGET_CONF_DIR.toString()); container.getArtifacts().add(ContainerSpecification.Artifact .newBuilder() .setSource(new Path(coreSitePath.toURI())) .setDest(new Path(TARGET_CONF_DIR, coreSitePath.getName())) .setCachable(true) .build()); container.getArtifacts().add(ContainerSpecification.Artifact .newBuilder() .setSource(new Path(hdfsSitePath.toURI())) .setDest(new Path(TARGET_CONF_DIR, hdfsSitePath.getName())) .setCachable(true) .build()); }
@Override public void configure(ContainerSpecification container) throws IOException { if(keytab != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(keytab) .setDest(TARGET_PATH) .setCachable(false) .build()); container.getDynamicConfiguration().setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(krb5Conf != null) { container.getArtifacts().add(ContainerSpecification.Artifact.newBuilder() .setSource(krb5Conf) .setDest(TARGET_PATH) .setCachable(true) .build()); container.getSystemProperties().setString(JAVA_SECURITY_KRB5_CONF, TARGET_PATH.getPath()); } }
@Override public void configure(ContainerSpecification container) throws IOException { if(ugi != null) { // overlay the Hadoop user identity (w/ tokens) container.getEnvironmentVariables().put("HADOOP_USER_NAME", ugi.getUserName()); } }
@Override public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs) throws IOException { java.nio.file.Path relativePath = sourceRoot.relativize(file); ContainerSpecification.Artifact.Builder artifact = ContainerSpecification.Artifact.newBuilder() .setSource(new Path(file.toUri())) .setDest(new Path(targetPath, relativePath.toString())) .setExecutable(Files.isExecutable(file)) .setCachable(true) .setExtract(false); env.getArtifacts().add(artifact.build()); return super.visitFile(file, attrs); } });