How to use

SliderUtils

in

org.apache.slider.common.tools

/**
 * Build the environment map from a role option map, finding all entries
 * beginning with "env.", adding them to a map of (prefix-removed)
 * env vars
 * @param roleOpts role options. This can be null, meaning the
 * role is undefined
 * @return a possibly empty map of environment variables.
 */
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
 return buildEnvMap(roleOpts, null);
}

/**
 * Release a tag associated with a container
 *
 * @param component
 * @param containerId
 */
public void releaseTag(String component, String containerId) {
 if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
  ConcurrentHashMap<String, String> compTags = allTags.get(component);
  if (compTags != null) {
   synchronized (compTags) {
    for (String key : compTags.keySet()) {
     if (compTags.get(key).equals(containerId)) {
      compTags.put(key, FREE);
     }
    }
   }
  }
 }
}

 private String getSubjectDN(String hostname, String containerId,
               String appName) {
  return String.format("/CN=%s%s%s",
             SliderUtils.extractDomainNameFromFQDN(hostname),
             containerId != null ? "/OU=" + containerId : "",
             appName != null ? "/OU=" + appName : "");
 }
}

/**
 * Get slider dependency parent dir in HDFS
 * 
 * @return the parent dir path of slider.tar.gz in HDFS
 */
public Path getDependencyPath() {
 String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR
   + SliderKeys.SLIDER_DEPENDENCY_DIR
   : SliderKeys.SLIDER_DEPENDENCY_DIR;
 Path dependencyPath = new Path(String.format(parentDir,
   SliderUtils.getSliderVersion()));
 return dependencyPath;
}

/**
 * Retrieve the version of the current Slider install
 * 
 * @return the version string of the Slider release
 */
public static String getSliderVersion() {
 if (isHdp()) {
  return getHdpVersion();
 } else {
  Properties props = SliderVersionInfo.loadVersionProperties();
  return props.getProperty(SliderVersionInfo.APP_VERSION);
 }
}

/**
 * Loads all dependency jars from the default path
 * @param providerResources map of provider resources to add these entries to
 * @param sliderFileSystem target filesystem
 * @param tempPath path in the cluster FS for temp files
 * @param libDir relative directory to place resources
 * @param libLocalSrcDir explicitly supplied local libs dir
 * @throws IOException
 * @throws SliderException
 */
public static void addAllDependencyJars(Map<String, LocalResource> providerResources,
                    SliderFileSystem sliderFileSystem,
                    Path tempPath,
                    String libDir,
                    String libLocalSrcDir)
  throws IOException, SliderException {
 String libSrcToUse = libLocalSrcDir;
 if (SliderUtils.isSet(libLocalSrcDir)) {
  File file = new File(libLocalSrcDir);
  if (!file.exists() || !file.isDirectory()) {
   throw new BadCommandArgumentsException("Supplied lib src dir %s is not valid", libLocalSrcDir);
  }
 }
 SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, libDir, libSrcToUse);
}

throws YarnException, IOException{
deployedClusterName = clustername;
validateClusterName(clustername);
verifyNoLiveClusters(clustername, "Launch");
Configuration config = getConfig();
lookupZKQuorum();
boolean clusterSecure = isHadoopClusterSecure(config);
  getApplicationDefinitionPath(appOperations));
 Path localConfDirPath = createLocalPath(confDir);
 remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
 log.debug("Slider configuration directory is {}; remote to be {}", 
   localConfDirPath, remoteConfPath);
 copyDirectory(config, localConfDirPath, remoteConfPath, null);
   sliderFileSystem.submitDirectory(remoteConfPath,
                   relativeConfDir);
  mergeMaps(localResources, submittedConfDir);
copyDirectory(config, snapshotConfPath, generatedConfDirPath,
  clusterPerms);
  extractImagePath(sliderFileSystem, internalOptions);
if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
 log.debug("Registered image path {}", imagePath);
 buildEnvMap(sliderAMResourceComponent));
ClasspathConstructor classpath = buildClasspath(relativeConfDir,

String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
  .getAppConfOperations(), roleGroup);
launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
  getStandardTokenMap(getAmState().getAppConfSnapshot(), roleName, roleGroup)));
String appHome = instanceDefinition.getAppConfOperations().
  getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
if (SliderUtils.isSet(appHome)) {
 scriptPath = new File(appHome, AgentKeys.AGENT_MAIN_SCRIPT).getPath();
String agentImage = instanceDefinition.getInternalOperations().
  get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
if (SliderUtils.isUnset(agentImage)) {
 agentImagePath =
   new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
if (SliderUtils.isSet(agentConf)) {
 LocalResource agentConfRes = fileSystem.createAmResource(fileSystem
                                .getFileSystem().resolvePath(new Path(agentConf)),
if (SliderUtils.isHadoopClusterSecure(getConfig())) {
 localizeServiceKeytabs(launcher, instanceDefinition, fileSystem);
if (SliderUtils.isSet(debugCmd)) {
 operation.add(ARG_DEBUG);
 operation.add(debugCmd);

 log.info("Conf dir {} does not exist.", confDir);
 File parentFile = confDir.getParentFile();
 log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile));
initAndAddService(sliderAMProvider);
InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
log.info("RM is at {}", rmSchedulerAddress);
yarnRPC = YarnRPC.create(serviceConf);
InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
 SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name()));
appAttemptID = appMasterContainerID.getApplicationAttemptId();
  if (SliderUtils.isUnset(appMasterProxiedUrl)) {
   log.warn("Proxied URL is not set in application report");
   appMasterProxiedUrl = appMasterTrackingUrl;

launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
if (SliderUtils.isSet(heap)) {
 String adjustedHeap = SliderUtils.translateTrailingHeapUnit(heap);
 launcher.setEnv(HBASE_HEAPSIZE, adjustedHeap);
if (SliderUtils.isSet(gcOpts)) {
 launcher.setEnv(HBASE_GC_OPTS, gcOpts);

 @Override
 public void validate() throws BadCommandArgumentsException, UsageException {
  super.validate();
  if (keytab != null && SliderUtils.isUnset(principal)) {
   throw new UsageException("Missing argument " + ARG_PRINCIPAL);
  }
  if (keytab == null && SliderUtils.isSet(principal)) {
   throw new UsageException("Missing argument " + ARG_KEYTAB);
  }
 }
}

launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
Map<String, String> env = SliderUtils.buildEnvMap(appComponent);
launcher.setEnv(ACCUMULO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
ConfTreeOperations appConf =
if (SliderUtils.isSet(heap)) {

private void createSummaryMetainfoFile(Path srcFile, Path destFile,
  boolean overwrite) throws IOException {
 FileSystem srcFs = srcFile.getFileSystem(getConfig());
 try (InputStream inputStreamJson = SliderUtils
   .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json");
   InputStream inputStreamXml = SliderUtils
     .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) {
  InputStream inputStream = null;
  Path summaryFileInFs = null;
  if (inputStreamJson != null) {
   inputStream = inputStreamJson;
   summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
     + ".metainfo.json");
   log.info("Found JSON metainfo file in package");
  } else if (inputStreamXml != null) {
   inputStream = inputStreamXml;
   summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
     + ".metainfo.xml");
   log.info("Found XML metainfo file in package");
  }
  if (inputStream != null) {
   try (FSDataOutputStream dataOutputStream = sliderFileSystem
     .getFileSystem().create(summaryFileInFs, overwrite)) {
    log.info("Creating summary metainfo file");
    IOUtils.copy(inputStream, dataOutputStream);
   }
  }
 }
}

/**
 * Get the URL log for a container
 * @param c container
 * @return the URL or "" if it cannot be determined
 */
public static String getLogsURLForContainer(Container c) {
 if (c==null) {
  return null;
 }
 String user = null;
 try {
  user = SliderUtils.getCurrentUser().getShortUserName();
 } catch (IOException ignored) {
 }
 String completedLogsUrl = "";
 String url = logServerURL;
 if (user != null && SliderUtils.isSet(url)) {
  completedLogsUrl = url
    + "/" + c.getNodeId() + "/" + c.getId() + "/ctx/" + user;
 }
 log.info("Completed log link = {}", completedLogsUrl);
 return completedLogsUrl;
}

/**
 * Submit a JAR containing and map it
 * @param providerResources provider map to build up
 * @param sliderFileSystem remote fs
 * @param libDir lib directory
 * @param srcPath copy jars from
 * @throws IOException, SliderException trouble copying to HDFS
 */
public static void putAllJars(Map<String, LocalResource> providerResources,
               SliderFileSystem sliderFileSystem,
               Path tempPath,
               String libDir,
               String srcPath) throws IOException, SliderException {
 log.info("Loading all dependencies from {}", srcPath);
 if (SliderUtils.isSet(srcPath)) {
  File srcFolder = new File(srcPath);
  FilenameFilter jarFilter = createJarFilter();
  File[] listOfJars = srcFolder.listFiles(jarFilter);
  if (listOfJars == null || listOfJars.length == 0) {
   return;
  }
  for (File jarFile : listOfJars) {
   LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
   providerResources.put(libDir + "/" + jarFile.getName(), res);
  }
 }
}

sortApplicationsByMostRecent(instances);
Map<String, ApplicationReport> reportMap =
  buildApplicationReportMap(instances, min, max);
log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
  persistentInstances.size(),
  String details = instanceDetailsToString(name, report,
    containers, version, components, verbose);
  print(details);

@Override //AMRMClientAsync
public void onError(Throwable e) {
 if (e instanceof InvalidResourceRequestException) {
  // stop the cluster
  LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
  ActionStopSlider stopSlider = new ActionStopSlider("stop",
    EXIT_EXCEPTION_THROWN, FinalApplicationStatus.FAILED,
    SliderUtils.extractFirstLine(e.getLocalizedMessage()));
  stopSlider.setExitReason(SliderExitReason.APP_ERROR);
  signalAMComplete(stopSlider);
 } else if (e instanceof InvalidApplicationMasterRequestException) {
  // halt the AM
  LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
  queue(new ActionHalt(EXIT_EXCEPTION_THROWN,
    SliderUtils.extractFirstLine(e.getLocalizedMessage())));
 } else {
  // ignore and log
  LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e);
 }
}

/**
 * Verify that a file exists
 * @param file file
 * @param errorlog log for output on an error
 * @throws FileNotFoundException
 */
public static void verifyFileExists(File file, Logger errorlog) throws
  FileNotFoundException {
 if (!file.exists()) {
  errorlog.warn("contents of {}: {}", file,
    listDir(file.getParentFile()));
  throw new FileNotFoundException(file.toString());
 }
 if (!file.isFile()) {
  throw new FileNotFoundException("Not a file: " + file.toString());
 }
}

@Test
public void testApplicationReportComparison() {
 List<ApplicationReport> instances = getApplicationReports();
 SliderUtils.sortApplicationsByMostRecent(instances);
 Assert.assertEquals(1000, instances.get(0).getStartTime());
 Assert.assertEquals(1000, instances.get(1).getStartTime());
 Assert.assertEquals(1000, instances.get(2).getStartTime());
 Assert.assertEquals(1000, instances.get(3).getStartTime());
 instances = getApplicationReports();
 SliderUtils.sortApplicationReport(instances);
 Assert.assertEquals(1000, instances.get(0).getStartTime());
 Assert.assertEquals(1000, instances.get(1).getStartTime());
 Assert.assertEquals(1000, instances.get(2).getStartTime());
 Assert.assertEquals(1000, instances.get(3).getStartTime());
 Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
          instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING);
 Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
          instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING);
 Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
          instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING);
 Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED);
}

SliderUtils.sortApplicationsByMostRecent(rmInstances);
Map<String, ApplicationReport> reportMap =
  SliderUtils.buildApplicationReportMap(rmInstances, minAppState,
    maxAppState);
log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",