Tabnine Logo
DefaultContainerExecutor
Code IndexAdd Tabnine to your IDE (free)

How to use
DefaultContainerExecutor
in
org.apache.hadoop.yarn.server.nodemanager

Best Java code snippets using org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor (Showing top 20 results out of 315)

origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

@Override
public boolean signalContainer(String user, String pid, Signal signal)
  throws IOException {
 LOG.debug("Sending signal " + signal.getValue() + " to pid " + pid
   + " as user " + user);
 if (!containerIsAlive(pid)) {
  return false;
 }
 try {
  killContainer(pid, signal);
 } catch (IOException e) {
  if (!containerIsAlive(pid)) {
   return false;
  }
  throw e;
 }
 return true;
}
origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

/**
 * Initialize the local directories for a particular user.
 * <ul>.mkdir
 * <li>$local.dir/usercache/$user</li>
 * </ul>
 */
void createUserLocalDirs(List<String> localDirs, String user)
  throws IOException {
 boolean userDirStatus = false;
 FsPermission userperms = new FsPermission(USER_PERM);
 for (String localDir : localDirs) {
  // create $local.dir/usercache/$user and its immediate parent
  try {
   createDir(getUserCacheDir(new Path(localDir), user), userperms, true, user);
  } catch (IOException e) {
   LOG.warn("Unable to create the user directory : " + localDir, e);
   continue;
  }
  userDirStatus = true;
 }
 if (!userDirStatus) {
  throw new IOException("Not able to initialize user directories "
    + "in any of the configured local directories for user " + user);
 }
}
origin: com.github.jiayuhan-it/hadoop-yarn-server-nodemanager

private Path getApplicationDir(Path base, String user, String appId) {
 return new Path(getAppcacheDir(base, user), appId);
}
origin: org.apache.hadoop/hadoop-yarn-server-nodemanager

/**
 * Create application log directories on all disks.
 *
 * @param appId the application ID
 * @param logDirs the target directories to create
 * @param user the user whose local cache directories should be initialized
 * @throws IOException if there's an issue initializing the application log
 * directories
 */
void createAppLogDirs(String appId, List<String> logDirs, String user)
  throws IOException {
 boolean appLogDirStatus = false;
 FsPermission appLogDirPerms = new
   FsPermission(getLogDirPermissions());
 for (String rootLogDir : logDirs) {
  // create $log.dir/$appid
  Path appLogDir = new Path(rootLogDir, appId);
  try {
   createDir(appLogDir, appLogDirPerms, true, user);
  } catch (IOException e) {
   LOG.warn("Unable to create the app-log directory : " + appLogDir, e);
   continue;
  }
  appLogDirStatus = true;
 }
 if (!appLogDirStatus) {
  throw new IOException("Not able to initialize app-log directories "
    + "in any of the configured local directories for app " + appId);
 }
}
origin: com.github.jiayuhan-it/hadoop-yarn-server-nodemanager

 Path appDir = new Path(appCacheDir, appIdStr);
 Path containerDir = new Path(appDir, containerIdStr);
 createDir(containerDir, dirPerm, true, user);
createContainerLogDirs(appIdStr, containerIdStr, logDirs, user);
createDir(tmpDir, dirPerm, false, user);
copyFile(nmPrivateTokensPath, tokenDst, user);
copyFile(nmPrivateContainerScriptPath, launchDst, user);
LocalWrapperScriptBuilder sb = getLocalWrapperScriptBuilder(
  containerIdStr, containerWorkDir); 
Path pidFile = getPidFilePath(containerId);
if (pidFile != null) {
 sb.writeLocalWrapperScript(launchDst, pidFile);
 setScriptExecutable(launchDst, user);
 setScriptExecutable(sb.getWrapperScriptPath(), user);
 shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(),
   containerIdStr, user, pidFile, container.getResource(),
   new File(containerWorkDir.toUri().getPath()),
   container.getLaunchContext().getEnvironment());
 if (isContainerActive(containerId)) {
  shExec.execute();
origin: com.github.jiayuhan-it/hadoop-yarn-server-nodemanager

@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
  InetSocketAddress nmAddr, String user, String appId, String locId,
  LocalDirsHandlerService dirsHandler)
  throws IOException, InterruptedException {
 List<String> localDirs = dirsHandler.getLocalDirs();
 List<String> logDirs = dirsHandler.getLogDirs();
 
 createUserLocalDirs(localDirs, user);
 createUserCacheDirs(localDirs, user);
 createAppDirs(localDirs, user, appId);
 createAppLogDirs(appId, logDirs, user);
 // randomly choose the local directory
 Path appStorageDir = getWorkingDir(localDirs, user, appId);
 String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
 Path tokenDst = new Path(appStorageDir, tokenFn);
 copyFile(nmPrivateContainerTokensPath, tokenDst, user);
 LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
 FileContext localizerFc = FileContext.getFileContext(
   lfs.getDefaultFileSystem(), getConf());
 localizerFc.setUMask(lfs.getUMask());
 localizerFc.setWorkingDirectory(appStorageDir);
 LOG.info("Localizer CWD set to " + appStorageDir + " = " 
   + localizerFc.getWorkingDirectory());
 ContainerLocalizer localizer =
   new ContainerLocalizer(localizerFc, user, appId, locId, 
     getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
 // TODO: DO it over RPC for maintaining similarity?
 localizer.runLocalization(nmAddr);
}
origin: io.hops/hadoop-yarn-server-nodemanager

List<String> logDirs = dirsHandler.getLogDirs();
createUserLocalDirs(localDirs, user, userFolder);
createUserCacheDirs(localDirs, user, userFolder);
createAppDirs(localDirs, user, appId, userFolder);
createAppLogDirs(appId, logDirs, user, userFolder);
Path appStorageDir = getWorkingDir(localDirs, user, appId, userFolder);
copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.getDefaultFileSystem(), getConf());
localizerFc.setUMask(lfs.getUMask());
localizerFc.setWorkingDirectory(appStorageDir);
  createContainerLocalizer(user, appId, locId, localDirs, localizerFc, userFolder);
origin: com.github.jiayuhan-it/hadoop-yarn-server-nodemanager

final Path appDir = getAppcacheDir(localDirPath, user);
try {
 createDir(appDir, appCachePerms, true, user);
 appcacheDirStatus = true;
} catch (IOException e) {
final Path distDir = getFileCacheDir(localDirPath, user);
try {
 createDir(distDir, fileperms, true, user);
 distributedCacheDirStatus = true;
} catch (IOException e) {
origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

/**
 * Initialize the local directories for a particular user.
 * <ul>
 * <li>$local.dir/usercache/$user/appcache/$appid</li>
 * </ul>
 * @param localDirs 
 */
void createAppDirs(List<String> localDirs, String user, String appId)
  throws IOException {
 boolean initAppDirStatus = false;
 FsPermission appperms = new FsPermission(APPDIR_PERM);
 for (String localDir : localDirs) {
  Path fullAppDir = getApplicationDir(new Path(localDir), user, appId);
  // create $local.dir/usercache/$user/appcache/$appId
  try {
   createDir(fullAppDir, appperms, true, user);
   initAppDirStatus = true;
  } catch (IOException e) {
   LOG.warn("Unable to create app directory " + fullAppDir.toString(), e);
  }
 }
 if (!initAppDirStatus) {
  throw new IOException("Not able to initialize app directories "
    + "in any of the configured local directories for app "
    + appId.toString());
 }
}
origin: io.hops/hadoop-yarn-server-nodemanager

/**
 * Create application log directories on all disks.
 */
void createAppLogDirs(String appId, List<String> logDirs, String user, String userFolder)
  throws IOException {
 boolean appLogDirStatus = false;
 FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM);
 for (String rootLogDir : logDirs) {
  // create $log.dir/$appid
  Path appLogDir = new Path(rootLogDir, userFolder + Path.SEPARATOR + appId);
  try {
   createDir(appLogDir, appLogDirPerms, true, user);
  } catch (IOException e) {
   LOG.warn("Unable to create the app-log directory : " + appLogDir, e);
   continue;
  }
  appLogDirStatus = true;
 }
 if (!appLogDirStatus) {
  throw new IOException("Not able to initialize app-log directories "
    + "in any of the configured local directories for app " + appId);
 }
}
origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

@Override
public boolean isContainerProcessAlive(String user, String pid)
  throws IOException {
 return containerIsAlive(pid);
}
origin: org.apache.hadoop/hadoop-yarn-server-nodemanager

 Path appDir = new Path(appCacheDir, appIdStr);
 Path containerDir = new Path(appDir, containerIdStr);
 createDir(containerDir, dirPerm, true, user);
createContainerLogDirs(appIdStr, containerIdStr, logDirs, user);
createDir(tmpDir, dirPerm, false, user);
copyFile(nmPrivateTokensPath, tokenDst, user);
copyFile(nmPrivateContainerScriptPath, launchDst, user);
LocalWrapperScriptBuilder sb = getLocalWrapperScriptBuilder(
  containerIdStr, containerWorkDir); 
Path pidFile = getPidFilePath(containerId);
if (pidFile != null) {
 sb.writeLocalWrapperScript(launchDst, pidFile);
 setScriptExecutable(launchDst, user);
 setScriptExecutable(sb.getWrapperScriptPath(), user);
 shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(),
   containerIdStr, user, pidFile, container.getResource(),
   new File(containerWorkDir.toUri().getPath()),
   container.getLaunchContext().getEnvironment());
 if (isContainerActive(containerId)) {
  shExec.execute();
origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
  InetSocketAddress nmAddr, String user, String appId, String locId,
  LocalDirsHandlerService dirsHandler)
  throws IOException, InterruptedException {
 List<String> localDirs = dirsHandler.getLocalDirs();
 List<String> logDirs = dirsHandler.getLogDirs();
 
 createUserLocalDirs(localDirs, user);
 createUserCacheDirs(localDirs, user);
 createAppDirs(localDirs, user, appId);
 createAppLogDirs(appId, logDirs, user);
 // randomly choose the local directory
 Path appStorageDir = getWorkingDir(localDirs, user, appId);
 String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
 Path tokenDst = new Path(appStorageDir, tokenFn);
 copyFile(nmPrivateContainerTokensPath, tokenDst, user);
 LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
 FileContext localizerFc = FileContext.getFileContext(
   lfs.getDefaultFileSystem(), getConf());
 localizerFc.setUMask(lfs.getUMask());
 localizerFc.setWorkingDirectory(appStorageDir);
 LOG.info("Localizer CWD set to " + appStorageDir + " = " 
   + localizerFc.getWorkingDirectory());
 ContainerLocalizer localizer =
   new ContainerLocalizer(localizerFc, user, appId, locId, 
     getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
 // TODO: DO it over RPC for maintaining similarity?
 localizer.runLocalization(nmAddr);
}
origin: org.apache.hadoop/hadoop-yarn-server-nodemanager

List<String> logDirs = dirsHandler.getLogDirs();
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
createAppLogDirs(appId, logDirs, user);
Path appStorageDir = getWorkingDir(localDirs, user, appId);
copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath
  + " to " + tokenDst);
  FileContext.getFileContext(lfs.getDefaultFileSystem(), getConf());
localizerFc.setUMask(lfs.getUMask());
localizerFc.setWorkingDirectory(appStorageDir);
  createContainerLocalizer(user, appId, locId, localDirs, localizerFc);
origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

final Path appDir = getAppcacheDir(localDirPath, user);
try {
 createDir(appDir, appCachePerms, true, user);
 appcacheDirStatus = true;
} catch (IOException e) {
final Path distDir = getFileCacheDir(localDirPath, user);
try {
 createDir(distDir, fileperms, true, user);
 distributedCacheDirStatus = true;
} catch (IOException e) {
origin: com.github.jiayuhan-it/hadoop-yarn-server-nodemanager

/**
 * Initialize the local directories for a particular user.
 * <ul>
 * <li>$local.dir/usercache/$user/appcache/$appid</li>
 * </ul>
 * @param localDirs 
 */
void createAppDirs(List<String> localDirs, String user, String appId)
  throws IOException {
 boolean initAppDirStatus = false;
 FsPermission appperms = new FsPermission(APPDIR_PERM);
 for (String localDir : localDirs) {
  Path fullAppDir = getApplicationDir(new Path(localDir), user, appId);
  // create $local.dir/usercache/$user/appcache/$appId
  try {
   createDir(fullAppDir, appperms, true, user);
   initAppDirStatus = true;
  } catch (IOException e) {
   LOG.warn("Unable to create app directory " + fullAppDir.toString(), e);
  }
 }
 if (!initAppDirStatus) {
  throw new IOException("Not able to initialize app directories "
    + "in any of the configured local directories for app "
    + appId.toString());
 }
}
origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

/**
 * Create application log directories on all disks.
 */
void createAppLogDirs(String appId, List<String> logDirs, String user)
  throws IOException {
 boolean appLogDirStatus = false;
 FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM);
 for (String rootLogDir : logDirs) {
  // create $log.dir/$appid
  Path appLogDir = new Path(rootLogDir, appId);
  try {
   createDir(appLogDir, appLogDirPerms, true, user);
  } catch (IOException e) {
   LOG.warn("Unable to create the app-log directory : " + appLogDir, e);
   continue;
  }
  appLogDirStatus = true;
 }
 if (!appLogDirStatus) {
  throw new IOException("Not able to initialize app-log directories "
    + "in any of the configured local directories for app " + appId);
 }
}
origin: com.github.jiayuhan-it/hadoop-yarn-server-nodemanager

@Override
public boolean isContainerProcessAlive(String user, String pid)
  throws IOException {
 return containerIsAlive(pid);
}
origin: org.apache.hadoop/hadoop-yarn-server-nodemanager

boolean containerLogDirStatus = false;
FsPermission containerLogDirPerms = new
  FsPermission(getLogDirPermissions());
for (String rootLogDir : logDirs) {
 Path containerLogDir = new Path(appLogDir, containerId);
 try {
  createDir(containerLogDir, containerLogDirPerms, true, user);
 } catch (IOException e) {
  LOG.warn("Unable to create the container-log directory : "
origin: ch.cern.hadoop/hadoop-yarn-server-nodemanager

 Path appDir = new Path(appCacheDir, appIdStr);
 Path containerDir = new Path(appDir, containerIdStr);
 createDir(containerDir, dirPerm, true, user);
createContainerLogDirs(appIdStr, containerIdStr, logDirs, user);
createDir(tmpDir, dirPerm, false, user);
copyFile(nmPrivateTokensPath, tokenDst, user);
copyFile(nmPrivateContainerScriptPath, launchDst, user);
LocalWrapperScriptBuilder sb = getLocalWrapperScriptBuilder(
  containerIdStr, containerWorkDir); 
Path pidFile = getPidFilePath(containerId);
if (pidFile != null) {
 sb.writeLocalWrapperScript(launchDst, pidFile);
 setScriptExecutable(launchDst, user);
 setScriptExecutable(sb.getWrapperScriptPath(), user);
 shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(),
   containerIdStr, user, pidFile, container.getResource(),
   new File(containerWorkDir.toUri().getPath()),
   container.getLaunchContext().getEnvironment());
 if (isContainerActive(containerId)) {
  shExec.execute();
org.apache.hadoop.yarn.server.nodemanagerDefaultContainerExecutor

Javadoc

The DefaultContainerExecuter class offers generic container execution services. Process execution is handled in a platform-independent way via ProcessBuilder.

Most used methods

  • buildCommandExecutor
    Create a new ShellCommandExecutor using the parameters.
  • containerIsAlive
    Returns true if the process with the specified pid is alive.
  • copyFile
    Copy a file using the #lfs FileContext.
  • createAppDirs
    Initialize the local directories for a particular user. * $local.dir/usercache/$user/appcache/$a
  • createAppLogDirs
    Create application log directories on all disks.
  • createContainerLogDirs
    Create application log directories on all disks.
  • createDir
    Use the #lfs FileContext to create the target directory.
  • createUserCacheDirs
    Initialize the local cache directories for a particular user. * $local.dir/usercache/$user *
  • createUserLocalDirs
    Initialize the local directories for a particular user. * * $local.dir/usercache/$user
  • getAppcacheDir
  • getApplicationDir
  • getConf
  • getApplicationDir,
  • getConf,
  • getDiskFreeSpace,
  • getFileCacheDir,
  • getLocalWrapperScriptBuilder,
  • getPaths,
  • getPidFilePath,
  • getRunCommand,
  • getUserCacheDir,
  • getWorkingDir

Popular in Java

  • Creating JSON documents from java classes using gson
  • setRequestProperty (URLConnection)
  • addToBackStack (FragmentTransaction)
  • scheduleAtFixedRate (ScheduledExecutorService)
  • BorderLayout (java.awt)
    A border layout lays out a container, arranging and resizing its components to fit in five regions:
  • ServerSocket (java.net)
    This class represents a server-side socket that waits for incoming client connections. A ServerSocke
  • Comparator (java.util)
    A Comparator is used to compare two objects to determine their ordering with respect to each other.
  • Dictionary (java.util)
    Note: Do not use this class since it is obsolete. Please use the Map interface for new implementatio
  • TreeMap (java.util)
    Walk the nodes of the tree left-to-right or right-to-left. Note that in descending iterations, next
  • Location (org.springframework.beans.factory.parsing)
    Class that models an arbitrary location in a Resource.Typically used to track the location of proble
  • Best IntelliJ plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now