protected List<String> getUserLocalDirs(List<String> localDirs) { List<String> userLocalDirs = new ArrayList<>(localDirs.size()); String user = container.getUser(); for (String localDir : localDirs) { String userLocalDir = localDir + Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + Path.SEPARATOR; userLocalDirs.add(userLocalDir); } return userLocalDirs; }
/** * For each of the requested resources for a container, determines the * appropriate {@link LocalResourcesTracker} and forwards a * {@link LocalResourceRequest} to that tracker. */ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); // create a loading cache for the file statuses LoadingCache<Path,Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig())); LocalizerContext ctxt = new LocalizerContext( c.getUser(), c.getContainerId(), c.getCredentials(), statCache, c.getUserFolder()); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId() .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); if (LOG.isDebugEnabled()) { LOG.debug("Localizing " + req.getPath() + " for container " + c.getContainerId()); } } } }
protected List<String> getUserFilecacheDirs(List<String> localDirs) { List<String> userFilecacheDirs = new ArrayList<>(localDirs.size()); String user = container.getUser(); for (String localDir : localDirs) { String userFilecacheDir = localDir + Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + Path.SEPARATOR + ContainerLocalizer.FILECACHE; userFilecacheDirs.add(userFilecacheDir); } return userFilecacheDirs; }
protected List<String> getApplicationLocalDirs(List<String> localDirs, String appIdStr) { List<String> applicationLocalDirs = new ArrayList<>(localDirs.size()); String user = container.getUser(); for (String localDir : localDirs) { String appLocalDir = localDir + Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + Path.SEPARATOR + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr; applicationLocalDirs.add(appLocalDir); } return applicationLocalDirs; }
protected List<String> getContainerLocalDirs(List<String> localDirs) { List<String> containerLocalDirs = new ArrayList<>(localDirs.size()); String user = container.getUser(); String appIdStr = app.getAppId().toString(); String relativeContainerLocalDir = ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + Path.SEPARATOR + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr + Path.SEPARATOR; for (String localDir : localDirs) { containerLocalDirs.add(localDir + Path.SEPARATOR + relativeContainerLocalDir); } return containerLocalDirs; }
/** * Perform any cleanup before the next launch of the container. * @param container container */ public void cleanupBeforeRelaunch(Container container) throws IOException, InterruptedException { if (container.getLocalizedResources() != null) { Map<Path, Path> symLinks = resolveSymLinks( container.getLocalizedResources(), container.getUser()); for (Map.Entry<Path, Path> symLink : symLinks.entrySet()) { LOG.debug("{} deleting {}", container.getContainerId(), symLink.getValue()); deleteAsUser(new DeletionAsUserContext.Builder() .setUser(container.getUser()) .setSubDir(symLink.getValue()) .build()); } } }
@Override public String getNetworkTagHexID(Container container) { String userNetworkTagID = this.networkTagMapping.getUserNetworkTagID( container.getUser()); if (userNetworkTagID != null) { return userNetworkTagID; } UserGroupInformation userUGI = UserGroupInformation.createRemoteUser( container.getUser()); List<Group> groups = this.networkTagMapping.getGroups(); for(Group group : groups) { if (userUGI.getGroups().contains(group.getGroupName())) { return group.getNetworkTagID(); } } return this.networkTagMapping.getDefaultNetworkTagID(); }
private void deleteAsUser(Path path) { try { exec.deleteAsUser(new DeletionAsUserContext.Builder() .setUser(container.getUser()) .setSubDir(path) .build()); } catch (Exception e) { LOG.warn("Failed to delete " + path, e); } }
@GET @Path("/containers") @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) public ContainersInfo getNodeContainers(@javax.ws.rs.core.Context HttpServletRequest hsr) { init(); ContainersInfo allContainers = new ContainersInfo(); for (Entry<ContainerId, Container> entry : this.nmContext.getContainers() .entrySet()) { if (entry.getValue() == null) { // just skip it continue; } ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(), uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser()); ApplicationId appId = entry.getKey().getApplicationAttemptId() .getApplicationId(); // Allow only application-owner/admin for any type of access on the // application. if (filterAppsByUser && !hasAccess(entry.getValue().getUser(), appId, hsr)) { continue; } allContainers.add(info); } return allContainers; }
private void reapDockerContainerNoPid(String user) throws IOException { String containerIdStr = container.getContainerTokenIdentifier().getContainerID().toString(); LOG.info("Unable to obtain pid, but docker container request detected. " + "Attempting to reap container " + containerIdStr); boolean result = exec.reapContainer( new ContainerReapContext.Builder() .setContainer(container) .setUser(container.getUser()) .build()); if (LOG.isDebugEnabled()) { LOG.debug("Sent signal to docker container " + containerIdStr + " as user " + user + ", result=" + (result ? "success" : "failed")); } } }
void reapContainer() throws IOException { launchLock.lock(); try { // Reap the container boolean result = exec.reapContainer( new ContainerReapContext.Builder() .setContainer(container) .setUser(container.getUser()) .build()); if (!result) { throw new IOException("Reap container failed for container " + container.getContainerId()); } cleanupContainerFiles(getContainerWorkDir()); } finally { launchLock.unlock(); } }
/** * For each of the requested resources for a container, determines the * appropriate {@link LocalResourcesTracker} and forwards a * {@link LocalResourceRequest} to that tracker. */ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); // create a loading cache for the file statuses LoadingCache<Path,Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig())); LocalizerContext ctxt = new LocalizerContext( c.getUser(), c.getContainerId(), c.getCredentials(), statCache); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId() .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); } } }
/** * For each of the requested resources for a container, determines the * appropriate {@link LocalResourcesTracker} and forwards a * {@link LocalResourceRequest} to that tracker. */ private void handleInitContainerResources( ContainerLocalizationRequestEvent rsrcReqs) { Container c = rsrcReqs.getContainer(); // create a loading cache for the file statuses LoadingCache<Path,Future<FileStatus>> statCache = CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig())); LocalizerContext ctxt = new LocalizerContext( c.getUser(), c.getContainerId(), c.getCredentials(), statCache); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = rsrcReqs.getRequestedResources(); for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e : rsrcs.entrySet()) { LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(), c.getContainerId().getApplicationAttemptId() .getApplicationId()); for (LocalResourceRequest req : e.getValue()) { tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt)); } } }
private Path deriveContainerWorkDir() throws IOException { final String containerWorkDirPath = ContainerLocalizer.USERCACHE + Path.SEPARATOR + container.getUser() + Path.SEPARATOR + ContainerLocalizer.APPCACHE + Path.SEPARATOR + app.getAppId().toString() + Path.SEPARATOR + container.getContainerId().toString(); final Path containerWorkDir = dirsHandler.getLocalPathForWrite( containerWorkDirPath, LocalDirAllocator.SIZE_UNKNOWN, false); return containerWorkDir; }
@SuppressWarnings("unchecked") protected void stopContainerInternal(ContainerId containerID) throws YarnException, IOException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); LOG.info("Stopping container with container Id: " + containerIDStr); if (container == null) { if (!nodeStatusUpdater.isContainerRecentlyStopped(containerID)) { throw RPCUtil.getRemoteException("Container " + containerIDStr + " is not handled by this NodeManager"); } } else { if (container.isRecovering()) { throw new NMNotYetReadyException("Container " + containerIDStr + " is recovering, try later"); } context.getNMStateStore().storeContainerKilled(containerID); container.sendKillEvent(ContainerExitStatus.KILLED_BY_APPMASTER, "Container killed by the ApplicationMaster."); NMAuditLogger.logSuccess(container.getUser(), AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID .getApplicationAttemptId().getApplicationId(), containerID); } }
@SuppressWarnings("unchecked") private void stopContainerInternal(NMTokenIdentifier nmTokenIdentifier, ContainerId containerID) throws YarnException, IOException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); LOG.info("Stopping container with container Id: " + containerIDStr); authorizeGetAndStopContainerRequest(containerID, container, true, nmTokenIdentifier); if (container == null) { if (!nodeStatusUpdater.isContainerRecentlyStopped(containerID)) { throw RPCUtil.getRemoteException("Container " + containerIDStr + " is not handled by this NodeManager"); } } else { context.getNMStateStore().storeContainerKilled(containerID); dispatcher.getEventHandler().handle( new ContainerKillEvent(containerID, ContainerExitStatus.KILLED_BY_APPMASTER, "Container killed by the ApplicationMaster.")); NMAuditLogger.logSuccess(container.getUser(), AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID .getApplicationAttemptId().getApplicationId(), containerID); } }
@SuppressWarnings("unchecked") private void stopContainerInternal(NMTokenIdentifier nmTokenIdentifier, ContainerId containerID) throws YarnException, IOException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); LOG.info("Stopping container with container Id: " + containerIDStr); authorizeGetAndStopContainerRequest(containerID, container, true, nmTokenIdentifier); if (container == null) { if (!nodeStatusUpdater.isContainerRecentlyStopped(containerID)) { throw RPCUtil.getRemoteException("Container " + containerIDStr + " is not handled by this NodeManager"); } } else { context.getNMStateStore().storeContainerKilled(containerID); dispatcher.getEventHandler().handle( new ContainerKillEvent(containerID, ContainerExitStatus.KILLED_BY_APPMASTER, "Container killed by the ApplicationMaster.")); NMAuditLogger.logSuccess(container.getUser(), AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID .getApplicationAttemptId().getApplicationId(), containerID); } }
private void prepareContainer(Map<Path, List<String>> localResources, List<String> containerLocalDirs) throws IOException { exec.prepareContainer(new ContainerPrepareContext.Builder() .setContainer(container) .setLocalizedResources(localResources) .setUser(container.getUser()) .setContainerLocalDirs(containerLocalDirs) .setCommands(container.getLaunchContext().getCommands()) .build()); }
@SuppressWarnings("unchecked") private void stopContainerInternal(NMTokenIdentifier nmTokenIdentifier, ContainerId containerID) throws YarnException, IOException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); LOG.info("Stopping container with container Id: " + containerIDStr); authorizeGetAndStopContainerRequest(containerID, container, true, nmTokenIdentifier); if (container == null) { if (!nodeStatusUpdater.isContainerRecentlyStopped(containerID)) { throw RPCUtil.getRemoteException("Container " + containerIDStr + " is not handled by this NodeManager"); } } else { if (container.isRecovering()) { throw new NMNotYetReadyException("Container " + containerIDStr + " is recovering, try later"); } context.getNMStateStore().storeContainerKilled(containerID); dispatcher.getEventHandler().handle( new ContainerKillEvent(containerID, ContainerExitStatus.KILLED_BY_APPMASTER, "Container killed by the ApplicationMaster.")); NMAuditLogger.logSuccess(container.getUser(), AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID .getApplicationAttemptId().getApplicationId(), containerID); } }