public static int getRunningContainers() { int count = 0; for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) { NodeManager nm = yarnCluster.getNodeManager(nmId); ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers(); count += containers.size(); } return count; }
ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers(); for (Map.Entry<ContainerId, Container> entry : containers.entrySet()) { String command = StringUtils.join(entry.getValue().getLaunchContext().getCommands(), " "); nodeManager.getNMContext().getContainerManager().stopContainers(scr);
@Override protected void serviceStart() throws Exception { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); LOG.info("Node ID assigned is : " + this.nodeId); this.httpPort = this.context.getHttpPort(); this.nodeManagerVersionId = YarnVersionInfo.getVersion(); try { // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. this.resourceTracker = getRMClient(); registerWithRM(); super.serviceStart(); startStatusUpdater(); } catch (Exception e) { String errorMessage = "Unexpected error starting NodeStatusUpdater"; LOG.error(errorMessage, e); throw new YarnRuntimeException(e); } }
/** * Finds the local directories that logs for the given container are stored * on. */ public static List<File> getContainerLogDirs(ContainerId containerId, String remoteUser, Context context) throws YarnException { Container container = context.getContainers().get(containerId); Application application = getApplicationForContainer(containerId, context); checkAccess(remoteUser, application, context); // It is not required to have null check for container ( container == null ) // and throw back exception.Because when container is completed, NodeManager // remove container information from its NMContext.Configuring log // aggregation to false, container log view request is forwarded to NM. NM // does not have completed container information,but still NM serve request for // reading container logs. if (container != null) { checkState(container.getContainerState()); } return getContainerLogDirs(containerId, context.getLocalDirsHandler()); }
this.context.getContainerTokenSecretManager().setMasterKey(masterKey); this.context.getNMTokenSecretManager().setMasterKey(masterKey); ((ContainerManagerImpl) this.context.getContainerManager()) .setBlockNewContainerRequests(false);
protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier( org.apache.hadoop.yarn.api.records.Token token, ContainerTokenIdentifier containerTokenIdentifier) throws YarnException, InvalidToken { byte[] password = context.getContainerTokenSecretManager().retrievePassword( containerTokenIdentifier); byte[] tokenPass = token.getPassword().array(); if (password == null || tokenPass == null || !Arrays.equals(password, tokenPass)) { throw new InvalidToken( "Invalid container token used for starting container on : " + context.getNodeId().toString()); } return containerTokenIdentifier; }
private void updateMasterKeys(NodeHeartbeatResponse response) { // See if the master-key has rolled over MasterKey updatedMasterKey = response.getContainerTokenMasterKey(); if (updatedMasterKey != null) { // Will be non-null only on roll-over on RM side context.getContainerTokenSecretManager().setMasterKey(updatedMasterKey); } updatedMasterKey = response.getNMTokenMasterKey(); if (updatedMasterKey != null) { context.getNMTokenSecretManager().setMasterKey(updatedMasterKey); } } };
NodeHeartbeatRequest.newInstance(nodeStatus, NodeStatusUpdaterImpl.this.context .getContainerTokenSecretManager().getCurrentKey(), NodeStatusUpdaterImpl.this.context .getNMTokenSecretManager().getCurrentKey(), nodeLabelsForHeartbeat, nodeAttributesForHeartbeat, NodeStatusUpdaterImpl.this.context .getRegisteringCollectors()); .getLogAggregationStatusForApps()); if (logAggregationReports != null && !logAggregationReports.isEmpty()) { ((NMContext) context).setSystemCrendentialsForApps( parseCredentials(systemCredentials)); context.getContainerManager().handleCredentialUpdate(); response.getContainerQueuingLimit(); if (queuingLimit != null) { context.getContainerManager().updateQueuingLimit(queuingLimit); if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { updateTimelineCollectorData(response);
@Override protected void serviceStart() throws Exception { // NodeId is only available during start, the following cannot be moved // anywhere else. this.nodeId = this.context.getNodeId(); super.serviceStart(); }
@Private @VisibleForTesting protected void updateNMTokenIdentifier(NMTokenIdentifier nmTokenIdentifier) throws InvalidToken { context.getNMTokenSecretManager().appAttemptStartContainer( nmTokenIdentifier); }
public void cleanUpApplicationsOnNMShutDown() { Map<ApplicationId, Application> applications = this.context.getApplications(); if (applications.isEmpty()) { return; if (this.context.getNMStateStore().canRecover() && !this.context.getDecommissioned()) {
boolean shouldAbort = context.getNMStateStore().canRecover() && !context.getDecommissioned();
@Override public void transition(ApplicationImpl app, ApplicationEvent event) { ApplicationId appId = event.getApplicationID(); app.context.getApplications().remove(appId); app.aclsManager.removeApplication(appId); try { app.context.getNMStateStore().removeApplication(appId); } catch (IOException e) { LOG.error("Unable to remove application from state store", e); } } }
/** * Get the status of the OPPORTUNISTIC containers. * @return the status of the OPPORTUNISTIC containers. */ private OpportunisticContainersStatus getOpportunisticContainersStatus() { OpportunisticContainersStatus status = this.context.getContainerManager().getOpportunisticContainersStatus(); return status; }
private List<ApplicationId> getRunningApplications() { List<ApplicationId> runningApplications = new ArrayList<ApplicationId>(); runningApplications.addAll(this.context.getApplications().keySet()); return runningApplications; }
public ApplicationImpl(Dispatcher dispatcher, String user, ApplicationId appId, Credentials credentials, Context context) { this.dispatcher = dispatcher; this.user = user; this.appId = appId; this.credentials = credentials; this.aclsManager = context.getApplicationACLsManager(); this.context = context; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); stateMachine = stateMachineFactory.make(this); }
public void init(AMRMProxyApplicationContext applicationContext) { super.init(applicationContext); initLocal(applicationContext.getNMCotext().getNodeStatusUpdater() .getRMIdentifier(), applicationContext.getApplicationAttemptId(), applicationContext.getNMCotext().getContainerAllocator(), applicationContext.getNMCotext().getNMTokenSecretManager(), applicationContext.getUser()); }
public ApplicationImpl(Dispatcher dispatcher, String user, FlowContext flowContext, ApplicationId appId, Credentials credentials, Context context, long recoveredLogInitedTime) { this.dispatcher = dispatcher; this.user = user; this.appId = appId; this.credentials = credentials; this.aclsManager = context.getApplicationACLsManager(); Configuration conf = context.getConf(); if (YarnConfiguration.timelineServiceV2Enabled(conf)) { if (flowContext == null) { throw new IllegalArgumentException("flow context cannot be null"); } this.flowContext = flowContext; if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { context.getNMTimelinePublisher().createTimelineClient(appId); } } this.context = context; this.appStateStore = context.getNMStateStore(); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); stateMachine = stateMachineFactory.make(this); setAppLogInitedTimestamp(recoveredLogInitedTime); }
public AuxServices(AuxiliaryLocalPathHandler auxiliaryLocalPathHandler, Context nmContext, DeletionService deletionService) { super(AuxServices.class.getName()); serviceMap = Collections.synchronizedMap(new HashMap<String,AuxiliaryService>()); serviceMetaData = Collections.synchronizedMap(new HashMap<String,ByteBuffer>()); this.auxiliaryLocalPathHandler = auxiliaryLocalPathHandler; this.dirsHandler = nmContext.getLocalDirsHandler(); this.delService = deletionService; this.userUGI = getRemoteUgi(); // Obtain services from configuration in init() }
private void stopRecoveryStore() throws IOException { if (null != nmStore) { nmStore.stop(); if (null != context) { if (context.getDecommissioned() && nmStore.canRecover()) { LOG.info("Removing state store due to decommission"); Configuration conf = getConfig(); Path recoveryRoot = new Path(conf.get(YarnConfiguration.NM_RECOVERY_DIR)); LOG.info("Removing state store at " + recoveryRoot + " due to decommission"); FileSystem recoveryFs = FileSystem.getLocal(conf); if (!recoveryFs.delete(recoveryRoot, true)) { LOG.warn("Unable to delete " + recoveryRoot); } } } } }