Refine search
for (Container allocatedContainer : allocatedContainers) { LOG.info("Launching shell command on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" if (allocatedContainer.getPriority().getPriority() == 0) { String supervisorHost = allocatedContainer.getNodeId().getHost(); startType = STARTType.SUPERVISOR; String containerPath = RegistryUtils.componentPath( JOYConstants.APP_TYPE, jstormMasterContext.instanceName, allocatedContainer.getId().getApplicationAttemptId().getApplicationId().toString(), allocatedContainer.getId().toString()); sr = new ServiceRecord(); sr.set(JOYConstants.HOST, supervisorHost); sr.set(YarnRegistryAttributes.YARN_ID, allocatedContainer.getId().toString()); sr.description = JOYConstants.CONTAINER; sr.set(YarnRegistryAttributes.YARN_PERSISTENCE, serviceRecord.set(JOYConstants.NIMBUS_CONTAINER, allocatedContainer.getId().toString()); registryOperations.bind(path, serviceRecord, BindFlags.OVERWRITE);
@Override public ApplicationId getApplicationId() { for (ChildData childData : instancesCache.getCurrentData()) { byte[] data = getWorkerData(childData, WORKER_PREFIX); if (data == null) continue; ServiceRecord sr = null; try { sr = encoder.fromBytes(childData.getPath(), data); } catch (IOException e) { LOG.error("Unable to decode data for zkpath: {}." + " Ignoring from current instances list..", childData.getPath()); continue; } String containerStr = sr.get(HiveConf.ConfVars.LLAP_DAEMON_CONTAINER_ID.varname); if (containerStr == null || containerStr.isEmpty()) continue; return ContainerId.fromString(containerStr).getApplicationAttemptId().getApplicationId(); } return null; } }
public Container createContainer(Resource capability, Priority priority, String hostname, int port, String nodeHttpAddress) { ContainerId containerId = ContainerId.newContainerId(customAppAttemptId, nextId.getAndIncrement()); NodeId nodeId = NodeId.newInstance(hostname, port); Container container = Container.newInstance(containerId, nodeId, nodeHttpAddress, capability, priority, null); return container; } }
supervisorContainer.getId().getApplicationAttemptId().getApplicationId().toString(), supervisorContainer.getId().toString()); try { if (jstormMaster.registryOperations.exists(containerPath)) { nimbusContainer.getId().getApplicationAttemptId().getApplicationId().toString(), nimbusContainer.getId().toString());
nimbusMap.put(container.getId().getContainerId(), container); supervisorMap.put(container.getId().getContainerId(), container); Long containerId = containerStatus.getContainerId().getContainerId(); if (nimbusMap.containsKey(containerId)) { Container nimbusContainer = nimbusMap.get(containerId); containerAsk = setupContainerAskForRM(nimbusContainer.getResource().getMemory(), nimbusContainer.getResource().getVirtualCores(), nimbusContainer.getPriority().getPriority(), nimbusContainer.getNodeId().getHost()); LOG.info("restart nimbus container" + ", containerId=" + containerStatus.getContainerId()); } else if (supervisorMap.containsKey(containerId)) { Container supervisorContainer = supervisorMap.get(containerId); containerAsk = setupContainerAskForRM(supervisorContainer.getResource().getMemory(), supervisorContainer.getResource().getVirtualCores(), supervisorContainer.getPriority().getPriority(), supervisorContainer.getNodeId().getHost()); LOG.info("restart supervisor container" + ", containerId=" + containerStatus.getContainerId()); JOYConstants.APP_TYPE, jstormMasterContext.instanceName, jstormMasterContext.appAttemptID.getApplicationId().toString(), containerStatus.getContainerId().toString()); try { if (registryOperations.exists(containerPath)) {
publishApplicationAttemptEvent(timelineClient, jstormMasterContext.appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, jstormMasterContext.domainId, appSubmitterUgi); container.getId().getApplicationAttemptId().getApplicationId().toString(), container.getId().toString()); ServiceRecord sr = null; try { if (!registryOperations.exists(containerPath)) { String contianerHost = container.getNodeId().getHost(); registryOperations.mknode(containerPath, true); sr = new ServiceRecord(); sr.set(JOYConstants.HOST, contianerHost); sr.set(YarnRegistryAttributes.YARN_ID, container.getId().toString()); sr.description = JOYConstants.CONTAINER; sr.set(YarnRegistryAttributes.YARN_PERSISTENCE, JOYConstants.APP_TYPE, jstormMasterContext.instanceName, jstormMasterContext.appAttemptID.getApplicationId().toString()); String instancePath = RegistryUtils.serviceclassPath( JOYConstants.APP_TYPE, jstormMasterContext.instanceName); LOG.info("Registering application " + jstormMasterContext.appAttemptID.getApplicationId().toString());
+ container.getId()); String localDir = jstormMasterContext.nimbusDataDirPrefix + container.getId().toString() + JOYConstants.BACKLASH + jstormMasterContext.instanceName; vargs.add(localDir); SlotPortsView slotPortsView = new SlotPortsView(jstormMasterContext.instanceName, container.getId(), registryOperations); slotPortsView.setMinPort(conf.getInt(JOYConstants.SUPERVISOR_MIN_PORT_KEY, JOYConstants.PORT_RANGE_MIN)); slotPortsView.setMaxPort(conf.getInt(JOYConstants.SUPERVISOR_MAX_PORT_KEY, JOYConstants.PORT_RANGE_MAX)); try { slotPortsStr = slotPortsView.getSupervisorSlotPorts(container.getResource().getMemory(), container.getResource().getVirtualCores(), container.getNodeId().getHost()); try { logviewPort = slotPortsView.getSupervisorSlotPorts(JOYConstants.DEFAULT_SUPERVISOR_MEMORY, JOYConstants.DEFAULT_SUPERVISOR_VCORES, container.getNodeId().getHost()); nimbusThriftPort = slotPortsView.getSupervisorSlotPorts(JOYConstants.DEFAULT_SUPERVISOR_MEMORY, JOYConstants.DEFAULT_SUPERVISOR_VCORES, container.getNodeId().getHost()); } catch (Exception e) { e.printStackTrace(); deployDst = jstormMasterContext.nimbusDataDirPrefix; String dstPath = deployDst + container.getId().toString(); this.container.getId().toString(), localDir, jstormMasterContext.deployPath, hadoopHome, javaHome, pythonHome, dstPath, slotPortsStr, jstormMasterContext.shellArgs, envs.get(JOYConstants.CLASS_PATH), JOYConstants.ExecShellStringPath, jstormMasterContext.appAttemptID.getApplicationId().toString(), logviewPort, nimbusThriftPort);
ContainerId containerId = ConverterUtils.toContainerId(envs .get(ApplicationConstants.Environment.CONTAINER_ID.name())); jstormMasterContext.appAttemptID = containerId.getApplicationAttemptId(); + jstormMasterContext.appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + jstormMasterContext.appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + jstormMasterContext.appAttemptID.getAttemptId());
@Override public String info() throws TException { StringBuffer sbRet = new StringBuffer(); sbRet.append("JstormOnYarn\n"); sbRet.append("Instance Name:" + jstormMasterContext.instanceName + "\n"); sbRet.append("Jstorm's location on hdfs:" + jstormMasterContext.deployPath + "\n"); if (jstormMasterContext.user != null) { sbRet.append("Jstorm's data path:" + jstormMasterContext.nimbusDataDirPrefix + jstormMasterContext.instanceName + "\n"); sbRet.append("Cluster userName:" + jstormMasterContext.user + "\n"); } sbRet.append("Nimbus Count:" + jstormMasterContext.nimbusContainers.size() + "\n"); sbRet.append("Supervisor Count:" + jstormMasterContext.supervisorContainers.size() + "\n"); sbRet.append("detail :\n"); sbRet.append("Type \tContainerId \tHost \tContainerMemory\tContainerVCores\n"); for (Container container : jstormMasterContext.nimbusContainers) { sbRet.append("Nimbus \t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } for (Container container : jstormMasterContext.supervisorContainers) { sbRet.append("Supervisor\t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } LOG.info("info is: " + sbRet.toString()); return sbRet.toString(); }
if (conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR).equals(xlearningConf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR))) { boardLogPath = new Path(xlearningConf.get("fs.defaultFS"), conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR) + "/" + applicationAttemptID.getApplicationId().toString()); } else { boardLogPath = new Path(conf.get("fs.defaultFS"), conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, for (Container container : workerContainers) { Map<String, String> containerMessage = new HashMap<>(); containerMessage.put(AMParams.CONTAINER_HTTP_ADDRESS, container.getNodeHttpAddress()); if (tfEvaluator && container.getId().toString().equals(tfEvaluatorContainerId)) { containerMessage.put(AMParams.CONTAINER_ROLE, XLearningConstants.EVALUATOR); } else { containerMessage.put(AMParams.CONTAINER_ROLE, XLearningConstants.WORKER); if (applicationContext.getContainerStatus(new XLearningContainerId(container.getId())) != null) { containerMessage.put(AMParams.CONTAINER_STATUS, applicationContext.getContainerStatus(new XLearningContainerId(container.getId())).toString()); } else { container.getNodeHttpAddress(), container.getId().toString(), userName)); logMessage.put(container.getId().toString(), containerMessage); containerMessage.put(AMParams.CONTAINER_LOG_ADDRESS, String.format("http://%s/node/containerlogs/%s/%s", container.getNodeHttpAddress(), container.getId().toString(), userName)); logMessage.put(container.getId().toString(), containerMessage);
private static String getApplicationId(ContainerId containerId) { return containerId.getApplicationAttemptId().getApplicationId().toString(); }
LOG.info("Canceling container: " + container.getId().toString()); amrmAsync.releaseAssignedContainer(container.getId()); amrmAsync.addContainerRequest(psContainerRequest); LOG.info("Canceling container: " + container.getId().toString()); LOG.info("Release container " + releaseContainer.getId().toString()); tfEvaluatorContainerId = container.getId().toString(); this.appendMessage("container " + container.getId().toString() + " status is " + applicationContext.getContainerStatus(new XLearningContainerId(container.getId())), false); this.appendMessage("container " + container.getId().toString() + " status is " + applicationContext.getContainerStatus(new XLearningContainerId(container.getId())), false); Path tmpResultPath = new Path(outputInfos.get(0).getDfsLocation() + "/_temporary/" + finishedContainer.getId().toString() + "/_temporary/0/_temporary/" + taskID); LOG.info("tmpResultPath is " + tmpResultPath.toString()); Path finalResultPath = new Path(outputInfos.get(0).getDfsLocation() + "/" + finishedContainer.getId().toString()); LOG.info("finalResultPath is " + finalResultPath.toString()); if (fs.exists(tmpResultPath)) { Path tmpResultPath = new Path(outputInfo.getDfsLocation() + "/_temporary/" + finishedContainer.getId().toString()); if (fs.exists(tmpResultPath)) { LOG.info("Move from " + tmpResultPath.toString() + " to " + finalResultPath); Path tmpResultPath = new Path(outputInfo.getDfsLocation() + "/_temporary/" + finishedContainer.getId().toString()); if (fs.exists(tmpResultPath)) { LOG.info("Move from " + tmpResultPath.toString() + " to " + finalResultPath);
private static void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); event.addEventInfo(JOYConstants.NODE, container.getNodeId().toString()); event.addEventInfo(JOYConstants.RESOURCES, container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } }
public static ContainerId newContainerId(int appId, int appAttemptId, long timestamp, long containerId) { ApplicationId applicationId = ApplicationId.newInstance(timestamp, appId); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, appAttemptId); return ContainerId.newContainerId(applicationAttemptId, containerId); } }
@Override public int compareTo(Container other) { if (this.getId().compareTo(other.getId()) == 0) { if (this.getNodeId().compareTo(other.getNodeId()) == 0) { return this.getResource().compareTo(other.getResource()); } else { return this.getNodeId().compareTo(other.getNodeId()); } } else { return this.getId().compareTo(other.getId()); } } }
@Override public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityId, String entityType) { if (ApplicationMaster.DSEntity.DS_CONTAINER.toString().equals(entityType)) { ContainerId containerId = ContainerId.fromString(entityId); ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); return toEntityGroupId(appId.toString()); } return null; }
@Override public int compareTo(ContainerId other) { int result = this.getApplicationAttemptId().compareTo( other.getApplicationAttemptId()); if (result == 0) { return Long.compare(getContainerId(), other.getContainerId()); } else { return result; } }
@Override public int hashCode() { // Generated by IntelliJ IDEA 13.1. int result = (int) (getContainerId() ^ (getContainerId() >>> 32)); result = 31 * result + getApplicationAttemptId().hashCode(); return result; }
@Override public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) { LOG.info("Container " + containerId.toString() + " started"); }