private String buildContainerCommand(Container container, String helixInstanceName) { String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName(); return new StringBuilder() .append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java") .append(" -Xmx").append(container.getResource().getMemory()).append("M") .append(" ").append(JvmUtils.formatJvmArguments(this.containerJvmArgs)) .append(" ").append(GobblinYarnTaskRunner.class.getName()) .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) .append(" ").append(this.applicationName) .append(" --").append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME) .append(" ").append(helixInstanceName) .append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append( containerProcessName).append(".").append(ApplicationConstants.STDOUT) .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append( containerProcessName).append(".").append(ApplicationConstants.STDERR) .toString(); }
@Override public String info() throws TException { StringBuffer sbRet = new StringBuffer(); sbRet.append("JstormOnYarn\n"); sbRet.append("Instance Name:" + jstormMasterContext.instanceName + "\n"); sbRet.append("Jstorm's location on hdfs:" + jstormMasterContext.deployPath + "\n"); if (jstormMasterContext.user != null) { sbRet.append("Jstorm's data path:" + jstormMasterContext.nimbusDataDirPrefix + jstormMasterContext.instanceName + "\n"); sbRet.append("Cluster userName:" + jstormMasterContext.user + "\n"); } sbRet.append("Nimbus Count:" + jstormMasterContext.nimbusContainers.size() + "\n"); sbRet.append("Supervisor Count:" + jstormMasterContext.supervisorContainers.size() + "\n"); sbRet.append("detail :\n"); sbRet.append("Type \tContainerId \tHost \tContainerMemory\tContainerVCores\n"); for (Container container : jstormMasterContext.nimbusContainers) { sbRet.append("Nimbus \t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } for (Container container : jstormMasterContext.supervisorContainers) { sbRet.append("Supervisor\t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } LOG.info("info is: " + sbRet.toString()); return sbRet.toString(); }
/** * Utility method to display YARN container information in a useful way for * log messages. * * @param container * @return */ public static String describeContainer(Container container) { StringBuilder buf = new StringBuilder() .append("[id: ") .append(container.getId()) .append(", host: ") .append(container.getNodeId().getHost()) .append(", priority: ") .append(container.getPriority()) .append(", memory: ") .append(container.getResource().getMemory()) .append(" MB, vcores: ") .append(container.getResource().getVirtualCores()) .append("]"); return buf.toString(); }
@Override public void onContainersAllocated(List<Container> containers) { for (Container acquiredContainer : containers) { LOG.info("Acquired container " + acquiredContainer.getId() + " on host " + acquiredContainer.getNodeId().getHost() + " , with the resource " + acquiredContainer.getResource().toString()); String host = acquiredContainer.getNodeId().getHost(); if (!blackHosts.contains(host)) { if (workerContainersAllocating.get()) { acquiredWorkerContainers.add(acquiredContainer); acquiredWorkerContainersCount.incrementAndGet(); } else { acquiredPsContainers.add(acquiredContainer); acquiredPsContainersCount.incrementAndGet(); } } else { LOG.info("Add container " + acquiredContainer.getId() + " to cancel list"); cancelContainers.add(acquiredContainer); } } LOG.info("Current acquired worker container " + acquiredWorkerContainersCount.get() + " / " + neededWorkerContainersCount + " ps container " + acquiredPsContainersCount.get() + " / " + neededPsContainersCount); }
/** * @param cont Container. * @return {@code True} if container satisfies requirements. */ private boolean checkContainer(Container cont) { // Check limit on running nodes. if (props.instances() <= containers.size()) return false; // Check host name if (props.hostnameConstraint() != null && props.hostnameConstraint().matcher(cont.getNodeId().getHost()).matches()) return false; // Check that slave satisfies min requirements. if (cont.getResource().getVirtualCores() < props.cpusPerNode() || cont.getResource().getMemory() < props.totalMemoryPerNode()) { log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}", new Object[]{cont.getNodeId().getHost(), cont.getResource().getVirtualCores(), cont.getResource().getMemory()}); return false; } return true; }
private static Container mockContainer(String host, int port, int containerId, Resource resource) { Container mockContainer = mock(Container.class); NodeId mockNodeId = NodeId.newInstance(host, port); ContainerId mockContainerId = ContainerId.newInstance( ApplicationAttemptId.newInstance( ApplicationId.newInstance(System.currentTimeMillis(), 1), 1 ), containerId ); when(mockContainer.getId()).thenReturn(mockContainerId); when(mockContainer.getNodeId()).thenReturn(mockNodeId); when(mockContainer.getResource()).thenReturn(resource); when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED); return mockContainer; }
String slotPortsStr = JOYConstants.EMPTY; try { slotPortsStr = slotPortsView.getSupervisorSlotPorts(container.getResource().getMemory(), container.getResource().getVirtualCores(), container.getNodeId().getHost());
if (nimbusMap.containsKey(containerId)) { Container nimbusContainer = nimbusMap.get(containerId); containerAsk = setupContainerAskForRM(nimbusContainer.getResource().getMemory(), nimbusContainer.getResource().getVirtualCores(), nimbusContainer.getPriority().getPriority(), nimbusContainer.getNodeId().getHost()); LOG.info("restart nimbus container" + ", containerId=" + containerStatus.getContainerId()); } else if (supervisorMap.containsKey(containerId)) { Container supervisorContainer = supervisorMap.get(containerId); containerAsk = setupContainerAskForRM(supervisorContainer.getResource().getMemory(), supervisorContainer.getResource().getVirtualCores(), supervisorContainer.getPriority().getPriority(), supervisorContainer.getNodeId().getHost()); LOG.info("restart supervisor container" + ", containerId=" + containerStatus.getContainerId());
c.getId(), c.getNodeId(), c.getResource().getVirtualCores(), c.getResource().getMemory()));
container.getResource(), containerIdStr, container.getNodeId().getHost());
private static void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); event.addEventInfo(JOYConstants.NODE, container.getNodeId().toString()); event.addEventInfo(JOYConstants.RESOURCES, container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } }
+ ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory" + allocatedContainer.getResource().getMemory() + ", containerResourceVirtualCores" + allocatedContainer.getResource().getVirtualCores());
if (task.container != null) { containerId = task.container.getId().toString(); Resource resource = task.container.getResource(); memoryMb = resource.getMemory(); vcores = resource.getVirtualCores();
@Override public void onContainersAllocated(List<Container> allocatedContainers) { LOG.info("Got response from RM for container ask, allocatedCnt=" + allocatedContainers.size()); for (Container allocatedContainer : allocatedContainers) { containers.put(allocatedContainer.getId(), allocatedContainer); state.registerContainer(allocatedContainer.getResource(), allocatedContainer); LOG.info("Launching shell command on a new container." + ", containerId=" + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory=" + allocatedContainer.getResource().getMemory() + ", containerResourceVirtualCores=" + allocatedContainer.getResource().getVirtualCores()); } }
public void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId("" + container.getId()); entity.setEntityType(ApplicationMaster.DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(ContainerEvents.CONTAINER_START.toString()); event.addEventInfo("Node", container.getNodeId().toString()); event.addEventInfo("Resources", container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } } }
public CSAssignment(FiCaSchedulerApp application, RMContainer excessReservation) { this(excessReservation.getContainer().getResource(), NodeType.NODE_LOCAL, excessReservation, application, SkippedType.NONE, false); }
/** * Update the resources of the node when releasing a container. * @param container Container to release. */ protected synchronized void updateResourceForReleasedContainer( Container container) { if (container.getExecutionType() == ExecutionType.GUARANTEED) { addUnallocatedResource(container.getResource()); --numContainers; } }
@Override protected synchronized void updateResourceForReleasedContainer( Container container) { super.updateResourceForReleasedContainer(container); if (killableContainers.containsKey(container.getId())) { Resources.subtractFrom(totalKillableResources, container.getResource()); killableContainers.remove(container.getId()); } }
@Override public void onContainersAllocated(List<Container> containers) { LOG.info("Allocated: " + containers.size() + " containers."); for (Container container : containers) { LOG.info("Launching a task in container" + ", containerId = " + container.getId() + ", containerNode = " + container.getNodeId().getHost() + ":" + container.getNodeId().getPort() + ", resourceRequest = " + container.getResource()); new ContainerLauncher(container, containerListener).run(); } }
public static Container getMockContainer( ContainerId containerId, NodeId nodeId, Resource resource, Priority priority) { Container container = mock(Container.class); when(container.getId()).thenReturn(containerId); when(container.getNodeId()).thenReturn(nodeId); when(container.getResource()).thenReturn(resource); when(container.getPriority()).thenReturn(priority); return container; }