Refine search
int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); for (String c : jstormClientContext.conf.getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { if (jstormClientContext.conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(JOYConstants.COLON); classPathEnv.append(System.getProperty(JOYConstants.JAVA_CLASS_PATH)); Resource capability = Resource.newInstance(jstormClientContext.amMemory, jstormClientContext.amVCores); appContext.setResource(capability); String tokenRenewer = jstormClientContext.conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException( Priority pri = Priority.newInstance(jstormClientContext.amPriority); appContext.setPriority(pri);
private void buildContainerRequest(String[] hostLocals) { if (conf.getBoolean(XLearningConfiguration.XLEARNING_HOST_LOCAL_ENABLE, XLearningConfiguration.DEFAULT_XLEARNING_HOST_LOCAL_ENABLE)) { XLearningConfiguration xlConf = new XLearningConfiguration(); String hostLocaldir = xlConf.get("fs.defaultFS") + conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR) + "/" + conf.get("hadoop.job.ugi").split(",")[0] + "/" + envs.get(XLearningConstants.Environment.XLEARNING_APP_NAME.toString()); Path hostLocalPath = new Path(hostLocaldir); Priority priority = Records.newRecord(Priority.class); priority.setPriority(appPriority); Resource workerCapability = Records.newRecord(Resource.class); workerCapability.setMemory(workerMemory); workerCapability.setVirtualCores(workerVCores); workerContainerRequest = new ContainerRequest(workerCapability, hostLocals, null, priority); LOG.info("Create worker container request: " + workerContainerRequest.toString()); Resource psCapability = Records.newRecord(Resource.class); psCapability.setMemory(psMemory); psCapability.setVirtualCores(psVCores); psContainerRequest = new ContainerRequest(psCapability, hostLocals, null, priority); LOG.info("Create ps container request: " + psContainerRequest.toString());
memoryPerInstance = si.getResource().getMemorySize() * 1024L * 1024L;
@Override public void limitContainerSize(Resource maxResource) throws AMException { if (taskSpec.containerSpec.memoryMb > maxResource.getMemory()) { LOG.warn(taskSpec.name + " requires " + taskSpec.containerSpec.memoryMb + " MB but the maximum YARN container size is " + maxResource.getMemory() + " MB"); taskSpec.containerSpec.memoryMb = maxResource.getMemory(); } if (taskSpec.containerSpec.vCores > maxResource.getVirtualCores()) { LOG.warn(taskSpec.name + " requires " + taskSpec.containerSpec.vCores + " vcores but the maximum YARN container size is " + maxResource.getVirtualCores() + " vcores"); taskSpec.containerSpec.vCores = maxResource.getVirtualCores(); } }
public Resource getCapability() { // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(memoryMb); capability.setVirtualCores(vCores); DoYUtil.callSetDiskIfExists(capability, disks); return capability; }
private void requestContainer(Optional<String> preferredNode) { Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory(); capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ? this.requestedContainerMemoryMbs : maxMemoryCapacity); int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores(); capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ? this.requestedContainerCores : maxCoreCapacity); String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null; this.amrmClientAsync.addContainerRequest( new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority)); }
try { boolean sendSerializedEvents = conf.getBoolean("mapreduce.tez.input.initializer.serialize.event.payload", true); boolean generateConsistentSplits = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS); LOG.info("GenerateConsistentSplitsInHive=" + generateConsistentSplits); String realInputFormatName = conf.get("mapred.input.format.class"); boolean groupingEnabled = userPayloadProto.getGroupingEnabled(); if (groupingEnabled) { totalResource = getContext().getTotalAvailableResource().getMemory(); taskResource = getContext().getVertexTaskResource().getMemory(); availableSlots = totalResource / taskResource; final long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); final long minGrouping = conf.getLong(
+ container.getId()); jstormMasterContext.nimbusDataDirPrefix = conf.get(JOYConstants.INSTANCE_DATA_DIR_KEY); String localDir = jstormMasterContext.nimbusDataDirPrefix + container.getId().toString() + JOYConstants.BACKLASH + jstormMasterContext.instanceName; vargs.add(localDir); SlotPortsView slotPortsView = new SlotPortsView(jstormMasterContext.instanceName, container.getId(), registryOperations); slotPortsView.setMinPort(conf.getInt(JOYConstants.SUPERVISOR_MIN_PORT_KEY, JOYConstants.PORT_RANGE_MIN)); slotPortsView.setMaxPort(conf.getInt(JOYConstants.SUPERVISOR_MAX_PORT_KEY, JOYConstants.PORT_RANGE_MAX)); String slotPortsStr = JOYConstants.EMPTY; try { slotPortsStr = slotPortsView.getSupervisorSlotPorts(container.getResource().getMemory(), container.getResource().getVirtualCores(), container.getNodeId().getHost());
jstormMasterContext.maxMemory = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capability of resources in this cluster " + jstormMasterContext.maxMemory); jstormMasterContext.maxVcores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capability of resources in this cluster " + jstormMasterContext.maxVcores); container.getId().getApplicationAttemptId().getApplicationId().toString(), container.getId().toString()); ServiceRecord sr = null; try { if (!registryOperations.exists(containerPath)) { String contianerHost = container.getNodeId().getHost(); registryOperations.mknode(containerPath, true); sr = new ServiceRecord(); if (container.getPriority().getPriority() == 0) jstormMasterContext.supervisorContainers.add(container); else if (container.getPriority().getPriority() == 1) { jstormMasterContext.nimbusContainers.add(container); jstormMasterContext.instanceName = conf.get(JOYConstants.INSTANCE_NAME_KEY); this.jstormMasterContext.user = conf.get(JOYConstants.JSTORM_YARN_USER); this.jstormMasterContext.password = conf.get(JOYConstants.JSTORM_YARN_PASSWORD); this.jstormMasterContext.oldPassword = conf.get(JOYConstants.JSTORM_YARN_OLD_PASSWORD);
public static Resource getContainerResource(Configuration conf) { int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) : conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB); int cpus = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCPUVCORES) : conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES); return Resource.newInstance(memory, cpus); }
@Override public String info() throws TException { StringBuffer sbRet = new StringBuffer(); sbRet.append("JstormOnYarn\n"); sbRet.append("Instance Name:" + jstormMasterContext.instanceName + "\n"); sbRet.append("Jstorm's location on hdfs:" + jstormMasterContext.deployPath + "\n"); if (jstormMasterContext.user != null) { sbRet.append("Jstorm's data path:" + jstormMasterContext.nimbusDataDirPrefix + jstormMasterContext.instanceName + "\n"); sbRet.append("Cluster userName:" + jstormMasterContext.user + "\n"); } sbRet.append("Nimbus Count:" + jstormMasterContext.nimbusContainers.size() + "\n"); sbRet.append("Supervisor Count:" + jstormMasterContext.supervisorContainers.size() + "\n"); sbRet.append("detail :\n"); sbRet.append("Type \tContainerId \tHost \tContainerMemory\tContainerVCores\n"); for (Container container : jstormMasterContext.nimbusContainers) { sbRet.append("Nimbus \t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } for (Container container : jstormMasterContext.supervisorContainers) { sbRet.append("Supervisor\t" + container.getId().toString() + "\t" + container.getNodeId().getHost() + "\t" + container.getResource().getMemory() + "\t " + container.getResource().getVirtualCores() + "\n"); } LOG.info("info is: " + sbRet.toString()); return sbRet.toString(); }
public RMApp submitApp(Credentials cred, ByteBuffer tokensConf) throws Exception { return submitApp(Resource.newInstance(200, 1), "app1", "user", null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true, false, false, null, 0, null, true, Priority.newInstance(0), null, null, tokensConf); }
public static Resource multiplyTo(Resource lhs, double by) { lhs.setMemory((int)(lhs.getMemory() * by)); lhs.setVirtualCores((int)(lhs.getVirtualCores() * by)); return lhs; }
@Override public Resource getTotalResources() { int memory = 0; int vcores = 0; readLock.lock(); try { int numInstancesFound = 0; for (LlapServiceInstance inst : activeInstances.getAll()) { Resource r = inst.getResource(); memory += r.getMemory(); vcores += r.getVirtualCores(); numInstancesFound++; } if (LOG.isDebugEnabled()) { LOG.debug("GetTotalResources: numInstancesFound={}, totalMem={}, totalVcores={}", numInstancesFound, memory, vcores); } } finally { readLock.unlock(); } return Resource.newInstance(memory, vcores); }
@SuppressWarnings("unchecked") // dispatcher not typed private void sendContainerMonitorStartEvent() { long pmemBytes = getResource().getMemory() * 1024 * 1024L; float pmemRatio = daemonConf.getFloat( YarnConfiguration.NM_VMEM_PMEM_RATIO, YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); long vmemBytes = (long) (pmemRatio * pmemBytes); int cpuVcores = getResource().getVirtualCores(); dispatcher.getEventHandler().handle( new ContainerStartMonitoringEvent(containerId, vmemBytes, pmemBytes, cpuVcores)); }
protected ResourceRequest createResourceRequest(long id, String resource, int memory, int vCores, int priority, ExecutionType execType, int containers) { ResourceRequest req = Records.newRecord(ResourceRequest.class); req.setAllocationRequestId(id); req.setResourceName(resource); req.setCapability(Resource.newInstance(memory, vCores)); req.setPriority(Priority.newInstance(priority)); req.setExecutionTypeRequest(ExecutionTypeRequest.newInstance(execType)); req.setNumContainers(containers); return req; }
final YarnResourceManagerCallbackHandler callbackHandler = new YarnResourceManagerCallbackHandler(); AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = mock(AMRMClientAsync.class); doReturn(Collections.singletonList(Collections.nCopies(numInitialTaskManagers, new AMRMClient.ContainerRequest(Resource.newInstance(1024 * 1024, 1), null, null, Priority.newInstance(0))))) .when(resourceManagerClient).getMatchingRequests(any(Priority.class), anyString(), any(Resource.class)); when(mockContainer.getId()).thenReturn( ContainerId.newInstance( ApplicationAttemptId.newInstance( 1), i)); when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance("container", 1234)); containerList.add(mockContainer);
ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class); amContainerLaunchContext.setLocalResources(appMasterLocalResources); amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory()))); if (UserGroupInformation.isSecurityEnabled()) { setupSecurityTokens(amContainerLaunchContext); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
public static int computeAvailableContainers(Resource available, Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) { if (resourceTypes.contains(SchedulerResourceTypes.CPU)) { return Math.min( calculateRatioOrMaxValue(available.getMemorySize(), required.getMemorySize()), calculateRatioOrMaxValue(available.getVirtualCores(), required .getVirtualCores())); } return calculateRatioOrMaxValue( available.getMemorySize(), required.getMemorySize()); }