public double getContainerCpuCores() { return taskResourceProfile.getCpuCores() * slotNum; }
public void addTo(ResourceProfile another) { this.cpuCores += another.getCpuCores(); this.heapMemoryInMB += another.getHeapMemoryInMB(); this.directMemoryInMB += another.getDirectMemoryInMB(); this.nativeMemoryInMB += another.getNativeMemoryInMB(); this.networkMemoryInMB += another.getNetworkMemoryInMB(); if (!extendedResources.isEmpty() || !another.extendedResources.isEmpty()) { for (Map.Entry<String, Resource> extendResource : another.extendedResources.entrySet()) { Resource rfValue = extendedResources.get(extendResource.getKey()); if (rfValue != null) { extendedResources.put(extendResource.getKey(), extendResource.getValue().merge(rfValue)); } else { extendedResources.put(extendResource.getKey(), extendResource.getValue()); } } } }
public ResourceProfile multiply(int multiplier) { Map<String, Resource> newExtendedResource = new HashMap<>(extendedResources.size()); for (Map.Entry<String, Resource> entry : extendedResources.entrySet()) { newExtendedResource.put(entry.getKey(), entry.getValue().multiply(multiplier)); } return new ResourceProfile( this.getCpuCores() * multiplier, this.getHeapMemoryInMB() * multiplier, this.getDirectMemoryInMB() * multiplier, this.getNativeMemoryInMB() * multiplier, this.getNetworkMemoryInMB() * multiplier, newExtendedResource); }
@Override public Collection<ResourceProfile> startNewWorker(ResourceProfile resourceProfile) { // Priority for worker containers - priorities are intra-application //TODO: set priority according to the resource allocated Priority priority = Priority.newInstance(generatePriority(resourceProfile)); int mem = resourceProfile.getMemoryInMB() < 0 ? defaultTaskManagerMemoryMB : resourceProfile.getMemoryInMB(); int vcore = resourceProfile.getCpuCores() < 1 ? defaultCpus : (int) resourceProfile.getCpuCores(); Resource capability = Resource.newInstance(mem, vcore); requestYarnContainer(capability, priority); return slotsPerWorker; }
/** * Check whether required resource profile can be matched * * @param required the required resource profile * @return true if the requirement is matched, otherwise false */ public boolean isMatching(ResourceProfile required) { return cpuCores >= required.getCpuCores() && heapMemoryInMB >= required.getHeapMemoryInMB() && directMemoryInMB >= required.getDirectMemoryInMB() && nativeMemoryInMB >= required.getNativeMemoryInMB(); }
/** * Check whether required resource profile can be matched. * * @param required the required resource profile * @return true if the requirement is matched, otherwise false */ public boolean isMatching(ResourceProfile required) { if (cpuCores >= required.getCpuCores() && heapMemoryInMB >= required.getHeapMemoryInMB() && directMemoryInMB >= required.getDirectMemoryInMB() && nativeMemoryInMB >= required.getNativeMemoryInMB() && networkMemoryInMB >= required.getNetworkMemoryInMB()) { for (Map.Entry<String, Resource> resource : required.extendedResources.entrySet()) { // Skip floating memory, floating memory will not be considered when findMatchingSlot in slot manager if (resource.getKey().equals(ResourceSpec.FLOATING_MANAGED_MEMORY_NAME)) { continue; } if (!extendedResources.containsKey(resource.getKey()) || !extendedResources.get(resource.getKey()).getResourceAggregateType().equals(resource.getValue().getResourceAggregateType()) || extendedResources.get(resource.getKey()).getValue() < resource.getValue().getValue()) { return false; } } return true; } return false; }
/** * Check whether required resource profile can be matched. * * @param required the required resource profile * @return true if the requirement is matched, otherwise false */ public boolean isMatching(ResourceProfile required) { if (cpuCores >= required.getCpuCores() && heapMemoryInMB >= required.getHeapMemoryInMB() && directMemoryInMB >= required.getDirectMemoryInMB() && nativeMemoryInMB >= required.getNativeMemoryInMB() && networkMemoryInMB >= required.getNetworkMemoryInMB()) { for (Map.Entry<String, Resource> resource : required.extendedResources.entrySet()) { if (!extendedResources.containsKey(resource.getKey()) || !extendedResources.get(resource.getKey()).getResourceAggregateType().equals(resource.getValue().getResourceAggregateType()) || extendedResources.get(resource.getKey()).getValue() < resource.getValue().getValue()) { return false; } } return true; } return false; }
/** * Check whether required resource profile can be matched. * * @param required the required resource profile * @return true if the requirement is matched, otherwise false */ public boolean isMatching(ResourceProfile required) { if (cpuCores >= required.getCpuCores() && heapMemoryInMB >= required.getHeapMemoryInMB() && directMemoryInMB >= required.getDirectMemoryInMB() && nativeMemoryInMB >= required.getNativeMemoryInMB() && networkMemoryInMB >= required.getNetworkMemoryInMB()) { for (Map.Entry<String, Resource> resource : required.extendedResources.entrySet()) { if (!extendedResources.containsKey(resource.getKey()) || !extendedResources.get(resource.getKey()).getResourceAggregateType().equals(resource.getValue().getResourceAggregateType()) || extendedResources.get(resource.getKey()).getValue() < resource.getValue().getValue()) { return false; } } return true; } return false; }
public static TaskManagerResourceDescription fromResourceProfile(ResourceProfile resourceProfile) { return new TaskManagerResourceDescription(resourceProfile.getCpuCores(), ((long) resourceProfile.getDirectMemoryInMB()) << 20, ((long) resourceProfile.getHeapMemoryInMB()) << 20, ((long) resourceProfile.getNativeMemoryInMB()) << 20, ((long) resourceProfile.getManagedMemoryInMB()) << 20, ((long) resourceProfile.getNetworkMemoryInMB()) << 20); }
public ResourceProfile getTaskResourceProfile() { int userHeap = taskResourceProfile.getHeapMemoryInMB(); if (taskManagerTotalMemoryMB > 0) { if (offHeap) { userHeap = getTotalHeapMemory() - taskManagerHeapMemorySizeMB; } else { userHeap = getTotalHeapMemory() - taskManagerHeapMemorySizeMB - getManagedMemorySize() - getFloatingManagedMemorySize(); } } return new ResourceProfile(taskResourceProfile.getCpuCores(), userHeap, taskResourceProfile.getDirectMemoryInMB(), taskResourceProfile.getNativeMemoryInMB(), taskResourceProfile.getNetworkMemoryInMB(), taskResourceProfile.getExtendedResources()); }
@Override public void serialize( ResourceProfile resourceProfile, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { jsonGenerator.writeStartObject(); jsonGenerator.writeNumberField(FIELD_NAME_RESOURCE_CPU_CORES, resourceProfile.getCpuCores()); jsonGenerator.writeNumberField(FIELD_NAME_RESOURCE_HEAP_MEMORY, convertMegabyteToByte(resourceProfile.getHeapMemoryInMB())); jsonGenerator.writeNumberField(FIELD_NAME_RESOURCE_DIRECT_MEMORY, convertMegabyteToByte(resourceProfile.getDirectMemoryInMB())); jsonGenerator.writeNumberField(FIELD_NAME_RESOURCE_NATIVE_MEMORY, convertMegabyteToByte(resourceProfile.getNativeMemoryInMB())); jsonGenerator.writeNumberField(FIELD_NAME_RESOURCE_NETWORK_MEMORY, convertMegabyteToByte(resourceProfile.getNetworkMemoryInMB())); jsonGenerator.writeNumberField(FIELD_NAME_RESOURCE_MANAGED_MEMORY, convertMegabyteToByte(resourceProfile.getManagedMemoryInMB())); jsonGenerator.writeEndObject(); }