@Override public Collection<ResourceProfile> startNewWorker(ResourceProfile resourceProfile) { Preconditions.checkArgument( ResourceProfile.UNKNOWN.equals(resourceProfile), "The YarnResourceManager does not support custom ResourceProfiles yet. It assumes that all containers have the same resources."); requestYarnContainer(); return slotsPerWorker; }
@Override public boolean equals(Object obj) { if (obj == this) { return true; } else if (obj != null && obj.getClass() == TaskManagerResource.class) { TaskManagerResource that = (TaskManagerResource) obj; return this.taskResourceProfile.equals(that.taskResourceProfile) && this.taskManagerNettyMemorySizeMB == that.taskManagerNettyMemorySizeMB && this.taskManagerNativeMemorySizeMB == that.taskManagerNativeMemorySizeMB && this.taskManagerHeapMemorySizeMB == that.taskManagerHeapMemorySizeMB && this.offHeap == that.offHeap && this.managedMemoryFraction == that.managedMemoryFraction && this.slotNum == that.slotNum && this.dynamicYoungHeapRatio == that.dynamicYoungHeapRatio && this.persistentYoungHeapRatio == that.persistentYoungHeapRatio; } else { return false; } }
@Nullable private PendingTaskManagerSlot findExactlyMatchingPendingTaskManagerSlot(ResourceProfile resourceProfile) { for (PendingTaskManagerSlot pendingTaskManagerSlot : pendingSlots.values()) { if (pendingTaskManagerSlot.getResourceProfile().equals(resourceProfile)) { return pendingTaskManagerSlot; } } return null; }
@Nullable private PendingTaskManagerSlot findExactlyMatchingPendingTaskManagerSlot(ResourceProfile resourceProfile) { for (PendingTaskManagerSlot pendingTaskManagerSlot : pendingSlots.values()) { if (pendingTaskManagerSlot.getResourceProfile().equals(resourceProfile)) { return pendingTaskManagerSlot; } } return null; }
@Override public Collection<ResourceProfile> startNewWorker(ResourceProfile resourceProfile) { Preconditions.checkArgument( ResourceProfile.UNKNOWN.equals(resourceProfile), "The YarnResourceManager does not support custom ResourceProfiles yet. It assumes that all containers have the same resources."); requestYarnContainer(); return slotsPerWorker; }
public ResourceProfile getTotalResource() { ResourceProfile totalResources = new ResourceProfile(ResourceProfile.EMTPY); for (Map.Entry<SlotID, TaskManagerSlot> entry : slots.entrySet()) { if (!entry.getValue().getResourceProfile().equals(ResourceProfile.UNKNOWN)) { totalResources.addTo(entry.getValue().getResourceProfile()); } } return totalResources; }
public ResourceProfile getAvailableResource() { ResourceProfile availableResources = new ResourceProfile(ResourceProfile.EMTPY); for (Map.Entry<SlotID, TaskManagerSlot> entry : slots.entrySet()) { if (entry.getValue().getState() == TaskManagerSlot.State.FREE && !entry.getValue().getResourceProfile().equals(ResourceProfile.UNKNOWN)) { availableResources.addTo(entry.getValue().getResourceProfile()); } } return availableResources; }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SlotStatus that = (SlotStatus) o; if (!slotID.equals(that.slotID)) { return false; } if (!resourceProfile.equals(that.resourceProfile)) { return false; } if (allocationID != null ? !allocationID.equals(that.allocationID) : that.allocationID != null) { return false; } if (allocationResourceProfile != null ? !allocationResourceProfile.equals(that.allocationResourceProfile) : that.allocationResourceProfile != null) { return false; } if (version != that.version) { return false; } return jobID != null ? jobID.equals(that.jobID) : that.jobID == null; }
public ResourceProfile getAvailableResourceOf(ResourceID resourceID) { ResourceProfile availableResources = new ResourceProfile(ResourceProfile.EMTPY); for (Map.Entry<SlotID, TaskManagerSlot> entry : slots.entrySet()) { if (entry.getValue().getState() == TaskManagerSlot.State.FREE && entry.getKey().getResourceID().equals(resourceID) && !entry.getValue().getResourceProfile().equals(ResourceProfile.UNKNOWN)) { availableResources.addTo(entry.getValue().getResourceProfile()); } } return availableResources; }
public ResourceProfile getTotalResourceOf(ResourceID resourceID) { ResourceProfile totalResources = new ResourceProfile(ResourceProfile.EMTPY); for (Map.Entry<SlotID, TaskManagerSlot> entry : slots.entrySet()) { if (entry.getKey().getResourceID().equals(resourceID) && !entry.getValue().getResourceProfile().equals(ResourceProfile.UNKNOWN)) { totalResources.addTo(entry.getValue().getResourceProfile()); } } return totalResources; }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SlotStatus that = (SlotStatus) o; if (!slotID.equals(that.slotID)) { return false; } if (!resourceProfile.equals(that.resourceProfile)) { return false; } if (allocationID != null ? !allocationID.equals(that.allocationID) : that.allocationID != null) { return false; } return jobID != null ? jobID.equals(that.jobID) : that.jobID == null; }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SlotStatus that = (SlotStatus) o; if (!slotID.equals(that.slotID)) { return false; } if (!resourceProfile.equals(that.resourceProfile)) { return false; } if (allocationID != null ? !allocationID.equals(that.allocationID) : that.allocationID != null) { return false; } return jobID != null ? jobID.equals(that.jobID) : that.jobID == null; }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } SlotStatus that = (SlotStatus) o; if (!slotID.equals(that.slotID)) { return false; } if (!resourceProfile.equals(that.resourceProfile)) { return false; } if (allocationID != null ? !allocationID.equals(that.allocationID) : that.allocationID != null) { return false; } return jobID != null ? jobID.equals(that.jobID) : that.jobID == null; }
private void recordAllocatedSlotAndResource(SlotID slotID, ResourceProfile resourceProfile) { // Use EMPTY instead of UNKNOWN to make resource calculation correctly. if (resourceProfile.equals(ResourceProfile.UNKNOWN)) { resourceProfile = new ResourceProfile(ResourceProfile.EMTPY); } Tuple2<Map<SlotID, ResourceProfile>, ResourceProfile> slotToResource = allocatedSlotsResource.get(slotID.getResourceID()); if (slotToResource != null) { slotToResource.f0.put(slotID, resourceProfile); slotToResource.f1 = slotToResource.f1.minus(resourceProfile); } else { Map<SlotID, ResourceProfile> allocated = new HashMap<>(); allocated.put(slotID, resourceProfile); slotToResource = new Tuple2<>(allocated, totalResourceOfTaskExecutor.minus(resourceProfile)); allocatedSlotsResource.put(slotID.getResourceID(), slotToResource); } }
private boolean hasEnoughResource(ResourceProfile allocationResourceProfile) { if (totalResource.equals(ResourceProfile.UNKNOWN)) { return true; } ResourceProfile remainResource = totalResource; for (TaskSlot taskSlot : taskSlots) { if (taskSlot.getAllocationId() != null) { remainResource = remainResource.minus(taskSlot.getAllocationResourceProfile()); } } return remainResource.isMatching(allocationResourceProfile); }
@Override protected TaskManagerSlot findMatchingSlot(SlotRequest slotRequest) { Iterator<Map.Entry<SlotID, TaskManagerSlot>> iterator = freeSlots.entrySet().iterator(); while (iterator.hasNext()) { TaskManagerSlot taskManagerSlot = iterator.next().getValue(); // sanity check Preconditions.checkState(taskManagerSlot.getState() == TaskManagerSlot.State.FREE, String.format("Slot %s is in state %s", taskManagerSlot.getSlotId(), taskManagerSlot.getState())); if (taskManagerSlot.getResourceProfile().equals(slotRequest.getResourceProfile()) && placementConstraintManager.check( slotRequest.getJobId(), allocationIdTags.get(slotRequest.getAllocationId()), getTaskExecutorSlotTags(taskManagerSlot.getSlotId()))) { freeSlots.remove(taskManagerSlot.getSlotId()); return taskManagerSlot; } } return null; }
/** * Generate the slot offer from this TaskSlot. * * @return The sot offer which this task slot can provide */ public SlotOffer generateSlotOffer() { Preconditions.checkState(TaskSlotState.ACTIVE == state || TaskSlotState.ALLOCATED == state, "The task slot is not in state active or allocated."); Preconditions.checkState(allocationId != null, "The task slot are not allocated"); if (resourceProfile.equals(ResourceProfile.UNKNOWN)) { return new SlotOffer(allocationId, index, allocationResourceProfile, tags); } return new SlotOffer(allocationId, index, resourceProfile, tags); }
@Override protected PendingSlotRequest findMatchingRequest(TaskManagerSlot taskManagerSlot) { for (PendingSlotRequest pendingSlotRequest : pendingSlotRequests.values()) { if (taskManagerSlot.getResourceProfile().equals(pendingSlotRequest.getResourceProfile()) && !pendingSlotRequest.isAssigned() && placementConstraintManager.check( pendingSlotRequest.getJobId(), allocationIdTags.get(pendingSlotRequest.getAllocationId()), getTaskExecutorSlotTags(taskManagerSlot.getSlotId()))) { return pendingSlotRequest; } } return null; }
newAllocationResourceProfile.equals(allocationResourceProfile) && new HashSet<>(newTags).equals(new HashSet<>(tags)); } else {