public static DefaultSlotPoolFactory fromConfiguration( @Nonnull Configuration configuration, @Nonnull RpcService rpcService) { final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration); final Time slotIdleTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT)); final SchedulingStrategy schedulingStrategy = selectSchedulingStrategy(configuration); return new DefaultSlotPoolFactory( rpcService, schedulingStrategy, SystemClock.getInstance(), rpcTimeout, slotIdleTimeout); }
/** * Check the available slots, release the slot that is idle for a long time. */ private void checkIdleSlot() { // The timestamp in SlotAndTimestamp is relative final long currentRelativeTimeMillis = clock.relativeTimeMillis(); final List<AllocatedSlot> expiredSlots = new ArrayList<>(availableSlots.size()); for (SlotAndTimestamp slotAndTimestamp : availableSlots.availableSlots.values()) { if (currentRelativeTimeMillis - slotAndTimestamp.timestamp > idleSlotTimeout.toMilliseconds()) { expiredSlots.add(slotAndTimestamp.slot); } } final FlinkException cause = new FlinkException("Releasing idle slot."); for (AllocatedSlot expiredSlot : expiredSlots) { final AllocationID allocationID = expiredSlot.getAllocationId(); if (availableSlots.tryRemove(allocationID)) { log.info("Releasing idle slot {}.", allocationID); releaseSlotToTaskManager(expiredSlot, cause); } } scheduleRunAsync(this::checkIdleSlot, idleSlotTimeout); }
pendingRequest.setTimestamp(clock.absoluteTimeMillis());
/** * Tries to fulfill with the given allocated slot a pending slot request or add the * allocated slot to the set of available slots if no matching request is available. * * @param allocatedSlot which shall be returned */ private void tryFulfillSlotRequestOrMakeAvailable(AllocatedSlot allocatedSlot) { Preconditions.checkState(!allocatedSlot.isUsed(), "Provided slot is still in use."); final PendingRequest pendingRequest = pollMatchingPendingRequest(allocatedSlot); if (pendingRequest != null) { log.debug("Fulfilling pending slot request [{}] early with returned slot [{}]", pendingRequest.getSlotRequestId(), allocatedSlot.getAllocationId()); allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot); pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot); } else { log.debug("Adding returned slot [{}] to available slots", allocatedSlot.getAllocationId()); availableSlots.add(allocatedSlot, clock.relativeTimeMillis()); } }
public static DefaultSlotPoolFactory fromConfiguration( @Nonnull Configuration configuration, @Nonnull RpcService rpcService) { final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration); final Time slotIdleTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT)); final SchedulingStrategy schedulingStrategy = selectSchedulingStrategy(configuration); return new DefaultSlotPoolFactory( rpcService, schedulingStrategy, SystemClock.getInstance(), rpcTimeout, slotIdleTimeout); }
/** * Tries to fulfill with the given allocated slot a pending slot request or add the * allocated slot to the set of available slots if no matching request is available. * * @param allocatedSlot which shall be returned */ private void tryFulfillSlotRequestOrMakeAvailable(AllocatedSlot allocatedSlot) { Preconditions.checkState(!allocatedSlot.isUsed(), "Provided slot is still in use."); final PendingRequest pendingRequest = pollMatchingPendingRequest(allocatedSlot); if (pendingRequest != null) { log.debug("Fulfilling pending request [{}] early with returned slot [{}]", pendingRequest.getSlotRequestId(), allocatedSlot.getAllocationId()); allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot); pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot); } else { log.debug("Adding returned slot [{}] to available slots", allocatedSlot.getAllocationId()); availableSlots.add(allocatedSlot, clock.relativeTimeMillis()); } }
@VisibleForTesting protected SlotPool(RpcService rpcService, JobID jobId, SchedulingStrategy schedulingStrategy) { this( rpcService, jobId, schedulingStrategy, SystemClock.getInstance(), AkkaUtils.getDefaultTimeout(), Time.milliseconds(JobManagerOptions.SLOT_IDLE_TIMEOUT.defaultValue())); }
/** * Tries to fulfill with the given allocated slot a pending slot request or add the * allocated slot to the set of available slots if no matching request is available. * * @param allocatedSlot which shall be returned */ private void tryFulfillSlotRequestOrMakeAvailable(AllocatedSlot allocatedSlot) { Preconditions.checkState(!allocatedSlot.isUsed(), "Provided slot is still in use."); final PendingRequest pendingRequest = pollMatchingPendingRequest(allocatedSlot); if (pendingRequest != null) { log.debug("Fulfilling pending slot request [{}] early with returned slot [{}]", pendingRequest.getSlotRequestId(), allocatedSlot.getAllocationId()); allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot); pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot); } else { log.debug("Adding returned slot [{}] to available slots", allocatedSlot.getAllocationId()); availableSlots.add(allocatedSlot, clock.relativeTimeMillis()); } }
@VisibleForTesting protected SlotPool(RpcService rpcService, JobID jobId, SchedulingStrategy schedulingStrategy) { this( rpcService, jobId, schedulingStrategy, SystemClock.getInstance(), AkkaUtils.getDefaultTimeout(), Time.milliseconds(JobManagerOptions.SLOT_IDLE_TIMEOUT.defaultValue()), true); }
final long currentRelativeTimeMillis = clock.relativeTimeMillis();
@VisibleForTesting protected SlotPool(RpcService rpcService, JobID jobId, SchedulingStrategy schedulingStrategy) { this( rpcService, jobId, schedulingStrategy, SystemClock.getInstance(), AkkaUtils.getDefaultTimeout(), Time.milliseconds(JobManagerOptions.SLOT_IDLE_TIMEOUT.defaultValue())); }
final long currentRelativeTimeMillis = clock.relativeTimeMillis();
public static DefaultSlotPoolFactory fromConfiguration( @Nonnull Configuration configuration, @Nonnull RpcService rpcService) { final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration); final Time slotIdleTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_IDLE_TIMEOUT)); final SchedulingStrategy schedulingStrategy = selectSchedulingStrategy(configuration); final Boolean enableSharedSlot = configuration.getBoolean(JobManagerOptions.SLOT_ENABLE_SHARED_SLOT); return new DefaultSlotPoolFactory( rpcService, schedulingStrategy, SystemClock.getInstance(), rpcTimeout, slotIdleTimeout, enableSharedSlot); }
availableSlots.add(slot, clock.relativeTimeMillis());
public SlotPool( RpcService rpcService, JobID jobId, Clock clock, Time slotRequestTimeout, Time resourceManagerAllocationTimeout, Time resourceManagerRequestTimeout) { super(rpcService); this.jobId = checkNotNull(jobId); this.clock = checkNotNull(clock); this.resourceManagerRequestsTimeout = checkNotNull(resourceManagerRequestTimeout); this.resourceManagerAllocationTimeout = checkNotNull(resourceManagerAllocationTimeout); this.registeredTaskManagers = new HashSet<>(); this.allocatedSlots = new AllocatedSlots(); this.availableSlots = new AvailableSlots(); this.pendingRequests = new HashMap<>(); this.waitingForResourceManager = new HashMap<>(); this.providerAndOwner = new ProviderAndOwner(getSelf(), slotRequestTimeout); }
availableSlots.add(taskManagerSlot, clock.relativeTimeMillis());