public static synchronized ResourceLimiterStats getInstance(BigtableInstanceName instanceName) { String key = instanceName.getInstanceName(); ResourceLimiterStats instance = stats.get(key); if (instance == null) { instance = new ResourceLimiterStats(); stats.put(key, instance); } return instance; }
/** * Mark an operation id, as returned by {@code registerOperationWithHeapSize}, as complete * * @param id a long. */ public void markCanBeCompleted(long id) { Long opId = id; completedOperationIds.offerLast(opId); Long start = starTimes.remove(opId); if (start != null) { stats.markRpcComplete(clock.nanoTime() - start); } }
@Override public void run() { long meanLatencyMs = getMeanMs(stats.getMutationTimer()); if (meanLatencyMs >= bulkMutationRpcTargetMs * 3) { // decrease at 30% of the maximum RPCs, with a minimum of 2.5% reduceParallelism(meanLatencyMs, absoluteMaxInFlightRpcs * 3 / 10); } else if (meanLatencyMs >= highTargetMs) { // decrease at 10% of the maximum RPCs, with a minimum of 2.5% reduceParallelism(meanLatencyMs, absoluteMaxInFlightRpcs / 10); } else if (getMeanMs(stats.getThrottlingTimer()) > 1) { if (meanLatencyMs <= lowTargetMs) { // if latency is low, and there was throttling of at least one millisecond, then // increase the parallelism so that new calls will not be throttled. // Increase parallelism at a slower than we decrease. The lower rate should help the // system maintain stability. increaseParallelism(meanLatencyMs, absoluteMaxInFlightRpcs / 20); } else if (currentInFlightMaxRpcs < absoluteMaxInFlightRpcs / 20 && meanLatencyMs <= (bulkMutationRpcTargetMs * 2)) { // For some reason, when parallelism is reduced latency tends to be artificially higher. // Increase slowly to ensure that the system restabilizes. increaseParallelism(meanLatencyMs, absoluteMaxInFlightRpcs / 50); } } }
/** * Register an operation with the given size before sending. * This call WILL BLOCK until resources are available. This method must * be paired with a call to {@code markCanBeCompleted} in order to make sure * resources are properly released. * * @param heapSize The serialized size of the RPC to be sent * @return A unique operation id * @throws java.lang.InterruptedException if any. */ public long registerOperationWithHeapSize(long heapSize) throws InterruptedException { long start = clock.nanoTime(); synchronized (this) { while (unsynchronizedIsFull()) { waitForCompletions(REGISTER_WAIT_MILLIS); } long waitComplete = clock.nanoTime(); stats.markThrottling(waitComplete - start); long operationId = operationSequenceGenerator.incrementAndGet(); pendingOperationsWithSize.put(operationId, heapSize); currentWriteBufferSize += heapSize; starTimes.put(operationId, waitComplete); return operationId; } }
private synchronized static ResourceLimiter initializeResourceLimiter(BigtableOptions options) { BigtableInstanceName instanceName = options.getInstanceName(); String key = instanceName.toString(); ResourceLimiter resourceLimiter = resourceLimiterMap.get(key); if (resourceLimiter == null) { int maxInflightRpcs = options.getBulkOptions().getMaxInflightRpcs(); long maxMemory = options.getBulkOptions().getMaxMemory(); ResourceLimiterStats stats = ResourceLimiterStats.getInstance(instanceName); resourceLimiter = new ResourceLimiter(stats, maxMemory, maxInflightRpcs); BulkOptions bulkOptions = options.getBulkOptions(); if (bulkOptions.isEnableBulkMutationThrottling()) { resourceLimiter.throttle(bulkOptions.getBulkMutationRpcTargetMs()); } resourceLimiterMap.put(key, resourceLimiter); } return resourceLimiter; }