@Override public long getMaintenanceIntervalMicros() { if (this.context.extras == null) { return this.getHost().getMaintenanceIntervalMicros(); } return this.context.extras.maintenanceInterval; }
/** * Exponential backoff rely on retry count stat. If this stat is not available * then we will fall back to constant delay for each retry. * To get exponential delay, multiply retry count's power of 2 with constant delay. * @param statNameRetryCount */ private long getExponentialDelay(String statNameRetryCount) { long delay = getHost().getMaintenanceIntervalMicros(); ServiceStats.ServiceStat stat = getStat(statNameRetryCount); if (stat != null && stat.latestValue > 0) { return (1 << ((long)stat.latestValue)) * delay; } return delay; }
/** * Exponential backoff rely on synch task retry count stat. If this stat is not available * then we will fall back to constant delay for each retry. * To get exponential delay, multiply retry count's power of 2 with constant delay. */ private long getExponentialDelay() { long delay = getHost().getMaintenanceIntervalMicros(); ServiceStats.ServiceStat stat = getStat(STAT_NAME_SYNCH_TASK_RETRY_COUNT); if (stat != null && stat.latestValue > 0) { return (1 << ((long)stat.latestValue)) * delay; } return delay; }
/** * Evaluates current node group state and returns true if requests should be process, * forwarded, etc * The conditions are: * * 1) The node group membership should have been stable (gossip did not produce * changes) for a specific period * * 2) The number of node group members in available stage is >= membershipQuorum */ public static boolean isNodeGroupAvailable(ServiceHost host, NodeGroupState localState) { // we invoke the isMembershipSettled first because it has low overhead, does not walk all // nodes in the group if (NodeGroupUtils.isMembershipSettled(host, host .getMaintenanceIntervalMicros(), localState) && NodeGroupUtils.hasMembershipQuorum(host, localState)) { return true; } return false; }
public void schedule(Service s, long now) { long interval = s.getMaintenanceIntervalMicros(); if (interval == 0) { interval = this.host.getMaintenanceIntervalMicros(); } if (interval < this.host.getMaintenanceCheckIntervalMicros()) { this.host.setMaintenanceCheckIntervalMicros(interval); } long nextExpirationMicros = Math.max(now, now + interval - SCHEDULING_EPSILON_MICROS); String selfLink = s.getSelfLink(); synchronized (this) { // To avoid double scheduling the same self-link // we lookup the self-link in our trackedServices map and remove // it before adding the new schedule. Long expiration = this.trackedServices.get(selfLink); if (expiration != null) { Set<String> services = this.nextExpiration.get(expiration); if (services != null) { services.remove(selfLink); } } this.trackedServices.put(selfLink, nextExpirationMicros); Set<String> services = this.nextExpiration.get(nextExpirationMicros); if (services == null) { services = new HashSet<>(); this.nextExpiration.put(nextExpirationMicros, services); } services.add(selfLink); } }
&& s.failedNotificationCount > ServiceSubscriber.NOTIFICATION_FAILURE_LIMIT) { if (now - s.initialFailedNotificationTimeMicros > getHost() .getMaintenanceIntervalMicros()) { getHost().log(Level.INFO, "removing subscriber, failed notifications: %d",
void retryOnDemandLoadConflict(Operation op, Service s) { if (!ServiceHost.isServiceIndexed(s)) { // service is stopped but it's not persistent, so it doesn't start/stop on-demand. // no point in retrying the operation. op.fail(new CancellationException("Service has stopped")); return; } op.removePragmaDirective(Operation.PRAGMA_DIRECTIVE_INDEX_CHECK); this.host.log(Level.WARNING, "ODL conflict: retrying %s (%d %s) on %s", op.getAction(), op.getId(), op.getContextId(), op.getUri().getPath()); long interval = Math.max(TimeUnit.SECONDS.toMicros(1), this.host.getMaintenanceIntervalMicros()); this.host.scheduleCore(() -> { this.host.handleRequest(null, op); }, interval, TimeUnit.MICROSECONDS); }
@Override public void handleStart(Operation post) { ServiceDocument initState = post.getBody(ServiceDocument.class); this.documentSelfLink = initState.documentSelfLink; this.documentExpirationTimeMicros = initState.documentExpirationTimeMicros; long ttl = this.documentExpirationTimeMicros - Utils.getSystemNowMicrosUtc(); if (ttl < 0) { logWarning("Task expiration is in the past, extending it"); // task has already expired. Add some more time instead of failing ttl = Math.max(getHost().getMaintenanceIntervalMicros() * 2, DEFAULT_TTL_MICROS); } super.toggleOption(ServiceOption.PERIODIC_MAINTENANCE, true); super.setMaintenanceIntervalMicros(ttl); post.complete(); }
@Override public void handleStart(Operation post) { ServiceDocument initState = post.getBody(ServiceDocument.class); this.documentSelfLink = initState.documentSelfLink; this.documentExpirationTimeMicros = initState.documentExpirationTimeMicros; long ttl = this.documentExpirationTimeMicros - Utils.getSystemNowMicrosUtc(); if (ttl < 0) { logWarning("Task expiration is in the past, extending it"); // task has already expired. Add some more time instead of failing ttl = Math.max(getHost().getMaintenanceIntervalMicros() * 2, DEFAULT_TTL_MICROS); } super.toggleOption(ServiceOption.PERIODIC_MAINTENANCE, true); super.setMaintenanceIntervalMicros(ttl); post.complete(); }
@Override public void handleStart(final Operation post) { super.setMaintenanceIntervalMicros(getHost().getMaintenanceIntervalMicros() * 5); // index service getUri() will be invoked on every load and save call for every operation, // so its worth caching (plus we only have a very small number of index services this.uri = post.getUri(); ExecutorService es = new ThreadPoolExecutor(QUERY_THREAD_COUNT, QUERY_THREAD_COUNT, 1, TimeUnit.MINUTES, new ArrayBlockingQueue<>(QUERY_EXECUTOR_WORK_QUEUE_CAPACITY), new NamedThreadFactory(getUri() + "/queries")); this.privateQueryExecutor = TracingExecutor.create(es, getHost().getTracer()); es = new ThreadPoolExecutor(UPDATE_THREAD_COUNT, UPDATE_THREAD_COUNT, 1, TimeUnit.MINUTES, new ArrayBlockingQueue<>(UPDATE_EXECUTOR_WORK_QUEUE_CAPACITY), new NamedThreadFactory(getUri() + "/updates")); this.privateIndexingExecutor = TracingExecutor.create(es, getHost().getTracer()); post.complete(); }
@Override public void handleStart(Operation post) { ServiceDocument initState = post.getBody(ServiceDocument.class); long interval = initState.documentExpirationTimeMicros - Utils.getSystemNowMicrosUtc(); if (interval < 0) { logWarning("Task expiration is in the past, extending it"); interval = getHost().getMaintenanceIntervalMicros() * 2; } super.toggleOption(ServiceOption.PERIODIC_MAINTENANCE, true); super.setMaintenanceIntervalMicros(interval); post.complete(); }
private void handleJoinFailure(Throwable e, JoinPeerRequest joinBody, NodeGroupState localState, long expirationMicros) { if (Utils.beforeNow(expirationMicros)) { logSevere("Failure joining peer %s due to %s, attempt expired, will not retry", joinBody.memberGroupReference, e.toString()); return; } // avoid rescheduling if the host is in the process of stopping if (getHost().isStopping()) { return; } getHost().scheduleCore(() -> { logWarning("Retrying GET to %s, due to %s", joinBody.memberGroupReference, e.toString()); handleJoinPost(joinBody, null, expirationMicros, localState, null); adjustStat(STAT_NAME_JOIN_RETRY_COUNT, 1); }, getHost().getMaintenanceIntervalMicros(), TimeUnit.MICROSECONDS); }
/** * Complete operation when target service for the given path becomes available in node group * * @see ServiceHost#registerForServiceAvailability(CompletionHandler, boolean, String...) * @see #checkServiceAvailability(CompletionHandler, ServiceHost, URI, String) */ public static void registerForReplicatedServiceAvailability(ServiceHost host, Operation op, String servicePath, String nodeSelectorPath) { CompletionHandler ch = (o, e) -> { if (e != null) { if (op.getExpirationMicrosUtc() < Utils.getSystemNowMicrosUtc()) { String msg = "Failed to check replicated service availability"; op.fail(new TimeoutException(msg)); return; } // service is not yet available, reschedule host.scheduleCore(() -> { registerForReplicatedServiceAvailability(host, op, servicePath, nodeSelectorPath); }, host.getMaintenanceIntervalMicros(), TimeUnit.MICROSECONDS); return; } op.complete(); }; host.checkReplicatedServiceAvailable(ch, servicePath, nodeSelectorPath); }
private boolean handleQueryRetry(QueryTask task, Operation directOp) { if (task.querySpec.expectedResultCount == null) { return false; } if (task.results.documentCount >= task.querySpec.expectedResultCount) { return false; } // Fail the task now if we would expire within the next maint interval. // Otherwise self patch can fail if the document has expired and clients // need a chance to GET the FAILED state. long exp = task.documentExpirationTimeMicros - getHost().getMaintenanceIntervalMicros(); if (exp < Utils.getSystemNowMicrosUtc()) { failTask(new TimeoutException(), directOp, (o, e) -> { scheduleTaskExpiration(task); }); return true; } getHost().scheduleCore(() -> { forwardQueryToDocumentIndexService(task, directOp); }, getMaintenanceIntervalMicros(), TimeUnit.MICROSECONDS); return true; }
public static void registerForReplicatedServiceAvailability(ServiceHost host, Operation op, String servicePath, String nodeSelectorPath, AuthorizationContext authorizationContext) { CompletionHandler ch = (o, e) -> { if (e != null) { if (op.getExpirationMicrosUtc() < Utils.getSystemNowMicrosUtc()) { String msg = "Failed to check replicated service availability"; op.fail(new TimeoutException(msg)); return; } // service is not yet available, reschedule host.scheduleCore(() -> { registerForReplicatedServiceAvailability(host, op, servicePath, nodeSelectorPath, authorizationContext); }, host.getMaintenanceIntervalMicros(), TimeUnit.MICROSECONDS); return; } op.complete(); }; URI serviceUri = UriUtils.buildUri(host, servicePath); checkReplicatedServiceAvailability(ch, host, serviceUri, nodeSelectorPath, authorizationContext); }
@Override public void handleMaintenance(Operation op) { if (this.pausedChannels.isEmpty()) { op.complete(); return; } try { long now = Utils.getSystemNowMicrosUtc(); for (NettyListenerChannelContext ctx : this.pausedChannels.values()) { Channel c = ctx.getChannel(); if (c.config().isAutoRead()) { continue; } if (now - ctx.getLastUseTimeMicros() < this.host.getMaintenanceIntervalMicros()) { continue; } this.host.log(Level.INFO, "Resuming paused channel %s, last use: %d", c, ctx.getLastUseTimeMicros()); c.config().setAutoRead(true); } op.complete(); } catch (Exception e) { op.fail(e); } }
@Override public void handleMaintenance(Operation op) { if (this.delayMaintenance) { try { logInfo("delaying maintenance on purpose"); Thread.sleep((2 * getHost().getMaintenanceIntervalMicros()) / 1000); } catch (Exception e1) { } } if (!op.hasBody()) { adjustStat(STAT_NAME_MAINTENANCE_FAILURE_COUNT, 1); op.fail(new IllegalStateException("missing body in maintenance op")); return; } ServiceMaintenanceRequest r = op.getBody(ServiceMaintenanceRequest.class); if (!ServiceMaintenanceRequest.KIND.equals(r.kind)) { adjustStat(STAT_NAME_MAINTENANCE_FAILURE_COUNT, 1); op.fail(new IllegalStateException("invalid body in maintenance op")); return; } if (!r.reasons.contains(MaintenanceReason.PERIODIC_SCHEDULE)) { adjustStat(STAT_NAME_MAINTENANCE_FAILURE_COUNT, 1); op.fail(new IllegalStateException("invalid reason")); return; } adjustStat(STAT_NAME_MAINTENANCE_SUCCESS_COUNT, 1); op.complete(); }
@Override public void handleMaintenance(Operation op) { if (this.delayMaintenance) { try { logInfo("delaying maintenance on purpose"); Thread.sleep((2 * getHost().getMaintenanceIntervalMicros()) / 1000); } catch (Exception e1) { } } if (!op.hasBody()) { adjustStat(STAT_NAME_MAINTENANCE_FAILURE_COUNT, 1); op.fail(new IllegalStateException("missing body in maintenance op")); return; } ServiceMaintenanceRequest r = op.getBody(ServiceMaintenanceRequest.class); if (!ServiceMaintenanceRequest.KIND.equals(r.kind)) { adjustStat(STAT_NAME_MAINTENANCE_FAILURE_COUNT, 1); op.fail(new IllegalStateException("invalid body in maintenance op")); return; } if (!r.reasons.contains(MaintenanceReason.PERIODIC_SCHEDULE)) { adjustStat(STAT_NAME_MAINTENANCE_FAILURE_COUNT, 1); op.fail(new IllegalStateException("invalid reason")); return; } adjustStat(STAT_NAME_MAINTENANCE_SUCCESS_COUNT, 1); op.complete(); }
if (!NodeGroupUtils.isMembershipSettled(getHost(), getHost().getMaintenanceIntervalMicros(), this.cachedGroupState)) { checkConvergence(membershipUpdateMicros, maintOp);
long now = Utils.getSystemNowMicrosUtc(); long actual = now - start[0]; long limit = Math.max(this.host.getMaintenanceIntervalMicros(), s.getMaintenanceIntervalMicros()); if (s.hasOption(ServiceOption.INSTRUMENTATION)) {