@Override public void sendResponse(Operation op, Object response) { if (!RESPONSE_RECEIVED.compareAndSet(this, FALSE, TRUE)) { throw new ResponseAlreadySentException("NormalResponse already responseReceived for callback: " + this + ", current-response: " + response); } if (response instanceof CallTimeoutResponse) { notifyCallTimeout(); } else if (response instanceof ErrorResponse || response instanceof Throwable) { notifyError(response); } else if (response instanceof NormalResponse) { NormalResponse normalResponse = (NormalResponse) response; notifyNormalResponse(normalResponse.getValue(), normalResponse.getBackupAcks()); } else { // there are no backups or the number of expected backups has returned; so signal the future that the result is ready complete(response); } }
/** * Checks if this Invocation has received a heartbeat in time. * * If the response is already set, or if a heartbeat has been received in time, then {@code false} is returned. * If no heartbeat has been received, then the future.set is called with HEARTBEAT_TIMEOUT and {@code true} is returned. * * Gets called from the monitor-thread. * * @return {@code true} if there is a timeout detected, {@code false} otherwise. */ boolean detectAndHandleTimeout(long heartbeatTimeoutMillis) { // skip if local and not BackupAwareOperation if (!(remote || op instanceof BackupAwareOperation)) { return false; } HeartbeatTimeout heartbeatTimeout = detectTimeout(heartbeatTimeoutMillis); if (heartbeatTimeout == TIMEOUT) { complete(HEARTBEAT_TIMEOUT); return true; } else { return false; } }
private void resetAndReInvoke() { if (!context.invocationRegistry.deregister(this)) { // another thread already did something else with this invocation return; } invokeCount = 0; pendingResponse = VOID; pendingResponseReceivedMillis = -1; backupsAcksExpected = 0; backupsAcksReceived = 0; lastHeartbeatMillis = 0; doInvoke(false); }
@SuppressFBWarnings(value = "VO_VOLATILE_INCREMENT", justification = "We have the guarantee that only a single thread at any given time can change the volatile field") private void doInvoke(boolean isAsync) { if (!engineActive()) { return; initInvocationTarget(); } catch (Exception e) { notifyError(initializationFailure); return; doInvokeRemote(); } else { doInvokeLocal(isAsync);
private void invoke0(boolean isAsync) { if (invokeCount > 0) { throw new IllegalStateException("This invocation is already in progress"); } else if (isActive()) { throw new IllegalStateException( "Attempt to reuse the same operation in multiple invocations. Operation is " + op); } try { setCallTimeout(op, callTimeoutMillis); setCallerAddress(op, context.thisAddress); op.setNodeEngine(context.nodeEngine); boolean isAllowed = context.operationExecutor.isInvocationAllowed(op, isAsync); if (!isAllowed && !isMigrationOperation(op)) { throw new IllegalThreadStateException(Thread.currentThread() + " cannot make remote call: " + op); } doInvoke(isAsync); } catch (Exception e) { handleInvocationException(e); } }
private void handleRetry(Object cause) { context.retryCount.inc(); if (invokeCount % LOG_INVOCATION_COUNT_MOD == 0) { Level level = invokeCount > LOG_MAX_INVOCATION_COUNT ? WARNING : FINEST; if (context.logger.isLoggable(level)) { context.logger.log(level, "Retrying invocation: " + toString() + ", Reason: " + cause); } } if (future.interrupted) { complete(INTERRUPTED); } else { try { InvocationRetryTask retryTask = new InvocationRetryTask(); if (invokeCount < MAX_FAST_INVOCATION_COUNT) { // fast retry for the first few invocations context.invocationMonitor.execute(retryTask); } else { // progressive retry delay long delayMillis = Math.min(1 << (invokeCount - MAX_FAST_INVOCATION_COUNT), tryPauseMillis); context.invocationMonitor.schedule(retryTask, delayMillis); } } catch (RejectedExecutionException e) { completeWhenRetryRejected(e); } } }
if (shouldFailOnIndeterminateOperationState()) { complete(new IndeterminateOperationStateException(this + " failed because backup acks missed.")); return true; resetAndReInvoke(); return false; complete(pendingResponse); return true;
complete(CALL_TIMEOUT); return; handleRetry("invocation timeout");
@Override public void run0() { if (logger.isFinestEnabled()) { logger.finest("Scanning all invocations"); } if (invocationRegistry.size() == 0) { return; } int backupTimeouts = 0; int normalTimeouts = 0; int invocationCount = 0; for (Entry<Long, Invocation> e : invocationRegistry.entrySet()) { invocationCount++; Invocation inv = e.getValue(); try { if (inv.detectAndHandleTimeout(invocationTimeoutMillis)) { normalTimeouts++; } else if (inv.detectAndHandleBackupTimeout(backupTimeoutMillis)) { backupTimeouts++; } } catch (Throwable t) { inspectOutOfMemoryError(t); logger.severe("Failed to check invocation:" + inv, t); } } backupTimeoutsCount.inc(backupTimeouts); normalTimeoutsCount.inc(normalTimeouts); log(invocationCount, backupTimeouts, normalTimeouts); }
void notifyBackupComplete() { int newBackupAcksCompleted = BACKUP_ACKS_RECEIVED.incrementAndGet(this); Object pendingResponse = this.pendingResponse; if (pendingResponse == VOID) { // no pendingResponse has been set, so we are done since the invocation on the primary needs to complete first return; } // if a pendingResponse is set, then the backupsAcksExpected has been set (so we can now safely read backupsAcksExpected) int backupAcksExpected = this.backupsAcksExpected; if (backupAcksExpected < newBackupAcksCompleted) { // the backups have not yet completed, so we are done return; } if (backupAcksExpected != newBackupAcksCompleted) { // we managed to complete one backup, but we were not the one completing the last backup, so we are done return; } // we are the lucky one since we just managed to complete the last backup for this invocation and since the // pendingResponse is set, we can set it on the future complete(pendingResponse); }
Invocation(Context context, Operation op, Runnable taskDoneCallback, int tryCount, long tryPauseMillis, long callTimeoutMillis, boolean deserialize) { this.context = context; this.op = op; this.taskDoneCallback = taskDoneCallback; this.tryCount = tryCount; this.tryPauseMillis = tryPauseMillis; this.callTimeoutMillis = getCallTimeoutMillis(callTimeoutMillis); this.future = new InvocationFuture(this, deserialize); }
@SuppressFBWarnings(value = "VO_VOLATILE_INCREMENT", justification = "We have the guarantee that only a single thread at any given time can change the volatile field") private void doInvoke(boolean isAsync) { if (!engineActive()) { return; initInvocationTarget(); } catch (Exception e) { notifyError(initializationFailure); return; doInvokeRemote(); } else { doInvokeLocal(isAsync);
private void invoke0(boolean isAsync) { if (invokeCount > 0) { throw new IllegalStateException("This invocation is already in progress"); } else if (isActive()) { throw new IllegalStateException( "Attempt to reuse the same operation in multiple invocations. Operation is " + op); } try { setCallTimeout(op, callTimeoutMillis); setCallerAddress(op, context.thisAddress); op.setNodeEngine(context.nodeEngine); boolean isAllowed = context.operationExecutor.isInvocationAllowed(op, isAsync); if (!isAllowed && !isMigrationOperation(op)) { throw new IllegalThreadStateException(Thread.currentThread() + " cannot make remote call: " + op); } doInvoke(isAsync); } catch (Exception e) { handleInvocationException(e); } }
private void handleRetry(Object cause) { context.retryCount.inc(); if (invokeCount % LOG_INVOCATION_COUNT_MOD == 0) { Level level = invokeCount > LOG_MAX_INVOCATION_COUNT ? WARNING : FINEST; if (context.logger.isLoggable(level)) { context.logger.log(level, "Retrying invocation: " + toString() + ", Reason: " + cause); } } if (future.interrupted) { complete(INTERRUPTED); } else { try { InvocationRetryTask retryTask = new InvocationRetryTask(); if (invokeCount < MAX_FAST_INVOCATION_COUNT) { // fast retry for the first few invocations context.invocationMonitor.execute(retryTask); } else { // progressive retry delay long delayMillis = Math.min(1 << (invokeCount - MAX_FAST_INVOCATION_COUNT), tryPauseMillis); context.invocationMonitor.schedule(retryTask, delayMillis); } } catch (RejectedExecutionException e) { completeWhenRetryRejected(e); } } }
if (shouldFailOnIndeterminateOperationState()) { complete(new IndeterminateOperationStateException(this + " failed because backup acks missed.")); return true; resetAndReInvoke(); return false; complete(pendingResponse); return true;
complete(CALL_TIMEOUT); return; handleRetry("invocation timeout");
@Override public void run0() { if (logger.isFinestEnabled()) { logger.finest("Scanning all invocations"); } if (invocationRegistry.size() == 0) { return; } int backupTimeouts = 0; int normalTimeouts = 0; int invocationCount = 0; for (Entry<Long, Invocation> e : invocationRegistry.entrySet()) { invocationCount++; Invocation inv = e.getValue(); try { if (inv.detectAndHandleTimeout(invocationTimeoutMillis)) { normalTimeouts++; } else if (inv.detectAndHandleBackupTimeout(backupTimeoutMillis)) { backupTimeouts++; } } catch (Throwable t) { inspectOutOfMemoryError(t); logger.severe("Failed to check invocation:" + inv, t); } } backupTimeoutsCount.inc(backupTimeouts); normalTimeoutsCount.inc(normalTimeouts); log(invocationCount, backupTimeouts, normalTimeouts); }
void notifyNormalResponse(Object value, int expectedBackups) { // if a regular response comes and there are backups, we need to wait for the backups // when the backups complete, the response will be send by the last backup or backup-timeout-handle mechanism kicks on if (expectedBackups > backupsAcksReceived) { // so the invocation has backups and since not all backups have completed, we need to wait // (it could be that backups arrive earlier than the response) this.pendingResponseReceivedMillis = Clock.currentTimeMillis(); this.backupsAcksExpected = expectedBackups; // it is very important that the response is set after the backupsAcksExpected is set, else the system // can assume the invocation is complete because there is a response and no backups need to respond this.pendingResponse = value; if (backupsAcksReceived != expectedBackups) { // we are done since not all backups have completed. Therefor we should not notify the future return; } } // we are going to notify the future that a response is available; this can happen when: // - we had a regular operation (so no backups we need to wait for) that completed // - we had a backup-aware operation that has completed, but also all its backups have completed complete(value); }
Invocation(Context context, Operation op, Runnable taskDoneCallback, int tryCount, long tryPauseMillis, long callTimeoutMillis, boolean deserialize) { this.context = context; this.op = op; this.taskDoneCallback = taskDoneCallback; this.tryCount = tryCount; this.tryPauseMillis = tryPauseMillis; this.callTimeoutMillis = getCallTimeoutMillis(callTimeoutMillis); this.future = new InvocationFuture(this, deserialize); }
@Override public void sendResponse(Operation op, Object response) { if (!RESPONSE_RECEIVED.compareAndSet(this, FALSE, TRUE)) { throw new ResponseAlreadySentException("NormalResponse already responseReceived for callback: " + this + ", current-response: " + response); } if (response instanceof CallTimeoutResponse) { notifyCallTimeout(); } else if (response instanceof ErrorResponse || response instanceof Throwable) { notifyError(response); } else if (response instanceof NormalResponse) { NormalResponse normalResponse = (NormalResponse) response; notifyNormalResponse(normalResponse.getValue(), normalResponse.getBackupAcks()); } else { // there are no backups or the number of expected backups has returned; so signal the future that the result is ready complete(response); } }