synchronized TxnRequestHandler nextRequestHandler(boolean hasIncompleteBatches) { if (!newPartitionsInTransaction.isEmpty()) enqueueRequest(addPartitionsToTransactionHandler()); TxnRequestHandler nextRequestHandler = pendingRequests.peek(); if (nextRequestHandler == null) return null; // Do not send the EndTxn until all batches have been flushed if (nextRequestHandler.isEndTxn() && hasIncompleteBatches) return null; pendingRequests.poll(); if (maybeTerminateRequestWithError(nextRequestHandler)) { log.trace("Not sending transactional request {} because we are in an error state", nextRequestHandler.requestBuilder()); return null; } if (nextRequestHandler.isEndTxn() && !transactionStarted) { nextRequestHandler.result.done(); if (currentState != State.FATAL_ERROR) { log.debug("Not sending EndTxn for completed transaction since no partitions " + "or offsets were successfully added"); completeTransaction(); } nextRequestHandler = pendingRequests.poll(); } if (nextRequestHandler != null) log.trace("Request {} dequeued for sending", nextRequestHandler.requestBuilder()); return nextRequestHandler; }
@Override public void onComplete(ClientResponse response) { if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) { fatalError(new RuntimeException("Detected more than one in-flight transactional request.")); } else { clearInFlightTransactionalRequestCorrelationId(); if (response.wasDisconnected()) { log.debug("Disconnected from {}. Will retry.", response.destination()); if (this.needsCoordinator()) lookupCoordinator(this.coordinatorType(), this.coordinatorKey()); reenqueue(); } else if (response.versionMismatch() != null) { fatalError(response.versionMismatch()); } else if (response.hasResponse()) { log.trace("Received transactional response {} for request {}", response.responseBody(), requestBuilder()); synchronized (TransactionManager.this) { handleResponse(response.responseBody()); } } else { fatalError(new KafkaException("Could not execute transactional request for unknown reasons")); } } }
return false; AbstractRequest.Builder<?> requestBuilder = nextRequestHandler.requestBuilder(); while (!forceClose) { Node targetNode = null; try { if (nextRequestHandler.needsCoordinator()) { targetNode = transactionManager.coordinator(nextRequestHandler.coordinatorType()); if (targetNode == null) { transactionManager.lookupCoordinator(nextRequestHandler); if (nextRequestHandler.isRetry()) time.sleep(nextRequestHandler.retryBackoffMs()); ClientRequest clientRequest = client.newClientRequest( targetNode.idString(), requestBuilder, now, true, requestTimeoutMs, nextRequestHandler); log.debug("Disconnect from {} while trying to send request {}. Going " + "to back off and retry.", targetNode, requestBuilder, e); if (nextRequestHandler.needsCoordinator()) {
@Test public void testAddPartitionToTransactionRetainsRetryBackoffWhenPartitionsAlreadyAdded() { long pid = 13131L; short epoch = 1; TopicPartition partition = new TopicPartition("foo", 0); doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(partition); assertTrue(transactionManager.hasPartitionsToAdd()); assertFalse(transactionManager.isPartitionAdded(partition)); assertTrue(transactionManager.isPartitionPendingAdd(partition)); prepareAddPartitionsToTxn(partition, Errors.NONE); sender.run(time.milliseconds()); assertTrue(transactionManager.isPartitionAdded(partition)); TopicPartition otherPartition = new TopicPartition("foo", 1); transactionManager.maybeAddPartitionToTransaction(otherPartition); prepareAddPartitionsToTxn(otherPartition, Errors.CONCURRENT_TRANSACTIONS); TransactionManager.TxnRequestHandler handler = transactionManager.nextRequestHandler(false); assertNotNull(handler); assertEquals(DEFAULT_RETRY_BACKOFF_MS, handler.retryBackoffMs()); }
@Test public void testAddPartitionToTransactionRetainsRetryBackoffForRegularRetriableError() { long pid = 13131L; short epoch = 1; TopicPartition partition = new TopicPartition("foo", 0); doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(partition); assertTrue(transactionManager.hasPartitionsToAdd()); assertFalse(transactionManager.isPartitionAdded(partition)); assertTrue(transactionManager.isPartitionPendingAdd(partition)); prepareAddPartitionsToTxn(partition, Errors.COORDINATOR_NOT_AVAILABLE); sender.run(time.milliseconds()); TransactionManager.TxnRequestHandler handler = transactionManager.nextRequestHandler(false); assertNotNull(handler); assertEquals(DEFAULT_RETRY_BACKOFF_MS, handler.retryBackoffMs()); }
@Test public void testAddPartitionToTransactionOverridesRetryBackoffForConcurrentTransactions() { long pid = 13131L; short epoch = 1; TopicPartition partition = new TopicPartition("foo", 0); doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(partition); assertTrue(transactionManager.hasPartitionsToAdd()); assertFalse(transactionManager.isPartitionAdded(partition)); assertTrue(transactionManager.isPartitionPendingAdd(partition)); prepareAddPartitionsToTxn(partition, Errors.CONCURRENT_TRANSACTIONS); sender.run(time.milliseconds()); TransactionManager.TxnRequestHandler handler = transactionManager.nextRequestHandler(false); assertNotNull(handler); assertEquals(20, handler.retryBackoffMs()); }
boolean needsCoordinator() { return coordinatorType() != null; }
synchronized void authenticationFailed(AuthenticationException e) { for (TxnRequestHandler request : pendingRequests) request.fatalError(e); }
@Test public void testEndTxnNotSentIfIncompleteBatches() { long pid = 13131L; short epoch = 1; doInitTransactions(pid, epoch); transactionManager.beginTransaction(); transactionManager.maybeAddPartitionToTransaction(tp0); prepareAddPartitionsToTxn(tp0, Errors.NONE); sender.run(time.milliseconds()); assertTrue(transactionManager.isPartitionAdded(tp0)); transactionManager.beginCommit(); assertNull(transactionManager.nextRequestHandler(true)); assertTrue(transactionManager.nextRequestHandler(false).isEndTxn()); }
void lookupCoordinator(TxnRequestHandler request) { lookupCoordinator(request.coordinatorType(), request.coordinatorKey()); }
private boolean maybeTerminateRequestWithError(TxnRequestHandler requestHandler) { if (hasError()) { if (hasAbortableError() && requestHandler instanceof FindCoordinatorHandler) // No harm letting the FindCoordinator request go through if we're expecting to abort return false; requestHandler.fail(lastError); return true; } return false; }