public TransactionManager(LogContext logContext, String transactionalId, int transactionTimeoutMs, long retryBackoffMs) { this.producerIdAndEpoch = new ProducerIdAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH); this.nextSequence = new HashMap<>(); this.lastAckedSequence = new HashMap<>(); this.transactionalId = transactionalId; this.log = logContext.logger(TransactionManager.class); this.transactionTimeoutMs = transactionTimeoutMs; this.transactionCoordinator = null; this.consumerGroupCoordinator = null; this.newPartitionsInTransaction = new HashSet<>(); this.pendingPartitionsInTransaction = new HashSet<>(); this.partitionsInTransaction = new HashSet<>(); this.pendingTxnOffsetCommits = new HashMap<>(); this.pendingRequests = new PriorityQueue<>(10, new Comparator<TxnRequestHandler>() { @Override public int compare(TxnRequestHandler o1, TxnRequestHandler o2) { return Integer.compare(o1.priority().priority, o2.priority().priority); } }); this.partitionsWithUnresolvedSequences = new HashSet<>(); this.inflightBatchesBySequence = new HashMap<>(); this.lastAckedOffset = new HashMap<>(); this.retryBackoffMs = retryBackoffMs; }
public boolean hasProducerId() { return producerIdAndEpoch.isValid(); }
private synchronized void startSequencesAtBeginning(TopicPartition topicPartition) { int sequence = 0; for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(topicPartition)) { log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), inFlightBatch.topicPartition, sequence); inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), sequence, inFlightBatch.isTransactional()); sequence += inFlightBatch.recordCount; } setNextSequence(topicPartition, sequence); lastAckedSequence.remove(topicPartition); }
private boolean shouldStopDrainBatchesForPartition(ProducerBatch first, TopicPartition tp) { ProducerIdAndEpoch producerIdAndEpoch = null; if (transactionManager != null) { if (!transactionManager.isSendToPartitionAllowed(tp)) return true; producerIdAndEpoch = transactionManager.producerIdAndEpoch(); if (!producerIdAndEpoch.isValid()) // we cannot send the batch until we have refreshed the producer id return true; if (!first.hasSequence() && transactionManager.hasUnresolvedSequence(first.topicPartition)) // Don't drain any new batches while the state of previous sequence numbers // is unknown. The previous batches would be unknown if they were aborted // on the client after being sent to the broker at least once. return true; int firstInFlightSequence = transactionManager.firstInFlightSequence(first.topicPartition); if (firstInFlightSequence != RecordBatch.NO_SEQUENCE && first.hasSequence() && first.baseSequence() != firstInFlightSequence) // If the queued batch already has an assigned sequence, then it is being retried. // In this case, we wait until the next immediate batch is ready and drain that. // We only move on when the next in line batch is complete (either successfully or due to // a fatal broker error). This effectively reduces our in flight request count to 1. return true; } return false; }
synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { if (!this.nextSequence.containsKey(batch.topicPartition)) // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just // reset due to a previous OutOfOrderSequenceException. return; log.debug("producerId: {}, send to partition {} failed fatally. Reducing future sequence numbers by {}", batch.producerId(), batch.topicPartition, batch.recordCount); int currentSequence = sequenceNumber(batch.topicPartition); currentSequence -= batch.recordCount; if (currentSequence < 0) throw new IllegalStateException("Sequence number for partition " + batch.topicPartition + " is going to become negative : " + currentSequence); setNextSequence(batch.topicPartition, currentSequence); for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(batch.topicPartition)) { if (inFlightBatch.baseSequence() < batch.baseSequence()) continue; int newSequence = inFlightBatch.baseSequence() - batch.recordCount; if (newSequence < 0) throw new IllegalStateException("Sequence number for batch with sequence " + inFlightBatch.baseSequence() + " for partition " + batch.topicPartition + " is going to become negative :" + newSequence); log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), batch.topicPartition, newSequence); inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional()); } }
@Test public void testIdempotentSplitBatchAndSend() throws Exception { TopicPartition tp = new TopicPartition("testSplitBatchAndSend", 1); TransactionManager txnManager = new TransactionManager(); ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(123456L, (short) 0); txnManager.setProducerIdAndEpoch(producerIdAndEpoch); testSplitBatchAndSend(txnManager, producerIdAndEpoch, tp); }
ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId(), producerEpoch()); for (ProducerBatch newBatch : batches) { newBatch.setProducerState(producerIdAndEpoch, sequence, isTransactional());
Errors error = initProducerIdResponse.error(); if (error == Errors.NONE) { ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch( initProducerIdResponse.producerId(), initProducerIdResponse.epoch()); transactionManager.setProducerIdAndEpoch(producerIdAndEpoch);
@Test public void testTransactionalSplitBatchAndSend() throws Exception { ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(123456L, (short) 0); TopicPartition tp = new TopicPartition("testSplitBatchAndSend", 1); TransactionManager txnManager = new TransactionManager(logContext, "testSplitBatchAndSend", 60000, 100); setupWithTransactionState(txnManager); doInitTransactions(txnManager, producerIdAndEpoch); txnManager.beginTransaction(); txnManager.maybeAddPartitionToTransaction(tp); client.prepareResponse(new AddPartitionsToTxnResponse(0, Collections.singletonMap(tp, Errors.NONE))); sender.run(time.milliseconds()); testSplitBatchAndSend(txnManager, producerIdAndEpoch, tp); }
@Override public void handleResponse(AbstractResponse response) { InitProducerIdResponse initProducerIdResponse = (InitProducerIdResponse) response; Errors error = initProducerIdResponse.error(); if (error == Errors.NONE) { ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(initProducerIdResponse.producerId(), initProducerIdResponse.epoch()); setProducerIdAndEpoch(producerIdAndEpoch); transitionTo(State.READY); lastError = null; result.done(); } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) { lookupCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId); reenqueue(); } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) { reenqueue(); } else if (error == Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED) { fatalError(error.exception()); } else { fatalError(new KafkaException("Unexpected error in InitProducerIdResponse; " + error.message())); } } }
final long producerId = 343434L; TransactionManager transactionManager = new TransactionManager(); transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0)); setupWithTransactionState(transactionManager);
@Test public void testResetWhenOutOfOrderSequenceReceived() throws InterruptedException { final long producerId = 343434L; TransactionManager transactionManager = new TransactionManager(); transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0)); setupWithTransactionState(transactionManager); int maxRetries = 10; Metrics m = new Metrics(); SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m); Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions); Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); // connect. sender.run(time.milliseconds()); // send. assertEquals(1, client.inFlightRequestCount()); assertEquals(1, sender.inFlightBatches(tp0).size()); client.respond(produceResponse(tp0, 0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, 0)); sender.run(time.milliseconds()); assertTrue(responseFuture.isDone()); assertEquals(0, sender.inFlightBatches(tp0).size()); assertFalse("Expected transaction state to be reset upon receiving an OutOfOrderSequenceException", transactionManager.hasProducerId()); }
final long producerId = 343434L; TransactionManager transactionManager = new TransactionManager(); transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0)); setupWithTransactionState(transactionManager);
final long producerId = 343434L; TransactionManager transactionManager = new TransactionManager(); transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0)); setupWithTransactionState(transactionManager); transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId + 1, (short) 0));
final long producerId = 343434L; TransactionManager transactionManager = new TransactionManager(); transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0)); setupWithTransactionState(transactionManager);