private synchronized void startSequencesAtBeginning(TopicPartition topicPartition) { int sequence = 0; for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(topicPartition)) { log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), inFlightBatch.topicPartition, sequence); inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), sequence, inFlightBatch.isTransactional()); sequence += inFlightBatch.recordCount; } setNextSequence(topicPartition, sequence); lastAckedSequence.remove(topicPartition); }
synchronized void adjustSequencesDueToFailedBatch(ProducerBatch batch) { if (!this.nextSequence.containsKey(batch.topicPartition)) // Sequence numbers are not being tracked for this partition. This could happen if the producer id was just // reset due to a previous OutOfOrderSequenceException. return; log.debug("producerId: {}, send to partition {} failed fatally. Reducing future sequence numbers by {}", batch.producerId(), batch.topicPartition, batch.recordCount); int currentSequence = sequenceNumber(batch.topicPartition); currentSequence -= batch.recordCount; if (currentSequence < 0) throw new IllegalStateException("Sequence number for partition " + batch.topicPartition + " is going to become negative : " + currentSequence); setNextSequence(batch.topicPartition, currentSequence); for (ProducerBatch inFlightBatch : inflightBatchesBySequence.get(batch.topicPartition)) { if (inFlightBatch.baseSequence() < batch.baseSequence()) continue; int newSequence = inFlightBatch.baseSequence() - batch.recordCount; if (newSequence < 0) throw new IllegalStateException("Sequence number for batch with sequence " + inFlightBatch.baseSequence() + " for partition " + batch.topicPartition + " is going to become negative :" + newSequence); log.info("Resetting sequence number of batch with current sequence {} for partition {} to {}", inFlightBatch.baseSequence(), batch.topicPartition, newSequence); inFlightBatch.resetProducerState(new ProducerIdAndEpoch(inFlightBatch.producerId(), inFlightBatch.producerEpoch()), newSequence, inFlightBatch.isTransactional()); } }
private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response) { if (transactionManager != null) { if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) { transactionManager .maybeUpdateLastAckedSequence(batch.topicPartition, batch.baseSequence() + batch.recordCount - 1); log.debug("ProducerId: {}; Set last ack'd sequence number for topic-partition {} to {}", batch.producerId(), batch.topicPartition, transactionManager.lastAckedSequence(batch.topicPartition)); } transactionManager.updateLastAckedOffset(response, batch); transactionManager.removeInFlightBatch(batch); } if (batch.done(response.baseOffset, response.logAppendTime, null)) { maybeRemoveFromInflightBatches(batch); this.accumulator.deallocate(batch); } }
ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId(), producerEpoch()); for (ProducerBatch newBatch : batches) { newBatch.setProducerState(producerIdAndEpoch, sequence, isTransactional());
if (transactionManager == null) { reenqueueBatch(batch, now); } else if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) { batch.topicPartition, batch.producerId(), batch.baseSequence()); reenqueueBatch(batch, now); } else { failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " + "batch but the producer id changed from " + batch.producerId() + " to " + transactionManager.producerIdAndEpoch().producerId + " in the mean time. This batch will be dropped."), false);
synchronized boolean canRetry(ProduceResponse.PartitionResponse response, ProducerBatch batch) { if (!hasProducerId(batch.producerId())) return false;
if (exception instanceof OutOfOrderSequenceException && !transactionManager.isTransactional() && transactionManager.hasProducerId(batch.producerId())) { log.error("The broker returned {} for topic-partition " + "{} at offset {}. This indicates data loss on the broker, and should be investigated.",