+ " must be numeric, was " + object); super.setTransactionTimeout(transactionTimeout); super.enableTransactionTimeoutWarnings(0.8);
/** * Disables the propagation of exceptions thrown when committing presumably timed out Kafka * transactions during recovery of the job. If a Kafka transaction is timed out, a commit will * never be successful. Hence, use this feature to avoid recovery loops of the Job. Exceptions * will still be logged to inform the user that data loss might have occurred. * * <p>Note that we use {@link System#currentTimeMillis()} to track the age of a transaction. * Moreover, only exceptions thrown during the recovery are caught, i.e., the producer will * attempt at least one commit of the transaction before giving up.</p> */ @Override public FlinkKafkaProducer011<IN> ignoreFailuresAfterTransactionTimeout() { super.ignoreFailuresAfterTransactionTimeout(); return this; }
/** * Initializes the connection to Kafka. */ @Override public void open(Configuration configuration) throws Exception { if (logFailuresOnly) { callback = new Callback() { @Override public void onCompletion(RecordMetadata metadata, Exception e) { if (e != null) { LOG.error("Error while sending record to Kafka: " + e.getMessage(), e); } acknowledgeMessage(); } }; } else { callback = new Callback() { @Override public void onCompletion(RecordMetadata metadata, Exception exception) { if (exception != null && asyncException == null) { asyncException = exception; } acknowledgeMessage(); } }; } super.open(configuration); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { // this is like the pre-commit of a 2-phase-commit transaction // we are ready to commit and remember the transaction checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot"); long checkpointId = context.getCheckpointId(); LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder); preCommit(currentTransactionHolder.handle); pendingCommitTransactions.put(checkpointId, currentTransactionHolder); LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder); state.clear(); state.add(new State<>( this.currentTransactionHolder, new ArrayList<>(pendingCommitTransactions.values()), userContext)); }
LOG.info("{} - restoring state", name()); for (State<TXN, CONTEXT> operatorState : state.get()) { userContext = operatorState.getContext(); for (TransactionHolder<TXN> recoveredTransaction : recoveredTransactions) { recoverAndCommitInternal(recoveredTransaction); LOG.info("{} committed recovered transaction {}", name(), recoveredTransaction); recoverAndAbort(operatorState.getPendingTransaction().handle); LOG.info("{} aborted recovered transaction {}", name(), operatorState.getPendingTransaction()); finishRecoveringContext(); recoveredUserContext = true; LOG.info("{} - no state to restore", name()); userContext = initializeUserContext(); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { super.snapshotState(context); nextTransactionalIdHintState.clear(); // To avoid duplication only first subtask keeps track of next transactional id hint. Otherwise all of the // subtasks would write exactly same information. if (getRuntimeContext().getIndexOfThisSubtask() == 0 && semantic == FlinkKafkaProducer.Semantic.EXACTLY_ONCE) { checkState(nextTransactionalIdHint != null, "nextTransactionalIdHint must be set for EXACTLY_ONCE"); long nextFreeTransactionalId = nextTransactionalIdHint.nextFreeTransactionalId; // If we scaled up, some (unknown) subtask must have created new transactional ids from scratch. In that // case we adjust nextFreeTransactionalId by the range of transactionalIds that could be used for this // scaling up. if (getRuntimeContext().getNumberOfParallelSubtasks() > nextTransactionalIdHint.lastParallelism) { nextFreeTransactionalId += getRuntimeContext().getNumberOfParallelSubtasks() * kafkaProducersPoolSize; } nextTransactionalIdHintState.add(new FlinkKafkaProducer.NextTransactionalIdHint( getRuntimeContext().getNumberOfParallelSubtasks(), nextFreeTransactionalId)); } }
@Override public void close() throws FlinkKafkaException { final FlinkKafkaProducer.KafkaTransactionState currentTransaction = currentTransaction(); if (currentTransaction != null) { // to avoid exceptions on aborting transactions with some pending records flush(currentTransaction); // normal abort for AT_LEAST_ONCE and NONE do not clean up resources because of producer reusing, thus // we need to close it manually switch (semantic) { case EXACTLY_ONCE: break; case AT_LEAST_ONCE: case NONE: currentTransaction.producer.close(); break; } } try { super.close(); } catch (Exception e) { asyncException = ExceptionUtils.firstOrSuppressed(e, asyncException); } // make sure we propagate pending errors checkErroneous(); pendingTransactions().forEach(transaction -> IOUtils.closeQuietly(transaction.getValue().producer) ); }
super.initializeState(context);
/** * Abort a transaction that was rejected by a coordinator after a failure. */ protected void recoverAndAbort(TXN transaction) { abort(transaction); }
/** * This method must be the only place to call {@link #beginTransaction()} to ensure that the * {@link TransactionHolder} is created at the same time. */ private TransactionHolder<TXN> beginTransactionInternal() throws Exception { return new TransactionHolder<>(beginTransaction(), clock.millis()); }
LOG.info("{} - restoring state", name()); for (State<TXN, CONTEXT> operatorState : state.get()) { userContext = operatorState.getContext(); for (TransactionHolder<TXN> recoveredTransaction : recoveredTransactions) { recoverAndCommitInternal(recoveredTransaction); LOG.info("{} committed recovered transaction {}", name(), recoveredTransaction); recoverAndAbort(operatorState.getPendingTransaction().handle); LOG.info("{} aborted recovered transaction {}", name(), operatorState.getPendingTransaction()); finishRecoveringContext(); recoveredUserContext = true; LOG.info("{} - no state to restore", name()); userContext = initializeUserContext(); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { // this is like the pre-commit of a 2-phase-commit transaction // we are ready to commit and remember the transaction checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot"); long checkpointId = context.getCheckpointId(); LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder); preCommit(currentTransactionHolder.handle); pendingCommitTransactions.put(checkpointId, currentTransactionHolder); LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder); state.clear(); state.add(new State<>( this.currentTransactionHolder, new ArrayList<>(pendingCommitTransactions.values()), userContext)); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { super.snapshotState(context); nextTransactionalIdHintState.clear(); // To avoid duplication only first subtask keeps track of next transactional id hint. Otherwise all of the // subtasks would write exactly same information. if (getRuntimeContext().getIndexOfThisSubtask() == 0 && semantic == Semantic.EXACTLY_ONCE) { checkState(nextTransactionalIdHint != null, "nextTransactionalIdHint must be set for EXACTLY_ONCE"); long nextFreeTransactionalId = nextTransactionalIdHint.nextFreeTransactionalId; // If we scaled up, some (unknown) subtask must have created new transactional ids from scratch. In that // case we adjust nextFreeTransactionalId by the range of transactionalIds that could be used for this // scaling up. if (getRuntimeContext().getNumberOfParallelSubtasks() > nextTransactionalIdHint.lastParallelism) { nextFreeTransactionalId += getRuntimeContext().getNumberOfParallelSubtasks() * kafkaProducersPoolSize; } nextTransactionalIdHintState.add(new NextTransactionalIdHint( getRuntimeContext().getNumberOfParallelSubtasks(), nextFreeTransactionalId)); } }
@Override public void close() throws FlinkKafka011Exception { final KafkaTransactionState currentTransaction = currentTransaction(); if (currentTransaction != null) { // to avoid exceptions on aborting transactions with some pending records flush(currentTransaction); // normal abort for AT_LEAST_ONCE and NONE do not clean up resources because of producer reusing, thus // we need to close it manually switch (semantic) { case EXACTLY_ONCE: break; case AT_LEAST_ONCE: case NONE: currentTransaction.producer.close(); break; } } try { super.close(); } catch (Exception e) { asyncException = ExceptionUtils.firstOrSuppressed(e, asyncException); } // make sure we propagate pending errors checkErroneous(); pendingTransactions().forEach(transaction -> IOUtils.closeQuietly(transaction.getValue().producer) ); }
super.initializeState(context);
@Override public void close() throws Exception { super.close(); if (currentTransactionHolder != null) { abort(currentTransactionHolder.handle); currentTransactionHolder = null; } }
/** * This method must be the only place to call {@link #beginTransaction()} to ensure that the * {@link TransactionHolder} is created at the same time. */ private TransactionHolder<TXN> beginTransactionInternal() throws Exception { return new TransactionHolder<>(beginTransaction(), clock.millis()); }
LOG.info("{} - restoring state", name()); for (State<TXN, CONTEXT> operatorState : state.get()) { userContext = operatorState.getContext(); for (TransactionHolder<TXN> recoveredTransaction : recoveredTransactions) { recoverAndCommitInternal(recoveredTransaction); LOG.info("{} committed recovered transaction {}", name(), recoveredTransaction); recoverAndAbort(operatorState.getPendingTransaction().handle); LOG.info("{} aborted recovered transaction {}", name(), operatorState.getPendingTransaction()); finishRecoveringContext(); recoveredUserContext = true; LOG.info("{} - no state to restore", name()); userContext = initializeUserContext(); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder);
+ " must be numeric, was " + object); super.setTransactionTimeout(transactionTimeout); super.enableTransactionTimeoutWarnings(0.8);
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { // this is like the pre-commit of a 2-phase-commit transaction // we are ready to commit and remember the transaction checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot"); long checkpointId = context.getCheckpointId(); LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder); preCommit(currentTransactionHolder.handle); pendingCommitTransactions.put(checkpointId, currentTransactionHolder); LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder); state.clear(); state.add(new State<>( this.currentTransactionHolder, new ArrayList<>(pendingCommitTransactions.values()), userContext)); }