@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { sessionIdsPerSnapshot.add(new Tuple2<>(context.getCheckpointId(), sessionIds)); sessionIds = new ArrayList<>(64); super.snapshotState(context); } }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(this.checkpointedState != null, "The " + getClass().getSimpleName() + " has not been properly initialized."); if (LOG.isDebugEnabled()) { LOG.debug("{} checkpointing: Messages: {}, checkpoint id: {}, timestamp: {}", idsForCurrentCheckpoint, context.getCheckpointId(), context.getCheckpointTimestamp()); } pendingCheckpoints.addLast(new Tuple2<>(context.getCheckpointId(), idsForCurrentCheckpoint)); idsForCurrentCheckpoint = new HashSet<>(64); this.checkpointedState.clear(); this.checkpointedState.add(SerializedCheckpointData.fromDeque(pendingCheckpoints, idSerializer)); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(bucketStates != null && maxPartCountersState != null, "sink has not been initialized"); buckets.snapshotState( context.getCheckpointId(), bucketStates, maxPartCountersState); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(this.checkpointedState != null, "The " + getClass().getSimpleName() + " has not been properly initialized."); if (LOG.isDebugEnabled()) { LOG.debug("{} checkpointing: Messages: {}, checkpoint id: {}, timestamp: {}", idsForCurrentCheckpoint, context.getCheckpointId(), context.getCheckpointTimestamp()); } pendingCheckpoints.addLast(new Tuple2<>(context.getCheckpointId(), idsForCurrentCheckpoint)); idsForCurrentCheckpoint = new HashSet<>(64); this.checkpointedState.clear(); this.checkpointedState.add(SerializedCheckpointData.fromDeque(pendingCheckpoints, idSerializer)); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkNotNull(restoredBucketStates, "The " + getClass().getSimpleName() + " has not been properly initialized."); int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); if (isWriterOpen) { bucketState.currentFile = currentPartPath.toString(); bucketState.currentFileValidLength = writer.flush(); } synchronized (bucketState.pendingFilesPerCheckpoint) { bucketState.pendingFilesPerCheckpoint.put(context.getCheckpointId(), bucketState.pendingFiles); } bucketState.pendingFiles = new ArrayList<>(); restoredBucketStates.clear(); restoredBucketStates.add(bucketState); if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx={}) checkpointed {}.", getClass().getSimpleName(), subtaskIdx, bucketState); } }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(this.checkpointedState != null, "The " + getClass().getSimpleName() + " has not been properly initialized."); if (LOG.isDebugEnabled()) { LOG.debug("{} checkpointing: Messages: {}, checkpoint id: {}, timestamp: {}", idsForCurrentCheckpoint, context.getCheckpointId(), context.getCheckpointTimestamp()); } pendingCheckpoints.addLast(new Tuple2<>(context.getCheckpointId(), idsForCurrentCheckpoint)); idsForCurrentCheckpoint = new ArrayList<>(64); this.checkpointedState.clear(); this.checkpointedState.add(SerializedCheckpointData.fromDeque(pendingCheckpoints, idSerializer)); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkNotNull(restoredBucketStates, "The operator has not been properly initialized."); restoredBucketStates.clear(); synchronized (state.bucketStates) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); for (Map.Entry<String, BucketState<T>> bucketStateEntry : state.bucketStates.entrySet()) { BucketState<T> bucketState = bucketStateEntry.getValue(); if (bucketState.isWriterOpen) { bucketState.currentFileValidLength = bucketState.writer.flush(); } synchronized (bucketState.pendingFilesPerCheckpoint) { bucketState.pendingFilesPerCheckpoint.put(context.getCheckpointId(), bucketState.pendingFiles); } bucketState.pendingFiles = new ArrayList<>(); } restoredBucketStates.add(state); if (LOG.isDebugEnabled()) { LOG.debug("{} idx {} checkpointed {}.", getClass().getSimpleName(), subtaskIdx, state); } } }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(this.checkpointedState != null, "The " + getClass().getSimpleName() + " has not been properly initialized."); if (LOG.isDebugEnabled()) { LOG.debug("{} checkpointing: Messages: {}, checkpoint id: {}, timestamp: {}", idsForCurrentCheckpoint, context.getCheckpointId(), context.getCheckpointTimestamp()); } pendingCheckpoints.addLast(new Tuple2<>(context.getCheckpointId(), idsForCurrentCheckpoint)); idsForCurrentCheckpoint = new HashSet<>(64); this.checkpointedState.clear(); this.checkpointedState.add(SerializedCheckpointData.fromDeque(pendingCheckpoints, idSerializer)); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { // this is like the pre-commit of a 2-phase-commit transaction // we are ready to commit and remember the transaction checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot"); long checkpointId = context.getCheckpointId(); LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder); preCommit(currentTransactionHolder.handle); pendingCommitTransactions.put(checkpointId, currentTransactionHolder); LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder); state.clear(); state.add(new State<>( this.currentTransactionHolder, new ArrayList<>(pendingCommitTransactions.values()), userContext)); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { if (context.getCheckpointId() > 5) { waitForCheckpointLatch.trigger(); failInCheckpointLatch.await(); if (!failedAlready.getAndSet(true)) { throw new RuntimeException("Failing on purpose."); } } }
pendingOffsetsToCommit.put(context.getCheckpointId(), restoredState); pendingOffsetsToCommit.put(context.getCheckpointId(), currentOffsets);
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { sessionIdsPerSnapshot.add(new Tuple2<>(context.getCheckpointId(), sessionIds)); sessionIds = new ArrayList<>(64); super.snapshotState(context); } }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { sessionIdsPerSnapshot.add(new Tuple2<>(context.getCheckpointId(), sessionIds)); sessionIds = new ArrayList<>(64); super.snapshotState(context); } }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { sessionIdsPerSnapshot.add(new Tuple2<>(context.getCheckpointId(), sessionIds)); sessionIds = new ArrayList<>(64); super.snapshotState(context); } }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(bucketStates != null && maxPartCountersState != null, "sink has not been initialized"); buckets.snapshotState( context.getCheckpointId(), bucketStates, maxPartCountersState); }
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { Preconditions.checkState(bucketStates != null && maxPartCountersState != null, "sink has not been initialized"); buckets.snapshotState( context.getCheckpointId(), bucketStates, maxPartCountersState); }
long checkpointId = functionSnapshotContext.getCheckpointId();
long checkpointId = functionSnapshotContext.getCheckpointId();
long checkpointId = functionSnapshotContext.getCheckpointId();
@Override public void snapshotState(FunctionSnapshotContext context) throws Exception { // this is like the pre-commit of a 2-phase-commit transaction // we are ready to commit and remember the transaction checkState(currentTransactionHolder != null, "bug: no transaction object when performing state snapshot"); long checkpointId = context.getCheckpointId(); LOG.debug("{} - checkpoint {} triggered, flushing transaction '{}'", name(), context.getCheckpointId(), currentTransactionHolder); preCommit(currentTransactionHolder.handle); pendingCommitTransactions.put(checkpointId, currentTransactionHolder); LOG.debug("{} - stored pending transactions {}", name(), pendingCommitTransactions); currentTransactionHolder = beginTransactionInternal(); LOG.debug("{} - started new transaction '{}'", name(), currentTransactionHolder); state.clear(); state.add(new State<>( this.currentTransactionHolder, new ArrayList<>(pendingCommitTransactions.values()), userContext)); }