/** * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always * be called by the same thread. * * @param checkpointId The Id of the checkpoint. * @param timestamp The timestamp of the checkpoint. * @param streamFactory The factory that we can use for writing our state to streams. * @param checkpointOptions Options for how to perform this checkpoint. * @return Future to the state handle of the snapshot data. * @throws Exception indicating a problem in the synchronous part of the checkpoint. */ @Nonnull @Override public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot( final long checkpointId, final long timestamp, @Nonnull final CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) throws Exception { long startTime = System.currentTimeMillis(); // flush everything into db before taking a snapshot writeBatchWrapper.flush(); RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy = CheckpointType.SAVEPOINT == checkpointOptions.getCheckpointType() ? savepointSnapshotStrategy : checkpointSnapshotStrategy; RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner = chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions); chosenSnapshotStrategy.logSyncCompleted(streamFactory, startTime); return snapshotRunner; }
private SupplierWithException<CheckpointStreamWithResultProvider, Exception> createCheckpointStreamSupplier( long checkpointId, CheckpointStreamFactory primaryStreamFactory, CheckpointOptions checkpointOptions) { return localRecoveryConfig.isLocalRecoveryEnabled() && (CheckpointType.SAVEPOINT != checkpointOptions.getCheckpointType()) ? () -> CheckpointStreamWithResultProvider.createDuplicatingStream( checkpointId, CheckpointedStateScope.EXCLUSIVE, primaryStreamFactory, localRecoveryConfig.getLocalStateDirectoryProvider()) : () -> CheckpointStreamWithResultProvider.createSimpleStream( CheckpointedStateScope.EXCLUSIVE, primaryStreamFactory); }
checkpointMetaData.getCheckpointId(), checkpointOptions.getCheckpointType(), getName());
/** * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always * be called by the same thread. * * @param checkpointId The Id of the checkpoint. * @param timestamp The timestamp of the checkpoint. * @param streamFactory The factory that we can use for writing our state to streams. * @param checkpointOptions Options for how to perform this checkpoint. * @return Future to the state handle of the snapshot data. * @throws Exception indicating a problem in the synchronous part of the checkpoint. */ @Nonnull @Override public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot( final long checkpointId, final long timestamp, @Nonnull final CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) throws Exception { long startTime = System.currentTimeMillis(); // flush everything into db before taking a snapshot writeBatchWrapper.flush(); RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy = CheckpointType.SAVEPOINT == checkpointOptions.getCheckpointType() ? savepointSnapshotStrategy : checkpointSnapshotStrategy; RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner = chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions); chosenSnapshotStrategy.logSyncCompleted(streamFactory, startTime); return snapshotRunner; }
/** * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always * be called by the same thread. * * @param checkpointId The Id of the checkpoint. * @param timestamp The timestamp of the checkpoint. * @param streamFactory The factory that we can use for writing our state to streams. * @param checkpointOptions Options for how to perform this checkpoint. * @return Future to the state handle of the snapshot data. * @throws Exception indicating a problem in the synchronous part of the checkpoint. */ @Nonnull @Override public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot( final long checkpointId, final long timestamp, @Nonnull final CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) throws Exception { long startTime = System.currentTimeMillis(); // flush everything into db before taking a snapshot writeBatchWrapper.flush(); RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy = CheckpointType.SAVEPOINT == checkpointOptions.getCheckpointType() ? savepointSnapshotStrategy : checkpointSnapshotStrategy; RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner = chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions); chosenSnapshotStrategy.logSyncCompleted(streamFactory, startTime); return snapshotRunner; }
/** * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always * be called by the same thread. * * @param checkpointId The Id of the checkpoint. * @param timestamp The timestamp of the checkpoint. * @param streamFactory The factory that we can use for writing our state to streams. * @param checkpointOptions Options for how to perform this checkpoint. * @return Future to the state handle of the snapshot data. * @throws Exception */ @Override public RunnableFuture<KeyedStateHandle> snapshot( final long checkpointId, final long timestamp, final CheckpointStreamFactory streamFactory, CheckpointOptions checkpointOptions) throws Exception { if (checkpointOptions.getCheckpointType() != CheckpointOptions.CheckpointType.SAVEPOINT && enableIncrementalCheckpointing) { return snapshotIncrementally(checkpointId, timestamp, streamFactory); } else { return snapshotFully(checkpointId, timestamp, streamFactory); } }
private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException { final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions(); final CheckpointType checkpointType = checkpointOptions.getCheckpointType(); final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ? null : checkpointOptions.getTargetLocation().getReferenceBytes(); final ByteBuffer buf = ByteBuffer.allocate(28 + (locationBytes == null ? 0 : locationBytes.length)); // we do not use checkpointType.ordinal() here to make the serialization robust // against changes in the enum (such as changes in the order of the values) final int typeInt; if (checkpointType == CheckpointType.CHECKPOINT) { typeInt = CHECKPOINT_TYPE_CHECKPOINT; } else if (checkpointType == CheckpointType.SAVEPOINT) { typeInt = CHECKPOINT_TYPE_SAVEPOINT; } else { throw new IOException("Unknown checkpoint type: " + checkpointType); } buf.putInt(CHECKPOINT_BARRIER_EVENT); buf.putLong(barrier.getId()); buf.putLong(barrier.getTimestamp()); buf.putInt(typeInt); if (locationBytes == null) { buf.putInt(-1); } else { buf.putInt(locationBytes.length); buf.put(locationBytes); } buf.flip(); return buf; }
private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException { final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions(); final CheckpointType checkpointType = checkpointOptions.getCheckpointType(); final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ? null : checkpointOptions.getTargetLocation().getReferenceBytes(); final ByteBuffer buf = ByteBuffer.allocate(28 + (locationBytes == null ? 0 : locationBytes.length)); // we do not use checkpointType.ordinal() here to make the serialization robust // against changes in the enum (such as changes in the order of the values) final int typeInt; if (checkpointType == CheckpointType.CHECKPOINT) { typeInt = CHECKPOINT_TYPE_CHECKPOINT; } else if (checkpointType == CheckpointType.SAVEPOINT) { typeInt = CHECKPOINT_TYPE_SAVEPOINT; } else { throw new IOException("Unknown checkpoint type: " + checkpointType); } buf.putInt(CHECKPOINT_BARRIER_EVENT); buf.putLong(barrier.getId()); buf.putLong(barrier.getTimestamp()); buf.putInt(typeInt); if (locationBytes == null) { buf.putInt(-1); } else { buf.putInt(locationBytes.length); buf.put(locationBytes); } buf.flip(); return buf; }
private static ByteBuffer serializeCheckpointBarrier(CheckpointBarrier barrier) throws IOException { final CheckpointOptions checkpointOptions = barrier.getCheckpointOptions(); final CheckpointType checkpointType = checkpointOptions.getCheckpointType(); final byte[] locationBytes = checkpointOptions.getTargetLocation().isDefaultReference() ? null : checkpointOptions.getTargetLocation().getReferenceBytes(); final ByteBuffer buf = ByteBuffer.allocate(28 + (locationBytes == null ? 0 : locationBytes.length)); // we do not use checkpointType.ordinal() here to make the serialization robust // against changes in the enum (such as changes in the order of the values) final int typeInt; if (checkpointType == CheckpointType.CHECKPOINT) { typeInt = CHECKPOINT_TYPE_CHECKPOINT; } else if (checkpointType == CheckpointType.SAVEPOINT) { typeInt = CHECKPOINT_TYPE_SAVEPOINT; } else { throw new IOException("Unknown checkpoint type: " + checkpointType); } buf.putInt(CHECKPOINT_BARRIER_EVENT); buf.putLong(barrier.getId()); buf.putLong(barrier.getTimestamp()); buf.putInt(typeInt); if (locationBytes == null) { buf.putInt(-1); } else { buf.putInt(locationBytes.length); buf.put(locationBytes); } buf.flip(); return buf; }
/** * Returns a checkpoint stream factory for the provided options. * * <p>For {@link CheckpointType#FULL_CHECKPOINT} this returns the shared * factory of this operator. * * <p>For {@link CheckpointType#SAVEPOINT} it creates a custom factory per * savepoint. * * @param checkpointOptions Options for the checkpoint * @return Checkpoint stream factory for the checkpoints * @throws IOException Failures while creating a new stream factory are forwarded */ @VisibleForTesting CheckpointStreamFactory getCheckpointStreamFactory(CheckpointOptions checkpointOptions) throws IOException { CheckpointType checkpointType = checkpointOptions.getCheckpointType(); if (checkpointType == CheckpointType.FULL_CHECKPOINT) { return checkpointStreamFactory; } else if (checkpointType == CheckpointType.SAVEPOINT) { return container.createSavepointStreamFactory(this, checkpointOptions.getTargetLocation()); } else { throw new IllegalStateException("Unknown checkpoint type " + checkpointType); } }
private SupplierWithException<CheckpointStreamWithResultProvider, Exception> createCheckpointStreamSupplier( long checkpointId, CheckpointStreamFactory primaryStreamFactory, CheckpointOptions checkpointOptions) { return localRecoveryConfig.isLocalRecoveryEnabled() && (CheckpointType.SAVEPOINT != checkpointOptions.getCheckpointType()) ? () -> CheckpointStreamWithResultProvider.createDuplicatingStream( checkpointId, CheckpointedStateScope.EXCLUSIVE, primaryStreamFactory, localRecoveryConfig.getLocalStateDirectoryProvider()) : () -> CheckpointStreamWithResultProvider.createSimpleStream( CheckpointedStateScope.EXCLUSIVE, primaryStreamFactory); }
CheckpointType checkpointType = checkpointOptions.getCheckpointType();
private SupplierWithException<CheckpointStreamWithResultProvider, Exception> createCheckpointStreamSupplier( long checkpointId, CheckpointStreamFactory primaryStreamFactory, CheckpointOptions checkpointOptions) { return localRecoveryConfig.isLocalRecoveryEnabled() && (CheckpointType.SAVEPOINT != checkpointOptions.getCheckpointType()) ? () -> CheckpointStreamWithResultProvider.createDuplicatingStream( checkpointId, CheckpointedStateScope.EXCLUSIVE, primaryStreamFactory, localRecoveryConfig.getLocalStateDirectoryProvider()) : () -> CheckpointStreamWithResultProvider.createSimpleStream( CheckpointedStateScope.EXCLUSIVE, primaryStreamFactory); }
/** * Test method to trigger a checkpoint/savepoint. * * @param timestamp The timestamp for the checkpoint. * @param options The checkpoint options. * @return A future to the completed checkpoint */ @VisibleForTesting @Internal public Future<CompletedCheckpoint> triggerCheckpoint(long timestamp, CheckpointOptions options) throws Exception { switch (options.getCheckpointType()) { case SAVEPOINT: return triggerSavepoint(timestamp, options.getTargetLocation()); case FULL_CHECKPOINT: CheckpointTriggerResult triggerResult = triggerCheckpoint(timestamp, checkpointProperties, checkpointDirectory, false); if (triggerResult.isSuccess()) { return triggerResult.getPendingCheckpoint().getCompletionFuture(); } else { Throwable cause = new Exception("Failed to trigger checkpoint: " + triggerResult.getFailureReason().message()); return FlinkCompletableFuture.completedExceptionally(cause); } default: throw new IllegalArgumentException("Unknown checkpoint type: " + options.getCheckpointType()); } }
checkpointMetaData.getCheckpointId(), checkpointOptions.getCheckpointType(), getName());
checkpointMetaData.getCheckpointId(), checkpointOptions.getCheckpointType(), getName());
checkpointMetaData.getCheckpointId(), checkpointOptions.getCheckpointType(), getName());