/** * Synchronously writes the new checkpoints to ZooKeeper and asynchronously removes older ones. * * @param checkpoint Completed checkpoint to add. */ @Override public void addCheckpoint(final CompletedCheckpoint checkpoint) throws Exception { checkNotNull(checkpoint, "Checkpoint"); final String path = checkpointIdToPath(checkpoint.getCheckpointID()); // Now add the new one. If it fails, we don't want to loose existing data. checkpointsInZooKeeper.addAndLock(path, checkpoint); completedCheckpoints.addLast(checkpoint); // Everything worked, let's remove a previous checkpoint if necessary. while (completedCheckpoints.size() > maxNumberOfCheckpointsToRetain) { final CompletedCheckpoint completedCheckpoint = completedCheckpoints.removeFirst(); tryRemoveCompletedCheckpoint(completedCheckpoint, CompletedCheckpoint::discardOnSubsume); } LOG.debug("Added {} to {}.", checkpoint, path); }
/** * Synchronously writes the new checkpoints to ZooKeeper and asynchronously removes older ones. * * @param checkpoint Completed checkpoint to add. */ @Override public void addCheckpoint(final CompletedCheckpoint checkpoint) throws Exception { checkNotNull(checkpoint, "Checkpoint"); final String path = checkpointIdToPath(checkpoint.getCheckpointID()); // Now add the new one. If it fails, we don't want to loose existing data. checkpointsInZooKeeper.addAndLock(path, checkpoint); completedCheckpoints.addLast(checkpoint); // Everything worked, let's remove a previous checkpoint if necessary. while (completedCheckpoints.size() > maxNumberOfCheckpointsToRetain) { final CompletedCheckpoint completedCheckpoint = completedCheckpoints.removeFirst(); tryRemoveCompletedCheckpoint(completedCheckpoint, CompletedCheckpoint::discardOnSubsume); } LOG.debug("Added {} to {}.", checkpoint, path); }
@Override public void shutdown(JobStatus jobStatus) throws Exception { if (jobStatus.isGloballyTerminalState()) { LOG.info("Shutting down"); for (CompletedCheckpoint checkpoint : completedCheckpoints) { tryRemoveCompletedCheckpoint( checkpoint, completedCheckpoint -> completedCheckpoint.discardOnShutdown(jobStatus)); } completedCheckpoints.clear(); String path = "/" + client.getNamespace(); LOG.info("Removing {} from ZooKeeper", path); ZKPaths.deleteChildren(client.getZookeeperClient().getZooKeeper(), path, true); } else { LOG.info("Suspending"); // Clear the local handles, but don't remove any state completedCheckpoints.clear(); // Release the state handle locks in ZooKeeper such that they can be deleted checkpointsInZooKeeper.releaseAll(); } }
@Override public void shutdown(JobStatus jobStatus) throws Exception { if (jobStatus.isGloballyTerminalState()) { LOG.info("Shutting down"); for (CompletedCheckpoint checkpoint : completedCheckpoints) { tryRemoveCompletedCheckpoint( checkpoint, completedCheckpoint -> completedCheckpoint.discardOnShutdown(jobStatus)); } completedCheckpoints.clear(); String path = "/" + client.getNamespace(); LOG.info("Removing {} from ZooKeeper", path); ZKPaths.deleteChildren(client.getZookeeperClient().getZooKeeper(), path, true); } else { LOG.info("Suspending"); // Clear the local handles, but don't remove any state completedCheckpoints.clear(); // Release the state handle locks in ZooKeeper such that they can be deleted checkpointsInZooKeeper.releaseAll(); } }