private void notifyCheckpoint(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception { if (toNotifyOnCheckpoint != null) { CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp); CheckpointMetrics checkpointMetrics = new CheckpointMetrics() .setBytesBufferedInAlignment(0L) .setAlignmentDurationNanos(0L); toNotifyOnCheckpoint.triggerCheckpointOnBarrier(checkpointMetaData, checkpointOptions, checkpointMetrics); } }
@Override public void tryHandleCheckpointException( CheckpointMetaData checkpointMetaData, Exception exception) throws Exception { environment.declineCheckpoint(checkpointMetaData.getCheckpointId(), exception); } }
@Override public void triggerCheckpoint(long checkpointId) throws FlinkException { // TODO - we need to see how to derive those. We should probably not encode this in the // TODO - source's trigger message, but do a handshake in this task between the trigger // TODO - message from the master, and the source's trigger notification final CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation(); final long timestamp = System.currentTimeMillis(); final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp); try { SourceStreamTask.super.triggerCheckpoint(checkpointMetaData, checkpointOptions); } catch (RuntimeException | FlinkException e) { throw e; } catch (Exception e) { throw new FlinkException(e.getMessage(), e); } } };
@Override public boolean triggerCheckpoint(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions) { final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot(); checkpointStateHandles.putSubtaskStateByOperatorID( OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()), new OperatorSubtaskState()); getEnvironment().acknowledgeCheckpoint( checkpointMetaData.getCheckpointId(), new CheckpointMetrics(), checkpointStateHandles); triggerCheckpointLatch.countDown(); return true; }
@Override protected void run() throws Exception { triggerCheckpointOnBarrier( new CheckpointMetaData( 11L, System.currentTimeMillis()), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetrics()); while (isRunning()) { Thread.sleep(1L); } }
private void checkpointState( CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, CheckpointMetrics checkpointMetrics) throws Exception { CheckpointStreamFactory storage = checkpointStorage.resolveCheckpointStorageLocation( checkpointMetaData.getCheckpointId(), checkpointOptions.getTargetLocation()); CheckpointingOperation checkpointingOperation = new CheckpointingOperation( this, checkpointMetaData, checkpointOptions, storage, checkpointMetrics); checkpointingOperation.executeCheckpointing(); }
@Override public void triggerCheckpointOnBarrier( CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, CheckpointMetrics checkpointMetrics) throws Exception { assertTrue("wrong checkpoint id", nextExpectedCheckpointId == -1L || nextExpectedCheckpointId == checkpointMetaData.getCheckpointId()); assertTrue(checkpointMetaData.getTimestamp() > 0); assertTrue(checkpointMetrics.getBytesBufferedInAlignment() >= 0); assertTrue(checkpointMetrics.getAlignmentDurationNanos() >= 0); nextExpectedCheckpointId++; lastReportedBytesBufferedInAlignment = checkpointMetrics.getBytesBufferedInAlignment(); }
@SuppressWarnings("deprecation") private void checkpointStreamOperator(StreamOperator<?> op) throws Exception { if (null != op) { OperatorSnapshotFutures snapshotInProgress = op.snapshotState( checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions, storageLocation); operatorSnapshotsInProgress.put(op.getOperatorID(), snapshotInProgress); } }
@Override public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) { taskStateManager.reportTaskStateSnapshots( new CheckpointMetaData(checkpointId, 0L), checkpointMetrics, subtaskState, null); }
@Override public CheckpointRecoveryFactory getCheckpointRecoveryFactory() { enter(); try { return new StandaloneCheckpointRecoveryFactory(); } finally { exit(); } }
protected OperatorStateBackend operatorStateBackend( String operatorIdentifierText, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates, CloseableRegistry backendCloseableRegistry) throws Exception { String logDescription = "operator state backend for " + operatorIdentifierText; BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> backendRestorer = new BackendRestorerProcedure<>( () -> stateBackend.createOperatorStateBackend(environment, operatorIdentifierText), backendCloseableRegistry, logDescription); return backendRestorer.createAndRestore( prioritizedOperatorSubtaskStates.getPrioritizedManagedOperatorState()); }
private void notifyCheckpoint(CheckpointBarrier checkpointBarrier) throws Exception { if (toNotifyOnCheckpoint != null) { CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointBarrier.getId(), checkpointBarrier.getTimestamp()); long bytesBuffered = currentBuffered != null ? currentBuffered.size() : 0L; CheckpointMetrics checkpointMetrics = new CheckpointMetrics() .setBytesBufferedInAlignment(bytesBuffered) .setAlignmentDurationNanos(latestAlignmentDurationNanos); toNotifyOnCheckpoint.triggerCheckpointOnBarrier( checkpointMetaData, checkpointBarrier.getCheckpointOptions(), checkpointMetrics); } }
@Override public Boolean call() throws Exception { for (int i = 0; i < numCheckpoints; i++) { long currentCheckpointId = checkpointId.getAndIncrement(); CheckpointMetaData checkpointMetaData = new CheckpointMetaData(currentCheckpointId, 0L); sourceTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation()); Thread.sleep(checkpointInterval); } return true; } }
private void logFailedCleanupAttempt() { LOG.debug("{} - asynchronous checkpointing operation for checkpoint {} has " + "already been completed. Thus, the state handles are not cleaned up.", owner.getName(), checkpointMetaData.getCheckpointId()); } }
@Override public void triggerCheckpointOnBarrier(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions, CheckpointMetrics checkpointMetrics) throws Exception { assertTrue("More checkpoints than expected", i < checkpointIDs.length); final long expectedId = checkpointIDs[i++]; if (expectedId >= 0) { assertEquals("wrong checkpoint id", expectedId, checkpointMetaData.getCheckpointId()); assertTrue(checkpointMetaData.getTimestamp() > 0); } else { fail("got 'triggerCheckpointOnBarrier()' when expecting an 'abortCheckpointOnBarrier()'"); } }
private void triggerCheckpoint( OneInputStreamTaskTestHarness<String, String> testHarness, OneInputStreamTask<String, String> streamTask) throws Exception { long checkpointId = 1L; CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, 1L); testHarness.taskStateManager.setWaitForReportLatch(new OneShotLatch()); while (!streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation())) {} testHarness.taskStateManager.getWaitForReportLatch().await(); long reportedCheckpointId = testHarness.taskStateManager.getReportedCheckpointId(); assertEquals(checkpointId, reportedCheckpointId); }
@Override public boolean matches(Object o) { return o != null && o.getClass() == CheckpointMetaData.class && ((CheckpointMetaData) o).getCheckpointId() == checkpointId; }
@Override public boolean matches(Object o) { return o != null && o.getClass() == CheckpointMetaData.class && ((CheckpointMetaData) o).getCheckpointId() == checkpointId; }
@Override public void acknowledgeCheckpoint( JobID lJobID, ExecutionAttemptID lExecutionAttemptID, long lCheckpointId, CheckpointMetrics lCheckpointMetrics, TaskStateSnapshot lSubtaskState) { Assert.assertEquals(jobID, lJobID); Assert.assertEquals(executionAttemptID, lExecutionAttemptID); Assert.assertEquals(checkpointMetaData.getCheckpointId(), lCheckpointId); Assert.assertEquals(checkpointMetrics, lCheckpointMetrics); jmReported.set(true); } };