public OperatorSnapshotFinalizer( @Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException { SnapshotResult<KeyedStateHandle> keyedManaged = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture()); SnapshotResult<KeyedStateHandle> keyedRaw = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture()); SnapshotResult<OperatorStateHandle> operatorManaged = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture()); SnapshotResult<OperatorStateHandle> operatorRaw = FutureUtils.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture()); jobManagerOwnedState = new OperatorSubtaskState( operatorManaged.getJobManagerOwnedSnapshot(), operatorRaw.getJobManagerOwnedSnapshot(), keyedManaged.getJobManagerOwnedSnapshot(), keyedRaw.getJobManagerOwnedSnapshot() ); taskLocalState = new OperatorSubtaskState( operatorManaged.getTaskLocalSnapshot(), operatorRaw.getTaskLocalSnapshot(), keyedManaged.getTaskLocalSnapshot(), keyedRaw.getTaskLocalSnapshot() ); }
return new OperatorSubtaskState( new StateObjectCollection<>(managedOperatorState), new StateObjectCollection<>(rawOperatorState),
return new OperatorSubtaskState( new StateObjectCollection<>(mergedManagedOperatorState), new StateObjectCollection<>(mergedRawOperatorState),
@Test public void testInitializeAfterOpenning() throws Throwable { expectedException.expect(IllegalStateException.class); expectedException.expectMessage(containsString("TestHarness has already been initialized.")); AbstractStreamOperatorTestHarness<Integer> result; result = new AbstractStreamOperatorTestHarness<>( new AbstractStreamOperator<Integer>() { }, 1, 1, 0); result.setup(); result.open(); result.initializeState(new OperatorSubtaskState()); } }
@Override public boolean triggerCheckpoint(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions) { final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot(); checkpointStateHandles.putSubtaskStateByOperatorID( OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()), new OperatorSubtaskState()); getEnvironment().acknowledgeCheckpoint( checkpointMetaData.getCheckpointId(), new CheckpointMetrics(), checkpointStateHandles); triggerCheckpointLatch.countDown(); return true; }
OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState( new StateObjectCollection<>(operatorStateBackend), new StateObjectCollection<>(operatorStateStream),
OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState( new OperatorStreamStateHandle( Collections.singletonMap(
numSubtasks).get(subtaskIndex); OperatorSubtaskState processedJmOpSubtaskState = new OperatorSubtaskState( new StateObjectCollection<>(nullToEmptyCollection(localManagedOperatorState)), new StateObjectCollection<>(nullToEmptyCollection(localRawOperatorState)),
OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState( StateObjectCollection.empty(), new StateObjectCollection<>(operatorStateHandles),
public static OperatorSubtaskState operatorSubtaskStateFrom( OperatorInstanceID instanceID, Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState, Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState, Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) { if (!subManagedOperatorState.containsKey(instanceID) && !subRawOperatorState.containsKey(instanceID) && !subManagedKeyedState.containsKey(instanceID) && !subRawKeyedState.containsKey(instanceID)) { return new OperatorSubtaskState(); } if (!subManagedKeyedState.containsKey(instanceID)) { checkState(!subRawKeyedState.containsKey(instanceID)); } return new OperatorSubtaskState( new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList()))); }
public static OperatorSubtaskState operatorSubtaskStateFrom( OperatorInstanceID instanceID, Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState, Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState, Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) { if (!subManagedOperatorState.containsKey(instanceID) && !subRawOperatorState.containsKey(instanceID) && !subManagedKeyedState.containsKey(instanceID) && !subRawKeyedState.containsKey(instanceID)) { return new OperatorSubtaskState(); } if (!subManagedKeyedState.containsKey(instanceID)) { checkState(!subRawKeyedState.containsKey(instanceID)); } return new OperatorSubtaskState( new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList()))); }
public static OperatorSubtaskState operatorSubtaskStateFrom( OperatorInstanceID instanceID, Map<OperatorInstanceID, List<OperatorStateHandle>> subManagedOperatorState, Map<OperatorInstanceID, List<OperatorStateHandle>> subRawOperatorState, Map<OperatorInstanceID, List<KeyedStateHandle>> subManagedKeyedState, Map<OperatorInstanceID, List<KeyedStateHandle>> subRawKeyedState) { if (!subManagedOperatorState.containsKey(instanceID) && !subRawOperatorState.containsKey(instanceID) && !subManagedKeyedState.containsKey(instanceID) && !subRawKeyedState.containsKey(instanceID)) { return new OperatorSubtaskState(); } if (!subManagedKeyedState.containsKey(instanceID)) { checkState(!subRawKeyedState.containsKey(instanceID)); } return new OperatorSubtaskState( new StateObjectCollection<>(subManagedOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawOperatorState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subManagedKeyedState.getOrDefault(instanceID, Collections.emptyList())), new StateObjectCollection<>(subRawKeyedState.getOrDefault(instanceID, Collections.emptyList()))); }
private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException { // Duration field has been removed from SubtaskState long ignoredDuration = dis.readLong(); int len = dis.readInt(); StreamStateHandle nonPartitionableState = len == 0 ? null : deserializeStreamStateHandle(dis); len = dis.readInt(); OperatorStateHandle operatorStateBackend = len == 0 ? null : deserializeOperatorStateHandle(dis); len = dis.readInt(); OperatorStateHandle operatorStateStream = len == 0 ? null : deserializeOperatorStateHandle(dis); KeyedStateHandle keyedStateBackend = deserializeKeyedStateHandle(dis); KeyedStateHandle keyedStateStream = deserializeKeyedStateHandle(dis); return new OperatorSubtaskState( nonPartitionableState, operatorStateBackend, operatorStateStream, keyedStateBackend, keyedStateStream); }
private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException { // Duration field has been removed from SubtaskState, do not remove long ignoredDuration = dis.readLong(); // for compatibility, do not remove int len = dis.readInt(); if (SavepointSerializers.FAIL_WHEN_LEGACY_STATE_DETECTED) { Preconditions.checkState(len == 0, "Legacy state (from Flink <= 1.1, created through the 'Checkpointed' interface) is " + "no longer supported starting from Flink 1.4. Please rewrite your job to use " + "'CheckpointedFunction' instead!"); } else { for (int i = 0; i < len; ++i) { // absorb bytes from stream and ignore result deserializeStreamStateHandle(dis); } } len = dis.readInt(); OperatorStateHandle operatorStateBackend = len == 0 ? null : deserializeOperatorStateHandle(dis); len = dis.readInt(); OperatorStateHandle operatorStateStream = len == 0 ? null : deserializeOperatorStateHandle(dis); KeyedStateHandle keyedStateBackend = deserializeKeyedStateHandle(dis); KeyedStateHandle keyedStateStream = deserializeKeyedStateHandle(dis); return new OperatorSubtaskState( operatorStateBackend, operatorStateStream, keyedStateBackend, keyedStateStream); }
private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException { // Duration field has been removed from SubtaskState, do not remove long ignoredDuration = dis.readLong(); // for compatibility, do not remove int len = dis.readInt(); if (SavepointSerializers.FAIL_WHEN_LEGACY_STATE_DETECTED) { Preconditions.checkState(len == 0, "Legacy state (from Flink <= 1.1, created through the 'Checkpointed' interface) is " + "no longer supported starting from Flink 1.4. Please rewrite your job to use " + "'CheckpointedFunction' instead!"); } else { for (int i = 0; i < len; ++i) { // absorb bytes from stream and ignore result deserializeStreamStateHandle(dis); } } len = dis.readInt(); OperatorStateHandle operatorStateBackend = len == 0 ? null : deserializeOperatorStateHandle(dis); len = dis.readInt(); OperatorStateHandle operatorStateStream = len == 0 ? null : deserializeOperatorStateHandle(dis); KeyedStateHandle keyedStateBackend = deserializeKeyedStateBackend(dis); KeyedStateHandle keyedStateStream = deserializeRawKeyedStateHandle(dis); return new OperatorSubtaskState( operatorStateBackend, operatorStateStream, keyedStateBackend, keyedStateStream); }
private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException { // Duration field has been removed from SubtaskState, do not remove long ignoredDuration = dis.readLong(); // for compatibility, do not remove int len = dis.readInt(); if (SavepointSerializers.FAIL_WHEN_LEGACY_STATE_DETECTED) { Preconditions.checkState(len == 0, "Legacy state (from Flink <= 1.1, created through the 'Checkpointed' interface) is " + "no longer supported starting from Flink 1.4. Please rewrite your job to use " + "'CheckpointedFunction' instead!"); } else { for (int i = 0; i < len; ++i) { // absorb bytes from stream and ignore result deserializeStreamStateHandle(dis); } } len = dis.readInt(); OperatorStateHandle operatorStateBackend = len == 0 ? null : deserializeOperatorStateHandle(dis); len = dis.readInt(); OperatorStateHandle operatorStateStream = len == 0 ? null : deserializeOperatorStateHandle(dis); KeyedStateHandle keyedStateBackend = deserializeKeyedStateHandle(dis); KeyedStateHandle keyedStateStream = deserializeKeyedStateHandle(dis); return new OperatorSubtaskState( operatorStateBackend, operatorStateStream, keyedStateBackend, keyedStateStream); }
private static OperatorSubtaskState deserializeSubtaskState(DataInputStream dis) throws IOException { // Duration field has been removed from SubtaskState, do not remove long ignoredDuration = dis.readLong(); // for compatibility, do not remove int len = dis.readInt(); if (SavepointSerializers.FAIL_WHEN_LEGACY_STATE_DETECTED) { Preconditions.checkState(len == 0, "Legacy state (from Flink <= 1.1, created through the 'Checkpointed' interface) is " + "no longer supported starting from Flink 1.4. Please rewrite your job to use " + "'CheckpointedFunction' instead!"); } else { for (int i = 0; i < len; ++i) { // absorb bytes from stream and ignore result deserializeStreamStateHandle(dis); } } len = dis.readInt(); OperatorStateHandle operatorStateBackend = len == 0 ? null : deserializeOperatorStateHandle(dis); len = dis.readInt(); OperatorStateHandle operatorStateStream = len == 0 ? null : deserializeOperatorStateHandle(dis); KeyedStateHandle keyedStateBackend = deserializeKeyedStateHandle(dis); KeyedStateHandle keyedStateStream = deserializeKeyedStateHandle(dis); return new OperatorSubtaskState( operatorStateBackend, operatorStateStream, keyedStateBackend, keyedStateStream); }
new OperatorSubtaskState( opHandle, subtaskState.getRawOperatorState(),
public OperatorSnapshotFinalizer( @Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException { SnapshotResult<KeyedStateHandle> keyedManaged = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture()); SnapshotResult<KeyedStateHandle> keyedRaw = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture()); SnapshotResult<OperatorStateHandle> operatorManaged = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture()); SnapshotResult<OperatorStateHandle> operatorRaw = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture()); jobManagerOwnedState = new OperatorSubtaskState( operatorManaged.getJobManagerOwnedSnapshot(), operatorRaw.getJobManagerOwnedSnapshot(), keyedManaged.getJobManagerOwnedSnapshot(), keyedRaw.getJobManagerOwnedSnapshot() ); taskLocalState = new OperatorSubtaskState( operatorManaged.getTaskLocalSnapshot(), operatorRaw.getTaskLocalSnapshot(), keyedManaged.getTaskLocalSnapshot(), keyedRaw.getTaskLocalSnapshot() ); }
public OperatorSnapshotFinalizer( @Nonnull OperatorSnapshotFutures snapshotFutures) throws ExecutionException, InterruptedException { SnapshotResult<KeyedStateHandle> keyedManaged = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getKeyedStateManagedFuture()); SnapshotResult<KeyedStateHandle> keyedRaw = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getKeyedStateRawFuture()); SnapshotResult<OperatorStateHandle> operatorManaged = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getOperatorStateManagedFuture()); SnapshotResult<OperatorStateHandle> operatorRaw = FutureUtil.runIfNotDoneAndGet(snapshotFutures.getOperatorStateRawFuture()); jobManagerOwnedState = new OperatorSubtaskState( operatorManaged.getJobManagerOwnedSnapshot(), operatorRaw.getJobManagerOwnedSnapshot(), keyedManaged.getJobManagerOwnedSnapshot(), keyedRaw.getJobManagerOwnedSnapshot() ); taskLocalState = new OperatorSubtaskState( operatorManaged.getTaskLocalSnapshot(), operatorRaw.getTaskLocalSnapshot(), keyedManaged.getTaskLocalSnapshot(), keyedRaw.getTaskLocalSnapshot() ); }