@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context .getOperatorStateStore() .getSerializableListState("message-acknowledging-source-state"); this.idsForCurrentCheckpoint = new HashSet<>(64); this.pendingCheckpoints = new ArrayDeque<>(); this.idsProcessedButNotAcknowledged = new HashSet<>(); if (context.isRestored()) { LOG.info("Restoring state for the {}.", getClass().getSimpleName()); List<SerializedCheckpointData[]> retrievedStates = new ArrayList<>(); for (SerializedCheckpointData[] entry : this.checkpointedState.get()) { retrievedStates.add(entry); } // given that the parallelism of the function is 1, we can only have at most 1 state Preconditions.checkArgument(retrievedStates.size() == 1, getClass().getSimpleName() + " retrieved invalid state."); pendingCheckpoints = SerializedCheckpointData.toDeque(retrievedStates.get(0), idSerializer); // build a set which contains all processed ids. It may be used to check if we have // already processed an incoming message. for (Tuple2<Long, Set<UId>> checkpoint : pendingCheckpoints) { idsProcessedButNotAcknowledged.addAll(checkpoint.f1); } } else { LOG.info("No state to restore for the {}.", getClass().getSimpleName()); } }
if (context.isRestored()) { LOG.info("Restoring state for the {}.", getClass().getSimpleName());
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "from-elements-state", IntSerializer.INSTANCE ) ); if (context.isRestored()) { List<Integer> retrievedStates = new ArrayList<>(); for (Integer entry : this.checkpointedState.get()) { retrievedStates.add(entry); } // given that the parallelism of the function is 1, we can only have 1 state Preconditions.checkArgument(retrievedStates.size() == 1, getClass().getSimpleName() + " retrieved invalid state."); this.numElementsToSkip = retrievedStates.get(0); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkArgument(this.restoredBucketStates == null, "The " + getClass().getSimpleName() + " has already been initialized."); try { initFileSystem(); } catch (IOException e) { LOG.error("Error while creating FileSystem when initializing the state of the RollingSink.", e); throw new RuntimeException("Error while creating FileSystem when initializing the state of the RollingSink.", e); } if (this.refTruncate == null) { this.refTruncate = reflectTruncate(fs); } OperatorStateStore stateStore = context.getOperatorStateStore(); restoredBucketStates = stateStore.getSerializableListState("rolling-states"); int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex); for (BucketState bucketState : restoredBucketStates.get()) { handleRestoredBucketState(bucketState); } if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) restored {}", getClass().getSimpleName(), subtaskIndex, bucketState); } } else { LOG.info("No state to restore for the {} (taskIdx= {}).", getClass().getSimpleName(), subtaskIndex); } }
@Override public void initializeState(FunctionInitializationContext context) { if (!context.isRestored()) { int updatedValue = allowedInitializeCallsWithoutRestore.decrementAndGet(); if (updatedValue < 0) { illegalRestores.getAndIncrement(); throw new RuntimeException("We are not allowed any more restores."); } } else { if (!afterMessWithZooKeeper.get()) { illegalRestores.getAndIncrement(); } else if (successfulRestores.getAndIncrement() > 0) { // already saw the one allowed successful restore illegalRestores.getAndIncrement(); } } } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkArgument(this.restoredBucketStates == null, "The operator has already been initialized."); try { initFileSystem(); } catch (IOException e) { LOG.error("Error while creating FileSystem when initializing the state of the BucketingSink.", e); throw new RuntimeException("Error while creating FileSystem when initializing the state of the BucketingSink.", e); } if (this.refTruncate == null) { this.refTruncate = reflectTruncate(fs); } OperatorStateStore stateStore = context.getOperatorStateStore(); restoredBucketStates = stateStore.getSerializableListState("bucket-states"); int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex); for (State<T> recoveredState : restoredBucketStates.get()) { handleRestoredBucketState(recoveredState); if (LOG.isDebugEnabled()) { LOG.debug("{} idx {} restored {}", getClass().getSimpleName(), subtaskIndex, recoveredState); } } } else { LOG.info("No state to restore for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { readyToFail = false; if (context.isRestored()) { isRunning = false; } else { isRunning = true; OperatorStateStore operatorStateStore = context.getOperatorStateStore(); for (int i = 0; i < numListStates; ++i) { ListStateDescriptor<String> listStateDescriptor = new ListStateDescriptor<>("test-list-state-" + i, String.class); ListState<String> unionListState = operatorStateStore.getUnionListState(listStateDescriptor); for (int j = 0; j < numPartitionsPerListState; ++j) { unionListState.add(String.valueOf(j)); } } } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "stateful-sequence-source-state", LongSerializer.INSTANCE ) ); this.valuesToEmit = new ArrayDeque<>(); if (context.isRestored()) { // upon restoring for (Long v : this.checkpointedState.get()) { this.valuesToEmit.add(v); } } else { // the first time the job is executed final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks(); final int taskIdx = getRuntimeContext().getIndexOfThisSubtask(); final long congruence = start + taskIdx; long totalNoOfElements = Math.abs(end - start + 1); final int baseSize = safeDivide(totalNoOfElements, stepSize); final int toCollect = (totalNoOfElements % stepSize > taskIdx) ? baseSize + 1 : baseSize; for (long collected = 0; collected < toCollect; collected++) { this.valuesToEmit.add(collected * stepSize + congruence); } } }
@Override public final void initializeState(FunctionInitializationContext context) throws Exception { OperatorStateStore stateStore = context.getOperatorStateStore(); ListState<Tuple2<KafkaTopicPartition, Long>> oldRoundRobinListState = stateStore.getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME); this.unionOffsetStates = stateStore.getUnionListState(new ListStateDescriptor<>( OFFSETS_STATE_NAME, TypeInformation.of(new TypeHint<Tuple2<KafkaTopicPartition, Long>>() {}))); if (context.isRestored() && !restoredFromOldState) { restoredState = new TreeMap<>(new KafkaTopicPartition.Comparator()); // migrate from 1.2 state, if there is any for (Tuple2<KafkaTopicPartition, Long> kafkaOffset : oldRoundRobinListState.get()) { restoredFromOldState = true; unionOffsetStates.add(kafkaOffset); } oldRoundRobinListState.clear(); if (restoredFromOldState && discoveryIntervalMillis != PARTITION_DISCOVERY_DISABLED) { throw new IllegalArgumentException( "Topic / partition discovery cannot be enabled if the job is restored from a savepoint from Flink 1.2.x."); } // populate actual holder for restored state for (Tuple2<KafkaTopicPartition, Long> kafkaOffset : unionOffsetStates.get()) { restoredState.put(kafkaOffset.f0, kafkaOffset.f1); } LOG.info("Setting restore state in the FlinkKafkaConsumer: {}", restoredState); } else { LOG.info("No restore state for FlinkKafkaConsumer."); } }
.getUnionListState(new ListStateDescriptor<>(UNION_STATE_NAME, IntSerializer.INSTANCE)); if (context.isRestored()) { Integer lastNumSubtasks = lastNumSubtasksBroadcastState.get(LAST_NUM_SUBTASKS_STATE_KEY); Preconditions.checkState(lastNumSubtasks != null);
@Override public void initializeState(FunctionInitializationContext context) throws Exception { final int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask(); this.buckets = bucketsBuilder.createBuckets(subtaskIndex); final OperatorStateStore stateStore = context.getOperatorStateStore(); bucketStates = stateStore.getListState(BUCKET_STATE_DESC); maxPartCountersState = stateStore.getUnionListState(MAX_PART_COUNTER_STATE_DESC); if (context.isRestored()) { buckets.initializeState(bucketStates, maxPartCountersState); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { serializableListState = context .getOperatorStateStore() .getListState(new ListStateDescriptor<>("test-state", IntSerializer.INSTANCE)); if (context.isRestored()) { Iterator<Integer> integers = serializableListState.get().iterator(); int act = integers.next(); Assert.assertEquals(42, act); Assert.assertFalse(integers.hasNext()); wasRestored = true; } }
if (context.isRestored()) { LOG.info("{} - restoring state", name()); for (State<TXN, CONTEXT> operatorState : state.get()) {
keyRanges = new ArrayList<>(); if (context.isRestored()) {
@Override public void initializeState(FunctionInitializationContext context) throws Exception { sumState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>("sumState", Integer.class)); if (context.isRestored()) { for (Integer value : sumState.get()) { runningSum += value; } } } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { canFail = false; position = 0L; isRestored = context.isRestored(); positionState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>("posState", Long.class)); if (isRestored) { for (Long value : positionState.get()) { position += value; } } } }
if (functionInitializationContext.isRestored()) { Iterable<MapperSchedulingAndFailureInfo> iterable = schedulingAndFailureState.get(); String taskNameWithSubtasks = runtimeContext.getTaskNameWithSubtasks();
@Override public void initializeState(FunctionInitializationContext context) throws Exception { ListState<String> unionListState = context.getOperatorStateStore().getUnionListState( CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR); if (context.isRestored()) { assertThat(unionListState.get(), containsInAnyOrder(CheckpointingParallelSourceWithUnionListState.CHECKPOINTED_STRINGS)); getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter()); getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1); } else { throw new RuntimeException( "This source should always be restored because it's only used when restoring from a savepoint."); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { ListState<String> unionListState = context.getOperatorStateStore().getListState( CheckpointingNonParallelSourceWithListState.STATE_DESCRIPTOR); if (context.isRestored()) { assertThat(unionListState.get(), containsInAnyOrder( CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING, CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_1, CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_2, CheckpointingNonParallelSourceWithListState.CHECKPOINTED_STRING_3)); getRuntimeContext().addAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR, new IntCounter()); getRuntimeContext().getAccumulator(SUCCESSFUL_RESTORE_CHECK_ACCUMULATOR).add(1); } else { throw new RuntimeException( "This source should always be restored because it's only used when restoring from a savepoint."); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { if (broadcast) { this.counterPartitions = context .getOperatorStateStore() .getUnionListState(new ListStateDescriptor<>("counter_partitions", IntSerializer.INSTANCE)); } else { this.counterPartitions = context .getOperatorStateStore() .getListState(new ListStateDescriptor<>("counter_partitions", IntSerializer.INSTANCE)); } if (context.isRestored()) { for (int v : counterPartitions.get()) { counter += v; } checkCorrectRestore[getRuntimeContext().getIndexOfThisSubtask()] = counter; } } }