@Override public void initializeState(FunctionInitializationContext context) throws Exception { readyToFail = false; if (context.isRestored()) { isRunning = false; } else { isRunning = true; OperatorStateStore operatorStateStore = context.getOperatorStateStore(); for (int i = 0; i < numListStates; ++i) { ListStateDescriptor<String> listStateDescriptor = new ListStateDescriptor<>("test-list-state-" + i, String.class); ListState<String> unionListState = operatorStateStore.getUnionListState(listStateDescriptor); for (int j = 0; j < numPartitionsPerListState; ++j) { unionListState.add(String.valueOf(j)); } } } }
@Override public void initialize(FunctionInitializationContext initializationContext) throws Exception { listOperatorState = initializationContext.getOperatorStateStore().getListState(listStateDescriptor); listKeyedState = initializationContext.getKeyedStateStore().getListState(listStateDescriptor); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { unionListState = context.getOperatorStateStore().getListState( STATE_DESCRIPTOR); }
ValueStateDescriptor<String> stateDescriptor = new ValueStateDescriptor<>("state", String.class); valueState = functionInitializationContext.getKeyedStateStore().getState(stateDescriptor); functionInitializationContext.getOperatorStateStore().getUnionListState(mapperInfoStateDescriptor); if (functionInitializationContext.isRestored()) { Iterable<MapperSchedulingAndFailureInfo> iterable = schedulingAndFailureState.get(); String taskNameWithSubtasks = runtimeContext.getTaskNameWithSubtasks();
@Override public void initializeState(FunctionInitializationContext context) { if (!context.isRestored()) { int updatedValue = allowedInitializeCallsWithoutRestore.decrementAndGet(); if (updatedValue < 0) { illegalRestores.getAndIncrement(); throw new RuntimeException("We are not allowed any more restores."); } } else { if (!afterMessWithZooKeeper.get()) { illegalRestores.getAndIncrement(); } else if (successfulRestores.getAndIncrement() > 0) { // already saw the one allowed successful restore illegalRestores.getAndIncrement(); } } } }
@Override public void initialize(FunctionInitializationContext initializationContext) { valueState = initializationContext.getKeyedStateStore().getState(valueStateDescriptor); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { unionListState = context.getOperatorStateStore().getUnionListState( STATE_DESCRIPTOR); }
@Override @Nonnull State createState(@Nonnull FunctionInitializationContext context) { return context.getKeyedStateStore().getReducingState(stateDesc); }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "from-elements-state", IntSerializer.INSTANCE ) ); if (context.isRestored()) { List<Integer> retrievedStates = new ArrayList<>(); for (Integer entry : this.checkpointedState.get()) { retrievedStates.add(entry); } // given that the parallelism of the function is 1, we can only have 1 state Preconditions.checkArgument(retrievedStates.size() == 1, getClass().getSimpleName() + " retrieved invalid state."); this.numElementsToSkip = retrievedStates.get(0); } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { ListStateDescriptor<Long> currentKeyDescriptor = new ListStateDescriptor<>("currentKey", Long.class); sourceCurrentKeyState = context.getOperatorStateStore().getListState(currentKeyDescriptor); currentKey = getRuntimeContext().getIndexOfThisSubtask(); Iterable<Long> iterable = sourceCurrentKeyState.get(); if (iterable != null) { Iterator<Long> iterator = iterable.iterator(); if (iterator.hasNext()) { currentKey = iterator.next(); Preconditions.checkState(!iterator.hasNext()); } } } }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { pojoClass = getRuntimeContext().getUserCodeClassLoader().loadClass(POJO_NAME); fieldA = pojoClass.getDeclaredField("a"); fieldA.setAccessible(true); if (hasBField) { fieldB = pojoClass.getDeclaredField("b"); fieldB.setAccessible(true); } if (keyed) { keyedValueState = context.getKeyedStateStore().getState( new ValueStateDescriptor<>("keyedValueState", (Class<Object>) pojoClass)); keyedMapState = context.getKeyedStateStore().getMapState( new MapStateDescriptor<>("keyedMapState", (Class<Object>) pojoClass, (Class<Object>) pojoClass)); keyedListState = context.getKeyedStateStore().getListState( new ListStateDescriptor<>("keyedListState", (Class<Object>) pojoClass)); ReduceFunction<Object> reduceFunction = new FirstValueReducer<>(); keyedReducingState = context.getKeyedStateStore().getReducingState( new ReducingStateDescriptor<>("keyedReducingState", reduceFunction, (Class<Object>) pojoClass)); } else { partitionableListState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>("partitionableListState", (Class<Object>) pojoClass)); unionListState = context.getOperatorStateStore().getUnionListState( new ListStateDescriptor<>("unionListState", (Class<Object>) pojoClass)); } } }
@Override @Nonnull State createState(@Nonnull FunctionInitializationContext context) { return context.getKeyedStateStore().getListState(stateDesc); }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkState(this.checkpointedState == null, "The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context .getOperatorStateStore() .getSerializableListState("message-acknowledging-source-state"); this.idsForCurrentCheckpoint = new HashSet<>(64); this.pendingCheckpoints = new ArrayDeque<>(); this.idsProcessedButNotAcknowledged = new HashSet<>(); if (context.isRestored()) { LOG.info("Restoring state for the {}.", getClass().getSimpleName()); List<SerializedCheckpointData[]> retrievedStates = new ArrayList<>(); for (SerializedCheckpointData[] entry : this.checkpointedState.get()) { retrievedStates.add(entry); } // given that the parallelism of the function is 1, we can only have at most 1 state Preconditions.checkArgument(retrievedStates.size() == 1, getClass().getSimpleName() + " retrieved invalid state."); pendingCheckpoints = SerializedCheckpointData.toDeque(retrievedStates.get(0), idSerializer); // build a set which contains all processed ids. It may be used to check if we have // already processed an incoming message. for (Tuple2<Long, Set<UId>> checkpoint : pendingCheckpoints) { idsProcessedButNotAcknowledged.addAll(checkpoint.f1); } } else { LOG.info("No state to restore for the {}.", getClass().getSimpleName()); } }
nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState( NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR); transactionalIdsGenerator = new TransactionalIdsGenerator(
@Override @Nonnull State createState(@Nonnull FunctionInitializationContext context) { return context.getKeyedStateStore().getFoldingState(stateDesc); }
"The " + getClass().getSimpleName() + " has already been initialized."); this.checkpointedState = context.getOperatorStateStore().getListState( new ListStateDescriptor<>( "file-monitoring-state", if (context.isRestored()) { LOG.info("Restoring state for the {}.", getClass().getSimpleName());
nextTransactionalIdHintState = context.getOperatorStateStore().getUnionListState( NEXT_TRANSACTIONAL_ID_HINT_DESCRIPTOR); transactionalIdsGenerator = new TransactionalIdsGenerator(
@Override @Nonnull State createState(FunctionInitializationContext context) { return context.getKeyedStateStore().getState(stateDesc); }
@Override public void initializeState(FunctionInitializationContext context) throws Exception { Preconditions.checkArgument(this.restoredBucketStates == null, "The " + getClass().getSimpleName() + " has already been initialized."); try { initFileSystem(); } catch (IOException e) { LOG.error("Error while creating FileSystem when initializing the state of the RollingSink.", e); throw new RuntimeException("Error while creating FileSystem when initializing the state of the RollingSink.", e); } if (this.refTruncate == null) { this.refTruncate = reflectTruncate(fs); } OperatorStateStore stateStore = context.getOperatorStateStore(); restoredBucketStates = stateStore.getSerializableListState("rolling-states"); int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask(); if (context.isRestored()) { LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex); for (BucketState bucketState : restoredBucketStates.get()) { handleRestoredBucketState(bucketState); } if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx= {}) restored {}", getClass().getSimpleName(), subtaskIndex, bucketState); } } else { LOG.info("No state to restore for the {} (taskIdx= {}).", getClass().getSimpleName(), subtaskIndex); } }
@SuppressWarnings("unchecked") @Override public void initializeState(FunctionInitializationContext fic) throws Exception { state = (ListState) fic.getOperatorStateStore() .getSerializableListState(storageName); writer = Iterables.getOnlyElement(state.get(), this.writer); } }