if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) { throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() + " can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() + ") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() + "). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism." if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) { executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism()); executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism()); } else { operatorState.getMaxParallelism() + ") with which the latest " + "checkpoint of the execution job vertex " + executionJobVertex + " has been taken and the current maximum parallelism (" +
if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) { throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() + " can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() + ") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() + "). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism." if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) { executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism()); executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism()); } else { operatorState.getMaxParallelism() + ") with which the latest " + "checkpoint of the execution job vertex " + executionJobVertex + " has been taken and the current maximum parallelism (" +
if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) { throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() + " can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() + ") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() + "). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism." if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) { executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism()); executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism()); } else { operatorState.getMaxParallelism() + ") with which the latest " + "checkpoint of the execution job vertex " + executionJobVertex + " has been taken and the current maximum parallelism (" +
dos.writeInt(operatorState.getMaxParallelism()); dos.writeInt(1);
dos.writeInt(operatorState.getMaxParallelism()); dos.writeInt(1);
dos.writeInt(operatorState.getMaxParallelism()); dos.writeInt(1);
dos.writeInt(operatorState.getMaxParallelism()); dos.writeInt(1);
dos.writeInt(operatorState.getMaxParallelism()); dos.writeInt(1);
/** * Read keyed states using the provided reader for further processing * * @return The DataSet containing the deseralized state elements */ public <K, V, O> DataSet<O> readKeyedStates(KeyedStateReader<K, V, O> reader) throws Exception { readKeyedStates(); KeyedBackendSerializationProxy<?> proxy = StateMetadataUtils.getKeyedBackendSerializationProxy(opState) .orElseThrow(() -> new IllegalStateException("Cannot read state of a stateless operator.")); reader.configure(opState.getMaxParallelism(), getKeySerializer(proxy), StateMetadataUtils.getSerializer(proxy, reader.getStateName()) .orElseThrow(() -> new IllegalArgumentException("Cannot find state " + reader.getStateName()))); DataSet<O> parsedState = allKeyedStateRows.flatMap(reader); readStates.add(reader.getStateName()); return parsedState; }
if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) { throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() + " can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() + ") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() + "). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism." if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) { executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism()); executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism()); } else { operatorState.getMaxParallelism() + ") with which the latest " + "checkpoint of the execution job vertex " + executionJobVertex + " has been taken and the current maximum parallelism (" +
/** * Adds a dataset of K-V pairs to the keyed state of the operator. This * operation assumes that a state with the same name is already defined and * the metadata is reused. * <p> * To define new states see * {@link #createNewValueState(String, DataSet, TypeSerializer)} * <p> * Keep in mind that any state rows for the same name already added (through * {@link #addKeyedStateRows(DataSet)}) will not be overwritten. * * @param stateName * @param newState */ @SuppressWarnings({ "rawtypes", "unchecked" }) public <K, V> void addValueState(String stateName, DataSet<Tuple2<K, V>> newState) { TypeSerializer<V> valueSerializer = (TypeSerializer<V>) (TypeSerializer) StateMetadataUtils .getSerializer(proxy, stateName) .orElseThrow( () -> new IllegalArgumentException("Cannot find state " + stateName)); if (StateMetadataUtils.isTtlState(valueSerializer)) { throw new RuntimeException("Writing of TTL states is not supported at the moment."); } addKeyedStateRows(newState .map(new ValueStateToKeyedStateRow<K, V>(stateName, getKeySerializer(), valueSerializer, baseOpState.getMaxParallelism()))); }
/** * Defines/redefines a value state with the given name and type. This can be * used to create new states of an operator or change the type of an already * existing state. * <p> * When redefining a pre-existing state make sure you haven't added that as * keyed state rows before. * * @param stateName * @param newState * @param valueSerializer */ public <K, V> void createNewValueState(String stateName, DataSet<Tuple2<K, V>> newState, TypeSerializer<V> valueSerializer) { metaSnapshots.put(stateName, new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, stateName, VoidNamespaceSerializer.INSTANCE, valueSerializer).snapshot()); updateProxy(); addKeyedStateRows(newState .map(new ValueStateToKeyedStateRow<K, V>(stateName, getKeySerializer(), valueSerializer, baseOpState.getMaxParallelism()))); }
if (executionJobVertex.getMaxParallelism() == operatorState.getMaxParallelism() || !executionJobVertex.isMaxParallelismConfigured()) { operatorStates.put(operatorState.getOperatorID(), operatorState); savepoint, operatorState.getOperatorID(), operatorState.getMaxParallelism(), executionJobVertex.getMaxParallelism());
int maxParallelism = baseOpState.getMaxParallelism(); int parallelism = baseOpState.getParallelism(); Path outDir = makeOutputDir();
if (executionJobVertex.getMaxParallelism() == operatorState.getMaxParallelism() || !executionJobVertex.isMaxParallelismConfigured()) { operatorStates.put(operatorState.getOperatorID(), operatorState); checkpointMetadata, operatorState.getOperatorID(), operatorState.getMaxParallelism(), executionJobVertex.getMaxParallelism());
if (executionJobVertex.getMaxParallelism() == operatorState.getMaxParallelism() || !executionJobVertex.isMaxParallelismConfigured()) { operatorStates.put(operatorState.getOperatorID(), operatorState); checkpointMetadata, operatorState.getOperatorID(), operatorState.getMaxParallelism(), executionJobVertex.getMaxParallelism());
if (executionJobVertex.getMaxParallelism() == operatorState.getMaxParallelism() || !executionJobVertex.isMaxParallelismConfigured()) { operatorStates.put(operatorState.getOperatorID(), operatorState); checkpointMetadata, operatorState.getOperatorID(), operatorState.getMaxParallelism(), executionJobVertex.getMaxParallelism());