new KeyedBackendSerializationProxy<>(rocksDBKeyedStateBackend.userCodeClassLoader); serializationProxy.read(currentStateHandleInView); rocksDBKeyedStateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; serializationProxy.getStateMetaInfoSnapshots(); currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());
new KeyedBackendSerializationProxy<>(userCodeClassLoader); serializationProxy.read(inView); if (!serializationProxy.getKeySerializerConfigSnapshot() .resolveSchemaCompatibility(keySerializer).isCompatibleAsIs()) { throw new StateMigrationException("The new key serializer must be compatible."); serializationProxy.getStateMetaInfoSnapshots(); keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(), serializationProxy.isUsingKeyGroupCompression()); } finally { if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
public static KeyedBackendSerializationProxy<?> getKeyedBackendSerializationProxy( StreamStateHandle streamStateHandle) { KeyedBackendSerializationProxy<Integer> serializationProxy = new KeyedBackendSerializationProxy<>( StateMetadataUtils.class.getClassLoader()); try (FSDataInputStream is = streamStateHandle.openInputStream()) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(is); serializationProxy.read(iw); return serializationProxy; } catch (IOException e) { throw new RuntimeException(e); } }
private void updateProxy() { if (proxy == null && keySerializer == null) { throw new IllegalStateException( "KeySerializer must be defined when adding state to a previously stateless operator. Use writer.setKeySerializer(...)"); } proxy = new KeyedBackendSerializationProxy<>( getKeySerializer(), new ArrayList<>(metaSnapshots.values()), proxy != null ? proxy.isUsingKeyGroupCompression() : true); }
public static Map<Integer, String> getStateIdMapping(KeyedBackendSerializationProxy<?> proxy) { Map<Integer, String> stateIdMapping = new HashMap<>(); int stateId = 0; for (StateMetaInfoSnapshot snapshot : proxy.getStateMetaInfoSnapshots()) { stateIdMapping.put(stateId, snapshot.getName()); stateId++; } return stateIdMapping; }
private void writeKVStateMetaData() throws IOException { List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> metaInfoSnapshots = new ArrayList<>(stateBackend.kvStateInformation.size()); int kvStateId = 0; for (Map.Entry<String, Tuple2<ColumnFamilyHandle, RegisteredKeyedBackendStateMetaInfo<?, ?>>> column : stateBackend.kvStateInformation.entrySet()) { metaInfoSnapshots.add(column.getValue().f1.snapshot()); //retrieve iterator for this k/v states readOptions = new ReadOptions(); readOptions.setSnapshot(snapshot); kvStateIterators.add( new Tuple2<>(stateBackend.db.newIterator(column.getValue().f0, readOptions), kvStateId)); ++kvStateId; } KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.getKeySerializer(), metaInfoSnapshots); serializationProxy.write(outputView); }
@SuppressWarnings({ "rawtypes", "unchecked" }) private <T> TypeSerializer<T> getKeySerializer() { return proxy != null ? (TypeSerializer) proxy.getKeySerializerConfigSnapshot().restoreSerializer() : (TypeSerializer) keySerializer; }
ByteArrayOutputStream os = new ByteArrayOutputStream(); DataOutputView bow = new DataOutputViewStreamWrapper(os); proxy.write(bow); .reduceGroup(new RocksDBSavepointWriter(maxParallelism, parallelism, HashBiMap.create(StateMetadataUtils.getStateIdMapping(proxy)).inverse(), proxy.isUsingKeyGroupCompression(), outDir, os.toByteArray()));
new KeyedBackendSerializationProxy<>(keySerializer, metaInfoSnapshots);
public static StreamCompressionDecorator getCompressionDecorator(KeyedBackendSerializationProxy<?> proxy) { return proxy.isUsingKeyGroupCompression() ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; }
@SuppressWarnings("unchecked") @Override public void read(DataInputView in) throws IOException { super.read(in); // only starting from version 3, we have the key serializer and its config snapshot written if (getReadVersion() >= 3) { Tuple2<TypeSerializer<?>, TypeSerializerConfigSnapshot> keySerializerAndConfig = TypeSerializerSerializationUtil.readSerializersAndConfigsWithResilience(in, userCodeClassLoader).get(0); this.keySerializer = (TypeSerializer<K>) keySerializerAndConfig.f0; this.keySerializerConfigSnapshot = keySerializerAndConfig.f1; } else { this.keySerializer = TypeSerializerSerializationUtil.tryReadSerializer(in, userCodeClassLoader); this.keySerializerConfigSnapshot = null; } int numKvStates = in.readShort(); stateMetaInfoSnapshots = new ArrayList<>(numKvStates); for (int i = 0; i < numKvStates; i++) { stateMetaInfoSnapshots.add( KeyedBackendStateMetaInfoSnapshotReaderWriters .getReaderForVersion(getReadVersion(), userCodeClassLoader) .readStateMetaInfo(in)); } } }
super.read(in); final int readVersion = getReadVersion(); checkSerializerPresence(keySerializer); for (int i = 0; i < numKvStates; i++) { RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?> snapshot = KeyedBackendStateMetaInfoSnapshotReaderWriters .getReaderForVersion(getReadVersion(), userCodeClassLoader) .readStateMetaInfo(in); checkSerializerPresence(snapshot.getNamespaceSerializer()); checkSerializerPresence(snapshot.getStateSerializer());
@Override public KeyGroupsStateHandle performOperation() throws Exception { long asyncStartTime = System.currentTimeMillis(); CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle(); DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(stream); serializationProxy.write(outView); long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) { int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos); keyGroupRangeOffsets[keyGroupPos] = stream.getPos(); outView.writeInt(keyGroupId); for (Map.Entry<String, StateTable<K, ?, ?>> kvState : stateTables.entrySet()) { outView.writeShort(kVStateToId.get(kvState.getKey())); cowStateStableSnapshots.get(kvState.getValue()).writeMappingsInKeyGroup(outView, keyGroupId); } } final StreamStateHandle streamStateHandle = closeStreamAndGetStateHandle(); if (asynchronousSnapshots) { LOG.info("Heap backend snapshot ({}, asynchronous part) in thread {} took {} ms.", streamFactory, Thread.currentThread(), (System.currentTimeMillis() - asyncStartTime)); } if (streamStateHandle == null) { return null; } KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets); final KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(offsets, streamStateHandle); return keyGroupsStateHandle; } };
private List<RegisteredKeyedBackendStateMetaInfo.Snapshot<?, ?>> readMetaData( StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); stateBackend.cancelStreamRegistry.registerClosable(inputStream); KeyedBackendSerializationProxy<T> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); // check for key serializer compatibility; this also reconfigures the // key serializer to be compatible, if it is required and is possible if (CompatibilityUtil.resolveCompatibilityResult( serializationProxy.getKeySerializer(), UnloadableDummyTypeSerializer.class, serializationProxy.getKeySerializerConfigSnapshot(), stateBackend.keySerializer) .isRequiresMigration()) { // TODO replace with state migration; note that key hash codes need to remain the same after migration throw new StateMigrationException("The new key serializer is not compatible to read previous keys. " + "Aborting now since state migration is currently not available"); } return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (inputStream != null) { stateBackend.cancelStreamRegistry.unregisterClosable(inputStream); inputStream.close(); } } }
@SuppressWarnings("unchecked") public static <T> Optional<TypeSerializer<T>> getSerializer(KeyedBackendSerializationProxy<?> proxy, String stateName) { for (StateMetaInfoSnapshot snapshot : proxy.getStateMetaInfoSnapshots()) { if (snapshot.getName().equals(stateName)) { return Optional .of((TypeSerializer<T>) snapshot .getTypeSerializerSnapshot(CommonSerializerKeys.VALUE_SERIALIZER) .restoreSerializer()); } } return Optional.empty(); }
private StreamStateHandle materializeMetaData() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream outputStream = null; try { outputStream = checkpointStreamFactory .createCheckpointStateOutputStream(checkpointId, checkpointTimestamp); closeableRegistry.registerClosable(outputStream); KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(stateBackend.keySerializer, stateMetaInfoSnapshots); DataOutputView out = new DataOutputViewStreamWrapper(outputStream); serializationProxy.write(out); closeableRegistry.unregisterClosable(outputStream); StreamStateHandle result = outputStream.closeAndGetHandle(); outputStream = null; return result; } finally { if (outputStream != null) { closeableRegistry.unregisterClosable(outputStream); outputStream.close(); } } }
private TypeSerializer<?> getKeySerializer(KeyedBackendSerializationProxy<?> proxy) { TypeSerializer<?> keySerializer = proxy.getKeySerializerConfigSnapshot().restoreSerializer(); if (keySerializer instanceof TupleSerializerBase) { TupleSerializerBase ts = (TupleSerializerBase) keySerializer; if (ts.getTupleClass().equals(Tuple1.class)) { return ts.getFieldSerializers()[0]; } } return keySerializer; }
super.read(in); final int readVersion = getReadVersion();
new KeyedBackendSerializationProxy<>(userCodeClassLoader); serializationProxy.read(inView); if (!serializationProxy.getKeySerializerConfigSnapshot() .resolveSchemaCompatibility(keySerializer).isCompatibleAsIs()) { throw new StateMigrationException("The new key serializer must be compatible."); serializationProxy.getStateMetaInfoSnapshots(); keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(), serializationProxy.isUsingKeyGroupCompression()); } finally { if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
new KeyedBackendSerializationProxy<>(stateBackend.userCodeClassLoader); DataInputView in = new DataInputViewStreamWrapper(inputStream); serializationProxy.read(in); stateBackend.checkKeySerializerSchemaCompatibility(serializationProxy.getKeySerializerSnapshot()); if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) { throw new StateMigrationException("The new key serializer must be compatible."); return serializationProxy.getStateMetaInfoSnapshots(); } finally { if (stateBackend.cancelStreamRegistry.unregisterCloseable(inputStream)) {