protected void skipTimestampIfTtlEnabled(DataInputViewStreamWrapper iw) throws IOException { if (ttlState) { iw.skipBytesToRead(Long.BYTES); } } }
private static <K> K getDeserializedSingleKey( DataInputViewStreamWrapper inputView, TypeSerializer<K> keySerializer, int serializedStateNameLength ) throws IOException { inputView.skipBytesToRead(GROUP_WRITE_BYTES + serializedStateNameLength + KEY_PREFIX_BYTE_LENGTH); return keySerializer.deserialize(inputView); }
private static <F, S> S getDeserializedSecondKey( DataInputViewStreamWrapper inputView, TypeSerializer<F> firstSerializer, TypeSerializer<S> secondeSerializer, int serializedStateNameLength) throws IOException { getDeserializedSingleKey(inputView, firstSerializer, serializedStateNameLength); // skip key prefix inputView.skipBytesToRead(KEY_PREFIX_BYTE_LENGTH); return secondeSerializer.deserialize(inputView); }
public static <V> V deserializeNextElement( DataInputViewStreamWrapper in, TypeSerializer<V> elementSerializer, boolean ttl) throws IOException { if (in.available() > 0) { if (ttl) { in.skipBytesToRead(Long.BYTES); } V element = elementSerializer.deserialize(in); if (in.available() > 0) { in.readByte(); } System.err.println(element); return element; } return null; } }
public static <K, N, MK> MK getDeserializedMapKeyForSubKeyedMapState( byte[] serializedBytes, TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, TypeSerializer<MK> mapKeySerializer, int serializedStateNameLength) throws IOException { ByteArrayInputStreamWithPos inputStream = new ByteArrayInputStreamWithPos(serializedBytes); DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(inputStream); getDeserializedSecondKey(inputView, keySerializer, namespaceSerializer, serializedStateNameLength); inputView.skipBytesToRead(KEY_PREFIX_BYTE_LENGTH); return mapKeySerializer.deserialize(inputView); }
public static <K, N> Pair<K, N> getDeserializedKeyAndNamespace( byte[] serializedBytes, TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, int serializedStateNameLenght) throws IOException { ByteArrayInputStreamWithPos inputStream = new ByteArrayInputStreamWithPos(serializedBytes); DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(inputStream); K key = getDeserializedSingleKey(inputView, keySerializer, serializedStateNameLenght); inputView.skipBytesToRead(KEY_PREFIX_BYTE_LENGTH); N namespace = namespaceSerializer.deserialize(inputView); return new DefaultPair<>(key, namespace); }
@Override public void flatMap(KeyedStateRow row, Collector<Tuple2<K, List<V>>> out) throws Exception { if (!stateName.equals(row.getStateName())) { return; } byte[] keyAndNamespaceBytes = row.getKeyAndNamespaceBytes(); byte[] valueBytes = row.getValueBytes(); K key; try (ByteArrayInputStreamWithPos keyIs = new ByteArrayInputStreamWithPos(keyAndNamespaceBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(keyIs); iw.skipBytesToRead(keygroupPrefixBytes); key = RocksDBUtils.readKey(keyDeserializer, keyIs, iw, false); } out.collect(Tuple2.of(key, RocksDBUtils.deserializeList(valueBytes, valueDeserializer, ttlState))); } }
@Override public void flatMap(KeyedStateRow row, Collector<Tuple2<K, V>> out) throws Exception { if (!stateName.equals(row.getStateName())) { return; } byte[] keyAndNamespaceBytes = row.getKeyAndNamespaceBytes(); byte[] valueBytes = row.getValueBytes(); K key; try (ByteArrayInputStreamWithPos keyIs = new ByteArrayInputStreamWithPos(keyAndNamespaceBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(keyIs); iw.skipBytesToRead(keygroupPrefixBytes); key = RocksDBUtils.readKey(keyDeserializer, keyIs, iw, false); } V value = null; try (ByteArrayInputStreamWithPos valIs = new ByteArrayInputStreamWithPos(valueBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(valIs); skipTimestampIfTtlEnabled(iw); value = valueDeserializer.deserialize(iw); } out.collect(Tuple2.of(key, value)); } }
@Override public void flatMap(KeyedStateRow row, Collector<Tuple2<K, V>> out) throws Exception { if (!stateName.equals(row.getStateName())) { return; } byte[] keyAndNamespaceBytes = row.getKeyAndNamespaceBytes(); byte[] valueBytes = row.getValueBytes(); K key; try (ByteArrayInputStreamWithPos keyIs = new ByteArrayInputStreamWithPos(keyAndNamespaceBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(keyIs); iw.skipBytesToRead(keygroupPrefixBytes); key = RocksDBUtils.readKey(keyDeserializer, keyIs, iw, false); } try (ByteArrayInputStreamWithPos valueIs = new ByteArrayInputStreamWithPos(valueBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(valueIs); V next = RocksDBUtils.deserializeNextElement(iw, valueDeserializer, ttlState); while (next != null) { out.collect(Tuple2.of(key, next)); next = RocksDBUtils.deserializeNextElement(iw, valueDeserializer, ttlState); } } } }
@SuppressWarnings("unchecked") @Override public void flatMap(KeyedStateRow row, Collector<Tuple3<K, MK, V>> out) throws Exception { if (!stateName.equals(row.getStateName())) { return; } byte[] keyAndNamespaceBytes = row.getKeyAndNamespaceBytes(); byte[] valueBytes = row.getValueBytes(); K key; MK mapKey; try (ByteArrayInputStreamWithPos keyIs = new ByteArrayInputStreamWithPos(keyAndNamespaceBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(keyIs); iw.skipBytesToRead(keygroupPrefixBytes); key = RocksDBUtils.readKey(keyDeserializer, keyIs, iw, false); VoidNamespaceSerializer.INSTANCE.deserialize(iw); mapKey = (MK) mapKeySerializer.deserialize(iw); } V value = null; try (ByteArrayInputStreamWithPos valIs = new ByteArrayInputStreamWithPos(valueBytes)) { DataInputViewStreamWrapper iw = new DataInputViewStreamWrapper(valIs); if (!iw.readBoolean()) { skipTimestampIfTtlEnabled(iw); value = valueDeserializer.deserialize(iw); } } if (value == null) { throw new RuntimeException("MapStates with null values are not supported at the moment."); } out.collect(Tuple3.of(key, mapKey, value)); }