private void writeObject(final ObjectOutputStream out) throws IOException { // write all the non-transient fields out.defaultWriteObject(); // write the non-serializable default value field if (defaultValue == null) { // we don't have a default value out.writeBoolean(false); } else { // we have a default value out.writeBoolean(true); byte[] serializedDefaultValue; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) { TypeSerializer<T> duplicateSerializer = serializer.duplicate(); duplicateSerializer.serialize(defaultValue, outView); outView.flush(); serializedDefaultValue = baos.toByteArray(); } catch (Exception e) { throw new IOException("Unable to serialize default value of type " + defaultValue.getClass().getSimpleName() + ".", e); } out.writeInt(serializedDefaultValue.length); out.write(serializedDefaultValue); } }
/** * Copied from HeapListState.getSerializedValue(Object, Object). */ private byte[] serializeInitValue(List<Long> toSerialize) throws IOException { TypeSerializer<Long> serializer = listStateDesc.getElementSerializer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); // write the same as RocksDB writes lists, with one ',' separator for (int i = 0; i < toSerialize.size(); i++) { serializer.serialize(toSerialize.get(i), view); if (i < toSerialize.size() - 1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); }
private void writeObject(final ObjectOutputStream out) throws IOException { // write all the non-transient fields out.defaultWriteObject(); // write the non-serializable default value field if (defaultValue == null) { // we don't have a default value out.writeBoolean(false); } else { // we have a default value out.writeBoolean(true); byte[] serializedDefaultValue; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) { TypeSerializer<T> duplicateSerializer = serializer.duplicate(); duplicateSerializer.serialize(defaultValue, outView); outView.flush(); serializedDefaultValue = baos.toByteArray(); } catch (Exception e) { throw new IOException("Unable to serialize default value of type " + defaultValue.getClass().getSimpleName() + ".", e); } out.writeInt(serializedDefaultValue.length); out.write(serializedDefaultValue); } }
private void writeObject(final ObjectOutputStream out) throws IOException { // write all the non-transient fields out.defaultWriteObject(); // write the non-serializable default value field if (defaultValue == null) { // we don't have a default value out.writeBoolean(false); } else { // we have a default value out.writeBoolean(true); byte[] serializedDefaultValue; try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(baos)) { TypeSerializer<T> duplicateSerializer = serializer.duplicate(); duplicateSerializer.serialize(defaultValue, outView); outView.flush(); serializedDefaultValue = baos.toByteArray(); } catch (Exception e) { throw new IOException("Unable to serialize default value of type " + defaultValue.getClass().getSimpleName() + ".", e); } out.writeInt(serializedDefaultValue.length); out.write(serializedDefaultValue); } }
@Override public void write(DataOutputView out) throws IOException { int num = this.aggNames.length; out.writeInt(num); ByteArrayOutputStream boas = new ByteArrayOutputStream(); DataOutputViewStreamWrapper bufferStream = new DataOutputViewStreamWrapper(boas); for (int i = 0; i < num; i++) { // aggregator name and type out.writeUTF(this.aggNames[i]); out.writeUTF(this.aggregates[i].getClass().getName()); // aggregator value indirect as a byte array this.aggregates[i].write(bufferStream); bufferStream.flush(); byte[] bytes = boas.toByteArray(); out.writeInt(bytes.length); out.write(bytes); boas.reset(); } bufferStream.close(); boas.close(); }
@Override public void write(DataOutputView out) throws IOException { int num = this.aggNames.length; out.writeInt(num); ByteArrayOutputStream boas = new ByteArrayOutputStream(); DataOutputViewStreamWrapper bufferStream = new DataOutputViewStreamWrapper(boas); for (int i = 0; i < num; i++) { // aggregator name and type out.writeUTF(this.aggNames[i]); out.writeUTF(this.aggregates[i].getClass().getName()); // aggregator value indirect as a byte array this.aggregates[i].write(bufferStream); bufferStream.flush(); byte[] bytes = boas.toByteArray(); out.writeInt(bytes.length); out.write(bytes); boas.reset(); } bufferStream.close(); boas.close(); }
@Override public byte[] getSerializedValue(K key, N namespace) throws Exception { Preconditions.checkState(namespace != null, "No namespace given."); Preconditions.checkState(key != null, "No key given."); ArrayList<V> result = stateTable.get(key, namespace); if (result == null) { return null; } TypeSerializer<V> serializer = stateDesc.getElementSerializer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); // write the same as RocksDB writes lists, with one ',' separator for (int i = 0; i < result.size(); i++) { serializer.serialize(result.get(i), view); if (i < result.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); }
@Override public void write(DataOutputView out) throws IOException { int num = this.aggNames.length; out.writeInt(num); ByteArrayOutputStream boas = new ByteArrayOutputStream(); DataOutputViewStreamWrapper bufferStream = new DataOutputViewStreamWrapper(boas); for (int i = 0; i < num; i++) { // aggregator name and type out.writeUTF(this.aggNames[i]); out.writeUTF(this.aggregates[i].getClass().getName()); // aggregator value indirect as a byte array this.aggregates[i].write(bufferStream); bufferStream.flush(); byte[] bytes = boas.toByteArray(); out.writeInt(bytes.length); out.write(bytes); boas.reset(); } bufferStream.close(); boas.close(); }
@Override public void write(DataOutputView out) throws IOException { int num = this.aggNames.length; out.writeInt(num); ByteArrayOutputStream boas = new ByteArrayOutputStream(); DataOutputViewStreamWrapper bufferStream = new DataOutputViewStreamWrapper(boas); for (int i = 0; i < num; i++) { // aggregator name and type out.writeUTF(this.aggNames[i]); out.writeUTF(this.aggregates[i].getClass().getName()); // aggregator value indirect as a byte array this.aggregates[i].write(bufferStream); bufferStream.flush(); byte[] bytes = boas.toByteArray(); out.writeInt(bytes.length); out.write(bytes); boas.reset(); } bufferStream.close(); boas.close(); }
@Override public void snapshotState(FSDataOutputStream out, long checkpointId, long timestamp) throws Exception { super.snapshotState(out, checkpointId, timestamp); // we write the panes with the key/value maps into the stream, as well as when this state // should have triggered and slided DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out); outView.writeLong(nextEvaluationTime); outView.writeLong(nextSlideTime); panes.writeToOutput(outView, keySerializer, stateTypeSerializer); outView.flush(); }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception { Preconditions.checkNotNull(serializedKeyAndNamespace); Preconditions.checkNotNull(safeKeySerializer); Preconditions.checkNotNull(safeNamespaceSerializer); Preconditions.checkNotNull(safeValueSerializer); Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace( serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1); if (result == null) { return null; } final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); // write the same as RocksDB writes lists, with one ',' separator for (int i = 0; i < result.size(); i++) { dupSerializer.serialize(result.get(i), view); if (i < result.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception { Preconditions.checkNotNull(serializedKeyAndNamespace); Preconditions.checkNotNull(safeKeySerializer); Preconditions.checkNotNull(safeNamespaceSerializer); Preconditions.checkNotNull(safeValueSerializer); Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace( serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1); if (result == null) { return null; } final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); // write the same as RocksDB writes lists, with one ',' separator for (int i = 0; i < result.size(); i++) { dupSerializer.serialize(result.get(i), view); if (i < result.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception { Preconditions.checkNotNull(serializedKeyAndNamespace); Preconditions.checkNotNull(safeKeySerializer); Preconditions.checkNotNull(safeNamespaceSerializer); Preconditions.checkNotNull(safeValueSerializer); Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace( serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1); if (result == null) { return null; } final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); // write the same as RocksDB writes lists, with one ',' separator for (int i = 0; i < result.size(); i++) { dupSerializer.serialize(result.get(i), view); if (i < result.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<List<E>> safeValueSerializer) throws Exception { K key = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, VoidNamespaceSerializer.INSTANCE).f0; ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); if (stateStorage.lazySerde()) { List<E> value = get(key); if (value == null) { return null; } final TypeSerializer<E> dupSerializer = ((ListSerializer<E>) safeValueSerializer).getElementSerializer(); for (int i = 0; i < value.size(); i++) { dupSerializer.serialize(value.get(i), view); if (i < value.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); } else { return getSerializedValue(key, baos, view, safeKeySerializer); } }
@Override public byte[] getSerializedValue( final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<E>> safeValueSerializer) throws Exception { Tuple2<K, N> tuple = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer); K key = tuple.f0; N namespace = tuple.f1; ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos); if (stateStorage.lazySerde()) { List<E> value = get(key, namespace); if (value == null) { return null; } final TypeSerializer<E> dupSerializer = ((ListSerializer<E>) safeValueSerializer).getElementSerializer(); for (int i = 0; i < value.size(); i++) { dupSerializer.serialize(value.get(i), view); if (i < value.size() -1) { view.writeByte(','); } } view.flush(); return baos.toByteArray(); } else { return getSerializedValue(key, namespace, baos, view, safeKeySerializer, safeNamespaceSerializer); } }