@Override public DoFn<?, ?> load(Thread key) throws Exception { DoFn<?, ?> fn = (DoFn<?, ?>) SerializableUtils.deserializeFromByteArray( original, "DoFn Copy in thread " + key.getName()); DoFnInvokers.invokerFor(fn).invokeSetup(); return fn; } }
private GroupedValues( GlobalCombineFn<? super InputT, ?, OutputT> fn, DisplayData.ItemSpec<? extends Class<?>> fnDisplayData, List<PCollectionView<?>> sideInputs) { this.fn = SerializableUtils.clone(fn); this.fnDisplayData = fnDisplayData; this.sideInputs = sideInputs; }
public DeserializingCacheLoader(DoFn<?, ?> original) { this.original = SerializableUtils.serializeToByteArray(original); }
public static <T extends Serializable> T clone(T value) { final Thread thread = Thread.currentThread(); final ClassLoader tccl = thread.getContextClassLoader(); ClassLoader loader = tccl; try { if (tccl.loadClass(value.getClass().getName()) != value.getClass()) { loader = value.getClass().getClassLoader(); } } catch (final NoClassDefFoundError | ClassNotFoundException e) { loader = value.getClass().getClassLoader(); } if (loader == null) { loader = tccl; // will likely fail but the best we can do } thread.setContextClassLoader(loader); @SuppressWarnings("unchecked") final T copy; try { copy = (T) deserializeFromByteArray(serializeToByteArray(value), value.toString()); } finally { thread.setContextClassLoader(tccl); } return copy; }
/** Verifies that the given {@code Coder<T>} can be correctly serialized and deserialized. */ public static <T> void coderSerializable(Coder<T> coder) { SerializableUtils.ensureSerializable(coder); }
@SuppressWarnings("unchecked") private void initializeState() throws Exception { checkState(state == State.UNINITIALIZED, "Already initialized"); checkState(fn == null, "Uninitialized but fn != null"); if (cloningBehavior.equals(CloningBehavior.DO_NOT_CLONE)) { fn = origFn; } else { fn = (DoFn<InputT, OutputT>) SerializableUtils.deserializeFromByteArray( SerializableUtils.serializeToByteArray(origFn), origFn.toString()); } fnInvoker = DoFnInvokers.invokerFor(fn); fnInvoker.invokeSetup(); }
private Unbounded(@Nullable String name, UnboundedSource<T, ?> source) { super(name); this.source = SerializableUtils.ensureSerializable(source); }
@Override public SerializableMatcher<?> deserialize( JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { ObjectNode node = jsonParser.readValueAsTree(); String matcher = node.get("matcher").asText(); byte[] in = BaseEncoding.base64().decode(matcher); return (SerializableMatcher<?>) SerializableUtils.deserializeFromByteArray(in, "SerializableMatcher"); } }
public static <InT, OutT extends Serializable> SerializableFunction<InT, OutT> clonesOf( final OutT base) { return input -> SerializableUtils.clone(base); }
@Override public void serialize( SerializableMatcher<?> matcher, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { byte[] out = SerializableUtils.serializeToByteArray(matcher); String encodedString = BaseEncoding.base64().encode(out); jsonGenerator.writeStartObject(); jsonGenerator.writeStringField("matcher", encodedString); jsonGenerator.writeEndObject(); } }
private Bounded(@Nullable String name, BoundedSource<T> source) { super(name); this.source = SerializableUtils.ensureSerializable(source); }
@Override public Coder fromCloudObject(CloudObject cloudObject) { String serializedCoder = Structs.getString(cloudObject, CODER_FIELD); String type = Structs.getString(cloudObject, TYPE_FIELD); return (Coder<?>) SerializableUtils.deserializeFromByteArray( StringUtils.jsonStringToByteArray(serializedCoder), type); }
public static <T extends Serializable> T ensureSerializable(T value) { return clone(value); }
@Override public CloudObject toCloudObject(Coder target, SdkComponents sdkComponents) { // CustomCoder is used as the "marker" for a java-serialized coder CloudObject cloudObject = CloudObject.forClass(CustomCoder.class); Structs.addString(cloudObject, TYPE_FIELD, target.getClass().getName()); Structs.addString( cloudObject, CODER_FIELD, StringUtils.byteArrayToJsonString(SerializableUtils.serializeToByteArray(target))); return cloudObject; }
/** * Serializes a Coder and verifies that it can be correctly deserialized. * * <p>Throws a RuntimeException if serialized Coder cannot be deserialized, or if the deserialized * instance is not equal to the original. * * @return the deserialized Coder */ public static Coder<?> ensureSerializable(Coder<?> coder) { // Make sure that Coders are java serializable as well since // they are regularly captured within DoFn's. Coder<?> copy = (Coder<?>) ensureSerializable((Serializable) coder); checkState( coder.equals(copy), "Coder not equal to original after serialization, indicating that the Coder may not " + "implement serialization correctly. Before: %s, after: %s", coder, copy); return copy; }
@VisibleForTesting static FileBasedSink<?, ?, ?> sinkFromProto(SdkFunctionSpec sinkProto) throws IOException { checkArgument( sinkProto.getSpec().getUrn().equals(CUSTOM_JAVA_FILE_BASED_SINK_URN), "Cannot extract %s instance from %s with URN %s", FileBasedSink.class.getSimpleName(), FunctionSpec.class.getSimpleName(), sinkProto.getSpec().getUrn()); byte[] serializedSink = sinkProto.getSpec().getPayload().toByteArray(); return (FileBasedSink<?, ?, ?>) SerializableUtils.deserializeFromByteArray( serializedSink, FileBasedSink.class.getSimpleName()); }
this.metricsAccum = metricsAccum; this.stepName = stepName; this.doFn = SerializableUtils.clone(doFn); this.options = options; this.mainOutputTag = mainOutputTag;
cloudSource.getSpec(), SERIALIZED_SOURCE, encodeBase64String(serializeToByteArray(source))); getDesiredNumUnboundedSourceSplits(options.as(DataflowPipelineOptions.class)); for (UnboundedSource<?, ?> split : unboundedSource.split(desiredNumSplits, options)) { encodedSplits.add(encodeBase64String(serializeToByteArray(split)));
private void assertSerializationClassLoader( final ClassLoader loader, final Serializable customLoaderSourceInstance) { final Serializable copy = SerializableUtils.ensureSerializable(customLoaderSourceInstance); assertEquals(loader, copy.getClass().getClassLoader()); assertEquals( copy.getClass().getClassLoader(), customLoaderSourceInstance.getClass().getClassLoader()); }
public static DoFnAndMainOutput doFnAndMainOutputTagFromProto(SdkFunctionSpec fnSpec) { checkArgument( fnSpec.getSpec().getUrn().equals(CUSTOM_JAVA_DO_FN_URN), "Expected %s to be %s with URN %s, but URN was %s", DoFn.class.getSimpleName(), FunctionSpec.class.getSimpleName(), CUSTOM_JAVA_DO_FN_URN, fnSpec.getSpec().getUrn()); byte[] serializedFn = fnSpec.getSpec().getPayload().toByteArray(); return (DoFnAndMainOutput) SerializableUtils.deserializeFromByteArray(serializedFn, "Custom DoFn And Main Output tag"); }