@Override public DoFn<?, ?> load(Thread key) throws Exception { DoFn<?, ?> fn = (DoFn<?, ?>) SerializableUtils.deserializeFromByteArray( original, "DoFn Copy in thread " + key.getName()); DoFnInvokers.invokerFor(fn).invokeSetup(); return fn; } }
public static <T extends Serializable> T clone(T value) { final Thread thread = Thread.currentThread(); final ClassLoader tccl = thread.getContextClassLoader(); ClassLoader loader = tccl; try { if (tccl.loadClass(value.getClass().getName()) != value.getClass()) { loader = value.getClass().getClassLoader(); } } catch (final NoClassDefFoundError | ClassNotFoundException e) { loader = value.getClass().getClassLoader(); } if (loader == null) { loader = tccl; // will likely fail but the best we can do } thread.setContextClassLoader(loader); @SuppressWarnings("unchecked") final T copy; try { copy = (T) deserializeFromByteArray(serializeToByteArray(value), value.toString()); } finally { thread.setContextClassLoader(tccl); } return copy; }
@Override public SerializableMatcher<?> deserialize( JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException { ObjectNode node = jsonParser.readValueAsTree(); String matcher = node.get("matcher").asText(); byte[] in = BaseEncoding.base64().decode(matcher); return (SerializableMatcher<?>) SerializableUtils.deserializeFromByteArray(in, "SerializableMatcher"); } }
@VisibleForTesting static FileBasedSink<?, ?, ?> sinkFromProto(SdkFunctionSpec sinkProto) throws IOException { checkArgument( sinkProto.getSpec().getUrn().equals(CUSTOM_JAVA_FILE_BASED_SINK_URN), "Cannot extract %s instance from %s with URN %s", FileBasedSink.class.getSimpleName(), FunctionSpec.class.getSimpleName(), sinkProto.getSpec().getUrn()); byte[] serializedSink = sinkProto.getSpec().getPayload().toByteArray(); return (FileBasedSink<?, ?, ?>) SerializableUtils.deserializeFromByteArray( serializedSink, FileBasedSink.class.getSimpleName()); }
@Override public Coder fromCloudObject(CloudObject cloudObject) { String serializedCoder = Structs.getString(cloudObject, CODER_FIELD); String type = Structs.getString(cloudObject, TYPE_FIELD); return (Coder<?>) SerializableUtils.deserializeFromByteArray( StringUtils.jsonStringToByteArray(serializedCoder), type); }
private static Coder<?> fromCustomCoder(RunnerApi.Coder protoCoder) throws IOException { return (Coder<?>) SerializableUtils.deserializeFromByteArray( protoCoder.getSpec().getSpec().getPayload().toByteArray(), "Custom Coder Bytes"); } }
public static DoFnAndMainOutput doFnAndMainOutputTagFromProto(SdkFunctionSpec fnSpec) { checkArgument( fnSpec.getSpec().getUrn().equals(CUSTOM_JAVA_DO_FN_URN), "Expected %s to be %s with URN %s, but URN was %s", DoFn.class.getSimpleName(), FunctionSpec.class.getSimpleName(), CUSTOM_JAVA_DO_FN_URN, fnSpec.getSpec().getUrn()); byte[] serializedFn = fnSpec.getSpec().getPayload().toByteArray(); return (DoFnAndMainOutput) SerializableUtils.deserializeFromByteArray(serializedFn, "Custom DoFn And Main Output tag"); }
/** * Converts a {@link org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec} into a {@link * ViewFn} using the URN. */ public static ViewFn<?, ?> viewFnFromProto(RunnerApi.SdkFunctionSpec viewFn) throws InvalidProtocolBufferException { RunnerApi.FunctionSpec spec = viewFn.getSpec(); checkArgument( spec.getUrn().equals(ParDoTranslation.CUSTOM_JAVA_VIEW_FN_URN), "Can't deserialize unknown %s type %s", ViewFn.class.getSimpleName(), spec.getUrn()); return (ViewFn<?, ?>) SerializableUtils.deserializeFromByteArray( spec.getPayload().toByteArray(), "Custom ViewFn"); }
/** * Converts a {@link org.apache.beam.model.pipeline.v1.RunnerApi.SdkFunctionSpec} into a {@link * WindowMappingFn} using the URN. */ public static WindowMappingFn<?> windowMappingFnFromProto( RunnerApi.SdkFunctionSpec windowMappingFn) throws InvalidProtocolBufferException { RunnerApi.FunctionSpec spec = windowMappingFn.getSpec(); checkArgument( spec.getUrn().equals(ParDoTranslation.CUSTOM_JAVA_WINDOW_MAPPING_FN_URN), "Can't deserialize unknown %s type %s", WindowMappingFn.class.getSimpleName(), spec.getUrn()); return (WindowMappingFn<?>) SerializableUtils.deserializeFromByteArray( spec.getPayload().toByteArray(), "Custom WinodwMappingFn"); } }
@SuppressWarnings("unchecked") private void initializeState() throws Exception { checkState(state == State.UNINITIALIZED, "Already initialized"); checkState(fn == null, "Uninitialized but fn != null"); if (cloningBehavior.equals(CloningBehavior.DO_NOT_CLONE)) { fn = origFn; } else { fn = (DoFn<InputT, OutputT>) SerializableUtils.deserializeFromByteArray( SerializableUtils.serializeToByteArray(origFn), origFn.toString()); } fnInvoker = DoFnInvokers.invokerFor(fn); fnInvoker.invokeSetup(); }
@Test public void testDeserializationError() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("unable to deserialize a bogus string"); SerializableUtils.deserializeFromByteArray( "this isn't legal".getBytes(Charsets.UTF_8), "a bogus string"); }
public static UnboundedSource<?, ?> unboundedSourceFromProto(ReadPayload payload) throws InvalidProtocolBufferException { checkArgument(payload.getIsBounded().equals(IsBounded.Enum.UNBOUNDED)); return (UnboundedSource<?, ?>) SerializableUtils.deserializeFromByteArray( payload.getSource().getSpec().getPayload().toByteArray(), "UnboundedSource"); }
/** * @deprecated Since {@link CreatePCollectionView} is not a part of the Beam model, there is no * SDK-agnostic specification. Using this method means your runner is tied to Java. */ @Deprecated public static <ElemT, ViewT> PCollectionView<ViewT> getView( AppliedPTransform< PCollection<ElemT>, PCollection<ElemT>, PTransform<PCollection<ElemT>, PCollection<ElemT>>> application) throws IOException { RunnerApi.PTransform transformProto = PTransformTranslation.toProto( application, Collections.emptyList(), SdkComponents.create(application.getPipeline().getOptions())); checkArgument( PTransformTranslation.CREATE_VIEW_TRANSFORM_URN.equals(transformProto.getSpec().getUrn()), "Illegal attempt to extract %s from transform %s with name \"%s\" and URN \"%s\"", PCollectionView.class.getSimpleName(), application.getTransform(), application.getFullName(), transformProto.getSpec().getUrn()); return (PCollectionView<ViewT>) SerializableUtils.deserializeFromByteArray( transformProto.getSpec().getPayload().toByteArray(), PCollectionView.class.getSimpleName()); }
public static BoundedSource<?> boundedSourceFromProto(ReadPayload payload) throws InvalidProtocolBufferException { checkArgument(payload.getIsBounded().equals(IsBounded.Enum.BOUNDED)); return (BoundedSource<?>) SerializableUtils.deserializeFromByteArray( payload.getSource().getSpec().getPayload().toByteArray(), "BoundedSource"); }
SerializableUtils.deserializeFromByteArray( combineFnSpec.getPayload().toByteArray(), Combine.CombineFn.class.getSimpleName());
} else if (s.equals(SERIALIZED_JAVA_WINDOWFN_URN)) { return (WindowFn<?, ?>) SerializableUtils.deserializeFromByteArray( windowFnSpec.getSpec().getPayload().toByteArray(), "WindowFn"); } else {
@Test public void testEncodedProto() throws Exception { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); components.registerPCollection(testPCollection); AppliedPTransform<?, ?, ?> appliedPTransform = AppliedPTransform.of( "foo", testPCollection.expand(), createViewTransform.getView().expand(), createViewTransform, p); FunctionSpec payload = PTransformTranslation.toProto(appliedPTransform, components).getSpec(); // Checks that the payload is what it should be PCollectionView<?> deserializedView = (PCollectionView<?>) SerializableUtils.deserializeFromByteArray( payload.getPayload().toByteArray(), PCollectionView.class.getSimpleName()); assertThat(deserializedView, Matchers.equalTo(createViewTransform.getView())); }
@Test public void testExtractionDirectFromTransform() throws Exception { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); components.registerPCollection(testPCollection); AppliedPTransform<?, ?, ?> appliedPTransform = AppliedPTransform.of( "foo", testPCollection.expand(), createViewTransform.getView().expand(), createViewTransform, p); CreatePCollectionViewTranslation.getView((AppliedPTransform) appliedPTransform); FunctionSpec payload = PTransformTranslation.toProto(appliedPTransform, components).getSpec(); // Checks that the payload is what it should be PCollectionView<?> deserializedView = (PCollectionView<?>) SerializableUtils.deserializeFromByteArray( payload.getPayload().toByteArray(), PCollectionView.class.getSimpleName()); assertThat(deserializedView, Matchers.equalTo(createViewTransform.getView())); } }
assertEquals( combineFn, SerializableUtils.deserializeFromByteArray( combineProto.getCombineFn().getSpec().getPayload().toByteArray(), "CombineFn"));
assertEquals( combineFn, SerializableUtils.deserializeFromByteArray( combineProto.getCombineFn().getSpec().getPayload().toByteArray(), "CombineFn"));