public DeserializingCacheLoader(DoFn<?, ?> original) { this.original = SerializableUtils.serializeToByteArray(original); }
@Override public void serialize( SerializableMatcher<?> matcher, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException { byte[] out = SerializableUtils.serializeToByteArray(matcher); String encodedString = BaseEncoding.base64().encode(out); jsonGenerator.writeStartObject(); jsonGenerator.writeStringField("matcher", encodedString); jsonGenerator.writeEndObject(); } }
public static <T extends Serializable> T clone(T value) { final Thread thread = Thread.currentThread(); final ClassLoader tccl = thread.getContextClassLoader(); ClassLoader loader = tccl; try { if (tccl.loadClass(value.getClass().getName()) != value.getClass()) { loader = value.getClass().getClassLoader(); } } catch (final NoClassDefFoundError | ClassNotFoundException e) { loader = value.getClass().getClassLoader(); } if (loader == null) { loader = tccl; // will likely fail but the best we can do } thread.setContextClassLoader(loader); @SuppressWarnings("unchecked") final T copy; try { copy = (T) deserializeFromByteArray(serializeToByteArray(value), value.toString()); } finally { thread.setContextClassLoader(tccl); } return copy; }
@Override public CloudObject toCloudObject(Coder target, SdkComponents sdkComponents) { // CustomCoder is used as the "marker" for a java-serialized coder CloudObject cloudObject = CloudObject.forClass(CustomCoder.class); Structs.addString(cloudObject, TYPE_FIELD, target.getClass().getName()); Structs.addString( cloudObject, CODER_FIELD, StringUtils.byteArrayToJsonString(SerializableUtils.serializeToByteArray(target))); return cloudObject; }
@SuppressWarnings("unchecked") private void initializeState() throws Exception { checkState(state == State.UNINITIALIZED, "Already initialized"); checkState(fn == null, "Uninitialized but fn != null"); if (cloningBehavior.equals(CloningBehavior.DO_NOT_CLONE)) { fn = origFn; } else { fn = (DoFn<InputT, OutputT>) SerializableUtils.deserializeFromByteArray( SerializableUtils.serializeToByteArray(origFn), origFn.toString()); } fnInvoker = DoFnInvokers.invokerFor(fn); fnInvoker.invokeSetup(); }
cloudSource.getSpec(), SERIALIZED_SOURCE, encodeBase64String(serializeToByteArray(source))); getDesiredNumUnboundedSourceSplits(options.as(DataflowPipelineOptions.class)); for (UnboundedSource<?, ?> split : unboundedSource.split(desiredNumSplits, options)) { encodedSplits.add(encodeBase64String(serializeToByteArray(split)));
private static SdkFunctionSpec toProto(String urn, Serializable serializable) { return SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(urn) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(serializable))) .build()) .build(); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, View.CreatePCollectionView<?, ?>> transform, SdkComponents components) { return FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload( ByteString.copyFrom( SerializableUtils.serializeToByteArray(transform.getTransform().getView()))) .build(); } }
@Test public void testSerializationError() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("unable to serialize"); SerializableUtils.serializeToByteArray(new UnserializableByJava()); }
public static SdkFunctionSpec translateWindowMappingFn( WindowMappingFn<?> windowMappingFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowMappingFn))) .build()) .build(); }
public static SdkFunctionSpec toProto( GlobalCombineFn<?, ?, ?> combineFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(combineFn))) .build()) .build(); } }
private static SdkFunctionSpec toProto(UnboundedSource<?, ?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_UNBOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }
@Test public void testJsonEscaping() { // Declare an arbitrary function and make sure we can serialize it DoFn<Integer, Integer> doFn = new DoFn<Integer, Integer>() { @ProcessElement public void processElement(@Element Integer element, OutputReceiver<Integer> r) { r.output(element + 1); } }; byte[] serializedBytes = serializeToByteArray(doFn); String serializedJson = byteArrayToJsonString(serializedBytes); assertArrayEquals(serializedBytes, jsonStringToByteArray(serializedJson)); }
private static SdkFunctionSpec toProto(BoundedSource<?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_BOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }
public static SdkFunctionSpec translateViewFn(ViewFn<?, ?> viewFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_VIEW_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(viewFn))) .build()) .build(); }
private static RunnerApi.Coder toCustomCoder(Coder<?> coder) throws IOException { RunnerApi.Coder.Builder coderBuilder = RunnerApi.Coder.newBuilder(); return coderBuilder .setSpec( SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_CODER_URN) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(coder))) .build())) .build(); }
PropertyNames.SERIALIZED_FN, byteArrayToJsonString( serializeToByteArray( DoFnInfo.forFn( fn, windowingStrategy, sideInputs, inputCoder, outputCoders, mainOutput))));
public static SdkFunctionSpec translateDoFn( DoFn<?, ?> fn, TupleTag<?> tag, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_DO_FN_URN) .setPayload( ByteString.copyFrom( SerializableUtils.serializeToByteArray(DoFnAndMainOutput.of(fn, tag)))) .build()) .build(); }
.setPayload( ByteString.copyFrom( SerializableUtils.serializeToByteArray( combine.getTransform().getFn()))) .build())
} else { stepContext.addInput( PropertyNames.SERIALIZED_FN, byteArrayToJsonString(serializeToByteArray(fn)));