@Override public SdkFunctionSpec translateDoFn(SdkComponents newComponents) { return ParDoTranslation.translateDoFn(fn, pke.getMainOutputTag(), newComponents); }
@Override public Map<PValue, ReplacementOutput> mapOutputs( Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) { return ReplacementOutputs.singleton(outputs, newOutput); } }
/** * Create a new ArtifactServiceStager with the specified buffer size. Useful for testing * multi-part uploads. * * @param bufferSize the maximum size of the artifact chunk, in bytes. */ static ArtifactServiceStager overChannel(Channel channel, int bufferSize) { return new ArtifactServiceStager(channel, bufferSize); }
@Test public void testEncodedProto() throws Exception { SdkComponents components = SdkComponents.create(); components.registerEnvironment(Environments.createDockerEnvironment("java")); RunnerApi.TestStreamPayload payload = TestStreamTranslation.payloadForTestStream(testStream, components); verifyTestStreamEncoding( testStream, payload, RehydratedComponents.forComponents(components.toComponents())); }
@Test public void testViewFnTranslation() throws Exception { SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); assertEquals( new TestViewFn(), PCollectionViewTranslation.viewFnFromProto( ParDoTranslation.translateViewFn(new TestViewFn(), sdkComponents))); }
public FlinkReduceFunction( CombineFnBase.GlobalCombineFn<?, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
@Override public Map<PValue, ReplacementOutput> mapOutputs( Map<TupleTag<?>, PValue> outputs, PCollectionTuple newOutput) { return ReplacementOutputs.tagged(outputs, newOutput); } }
@Override public Map<String, SideInput> translateSideInputs(SdkComponents components) { return ParDoTranslation.translateSideInputs(pke.getSideInputs(), components); }
@Override public void onError(Throwable t) { if (writer != null) { writer.onError(t); } onCompleted(); }
/** * Returns a {@link PTransformMatcher} that matches a {@link PTransform} if the class of the * {@link PTransform} is equal to the {@link Class} provided ot this matcher. */ public static PTransformMatcher classEqualTo(Class<? extends PTransform> clazz) { return new EqualClassPTransformMatcher(clazz); }
@SuppressWarnings("rawtypes") public static < InputT extends PInput, OutputT extends POutput, TransformT extends PTransform<InputT, OutputT>> UnsupportedOverrideFactory<InputT, OutputT, TransformT> withMessage(String message) { return new UnsupportedOverrideFactory<>(message); }
@Override public Reader createReader(PipelineOptions options, Checkpoint<T> checkpoint) throws IOException { if (checkpoint == null) { return new Reader(null /* residualElements */, boundedSource, options); } else { return new Reader(checkpoint.residualElements, checkpoint.residualSource, options); } }
@Override public StreamObserver<ArtifactApi.PutArtifactRequest> putArtifact( StreamObserver<ArtifactApi.PutArtifactResponse> responseObserver) { return new BufferingObserver(responseObserver); }
@Override public Map<PValue, ReplacementOutput> mapOutputs( Map<TupleTag<?>, PValue> outputs, PCollection<OutputT> newOutput) { return ReplacementOutputs.singleton(outputs, newOutput); } }
public FlinkReduceFunction( CombineFnBase.GlobalCombineFn<?, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
@Override public Map<PValue, ReplacementOutput> mapOutputs( Map<TupleTag<?>, PValue> outputs, PCollectionTuple newOutput) { return ReplacementOutputs.tagged(outputs, newOutput); } }
@Override public Map<PValue, ReplacementOutput> mapOutputs( Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) { return ReplacementOutputs.singleton(outputs, newOutput); } }
public ApexReadUnboundedInputOperator( UnboundedSource<OutputT, CheckpointMarkT> source, ApexPipelineOptions options) { this.pipelineOptions = new SerializablePipelineOptions(options); this.source = source; this.isBoundedSource = false; }
@Override public Map<PValue, ReplacementOutput> mapOutputs( Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) { return ReplacementOutputs.singleton(outputs, newOutput); }
@Override public Map<PValue, ReplacementOutput> mapOutputs( Map<TupleTag<?>, PValue> outputs, PCollection<T> newOutput) { return ReplacementOutputs.singleton(outputs, newOutput); }