@SuppressWarnings("ConstantConditions") public JavaRDD<WindowedValue<T>> getRDD() { if (rdd == null) { WindowedValue.ValueOnlyWindowedValueCoder<T> windowCoder = WindowedValue.getValueOnlyCoder(coder); rdd = jsc.parallelize(CoderHelpers.toByteArrays(windowedValues, windowCoder)) .map(CoderHelpers.fromByteFunction(windowCoder)); } return rdd; }
@Test public void testValueOnlyWindowedValueCoderIsSerializableWithWellKnownCoderType() { CoderProperties.coderSerializable(WindowedValue.getValueOnlyCoder(GlobalWindow.Coder.INSTANCE)); }
@Test(expected = Exception.class) public void parDoBaseClassPipelineOptionsNullTest() { TupleTag<String> mainTag = new TupleTag<>("main-output"); Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); new DoFnOperator<>( new TestDoFn(), "stepName", coder, null, Collections.emptyMap(), mainTag, Collections.emptyList(), new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, coder), WindowingStrategy.globalDefault(), new HashMap<>(), Collections.emptyList(), null, null, /* key coder */ null /* key selector */); }
@Test(expected = Exception.class) public void parDoBaseClassPipelineOptionsNullTest() { TupleTag<String> mainTag = new TupleTag<>("main-output"); Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); new DoFnOperator<>( new TestDoFn(), "stepName", coder, null, Collections.emptyMap(), mainTag, Collections.emptyList(), new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, coder), WindowingStrategy.globalDefault(), new HashMap<>(), Collections.emptyList(), null, null, /* key coder */ null /* key selector */); }
@Test(expected = Exception.class) public void parDoBaseClassPipelineOptionsNullTest() { TupleTag<String> mainTag = new TupleTag<>("main-output"); Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>( new TestDoFn(), "stepName", coder, mainTag, Collections.<TupleTag<?>>emptyList(), new DoFnOperator.MultiOutputOutputManagerFactory<>(mainTag, coder), WindowingStrategy.globalDefault(), new HashMap<Integer, PCollectionView<?>>(), Collections.<PCollectionView<?>>emptyList(), null, null); }
WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); IdentityDoFn<String> doFn = new IdentityDoFn<String>() {
@Test @SuppressWarnings("unchecked") public void testSingleOutput() throws Exception { Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); TupleTag<String> outputTag = new TupleTag<>("main-output"); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>( new IdentityDoFn<>(), "stepName", coder, null, Collections.emptyMap(), outputTag, Collections.emptyList(), new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, coder), WindowingStrategy.globalDefault(), new HashMap<>(), /* side-input mapping */ Collections.emptyList(), /* side inputs */ PipelineOptionsFactory.as(FlinkPipelineOptions.class), null, null); OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness = new OneInputStreamOperatorTestHarness<>(doFnOperator); testHarness.open(); testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("Hello"))); assertThat( stripStreamRecordFromWindowedValue(testHarness.getOutput()), contains(WindowedValue.valueInGlobalWindow("Hello"))); testHarness.close(); }
Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>(
@Test @SuppressWarnings("unchecked") public void testSingleOutput() throws Exception { Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); TupleTag<String> outputTag = new TupleTag<>("main-output"); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>( new IdentityDoFn<>(), "stepName", coder, null, Collections.emptyMap(), outputTag, Collections.emptyList(), new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, coder), WindowingStrategy.globalDefault(), new HashMap<>(), /* side-input mapping */ Collections.emptyList(), /* side inputs */ PipelineOptionsFactory.as(FlinkPipelineOptions.class), null, null); OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness = new OneInputStreamOperatorTestHarness<>(doFnOperator); testHarness.open(); testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("Hello"))); assertThat( stripStreamRecordFromWindowedValue(testHarness.getOutput()), contains(WindowedValue.valueInGlobalWindow("Hello"))); testHarness.close(); }
Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>( new TestDoFn(),
@Test @SuppressWarnings("unchecked") public void testSingleOutput() throws Exception { Coder<WindowedValue<String>> coder = WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()); TupleTag<String> outputTag = new TupleTag<>("main-output"); DoFnOperator<String, String> doFnOperator = new DoFnOperator<>( new IdentityDoFn<String>(), "stepName", coder, outputTag, Collections.<TupleTag<?>>emptyList(), new DoFnOperator.MultiOutputOutputManagerFactory<>(outputTag, coder), WindowingStrategy.globalDefault(), new HashMap<Integer, PCollectionView<?>>(), /* side-input mapping */ Collections.<PCollectionView<?>>emptyList(), /* side inputs */ PipelineOptionsFactory.as(FlinkPipelineOptions.class), null); OneInputStreamOperatorTestHarness<WindowedValue<String>, WindowedValue<String>> testHarness = new OneInputStreamOperatorTestHarness<>(doFnOperator); testHarness.open(); testHarness.processElement(new StreamRecord<>(WindowedValue.valueInGlobalWindow("Hello"))); assertThat( this.<String>stripStreamRecordFromWindowedValue(testHarness.getOutput()), contains(WindowedValue.valueInGlobalWindow("Hello"))); testHarness.close(); }
@Override public void translateNode( PTransform<PCollection<KV<K, InputT>>, PCollection<KeyedWorkItem<K, InputT>>> transform, FlinkStreamingTranslationContext context) { PCollection<KV<K, InputT>> input = context.getInput(transform); KvCoder<K, InputT> inputKvCoder = (KvCoder<K, InputT>) input.getCoder(); SingletonKeyedWorkItemCoder<K, InputT> workItemCoder = SingletonKeyedWorkItemCoder.of( inputKvCoder.getKeyCoder(), inputKvCoder.getValueCoder(), input.getWindowingStrategy().getWindowFn().windowCoder()); WindowedValue.ValueOnlyWindowedValueCoder<SingletonKeyedWorkItem<K, InputT>> windowedWorkItemCoder = WindowedValue.getValueOnlyCoder(workItemCoder); CoderTypeInformation<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> workItemTypeInfo = new CoderTypeInformation<>(windowedWorkItemCoder); DataStream<WindowedValue<KV<K, InputT>>> inputDataStream = context.getInputDataStream(input); DataStream<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> workItemStream = inputDataStream .flatMap(new ToKeyedWorkItemInGlobalWindow<>()) .returns(workItemTypeInfo) .name("ToKeyedWorkItem"); KeyedStream<WindowedValue<SingletonKeyedWorkItem<K, InputT>>, ByteBuffer> keyedWorkItemStream = workItemStream.keyBy(new WorkItemKeySelector<>(inputKvCoder.getKeyCoder())); context.setOutputDataStream(context.getOutput(transform), keyedWorkItemStream); } }
@Override public void translateNode( PTransform<PCollection<KV<K, InputT>>, PCollection<KeyedWorkItem<K, InputT>>> transform, FlinkStreamingTranslationContext context) { PCollection<KV<K, InputT>> input = context.getInput(transform); KvCoder<K, InputT> inputKvCoder = (KvCoder<K, InputT>) input.getCoder(); SingletonKeyedWorkItemCoder<K, InputT> workItemCoder = SingletonKeyedWorkItemCoder.of( inputKvCoder.getKeyCoder(), inputKvCoder.getValueCoder(), input.getWindowingStrategy().getWindowFn().windowCoder()); WindowedValue.ValueOnlyWindowedValueCoder<SingletonKeyedWorkItem<K, InputT>> windowedWorkItemCoder = WindowedValue.getValueOnlyCoder(workItemCoder); CoderTypeInformation<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> workItemTypeInfo = new CoderTypeInformation<>(windowedWorkItemCoder); DataStream<WindowedValue<KV<K, InputT>>> inputDataStream = context.getInputDataStream(input); DataStream<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> workItemStream = inputDataStream .flatMap(new ToKeyedWorkItemInGlobalWindow<>()) .returns(workItemTypeInfo) .name("ToKeyedWorkItem"); KeyedStream<WindowedValue<SingletonKeyedWorkItem<K, InputT>>, ByteBuffer> keyedWorkItemStream = workItemStream.keyBy(new WorkItemKeySelector<>(inputKvCoder.getKeyCoder())); context.setOutputDataStream(context.getOutput(transform), keyedWorkItemStream); } }
@Test public void unkeyedInputWithKeyPreserving() { PCollection<KV<String, Iterable<WindowedValue<KV<String, Integer>>>>> input = p.apply( Create.of( KV.of( "hello", (Iterable<WindowedValue<KV<String, Integer>>>) Collections.<WindowedValue<KV<String, Integer>>>emptyList())) .withCoder( KvCoder.of( StringUtf8Coder.of(), IterableCoder.of( WindowedValue.getValueOnlyCoder( KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))))); PCollection<KeyedWorkItem<String, KV<String, Integer>>> unkeyed = input .apply(ParDo.of(new ParDoMultiOverrideFactory.ToKeyedWorkItem<>())) .setCoder( KeyedWorkItemCoder.of( StringUtf8Coder.of(), KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()), GlobalWindow.Coder.INSTANCE)); p.traverseTopologically(visitor); assertThat(visitor.getKeyedPValues(), not(hasItem(unkeyed))); }
@Test public void testSerialization() { WindowedValue.ValueOnlyWindowedValueCoder<Integer> coder = WindowedValue.getValueOnlyCoder(VarIntCoder.of());
stepContext.addEncodingInput(WindowedValue.getValueOnlyCoder(VoidCoder.of())); stepContext.addInput(PropertyNames.PARALLEL_INPUT, context.getInput(transform));
WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());
KvCoder.of( StringUtf8Coder.of(), WindowedValue.getValueOnlyCoder( KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())))));
WindowedValue.getValueOnlyCoder(StringUtf8Coder.of());