@Override public PCollection<T> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, coder); }
@Override public PCollection<T> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, coder); }
@Override public final PCollection<T> expand(PBegin input) { source.validate(); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, source.getOutputCoder()); }
@Override public final PCollection<ValueWithRecordId<T>> expand(PInput input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, ValueWithRecordId.ValueWithRecordIdCoder.of(source.getOutputCoder())); }
@Override public final PCollection<T> expand(PBegin input) { source.validate(); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, source.getOutputCoder()); }
/** * Applies a window to the input collection if one hasn't already been specified. * * @return the input collection if it already has been windowed, otherwise a the same collection inside a default * window. */ public static <T> PCollection<T> ofDefaultWindow(PCollection<T> in) { if (in.getWindowingStrategy() != WindowingStrategy.globalDefault() && in.getWindowingStrategy() != null) return in; return in.apply("ApplyDefaultWindow", Window.<T> into(FixedWindows.of(DEFAULT_WINDOW_SIZE))); }
@Override public PCollection<T> expand(PBegin input) { runner.setClockSupplier(new TestClockSupplier()); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, original.getValueCoder()); } }
/** * Applies a window to the input collection if one hasn't already been specified. * * @return the input collection if it already has been windowed, otherwise a the same collection inside a default * window. */ public static <T> PCollection<T> ofDefaultWindow(PCollection<T> in) { if (in.getWindowingStrategy() != WindowingStrategy.globalDefault() && in.getWindowingStrategy() != null) return in; return in.apply("ApplyDefaultWindow", Window.<T> into(FixedWindows.of(DEFAULT_WINDOW_SIZE))); }
@Test public void onGuaranteedFiringFiresAfterTrigger() throws Exception { CountDownLatch latch = new CountDownLatch(1); executor.callOnGuaranteedFiring( create, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, BoundedWindow.TIMESTAMP_MAX_VALUE); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(true)); }
@Test public void onGuaranteedFiringFiresAfterTrigger() throws Exception { CountDownLatch latch = new CountDownLatch(1); executor.callOnGuaranteedFiring( create, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, BoundedWindow.TIMESTAMP_MAX_VALUE); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(true)); }
@Override public PCollection<KV<K1, Iterable<KV<K2, V>>>> expand(PCollection<KV<K1, KV<K2, V>>> input) { @SuppressWarnings("unchecked") KvCoder<K1, KV<K2, V>> inputCoder = (KvCoder<K1, KV<K2, V>>) input.getCoder(); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getValueCoder()))); } }
@Test public void callAfterOutputMustHaveBeenProducedAlreadyAfterCallsImmediately() throws Exception { TransformResult<?> finishedResult = StepTransformResult.withoutHold(createdProducer).build(); context.handleResult(null, ImmutableList.of(), finishedResult); final CountDownLatch callLatch = new CountDownLatch(1); context.extractFiredTimers(); Runnable callback = callLatch::countDown; context.scheduleAfterOutputWouldBeProduced( downstream, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), callback); assertThat(callLatch.await(1, TimeUnit.SECONDS), is(true)); }
@Test public void callAfterOutputMustHaveBeenProducedAlreadyAfterCallsImmediately() throws Exception { TransformResult<?> finishedResult = StepTransformResult.withoutHold(createdProducer).build(); context.handleResult(null, ImmutableList.of(), finishedResult); final CountDownLatch callLatch = new CountDownLatch(1); context.extractFiredTimers(); Runnable callback = callLatch::countDown; context.scheduleAfterOutputWouldBeProduced( downstream, GlobalWindow.INSTANCE, WindowingStrategy.globalDefault(), callback); assertThat(callLatch.await(1, TimeUnit.SECONDS), is(true)); }
@Override public PCollection<Integer> expand(PCollection<Integer> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded(), input.getCoder()); } }
@Override public PCollection<KeyedWorkItem<KeyT, InputT>> expand(PCollection<KV<KeyT, InputT>> input) { KvCoder<KeyT, InputT> kvCoder = (KvCoder<KeyT, InputT>) input.getCoder(); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded(), KeyedWorkItemCoder.of( kvCoder.getKeyCoder(), kvCoder.getValueCoder(), input.getWindowingStrategy().getWindowFn().windowCoder())); }
@Test public void groupByKeyProducesKeyedOutput() { PCollection<KV<String, Iterable<Integer>>> keyed = p.apply(Create.of(KV.of("foo", 3))) .apply(new DirectGroupByKeyOnly<>()) .apply( new DirectGroupAlsoByWindow<>( WindowingStrategy.globalDefault(), WindowingStrategy.globalDefault())); p.traverseTopologically(visitor); assertThat(visitor.getKeyedPValues(), hasItem(keyed)); }
@Override public PCollection<KV<String, Integer>> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); } });
@Override public PCollection<KV<String, Integer>> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); } })
@Override public PCollection<KV<String, Integer>> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); } });
@Test public void testOfThenHas() { PCollection<Integer> pCollection = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of()); TupleTag<Integer> tag = new TupleTag<>(); assertTrue(PCollectionTuple.of(tag, pCollection).has(tag)); }