@Override public WindowedValue<T> decode(InputStream inStream, Context context) throws CoderException, IOException { T value = valueCoder.decode(inStream, context); return WindowedValue.valueInGlobalWindow(value); }
@Override public WindowedValue<byte[]> nextRecord(WindowedValue<byte[]> windowedValue) { checkState(availableOutput); availableOutput = false; if (windowedValue != null) { return windowedValue; } return WindowedValue.valueInGlobalWindow(new byte[0]); }
@Override public WindowedValue<byte[]> nextRecord(WindowedValue<byte[]> windowedValue) { checkState(availableOutput); availableOutput = false; if (windowedValue != null) { return windowedValue; } return WindowedValue.valueInGlobalWindow(new byte[0]); }
@Override public void close() { final Iterator<Map.Entry<K, A>> iterator = keyToAcuumulator.entrySet().iterator(); while (iterator.hasNext()) { final Map.Entry<K, A> entry = iterator.next(); final K key = entry.getKey(); final A accum = entry.getValue(); final O output = combineFnRunner.extractOutput(accum, null, null, null); outputCollector.emit(WindowedValue.valueInGlobalWindow(KV.of(key, output))); iterator.remove(); // for eager garbage collection } }
/** * Returns a {@code WindowedValue} with the given value and timestamp, {@code GlobalWindow} and * default pane. */ public static <T> WindowedValue<T> timestampedValueInGlobalWindow(T value, Instant timestamp) { if (BoundedWindow.TIMESTAMP_MIN_VALUE.equals(timestamp)) { return valueInGlobalWindow(value); } else { return new TimestampedValueInGlobalWindow<>(value, timestamp, PaneInfo.NO_FIRING); } }
@Override public void fireTimer(InternalTimer<?, TimerInternals.TimerData> timer) { doFnRunner.processElement(WindowedValue.valueInGlobalWindow( KeyedWorkItems.<String, KV<InputT, RestrictionT>>timersWorkItem( (String) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
@Override public Collection<CommittedBundle<ImpulseShard>> getInitialInputs( PTransformNode transform, int targetParallelism) { return Collections.singleton( bundleFactory .<ImpulseShard>createRootBundle() .add(WindowedValue.valueInGlobalWindow(new ImpulseShard())) .commit(BoundedWindow.TIMESTAMP_MIN_VALUE)); } }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { doFnRunner.processElement(WindowedValue.valueInGlobalWindow( KeyedWorkItems.<K, InputT>timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); }
@Override public void fireTimer(InternalTimer<?, TimerData> timer) { doFnRunner.processElement( WindowedValue.valueInGlobalWindow( KeyedWorkItems.timersWorkItem( (K) keyedStateInternals.getKey(), Collections.singletonList(timer.getNamespace())))); } }
@Override public Collection<CommittedBundle<ImpulseShard>> getInitialInputs( AppliedPTransform<PBegin, PCollection<byte[]>, PTransform<PBegin, PCollection<byte[]>>> transform, int targetParallelism) { return Collections.singleton( ctxt.<ImpulseShard>createRootBundle() .add(WindowedValue.valueInGlobalWindow(new ImpulseShard())) .commit(BoundedWindow.TIMESTAMP_MIN_VALUE)); } }
@Override public void processElement(WindowedValue<ImpulseShard> element) throws Exception { PCollection<byte[]> outputPCollection = (PCollection<byte[]>) Iterables.getOnlyElement(transform.getOutputs().values()); result.addOutput( ctxt.createBundle(outputPCollection).add(WindowedValue.valueInGlobalWindow(new byte[0]))); }
@Test public void rootBundleSucceeds() { UncommittedBundle<byte[]> root = factory.createRootBundle(); byte[] array = new byte[] {0, 1, 2}; root.add(WindowedValue.valueInGlobalWindow(array)); CommittedBundle<byte[]> committed = root.commit(Instant.now()); assertThat( committed.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(array))); }
@Test public void addAfterCommitShouldThrowException() { UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); bundle.add(WindowedValue.valueInGlobalWindow(1)); CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now()); assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1))); thrown.expect(IllegalStateException.class); thrown.expectMessage("3"); thrown.expectMessage("committed"); bundle.add(WindowedValue.valueInGlobalWindow(3)); }
@Test public void addAfterCommitShouldThrowException() { UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); bundle.add(WindowedValue.valueInGlobalWindow(1)); CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now()); assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1))); thrown.expect(IllegalStateException.class); thrown.expectMessage("3"); thrown.expectMessage("committed"); bundle.add(WindowedValue.valueInGlobalWindow(3)); }
@Test public void rootBundleSucceedsIgnoresCoder() { WindowedValue<Record> one = WindowedValue.valueInGlobalWindow(new Record()); WindowedValue<Record> two = WindowedValue.valueInGlobalWindow(new Record()); CommittedBundle<Record> root = factory.<Record>createRootBundle().add(one).add(two).commit(Instant.now()); assertThat(root.getElements(), containsInAnyOrder(one, two)); }
@Test public void commitAfterCommitShouldThrowException() { UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); bundle.add(WindowedValue.valueInGlobalWindow(1)); CommittedBundle<Integer> firstCommit = bundle.commit(Instant.now()); assertThat(firstCommit.getElements(), containsInAnyOrder(WindowedValue.valueInGlobalWindow(1))); thrown.expect(IllegalStateException.class); thrown.expectMessage("committed"); bundle.commit(Instant.now()); }
@Test public void bundleDecodeFailsAddFails() { PCollection<Record> pc = p.apply(Create.empty(new RecordNoDecodeCoder())); UncommittedBundle<Record> bundle = factory.createBundle(pc); thrown.expect(UserCodeException.class); thrown.expectCause(isA(CoderException.class)); thrown.expectMessage("Decode not allowed"); bundle.add(WindowedValue.valueInGlobalWindow(new Record())); }
@Test public void bundleEncodeFailsAddFails() { PCollection<Record> pc = p.apply(Create.empty(new RecordNoEncodeCoder())); UncommittedBundle<Record> bundle = factory.createBundle(pc); thrown.expect(UserCodeException.class); thrown.expectCause(isA(CoderException.class)); thrown.expectMessage("Encode not allowed"); bundle.add(WindowedValue.valueInGlobalWindow(new Record())); }
@Test public void keyedBundleDecodeFailsAddFails() { PCollection<Record> pc = p.apply(Create.empty(new RecordNoDecodeCoder())); UncommittedBundle<Record> bundle = factory.createKeyedBundle(StructuralKey.of("foo", StringUtf8Coder.of()), pc); thrown.expect(UserCodeException.class); thrown.expectCause(isA(CoderException.class)); thrown.expectMessage("Decode not allowed"); bundle.add(WindowedValue.valueInGlobalWindow(new Record())); }
@Test public void keyedBundleEncodeFailsAddFails() { PCollection<Record> pc = p.apply(Create.empty(new RecordNoEncodeCoder())); UncommittedBundle<Record> bundle = factory.createKeyedBundle(StructuralKey.of("foo", StringUtf8Coder.of()), pc); thrown.expect(UserCodeException.class); thrown.expectCause(isA(CoderException.class)); thrown.expectMessage("Encode not allowed"); bundle.add(WindowedValue.valueInGlobalWindow(new Record())); }