@Override public void processElement(WindowedValue<InputT> compressedElem) { if (observesWindow) { for (WindowedValue<InputT> elem : compressedElem.explodeWindows()) { invokeProcessElement(elem); } } else { invokeProcessElement(compressedElem); } }
final K key, Iterable<WindowedValue<V>> elements, final TimerInternals timerInternals) { return StreamSupport.stream(elements.spliterator(), false) .flatMap(wv -> StreamSupport.stream(wv.explodeWindows().spliterator(), false)) .filter( input -> {
final K key, Iterable<WindowedValue<V>> elements, final TimerInternals timerInternals) { return StreamSupport.stream(elements.spliterator(), false) .flatMap(wv -> StreamSupport.stream(wv.explodeWindows().spliterator(), false)) .filter( input -> {
@Override public Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedValue<InputT> elem) { if (views.isEmpty()) { // When there are no side inputs, we can preserve the compressed representation. underlying.processElement(elem); return Collections.emptyList(); } ImmutableList.Builder<WindowedValue<InputT>> pushedBack = ImmutableList.builder(); for (WindowedValue<InputT> windowElem : elem.explodeWindows()) { BoundedWindow mainInputWindow = Iterables.getOnlyElement(windowElem.getWindows()); if (isReady(mainInputWindow)) { // When there are any side inputs, we have to process the element in each window // individually, to disambiguate access to per-window side inputs. underlying.processElement(windowElem); } else { notReadyWindows.add(mainInputWindow); pushedBack.add(windowElem); } } return pushedBack.build(); }
@Override public void processElement(WindowedValue<InputT> input) { // StatefulDoFnRunner always observes windows, so we need to explode for (WindowedValue<InputT> value : input.explodeWindows()) { BoundedWindow window = value.getWindows().iterator().next(); if (isLate(window)) { // The element is too late for this window. droppedDueToLateness.inc(); WindowTracing.debug( "StatefulDoFnRunner.processElement: Dropping element at {}; window:{} " + "since too far behind inputWatermark:{}", input.getTimestamp(), window, cleanupTimer.currentInputWatermarkTime()); } else { cleanupTimer.setForWindow(window); doFnRunner.processElement(value); } } }
@Override public void processElement(WindowedValue<InputT> compressedElement) throws Exception { for (WindowedValue<InputT> element : compressedElement.explodeWindows()) { Collection<? extends BoundedWindow> windows = assignWindows(windowFn, element); outputBundle.add( WindowedValue.of( element.getValue(), element.getTimestamp(), windows, element.getPane())); } }
@Override public void processElement(WindowedValue<InputT> compressedElement) throws Exception { for (WindowedValue<InputT> element : compressedElement.explodeWindows()) { Collection<? extends BoundedWindow> windows = assignWindows(windowFn, element); outputBundle.add( WindowedValue.of( element.getValue(), element.getTimestamp(), windows, element.getPane())); } }
@Override public final void process( ApexStreamTuple<WindowedValue<KV<K, V>>> tuple, OutputEmitter<ApexStreamTuple<? extends WindowedValue<?>>> outputEmitter) { if (tuple instanceof ApexStreamTuple.WatermarkTuple) { outputEmitter.emit(tuple); } else { for (WindowedValue<KV<K, V>> in : tuple.getValue().explodeWindows()) { KeyedWorkItem<K, V> kwi = KeyedWorkItems.elementsWorkItem( in.getValue().getKey(), Collections.singletonList(in.withValue(in.getValue().getValue()))); outputEmitter.emit(ApexStreamTuple.DataTuple.of(in.withValue(kwi))); } } } }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(in.withValue(workItem)); } } }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(WindowedValue.valueInGlobalWindow(workItem)); } } }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(in.withValue(workItem)); } } }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(WindowedValue.valueInGlobalWindow(workItem)); } } }
@Override public void flatMap( WindowedValue<KV<K, InputT>> inWithMultipleWindows, Collector<WindowedValue<SingletonKeyedWorkItem<K, InputT>>> out) throws Exception { // we need to wrap each one work item per window for now // since otherwise the PushbackSideInputRunner will not correctly // determine whether side inputs are ready // // this is tracked as https://issues.apache.org/jira/browse/BEAM-1850 for (WindowedValue<KV<K, InputT>> in : inWithMultipleWindows.explodeWindows()) { SingletonKeyedWorkItem<K, InputT> workItem = new SingletonKeyedWorkItem<>( in.getValue().getKey(), in.withValue(in.getValue().getValue())); out.collect(in.withValue(workItem)); } } }
@Test public void testExplodeWindowsInOneWindowEquals() { Instant now = Instant.now(); BoundedWindow window = new IntervalWindow(now.minus(1000L), now.plus(1000L)); WindowedValue<String> value = WindowedValue.of("foo", now, window, PaneInfo.ON_TIME_AND_ONLY_FIRING); assertThat(Iterables.getOnlyElement(value.explodeWindows()), equalTo(value)); }
@Test public void processElementSideInputNotReadyMultipleWindows() { when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class))) .thenReturn(false); SimplePushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.of(singletonView)); WindowedValue<Integer> multiWindow = WindowedValue.of( 2, new Instant(-2), ImmutableList.of( new IntervalWindow(new Instant(-500L), new Instant(0L)), new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)), GlobalWindow.INSTANCE), PaneInfo.ON_TIME_AND_ONLY_FIRING); Iterable<WindowedValue<Integer>> multiWindowPushback = runner.processElementInReadyWindows(multiWindow); assertThat(multiWindowPushback, equalTo(multiWindow.explodeWindows())); assertThat(underlying.inputElems, emptyIterable()); }
private Iterable<WindowedValue<AccumT>> createAccumulator(WindowedValue<InputT> input) { Iterable<WindowedValue<InputT>> sortedInputs = sortByWindows(input.explodeWindows());
@Test public void processElementSideInputReadyAllWindows() { when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class))) .thenReturn(true); ImmutableList<PCollectionView<?>> views = ImmutableList.of(singletonView); SimplePushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(views); WindowedValue<Integer> multiWindow = WindowedValue.of( 2, new Instant(-2), ImmutableList.of( new IntervalWindow(new Instant(-500L), new Instant(0L)), new IntervalWindow(BoundedWindow.TIMESTAMP_MIN_VALUE, new Instant(250L)), GlobalWindow.INSTANCE), PaneInfo.ON_TIME_AND_ONLY_FIRING); Iterable<WindowedValue<Integer>> multiWindowPushback = runner.processElementInReadyWindows(multiWindow); assertThat(multiWindowPushback, emptyIterable()); assertThat( underlying.inputElems, containsInAnyOrder(ImmutableList.copyOf(multiWindow.explodeWindows()).toArray())); }
Iterable<WindowedValue<KV<K, InputT>>> sortedInputs = sortByWindows(wkvi.explodeWindows());
return null; return input.explodeWindows(); }) .filter(
@Test public void testExplodeWindowsManyWindowsMultipleWindowedValues() { Instant now = Instant.now(); BoundedWindow centerWindow = new IntervalWindow(now.minus(1000L), now.plus(1000L)); BoundedWindow pastWindow = new IntervalWindow(now.minus(1500L), now.plus(500L)); BoundedWindow futureWindow = new IntervalWindow(now.minus(500L), now.plus(1500L)); BoundedWindow futureFutureWindow = new IntervalWindow(now, now.plus(2000L)); PaneInfo pane = PaneInfo.createPane(false, false, Timing.ON_TIME, 3L, 0L); WindowedValue<String> value = WindowedValue.of( "foo", now, ImmutableList.of(pastWindow, centerWindow, futureWindow, futureFutureWindow), pane); assertThat( value.explodeWindows(), containsInAnyOrder( WindowedValue.of("foo", now, futureFutureWindow, pane), WindowedValue.of("foo", now, futureWindow, pane), WindowedValue.of("foo", now, centerWindow, pane), WindowedValue.of("foo", now, pastWindow, pane))); } }