/** Extract the windows associated with the values. */ private Set<W> collectWindows(Iterable<WindowedValue<InputT>> values) throws Exception { Set<W> windows = new HashSet<>(); for (WindowedValue<?> value : values) { for (BoundedWindow untypedWindow : value.getWindows()) { @SuppressWarnings("unchecked") W window = (W) untypedWindow; windows.add(window); } } return windows; }
private Set<W> collectWindows(Iterable<WindowedValue<KV<K, InputT>>> values) { Set<W> windows = new HashSet<>(); for (WindowedValue<?> value : values) { for (BoundedWindow untypedWindow : value.getWindows()) { @SuppressWarnings("unchecked") W window = (W) untypedWindow; windows.add(window); } } return windows; } }
private Set<W> collectWindows(Iterable<WindowedValue<KV<K, InputT>>> values) { Set<W> windows = new HashSet<>(); for (WindowedValue<?> value : values) { for (BoundedWindow untypedWindow : value.getWindows()) { @SuppressWarnings("unchecked") W window = (W) untypedWindow; windows.add(window); } } return windows; } }
FlinkAssignContext(WindowFn<InputT, W> fn, WindowedValue<InputT> value) { fn.super(); if (Iterables.size(value.getWindows()) != 1) { throw new IllegalArgumentException( String.format( "%s passed to window assignment must be in a single window, but it was in %s: %s", WindowedValue.class.getSimpleName(), Iterables.size(value.getWindows()), value.getWindows())); } this.value = value; }
FlinkAssignContext(WindowFn<InputT, W> fn, WindowedValue<InputT> value) { fn.super(); if (Iterables.size(value.getWindows()) != 1) { throw new IllegalArgumentException( String.format( "%s passed to window assignment must be in a single window, but it was in %s: %s", WindowedValue.class.getSimpleName(), Iterables.size(value.getWindows()), value.getWindows())); } this.value = value; }
FlinkAssignContext(WindowFn<InputT, W> fn, WindowedValue<InputT> value) { fn.super(); if (Iterables.size(value.getWindows()) != 1) { throw new IllegalArgumentException( String.format( "%s passed to window assignment must be in a single window, but it was in %s: %s", WindowedValue.class.getSimpleName(), Iterables.size(value.getWindows()), value.getWindows())); } this.value = value; }
@Override public BoundedWindow window() { return Iterables.getOnlyElement(value.getWindows()); } }
@Override public <T> T sideInput(PCollectionView<T> view) { checkNotNull(input, "Input in SparkCombineContext must not be null!"); //validate element window. final Collection<? extends BoundedWindow> elementWindows = input.getWindows(); checkState( elementWindows.size() == 1, "sideInput can only be called when the main " + "input element is in exactly one window"); return sideInputReader.get(view, elementWindows.iterator().next()); } }
protected static <T> Iterable<WindowedValue<T>> sortByWindows(Iterable<WindowedValue<T>> iter) { List<WindowedValue<T>> sorted = Lists.newArrayList(iter); sorted.sort(Comparator.comparing(o -> Iterables.getOnlyElement(o.getWindows()).maxTimestamp())); return sorted; }
@Override public int compare( WindowedValue<KV<K, InputT>> o1, WindowedValue<KV<K, InputT>> o2) { return Iterables.getOnlyElement(o1.getWindows()).maxTimestamp() .compareTo(Iterables.getOnlyElement(o2.getWindows()).maxTimestamp()); } });
@Override protected boolean matchesSafely(WindowedValue<? extends T> windowedValue) { return valueMatcher.matches(windowedValue.getValue()) && timestampMatcher.matches(windowedValue.getTimestamp()) && windowsMatcher.matches(windowedValue.getWindows()) && paneInfoMatcher.matches(windowedValue.getPane()); } }
@Override public <T> void output(TupleTag<T> tag, WindowedValue<T> output) { collector.collect( WindowedValue.of( new RawUnionValue(outputMap.get(tag), output.getValue()), output.getTimestamp(), output.getWindows(), output.getPane())); } }
@Override public KV<Instant, WindowedValue<KV<K, V>>> map(WindowedValue<KV<K, V>> wv) { BoundedWindow window = Iterables.getOnlyElement(wv.getWindows()); Instant timestamp = timestampCombiner.assign(window, windowFn.getOutputTime(wv.getTimestamp(), window)); return KV.of(timestamp, wv); } }
@Override @SuppressWarnings("unchecked") public <T> void output(TupleTag<T> tag, WindowedValue<T> output) { collector.collect( WindowedValue.of( new RawUnionValue(0 /* single output */, output.getValue()), output.getTimestamp(), output.getWindows(), output.getPane())); } }
@Override @SuppressWarnings("unchecked") public <T> void output(TupleTag<T> tag, WindowedValue<T> output) { collector.collect( WindowedValue.of(new RawUnionValue(0 /* single output */, output.getValue()), output.getTimestamp(), output.getWindows(), output.getPane())); } }
@Override @SuppressWarnings("unchecked") public <T> void output(TupleTag<T> tag, WindowedValue<T> output) { collector.collect( WindowedValue.of( new RawUnionValue(0 /* single output */, output.getValue()), output.getTimestamp(), output.getWindows(), output.getPane())); } }
@Override public WindowedValue<KV<K, OutputT>> call(WindowedValue<KV<K, Iterable<InputT>>> windowedKv) throws Exception { return WindowedValue.of( KV.of(windowedKv.getValue().getKey(), fn.apply(windowedKv)), windowedKv.getTimestamp(), windowedKv.getWindows(), windowedKv.getPane()); } }
/** * Returns a collection of {@link WindowedValue WindowedValues} identical to this one, except each * is in exactly one of the windows that this {@link WindowedValue} is in. */ public Iterable<WindowedValue<T>> explodeWindows() { ImmutableList.Builder<WindowedValue<T>> windowedValues = ImmutableList.builder(); for (BoundedWindow w : getWindows()) { windowedValues.add(of(getValue(), getTimestamp(), w, getPane())); } return windowedValues.build(); }
@Override public void encode(WindowedValue<T> windowedElem, OutputStream outStream, Context context) throws CoderException, IOException { InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); windowsCoder.encode(windowedElem.getWindows(), outStream); PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream); valueCoder.encode(windowedElem.getValue(), outStream, context); }
@Override public void registerByteSizeObserver(WindowedValue<T> value, ElementByteSizeObserver observer) throws Exception { InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer); windowsCoder.registerByteSizeObserver(value.getWindows(), observer); PaneInfoCoder.INSTANCE.registerByteSizeObserver(value.getPane(), observer); valueCoder.registerByteSizeObserver(value.getValue(), observer); }