@Override public void processElement(WindowedValue<Iterable<InT>> element) { for (InT input : element.getValue()) { elements.add(element.withValue(input)); } }
Iterable<WindowedValue<OutputT>> extractOutput(Iterable<WindowedValue<AccumT>> wvas) { return StreamSupport.stream(wvas.spliterator(), false) .map( wva -> { if (wva == null) { return null; } return wva.withValue(combineFn.extractOutput(wva.getValue(), ctxtForInput(wva))); }) .collect(Collectors.toList()); } }
Iterable<WindowedValue<OutputT>> extractOutput(Iterable<WindowedValue<KV<K, AccumT>>> wkvas) { return StreamSupport.stream(wkvas.spliterator(), false) .map( wkva -> { if (wkva == null) { return null; } AccumT accumulator = wkva.getValue().getValue(); return wkva.withValue(combineFn.extractOutput(accumulator, ctxtForInput(wkva))); }) .collect(Collectors.toList()); } }
@Override public WindowedValue<KV<Void, T>> map(WindowedValue<T> value) { return value.withValue(KV.of(null, value.getValue())); } }
@Override public WindowedValue<KV<Void, T>> map(WindowedValue<T> value) { return value.withValue(KV.of(null, value.getValue())); } }
@Override public void processElement(WindowedValue<KV<K, V>> element) { KV<K, V> kv = element.getValue(); K key = kv.getKey(); StructuralKey<K> groupingKey = StructuralKey.of(key, keyCoder); List<WindowedValue<V>> values = groupingMap.computeIfAbsent(groupingKey, k -> new ArrayList<>()); values.add(element.withValue(kv.getValue())); }
@Override public void processElement(WindowedValue<KV<K, V>> element) { KV<K, V> kv = element.getValue(); K key = kv.getKey(); StructuralKey<K> groupingKey = StructuralKey.of(key, keyCoder); List<WindowedValue<V>> values = groupingMap.computeIfAbsent(groupingKey, k -> new ArrayList<>()); values.add(element.withValue(kv.getValue())); }
@Override public void flatMap( WindowedValue<ValueWithRecordId<T>> value, Collector<WindowedValue<T>> collector) throws Exception { collector.collect(value.withValue(value.getValue().getValue())); } }
@Override public void flatMap( WindowedValue<ValueWithRecordId<T>> value, Collector<WindowedValue<T>> collector) throws Exception { collector.collect(value.withValue(value.getValue().getValue())); } }
@Override public void flatMap(WindowedValue<ValueWithRecordId<T>> value, Collector<WindowedValue<T>> collector) throws Exception { collector.collect(value.withValue(value.getValue().getValue())); }
@Override public void onData(final WindowedValue<T> element) { outputCollector.emit(element.withValue(new SideInputElement<>(index, element.getValue()))); }
/** Extract window from a {@link KV} with {@link WindowedValue} value. */ static <K, V> Function<KV<K, WindowedValue<V>>, WindowedValue<KV<K, V>>> toKVByWindowInValue() { return kv -> { WindowedValue<V> wv = kv.getValue(); return wv.withValue(KV.of(kv.getKey(), wv.getValue())); }; }
@Override @SuppressWarnings("unchecked") public void flatMap( WindowedValue<RawUnionValue> windowedValue, Collector<WindowedValue<T>> collector) throws Exception { int unionTag = windowedValue.getValue().getUnionTag(); if (unionTag == ourOutputTag) { collector.collect( (WindowedValue<T>) windowedValue.withValue(windowedValue.getValue().getValue())); } } }
@Override @SuppressWarnings("unchecked") public void flatMap( WindowedValue<RawUnionValue> windowedValue, Collector<WindowedValue<T>> collector) throws Exception { int unionTag = windowedValue.getValue().getUnionTag(); if (unionTag == ourOutputTag) { collector.collect( (WindowedValue<T>) windowedValue.withValue(windowedValue.getValue().getValue())); } } }
@Override protected void addSideInputValue(StreamRecord<RawUnionValue> streamRecord) { @SuppressWarnings("unchecked") WindowedValue<KV<Void, Iterable<?>>> value = (WindowedValue<KV<Void, Iterable<?>>>) streamRecord.getValue().getValue(); PCollectionView<?> sideInput = sideInputTagMapping.get(streamRecord.getValue().getUnionTag()); sideInputHandler.addSideInputValue(sideInput, value.withValue(value.getValue().getValue())); }
@Override @SuppressWarnings("unchecked") public void flatMap( WindowedValue<RawUnionValue> windowedValue, Collector<WindowedValue<T>> collector) throws Exception { int unionTag = windowedValue.getValue().getUnionTag(); if (unionTag == ourOutputTag) { collector.collect( (WindowedValue<T>) windowedValue.withValue(windowedValue.getValue().getValue())); } } }
@Override protected void addSideInputValue(StreamRecord<RawUnionValue> streamRecord) { @SuppressWarnings("unchecked") WindowedValue<KV<Void, Iterable<?>>> value = (WindowedValue<KV<Void, Iterable<?>>>) streamRecord.getValue().getValue(); PCollectionView<?> sideInput = sideInputTagMapping.get(streamRecord.getValue().getUnionTag()); sideInputHandler.addSideInputValue(sideInput, value.withValue(value.getValue().getValue())); }
@Override public void processElement( StreamRecord<WindowedValue<ValueWithRecordId<T>>> streamRecord) throws Exception { ByteBuffer currentKey = keyedStateBackend.getCurrentKey(); int groupIndex = keyedStateBackend.getCurrentKeyGroupIndex(); if (shouldOutput(groupIndex, currentKey)) { WindowedValue<ValueWithRecordId<T>> value = streamRecord.getValue(); output.collect(streamRecord.replace(value.withValue(value.getValue().getValue()))); } }
/** * Retrieve the values that have been output to this time, and clear out the output accumulator. */ public List<WindowedValue<OutputT>> extractOutput() { ImmutableList<WindowedValue<OutputT>> result = FluentIterable.from(testOutputter.outputs) .transform(input -> input.withValue(input.getValue().getValue())) .toList(); testOutputter.outputs.clear(); return result; }
@Override public void processElement(WindowedValue<KeyedWorkItem<K, InputT>> elem) { Iterable<WindowedValue<InputT>> nonLateElements = lateDataFilter.filter(elem.getValue().key(), elem.getValue().elementsIterable()); KeyedWorkItem<K, InputT> keyedWorkItem = KeyedWorkItems.workItem( elem.getValue().key(), elem.getValue().timersIterable(), nonLateElements); doFnRunner.processElement(elem.withValue(keyedWorkItem)); }