Coder<KV<Void, Object>> kvCoder = KvCoder.of(VoidCoder.of(), coder.getValueCoder()); kvCoders.add(coder.withValueCoder(kvCoder)); coder.withValueCoder(KvCoder.of(VoidCoder.of(), IterableCoder.of(coder.getValueCoder()))); viewCoders.add(viewCoder);
KvCoder<K, V> inputElementCoder = (KvCoder<K, V>) inputCoder.getValueCoder();
KvCoder<K, V> inputElementCoder = (KvCoder<K, V>) inputCoder.getValueCoder();
Coder<KV<Void, Object>> kvCoder = KvCoder.of(VoidCoder.of(), coder.getValueCoder()); kvCoders.add(coder.withValueCoder(kvCoder)); coder.withValueCoder(KvCoder.of(VoidCoder.of(), IterableCoder.of(coder.getValueCoder()))); viewCoders.add(viewCoder);
String operatorName, StreamingTranslationContext context) { KvCoder<K, V> inputElementCoder = (KvCoder<K, V>) windowedInputCoder.getValueCoder();
coder = wvCoder.withValueCoder(IterableCoder.of(wvCoder.getValueCoder()));
String operatorName, StreamingTranslationContext context) { KvCoder<K, V> inputElementCoder = (KvCoder<K, V>) windowedInputCoder.getValueCoder();
coder = wvCoder.withValueCoder(IterableCoder.of(wvCoder.getValueCoder()));
key = ((KeyedWorkItem) value).key(); @SuppressWarnings({"rawtypes", "unchecked"}) KeyedWorkItemCoder<Object, ?> kwiCoder = (KeyedWorkItemCoder) wvCoder.getValueCoder(); keyCoder = kwiCoder.getKeyCoder(); } else { key = ((KV) value).getKey(); @SuppressWarnings({"rawtypes", "unchecked"}) KvCoder<Object, ?> kwiCoder = (KvCoder) wvCoder.getValueCoder(); keyCoder = kwiCoder.getKeyCoder();
checkArgument(windowedValueCoder instanceof WindowedValue.WindowedValueCoder); Coder<KV<K, Iterable<V>>> outputKvCoder = ((WindowedValueCoder<KV<K, Iterable<V>>>) windowedValueCoder).getValueCoder(); checkArgument(outputKvCoder instanceof KvCoder); Coder<Iterable<V>> iterVCoder = ((KvCoder<K, Iterable<V>>) outputKvCoder).getValueCoder();
private Coder<K> getKeyCoder(PTransformNode application) { PCollectionNode inputPCollection = getOnlyElement(graph.getPerElementInputs(application)); try { // We know the type restrictions on the input PCollection, and the restrictions on the // Wire coder Builder builder = GroupByKeyOnlyEvaluatorFactory.this.components.toBuilder(); String wireCoderId = WireCoders.addRunnerWireCoder(inputPCollection, builder); Coder<WindowedValue<KV<K, V>>> wireCoder = (Coder<WindowedValue<KV<K, V>>>) RehydratedComponents.forComponents(builder.build()).getCoder(wireCoderId); checkArgument( wireCoder instanceof WindowedValue.WindowedValueCoder, "Wire %s must be a %s", Coder.class.getSimpleName(), WindowedValueCoder.class.getSimpleName()); WindowedValueCoder<KV<K, V>> windowedValueCoder = (WindowedValueCoder<KV<K, V>>) wireCoder; checkArgument( windowedValueCoder.getValueCoder() instanceof KvCoder, "Input elements to %s must be encoded with a %s", DirectGroupByKey.DirectGroupByKeyOnly.class.getSimpleName(), KvCoder.class.getSimpleName()); KvCoder<K, V> kvCoder = (KvCoder<K, V>) windowedValueCoder.getValueCoder(); return kvCoder.getKeyCoder(); } catch (IOException e) { throw new RuntimeException(e); } }
public Coder<V> getValueCoder(Coder<KV<K, Iterable<WindowedValue<V>>>> inputCoder) { // Coder<Iterable<...>> --> IterableCoder<...> Coder<Iterable<WindowedValue<V>>> iterableWindowedValueCoder = getKvCoder(inputCoder).getValueCoder(); checkArgument( iterableWindowedValueCoder instanceof IterableCoder, "%s requires a %s<..., %s> but got a %s", getClass().getSimpleName(), KvCoder.class.getSimpleName(), IterableCoder.class.getSimpleName(), iterableWindowedValueCoder); IterableCoder<WindowedValue<V>> iterableCoder = (IterableCoder<WindowedValue<V>>) iterableWindowedValueCoder; // Coder<WindowedValue<...>> --> WindowedValueCoder<...> Coder<WindowedValue<V>> iterableElementCoder = iterableCoder.getElemCoder(); checkArgument( iterableElementCoder instanceof WindowedValueCoder, "%s requires a %s<..., %s<%s>> but got a %s", getClass().getSimpleName(), KvCoder.class.getSimpleName(), IterableCoder.class.getSimpleName(), WindowedValueCoder.class.getSimpleName(), iterableElementCoder); WindowedValueCoder<V> windowedValueCoder = (WindowedValueCoder<V>) iterableElementCoder; return windowedValueCoder.getValueCoder(); }
@Override public PCollection<KV<K, Iterable<V>>> expand( PCollection<KV<K, Iterable<WindowedValue<V>>>> input) { @SuppressWarnings("unchecked") KvCoder<K, Iterable<WindowedValue<V>>> inputKvCoder = (KvCoder<K, Iterable<WindowedValue<V>>>) input.getCoder(); Coder<K> keyCoder = inputKvCoder.getKeyCoder(); Coder<Iterable<WindowedValue<V>>> inputValueCoder = inputKvCoder.getValueCoder(); IterableCoder<WindowedValue<V>> inputIterableValueCoder = (IterableCoder<WindowedValue<V>>) inputValueCoder; Coder<WindowedValue<V>> inputIterableElementCoder = inputIterableValueCoder.getElemCoder(); WindowedValueCoder<V> inputIterableWindowedValueCoder = (WindowedValueCoder<V>) inputIterableElementCoder; Coder<V> inputIterableElementValueCoder = inputIterableWindowedValueCoder.getValueCoder(); Coder<Iterable<V>> outputValueCoder = IterableCoder.of(inputIterableElementValueCoder); Coder<KV<K, Iterable<V>>> outputKvCoder = KvCoder.of(keyCoder, outputValueCoder); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), windowingStrategy, input.isBounded(), outputKvCoder); } }