@Override public FullWindowedValueCoder fromCloudObject(CloudObject object) { List<Coder<?>> components = getComponents(object); checkArgument(components.size() == 2, "Expecting 2 components, got " + components.size()); @SuppressWarnings("unchecked") Coder<? extends BoundedWindow> window = (Coder<? extends BoundedWindow>) components.get(1); return FullWindowedValueCoder.of(components.get(0), window); }
if (pc.getWindowingStrategy() != null) { coder = FullWindowedValueCoder.of( pc.getCoder(), pc.getWindowingStrategy().getWindowFn().windowCoder());
Iterable<WindowedValue<T>> getValues(PCollection<T> pcollection) { if (windowedValues == null) { WindowFn<?, ?> windowFn = pcollection.getWindowingStrategy().getWindowFn(); Coder<? extends BoundedWindow> windowCoder = windowFn.windowCoder(); final WindowedValue.WindowedValueCoder<T> windowedValueCoder; if (windowFn instanceof GlobalWindows) { windowedValueCoder = WindowedValue.ValueOnlyWindowedValueCoder.of(pcollection.getCoder()); } else { windowedValueCoder = WindowedValue.FullWindowedValueCoder.of(pcollection.getCoder(), windowCoder); } JavaRDDLike<byte[], ?> bytesRDD = rdd.map(CoderHelpers.toByteFunction(windowedValueCoder)); List<byte[]> clientBytes = bytesRDD.collect(); windowedValues = clientBytes .stream() .map(bytes -> CoderHelpers.fromByteArray(bytes, windowedValueCoder)) .collect(Collectors.toList()); } return windowedValues; }
private static Coder<?> instantiateCoder(CommonCoder coder) { List<Coder<?>> components = new ArrayList<>(); for (CommonCoder innerCoder : coder.getComponents()) { components.add(instantiateCoder(innerCoder)); } String s = coder.getUrn(); if (s.equals(getUrn(StandardCoders.Enum.BYTES))) { return ByteArrayCoder.of(); } else if (s.equals(getUrn(StandardCoders.Enum.KV))) { return KvCoder.of(components.get(0), components.get(1)); } else if (s.equals(getUrn(StandardCoders.Enum.VARINT))) { return VarLongCoder.of(); } else if (s.equals(getUrn(StandardCoders.Enum.INTERVAL_WINDOW))) { return IntervalWindowCoder.of(); } else if (s.equals(getUrn(StandardCoders.Enum.ITERABLE))) { return IterableCoder.of(components.get(0)); } else if (s.equals(getUrn(StandardCoders.Enum.TIMER))) { return Timer.Coder.of(components.get(0)); } else if (s.equals(getUrn(StandardCoders.Enum.GLOBAL_WINDOW))) { return GlobalWindow.Coder.INSTANCE; } else if (s.equals(getUrn(StandardCoders.Enum.WINDOWED_VALUE))) { return WindowedValue.FullWindowedValueCoder.of( components.get(0), (Coder<BoundedWindow>) components.get(1)); } else { throw new IllegalStateException("Unknown coder URN: " + coder.getUrn()); } }
private static <K, V, OutputT> JavaPairRDD<TupleTag<?>, WindowedValue<?>> statefulParDoTransform( KvCoder<K, V> kvCoder, Coder<? extends BoundedWindow> windowCoder, JavaRDD<WindowedValue<KV<K, V>>> kvInRDD, MultiDoFnFunction<KV<K, V>, OutputT> doFnFunction) { Coder<K> keyCoder = kvCoder.getKeyCoder(); final WindowedValue.WindowedValueCoder<V> wvCoder = WindowedValue.FullWindowedValueCoder.of(kvCoder.getValueCoder(), windowCoder); JavaRDD<WindowedValue<KV<K, Iterable<WindowedValue<V>>>>> groupRDD = GroupCombineFunctions.groupByKeyOnly(kvInRDD, keyCoder, wvCoder, null); return groupRDD .map( input -> { final K key = input.getValue().getKey(); Iterable<WindowedValue<V>> value = input.getValue().getValue(); return FluentIterable.from(value) .transform( windowedValue -> windowedValue.withValue(KV.of(key, windowedValue.getValue()))) .iterator(); }) .flatMapToPair(doFnFunction); }
private static <K, V, W extends BoundedWindow> TransformEvaluator<Reshuffle<K, V>> reshuffle() { return new TransformEvaluator<Reshuffle<K, V>>() { @Override public void evaluate(Reshuffle<K, V> transform, EvaluationContext context) { @SuppressWarnings("unchecked") JavaRDD<WindowedValue<KV<K, V>>> inRDD = ((BoundedDataset<KV<K, V>>) context.borrowDataset(transform)).getRDD(); @SuppressWarnings("unchecked") final WindowingStrategy<?, W> windowingStrategy = (WindowingStrategy<?, W>) context.getInput(transform).getWindowingStrategy(); @SuppressWarnings("unchecked") final KvCoder<K, V> coder = (KvCoder<K, V>) context.getInput(transform).getCoder(); @SuppressWarnings("unchecked") final WindowFn<Object, W> windowFn = (WindowFn<Object, W>) windowingStrategy.getWindowFn(); final Coder<K> keyCoder = coder.getKeyCoder(); final WindowedValue.WindowedValueCoder<V> wvCoder = WindowedValue.FullWindowedValueCoder.of(coder.getValueCoder(), windowFn.windowCoder()); JavaRDD<WindowedValue<KV<K, V>>> reshuffled = GroupCombineFunctions.reshuffle(inRDD, keyCoder, wvCoder); context.putDataset(transform, new BoundedDataset<>(reshuffled)); } @Override public String toNativeString() { return "repartition(...)"; } }; }
@Override public PCollection<KV<Integer, Iterable<KV<KV<K, W>, WindowedValue<V>>>>> expand( PCollection<KV<K, V>> input) { @SuppressWarnings("unchecked") Coder<W> windowCoder = (Coder<W>) input.getWindowingStrategy().getWindowFn().windowCoder(); @SuppressWarnings("unchecked") KvCoder<K, V> inputCoder = (KvCoder<K, V>) input.getCoder(); PCollection<KV<Integer, KV<KV<K, W>, WindowedValue<V>>>> keyedByHash; keyedByHash = input.apply(ParDo.of(new GroupByKeyHashAndSortByKeyAndWindowDoFn<K, V, W>(coder))); keyedByHash.setCoder( KvCoder.of( VarIntCoder.of(), KvCoder.of( KvCoder.of(inputCoder.getKeyCoder(), windowCoder), FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder)))); return keyedByHash.apply(new GroupByKeyAndSortValuesOnly<>()); } }
/** Transforms the input {@link PCollection} into a singleton {@link Map} per window. */ private <W extends BoundedWindow> PCollection<?> applyForSingletonFallback( PCollection<KV<K, V>> input) { @SuppressWarnings("unchecked") Coder<W> windowCoder = (Coder<W>) input.getWindowingStrategy().getWindowFn().windowCoder(); @SuppressWarnings({"rawtypes", "unchecked"}) KvCoder<K, V> inputCoder = (KvCoder) input.getCoder(); @SuppressWarnings({"unchecked", "rawtypes"}) Coder<Function<Iterable<WindowedValue<V>>, Iterable<V>>> transformCoder = (Coder) SerializableCoder.of(IterableWithWindowedValuesToIterable.class); Coder<TransformedMap<K, Iterable<WindowedValue<V>>, Iterable<V>>> finalValueCoder = TransformedMapCoder.of( transformCoder, MapCoder.of( inputCoder.getKeyCoder(), IterableCoder.of( FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder)))); return BatchViewAsSingleton.applyForSingleton( runner, input, new ToMultimapDoFn<>(windowCoder), finalValueCoder, view); }
/** Transforms the input {@link PCollection} into a singleton {@link Map} per window. */ private <W extends BoundedWindow> PCollection<?> applyForSingletonFallback( PCollection<KV<K, V>> input) { @SuppressWarnings("unchecked") Coder<W> windowCoder = (Coder<W>) input.getWindowingStrategy().getWindowFn().windowCoder(); @SuppressWarnings({"rawtypes", "unchecked"}) KvCoder<K, V> inputCoder = (KvCoder) input.getCoder(); @SuppressWarnings({"unchecked", "rawtypes"}) Coder<Function<WindowedValue<V>, V>> transformCoder = (Coder) SerializableCoder.of(WindowedValueToValue.class); Coder<TransformedMap<K, WindowedValue<V>, V>> finalValueCoder = TransformedMapCoder.of( transformCoder, MapCoder.of( inputCoder.getKeyCoder(), FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder))); return BatchViewAsSingleton.applyForSingleton( runner, input, new ToMapDoFn<>(windowCoder), finalValueCoder, view); } }
@Override public PCollection<KV<Integer, Iterable<KV<W, WindowedValue<T>>>>> expand( PCollection<T> input) { @SuppressWarnings("unchecked") Coder<W> windowCoder = (Coder<W>) input.getWindowingStrategy().getWindowFn().windowCoder(); PCollection<KV<Integer, KV<W, WindowedValue<T>>>> rval = input.apply( ParDo.of(new UseWindowHashAsKeyAndWindowAsSortKeyDoFn<T, W>(ismCoderForHash))); rval.setCoder( KvCoder.of( VarIntCoder.of(), KvCoder.of(windowCoder, FullWindowedValueCoder.of(input.getCoder(), windowCoder)))); return rval.apply(new GroupByKeyAndSortValuesOnly<>()); } }
@Test public void windowedValueCoderComponentsToConstructor() throws IOException { FullWindowedValueCoder<Iterable<KV<String, Integer>>> javaCoder = FullWindowedValueCoder.of( IterableCoder.of(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())), IntervalWindowCoder.of()); MessageWithComponents coderAndComponents = CoderTranslation.toProto(javaCoder); WindowedValueCoderComponents windowedValueCoderComponents = ModelCoders.getWindowedValueCoderComponents(coderAndComponents.getCoder()); Coder windowedCoder = ModelCoders.windowedValueCoder( windowedValueCoderComponents.elementCoderId(), windowedValueCoderComponents.windowCoderId()); assertThat(windowedCoder, equalTo(coderAndComponents.getCoder())); }
static <T> IsmRecordCoder<WindowedValue<T>> coderForListLike( Coder<? extends BoundedWindow> windowCoder, Coder<T> valueCoder) { // TODO: swap to use a variable length long coder which has values which compare // the same as their byte representation compare lexicographically within the key coder return IsmRecordCoder.of( 1, // We hash using only the window 0, // There are no metadata records ImmutableList.of(windowCoder, BigEndianLongCoder.of()), FullWindowedValueCoder.of(valueCoder, windowCoder)); } }
static <V> IsmRecordCoder<WindowedValue<V>> coderForMapLike( Coder<? extends BoundedWindow> windowCoder, Coder<?> keyCoder, Coder<V> valueCoder) { // TODO: swap to use a variable length long coder which has values which compare // the same as their byte representation compare lexicographically within the key coder return IsmRecordCoder.of( 1, // We use only the key for hashing when producing value records 2, // Since the key is not present, we add the window to the hash when // producing metadata records ImmutableList.of(MetadataKeyCoder.of(keyCoder), windowCoder, BigEndianLongCoder.of()), FullWindowedValueCoder.of(valueCoder, windowCoder)); } }
private static <K, InputT> FullWindowedValueCoder<KV<K, Iterable<InputT>>> windowedValueKeyValueCoderOf( final Coder<K> keyCoder, final Coder<InputT> iCoder, final Coder<? extends BoundedWindow> wCoder) { return FullWindowedValueCoder.of(KvCoder.of(keyCoder, IterableCoder.of(iCoder)), wCoder); }
/** * Returns the {@code Coder} to use for a {@code WindowedValue<T>}, using the given valueCoder and * windowCoder. */ public static <T> FullWindowedValueCoder<T> getFullCoder( Coder<T> valueCoder, Coder<? extends BoundedWindow> windowCoder) { return FullWindowedValueCoder.of(valueCoder, windowCoder); }
private SingletonKeyedWorkItemCoder( Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) { this.keyCoder = keyCoder; this.elemCoder = elemCoder; this.windowCoder = windowCoder; valueCoder = WindowedValue.FullWindowedValueCoder.of(elemCoder, windowCoder); }
private SingletonKeyedWorkItemCoder( Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) { this.keyCoder = keyCoder; this.elemCoder = elemCoder; this.windowCoder = windowCoder; valueCoder = WindowedValue.FullWindowedValueCoder.of(elemCoder, windowCoder); }
private SingletonKeyedWorkItemCoder( Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) { this.keyCoder = keyCoder; this.elemCoder = elemCoder; this.windowCoder = windowCoder; valueCoder = WindowedValue.FullWindowedValueCoder.of(elemCoder, windowCoder); }
static <T> IsmRecordCoder<WindowedValue<T>> coderForSingleton( Coder<? extends BoundedWindow> windowCoder, Coder<T> valueCoder) { return IsmRecordCoder.of( 1, // We hash using only the window 0, // There are no metadata records ImmutableList.of(windowCoder), FullWindowedValueCoder.of(valueCoder, windowCoder)); } }
private KeyedWorkItemCoder( Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) { this.keyCoder = keyCoder; this.elemCoder = elemCoder; this.windowCoder = windowCoder; this.timersCoder = IterableCoder.of(TimerDataCoder.of(windowCoder)); this.elemsCoder = IterableCoder.of(FullWindowedValueCoder.of(elemCoder, windowCoder)); }