return TimestampedValueCoder.of( inferCoderFromObject(coderRegistry, schemaRegistry, ((TimestampedValue) o).getValue())); } else if (o instanceof List) {
KvCoder.of( inputCoder.getKeyCoder(), TimestampedValue.TimestampedValueCoder.of(inputCoder.getValueCoder()))) .apply("Latest Value", Combine.perKey(new LatestFn<>())) .setCoder(inputCoder);
ValuesSource<TimestampedValue<String>> valuesSource = new ValuesSource<>( TEST_VALUES, TimestampedValue.TimestampedValueCoder.of(StringUtf8Coder.of())); SourceForTest<TimestampedValue<String>> sourceForTest = new SourceForTest<>(options, valuesSource);
input, Create.of(timestampedElements).withCoder(TimestampedValueCoder.of(coder)));
@Override public PCollection<KV<K, TimestampedValue<V>>> expand(PCollection<KV<K, V>> input) { KvCoder<K, V> coder = (KvCoder<K, V>) input.getCoder(); return input .apply( ParDo.of( new DoFn<KV<K, V>, KV<K, TimestampedValue<V>>>() { @ProcessElement public void processElement( @Element KV<K, V> element, @Timestamp Instant timestamp, OutputReceiver<KV<K, TimestampedValue<V>>> r) { r.output( KV.of( element.getKey(), TimestampedValue.of(element.getValue(), timestamp))); } })) .setCoder( KvCoder.of(coder.getKeyCoder(), TimestampedValueCoder.of(coder.getValueCoder()))); } }
@Override public PCollection<T> expand(PCollection<T> input) { Coder<T> inputCoder = input.getCoder(); return input .apply( "Reify Timestamps", ParDo.of( new DoFn<T, TimestampedValue<T>>() { @ProcessElement public void processElement( @Element T element, @Timestamp Instant timestamp, OutputReceiver<TimestampedValue<T>> r) { r.output(TimestampedValue.of(element, timestamp)); } })) .setCoder(TimestampedValue.TimestampedValueCoder.of(inputCoder)) .apply("Latest Value", Combine.globally(new LatestFn<>())) .setCoder(NullableCoder.of(inputCoder)); } }
@Override public PCollection<KV<K, V>> expand(PCollection<KV<K, TimestampedValue<V>>> input) { KvCoder<K, TimestampedValue<V>> kvCoder = (KvCoder<K, TimestampedValue<V>>) input.getCoder(); TimestampedValueCoder<V> tvCoder = (TimestampedValueCoder<V>) kvCoder.getValueCoder(); return input .apply( ParDo.of( new DoFn<KV<K, TimestampedValue<V>>, KV<K, V>>() { @Override public Duration getAllowedTimestampSkew() { return Duration.millis(Long.MAX_VALUE); } @ProcessElement public void processElement( @Element KV<K, TimestampedValue<V>> kv, OutputReceiver<KV<K, V>> r) { r.outputWithTimestamp( KV.of(kv.getKey(), kv.getValue().getValue()), kv.getValue().getTimestamp()); } })) .setCoder(KvCoder.of(kvCoder.getKeyCoder(), tvCoder.getValueCoder())); } }
@Override public PCollection<TimestampedValue<T>> expand(PCollection<T> input) { return input .apply( ParDo.of( new DoFn<T, TimestampedValue<T>>() { @ProcessElement public void processElement( @Element T element, @Timestamp Instant timestamp, OutputReceiver<TimestampedValue<T>> r) { r.output(TimestampedValue.of(element, timestamp)); } })) .setCoder(TimestampedValueCoder.of(input.getCoder())); } }
@Test public void testDefaultCoderHandlesNull() throws CannotProvideCoderException { Latest.LatestFn<Long> fn = new Latest.LatestFn<>(); CoderRegistry registry = CoderRegistry.createDefault(); TimestampedValue.TimestampedValueCoder<Long> inputCoder = TimestampedValue.TimestampedValueCoder.of(VarLongCoder.of()); assertThat( "Default output coder should handle null values", fn.getDefaultOutputCoder(registry, inputCoder), instanceOf(NullableCoder.class)); assertThat( "Default accumulator coder should handle null values", fn.getAccumulatorCoder(registry, inputCoder), instanceOf(NullableCoder.class)); } }
public static <T> TimestampedValueCoder<T> of(Coder<T> valueCoder) { return new TimestampedValueCoder<>(valueCoder); }
CheckpointCoder(Coder<T> elemCoder) { this.elemsCoder = NullableCoder.of(ListCoder.of(TimestampedValue.TimestampedValueCoder.of(elemCoder))); this.elemCoder = elemCoder; this.sourceCoder = NullableCoder.of(SerializableCoder.of(BoundedSource.class)); }
@Override public Coder<T> getDefaultOutputCoder( CoderRegistry registry, Coder<TimestampedValue<T>> inputCoder) throws CannotProvideCoderException { checkState( inputCoder instanceof TimestampedValue.TimestampedValueCoder, "inputCoder must be a TimestampedValueCoder, but was %s", inputCoder); TimestampedValue.TimestampedValueCoder<T> inputTVCoder = (TimestampedValue.TimestampedValueCoder<T>) inputCoder; return NullableCoder.of(inputTVCoder.getValueCoder()); }
ExactlyOnceWriter(Write<K, V> spec, Coder<KV<K, V>> elemCoder) { this.spec = spec; this.outOfOrderBufferSpec = StateSpecs.bag(KvCoder.of(BigEndianLongCoder.of(), TimestampedValueCoder.of(elemCoder))); }
private GrowthStateCoder( Coder<OutputT> outputCoder, Coder<TerminationStateT> terminationStateCoder) { this.outputCoder = outputCoder; this.terminationStateCoder = terminationStateCoder; this.completedCoder = MapCoder.of(HASH_CODE_CODER, INSTANT_CODER); this.timestampedOutputCoder = TimestampedValue.TimestampedValueCoder.of(outputCoder); }
@Override public void verifyDeterministic() throws NonDeterministicException { verifyDeterministic( this, "TimestampedValueCoder requires a deterministic valueCoder", valueCoder); }
ExactlyOnceWriter(Write<K, V> spec, Coder<KV<K, V>> elemCoder) { this.spec = spec; this.outOfOrderBufferSpec = StateSpecs.bag(KvCoder.of(BigEndianLongCoder.of(), TimestampedValueCoder.of(elemCoder))); }