@Override public Instant timestamp() { return timestampedValue.getTimestamp(); }
private Instant getTimestamp(Object value) { if (value instanceof TimestampedValue) { return ((TimestampedValue) value).getTimestamp(); } else { return Instant.now(); } } }
/** * Enqueue next micro-batch elements. This is backed by a {@link Queue} so stream input order * would keep the population order (FIFO). */ @SafeVarargs public final CreateStream<T> nextBatch(TimestampedValue<T>... batchElements) { // validate timestamps if timestamped elements. for (final TimestampedValue<T> timestampedValue : batchElements) { checkArgument( timestampedValue.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE), "Elements must have timestamps before %s. Got: %s", BoundedWindow.TIMESTAMP_MAX_VALUE, timestampedValue.getTimestamp()); } batches.offer(Arrays.asList(batchElements)); return this; }
@VisibleForTesting synchronized Instant getWatermark() { // Future elements that can be claimed in this restriction come either from // "pending" or from future polls, so the total watermark is // min(watermark for future polling, earliest remaining pending element) return Ordering.natural() .nullsLast() .min( pollWatermark, pending.isEmpty() ? null : pending.values().iterator().next().getTimestamp()); }
@Override public TimestampedValue<T> addInput( TimestampedValue<T> accumulator, TimestampedValue<T> input) { checkNotNull(accumulator, "accumulator must be non-null"); checkNotNull(input, "input must be non-null"); if (input.getTimestamp().isBefore(accumulator.getTimestamp())) { return accumulator; } else { return input; } }
@ProcessElement public void processElement(@Element TimestampedValue<T> element, OutputReceiver<T> r) { r.outputWithTimestamp(element.getValue(), element.getTimestamp()); } }
/** @deprecated Use {@link TestPipeline} with the {@code DirectRunner}. */ @Deprecated public void processTimestampedElement(TimestampedValue<InputT> element) throws Exception { checkNotNull(element, "Timestamped element cannot be null"); processWindowedElement(element.getValue(), element.getTimestamp(), GlobalWindow.INSTANCE); }
@Override public Object structuralValue(TimestampedValue<T> value) { Object structuralValue = valueCoder.structuralValue(value.getValue()); return TimestampedValue.of(structuralValue, value.getTimestamp()); }
/** * Adds the specified elements to the source with the provided timestamps. * * @return A {@link TestStream.Builder} like this one that will add the provided elements after * all earlier events have completed. */ @SafeVarargs public final Builder<T> addElements( TimestampedValue<T> element, TimestampedValue<T>... elements) { checkArgument( element.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE), "Elements must have timestamps before %s. Got: %s", BoundedWindow.TIMESTAMP_MAX_VALUE, element.getTimestamp()); for (TimestampedValue<T> multiElement : elements) { checkArgument( multiElement.getTimestamp().isBefore(BoundedWindow.TIMESTAMP_MAX_VALUE), "Elements must have timestamps before %s. Got: %s", BoundedWindow.TIMESTAMP_MAX_VALUE, multiElement.getTimestamp()); } ImmutableList<Event<T>> newEvents = ImmutableList.<Event<T>>builder() .addAll(events) .add(ElementEvent.add(element, elements)) .build(); return new Builder<>(coder, newEvents, currentWatermark); }
@Override public void encode(TimestampedValue<T> windowedElem, OutputStream outStream) throws IOException { valueCoder.encode(windowedElem.getValue(), outStream); InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); }
@ProcessElement public void processElement( @Element KV<K, TimestampedValue<V>> kv, OutputReceiver<KV<K, V>> r) { r.outputWithTimestamp( KV.of(kv.getKey(), kv.getValue().getValue()), kv.getValue().getTimestamp()); } }))
/** * Runs the {@link WindowFn} over the provided input, returning a map of windows to the timestamps * in those windows. This version allows to pass a list of {@link TimestampedValue} in case the * values are used to assign windows. */ public static <T, W extends BoundedWindow> Map<W, Set<String>> runWindowFnWithValue( WindowFn<T, W> windowFn, List<TimestampedValue<T>> timestampedValues) throws Exception { final TestWindowSet<W, String> windowSet = new TestWindowSet<>(); for (final TimestampedValue<T> element : timestampedValues) { for (W window : assignedWindowsWithValue(windowFn, element)) { windowSet.put(window, timestampValue(element.getTimestamp().getMillis())); } TestMergeContext<T, W> mergeContext = new TestMergeContext<>(windowSet, windowFn); windowFn.mergeWindows(mergeContext); } Map<W, Set<String>> actual = new HashMap<>(); for (W window : windowSet.windows()) { actual.put(window, windowSet.get(window)); } return actual; }
@Test public void testAtMinimumTimestamp() { TimestampedValue<String> tsv = TimestampedValue.atMinimumTimestamp("foobar"); assertEquals(BoundedWindow.TIMESTAMP_MIN_VALUE, tsv.getTimestamp()); }
@SafeVarargs private final <T> CommittedBundle<T> timestampedBundle( PCollection<T> pc, TimestampedValue<T>... values) { UncommittedBundle<T> bundle = bundleFactory.createBundle(pc); for (TimestampedValue<T> value : values) { bundle.add( WindowedValue.timestampedValueInGlobalWindow(value.getValue(), value.getTimestamp())); } return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE); }
public void fireTimers(W window, TimestampedValue<TimeDomain>... timers) throws Exception { ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); ArrayList<TimerData> timerData = new ArrayList<>(timers.length); for (TimestampedValue<TimeDomain> timer : timers) { timerData.add( TimerData.of( StateNamespaces.window(windowFn.windowCoder(), window), timer.getTimestamp(), timer.getValue())); } runner.onTimers(timerData); runner.persist(); }
/** * Assigns the given {@code timestampedValue} to windows using the specified {@code windowFn}, and * verifies that result of {@code windowFn.getOutputTimestamp} for each window is within the * proper bound. This version allows passing a {@link TimestampedValue} in case the value is * needed to assign windows. */ public static <T, W extends BoundedWindow> void validateNonInterferingOutputTimesWithValue( WindowFn<T, W> windowFn, TimestampedValue<T> timestampedValue) throws Exception { Collection<W> windows = assignedWindowsWithValue(windowFn, timestampedValue); Instant instant = timestampedValue.getTimestamp(); for (W window : windows) { Instant outputTimestamp = windowFn.getOutputTime(instant, window); assertFalse( "getOutputTime must be greater than or equal to input timestamp", outputTimestamp.isBefore(instant)); assertFalse( "getOutputTime must be less than or equal to the max timestamp", outputTimestamp.isAfter(window.maxTimestamp())); } }
public final void injectElements(List<TimestampedValue<InputT>> values) throws Exception { for (TimestampedValue<InputT> value : values) { WindowTracing.trace("TriggerTester.injectElements: {}", value); } Iterable<WindowedValue<InputT>> inputs = values .stream() .map( input -> { try { InputT value = input.getValue(); Instant timestamp = input.getTimestamp(); Collection<W> windows = windowFn.assignWindows( new TestAssignContext<W>( windowFn, value, timestamp, GlobalWindow.INSTANCE)); return WindowedValue.of(value, timestamp, windows, PaneInfo.NO_FIRING); } catch (Exception e) { throw new RuntimeException(e); } }) .collect(Collectors.toList()); ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); runner.processElements( new LateDataDroppingDoFnRunner.LateDataFilter(objectStrategy, timerInternals) .filter(KEY, inputs)); // Persist after each bundle. runner.persist(); }
Future<RecordMetadata> sendRecord(TimestampedValue<KV<K, V>> record, Counter sendCounter) { try { Long timestampMillis = spec.getPublishTimestampFunction() != null ? spec.getPublishTimestampFunction() .getTimestamp(record.getValue(), record.getTimestamp()) .getMillis() : null; Future<RecordMetadata> result = producer.send( new ProducerRecord<>( spec.getTopic(), null, timestampMillis, record.getValue().getKey(), record.getValue().getValue())); sendCounter.inc(); return result; } catch (KafkaException e) { ProducerSpEL.abortTransaction(producer); throw e; } }
@Test public void testValues() { Instant now = Instant.now(); TimestampedValue<String> tsv = TimestampedValue.of("foobar", now); assertEquals(now, tsv.getTimestamp()); assertEquals("foobar", tsv.getValue()); }
Future<RecordMetadata> sendRecord(TimestampedValue<KV<K, V>> record, Counter sendCounter) { try { Long timestampMillis = spec.getPublishTimestampFunction() != null ? spec.getPublishTimestampFunction() .getTimestamp(record.getValue(), record.getTimestamp()) .getMillis() : null; Future<RecordMetadata> result = producer.send( new ProducerRecord<>( spec.getTopic(), null, timestampMillis, record.getValue().getKey(), record.getValue().getValue())); sendCounter.inc(); return result; } catch (KafkaException e) { ProducerSpEL.abortTransaction(producer); throw e; } }