@Override public T extractOutput(TimestampedValue<T> accumulator) { return accumulator.getValue(); } }
@Override public T element() { return timestampedValue.getValue(); }
@ProcessElement public void processElement(@Element TimestampedValue<T> element, OutputReceiver<T> r) { r.outputWithTimestamp(element.getValue(), element.getTimestamp()); } }
/** @deprecated Use {@link TestPipeline} with the {@code DirectRunner}. */ @Deprecated public void processTimestampedElement(TimestampedValue<InputT> element) throws Exception { checkNotNull(element, "Timestamped element cannot be null"); processWindowedElement(element.getValue(), element.getTimestamp(), GlobalWindow.INSTANCE); }
@Override public Object structuralValue(TimestampedValue<T> value) { Object structuralValue = valueCoder.structuralValue(value.getValue()); return TimestampedValue.of(structuralValue, value.getTimestamp()); }
@Override public void encode(TimestampedValue<T> windowedElem, OutputStream outStream) throws IOException { valueCoder.encode(windowedElem.getValue(), outStream); InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); }
Future<RecordMetadata> sendRecord(TimestampedValue<KV<K, V>> record, Counter sendCounter) { try { Long timestampMillis = spec.getPublishTimestampFunction() != null ? spec.getPublishTimestampFunction() .getTimestamp(record.getValue(), record.getTimestamp()) .getMillis() : null; Future<RecordMetadata> result = producer.send( new ProducerRecord<>( spec.getTopic(), null, timestampMillis, record.getValue().getKey(), record.getValue().getValue())); sendCounter.inc(); return result; } catch (KafkaException e) { ProducerSpEL.abortTransaction(producer); throw e; } }
Future<RecordMetadata> sendRecord(TimestampedValue<KV<K, V>> record, Counter sendCounter) { try { Long timestampMillis = spec.getPublishTimestampFunction() != null ? spec.getPublishTimestampFunction() .getTimestamp(record.getValue(), record.getTimestamp()) .getMillis() : null; Future<RecordMetadata> result = producer.send( new ProducerRecord<>( spec.getTopic(), null, timestampMillis, record.getValue().getKey(), record.getValue().getValue())); sendCounter.inc(); return result; } catch (KafkaException e) { ProducerSpEL.abortTransaction(producer); throw e; } }
private String tryClaimNextPending(GrowthTracker<String, ?, ?> tracker) { assertTrue(tracker.hasPending()); Map.Entry<HashCode, TimestampedValue<String>> entry = tracker.getNextPending(); tracker.tryClaim(entry.getKey()); return entry.getValue().getValue(); }
@ProcessElement public void processElement( @Element KV<K, TimestampedValue<V>> kv, OutputReceiver<KV<K, V>> r) { r.outputWithTimestamp( KV.of(kv.getKey(), kv.getValue().getValue()), kv.getValue().getTimestamp()); } }))
@Test public void testNullValue() { TimestampedValue<String> tsv = TimestampedValue.atMinimumTimestamp(null); assertEquals(null, tsv.getValue()); }
@Test public void testExtractOutput() { assertEquals(TV.getValue(), fn.extractOutput(TV)); }
@SafeVarargs private final <T> CommittedBundle<T> timestampedBundle( PCollection<T> pc, TimestampedValue<T>... values) { UncommittedBundle<T> bundle = bundleFactory.createBundle(pc); for (TimestampedValue<T> value : values) { bundle.add( WindowedValue.timestampedValueInGlobalWindow(value.getValue(), value.getTimestamp())); } return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE); }
public void fireTimers(W window, TimestampedValue<TimeDomain>... timers) throws Exception { ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); ArrayList<TimerData> timerData = new ArrayList<>(timers.length); for (TimestampedValue<TimeDomain> timer : timers) { timerData.add( TimerData.of( StateNamespaces.window(windowFn.windowCoder(), window), timer.getTimestamp(), timer.getValue())); } runner.onTimers(timerData); runner.persist(); }
public final void injectElements(List<TimestampedValue<InputT>> values) throws Exception { for (TimestampedValue<InputT> value : values) { WindowTracing.trace("TriggerTester.injectElements: {}", value); } Iterable<WindowedValue<InputT>> inputs = values .stream() .map( input -> { try { InputT value = input.getValue(); Instant timestamp = input.getTimestamp(); Collection<W> windows = windowFn.assignWindows( new TestAssignContext<W>( windowFn, value, timestamp, GlobalWindow.INSTANCE)); return WindowedValue.of(value, timestamp, windows, PaneInfo.NO_FIRING); } catch (Exception e) { throw new RuntimeException(e); } }) .collect(Collectors.toList()); ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner(); runner.processElements( new LateDataDroppingDoFnRunner.LateDataFilter(objectStrategy, timerInternals) .filter(KEY, inputs)); // Persist after each bundle. runner.persist(); }
@Test public void testValues() { Instant now = Instant.now(); TimestampedValue<String> tsv = TimestampedValue.of("foobar", now); assertEquals(now, tsv.getTimestamp()); assertEquals("foobar", tsv.getValue()); }
@Override public void processElement(WindowedValue<TestStreamIndex<T>> element) throws Exception { TestStreamIndex<T> streamIndex = element.getValue(); List<Event<T>> events = streamIndex.getTestStream().getEvents(); int index = streamIndex.getIndex(); Instant watermark = element.getTimestamp(); Event<T> event = events.get(index); if (event.getType().equals(EventType.ELEMENT)) { UncommittedBundle<T> bundle = context.createBundle( (PCollection<T>) Iterables.getOnlyElement(application.getOutputs().values())); for (TimestampedValue<T> elem : ((ElementEvent<T>) event).getElements()) { bundle.add( WindowedValue.timestampedValueInGlobalWindow(elem.getValue(), elem.getTimestamp())); } resultBuilder.addOutput(bundle); } if (event.getType().equals(EventType.WATERMARK)) { watermark = ((WatermarkEvent<T>) event).getWatermark(); } if (event.getType().equals(EventType.PROCESSING_TIME)) { ((TestClock) context.getClock()) .advance(((ProcessingTimeEvent<T>) event).getProcessingTimeAdvance()); } TestStreamIndex<T> next = streamIndex.next(); if (next.getIndex() < events.size()) { resultBuilder.addUnprocessedElements( Collections.singleton(WindowedValue.timestampedValueInGlobalWindow(next, watermark))); } }
.setEncodedElement( ByteString.copyFrom( CoderUtils.encodeToByteArray(coder, element.getValue()))));
InputT value = input.getValue(); Instant timestamp = input.getTimestamp(); Collection<W> assignedWindows =
input1 -> { for (TimestampedValue<TimestampedValue<String>> elem : input1) { Instant originalTimestamp = elem.getValue().getTimestamp(); Instant afterReshuffleTimestamp = elem.getTimestamp(); assertThat(