/** * A terse alias for producing timestamped longs in the {@link GlobalWindow}, where the timestamp * is the epoch offset by the value of the element. */ private static WindowedValue<Long> tgw(Long elem) { return WindowedValue.timestampedValueInGlobalWindow(elem, new Instant(elem)); }
@Override public Object readCurrent() { try { if (!isStarted) { isStarted = true; isCurrentAvailable = reader.start(); } else { isCurrentAvailable = reader.advance(); } } catch (final Exception e) { throw new RuntimeException(e); } if (isCurrentAvailable) { final O elem = reader.getCurrent(); return WindowedValue.timestampedValueInGlobalWindow(elem, reader.getCurrentTimestamp()); } else { throw new NoSuchElementException(); } }
@Override public Message read() { Message message = null; try { if (available) { T data = reader.getCurrent(); org.joda.time.Instant timestamp = reader.getCurrentTimestamp(); message = new DefaultMessage( WindowedValue.timestampedValueInGlobalWindow(data, timestamp), timestamp.getMillis()); } available = reader.advance(); } catch (Exception e) { close(); throw new RuntimeException(e); } return message; }
private boolean tryProduceNext() { try (Closeable ignored = MetricsEnvironment.scopedMetricsContainer(metricsContainer)) { if (closed) { return FAILED_TO_OBTAIN_NEXT; } else { checkState(next == null, "unexpected non-null value for next"); if (seekNext()) { next = WindowedValue.timestampedValueInGlobalWindow( reader.getCurrent(), reader.getCurrentTimestamp()); return SUCCESSFULLY_OBTAINED_NEXT; } else { close(); return FAILED_TO_OBTAIN_NEXT; } } } catch (final Exception e) { throw new RuntimeException("Failed to read data.", e); } }
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testAssignWindowsGlobal() { WindowFn slidingWindows = new GlobalWindows(); WindowAssignTranslator.AssignWindows<String> assignWindows = new WindowAssignTranslator.AssignWindows(slidingWindows); String value = "v1"; Instant timestamp = new Instant(1); WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow(value, timestamp); ArrayList<WindowedValue<String>> expected = new ArrayList<>(); expected.add(WindowedValue.timestampedValueInGlobalWindow(value, timestamp)); Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue); assertThat(expected, equalTo(Lists.newArrayList(result))); } }
@SafeVarargs private final <T> CommittedBundle<T> timestampedBundle( PCollection<T> pc, TimestampedValue<T>... values) { UncommittedBundle<T> bundle = bundleFactory.createBundle(pc); for (TimestampedValue<T> value : values) { bundle.add( WindowedValue.timestampedValueInGlobalWindow(value.getValue(), value.getTimestamp())); } return bundle.commit(BoundedWindow.TIMESTAMP_MAX_VALUE); }
@Override public void processElement(WindowedValue<BoundedSourceShard<OutputT>> element) throws Exception { BoundedSource<OutputT> source = element.getValue().getSource(); try (final BoundedReader<OutputT> reader = source.createReader(options)) { boolean contentsRemaining = reader.start(); Future<BoundedSource<OutputT>> residualFuture = startDynamicSplitThread(source, reader); UncommittedBundle<OutputT> output = evaluationContext.createBundle(outputPCollection); while (contentsRemaining) { output.add( WindowedValue.timestampedValueInGlobalWindow( reader.getCurrent(), reader.getCurrentTimestamp())); contentsRemaining = reader.advance(); } resultBuilder.addOutput(output); try { BoundedSource<OutputT> residual = residualFuture.get(); if (residual != null) { resultBuilder.addUnprocessedElements( element.withValue(BoundedSourceShard.of(residual))); } } catch (ExecutionException exex) { // Un-and-rewrap the exception thrown by attempting to split throw UserCodeException.wrap(exex.getCause()); } } }
@Test public void getElementsAfterAddShouldReturnAddedElements() { WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1); WindowedValue<Integer> secondValue = WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L)); afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue)); }
@Test public void getElementsAfterAddShouldReturnAddedElements() { WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1); WindowedValue<Integer> secondValue = WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L)); afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue)); }
@Test public void addElementsAtEndOfTimeThrows() { Instant timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE; WindowedValue<Integer> value = WindowedValue.timestampedValueInGlobalWindow(1, timestamp); UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage(timestamp.toString()); bundle.add(value); }
@Test public void addElementsAtEndOfTimeThrows() { Instant timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE; WindowedValue<Integer> value = WindowedValue.timestampedValueInGlobalWindow(1, timestamp); UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage(timestamp.toString()); bundle.add(value); }
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testAssignWindowsWithSessions() { WindowFn slidingWindows = Sessions.withGapDuration(Duration.millis(10)); WindowAssignTranslator.AssignWindows<String> assignWindows = new WindowAssignTranslator.AssignWindows(slidingWindows); String value = "v1"; Instant timestamp = new Instant(1); WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow(value, timestamp); ArrayList<WindowedValue<String>> expected = new ArrayList<>(); expected.add( WindowedValue.of( value, timestamp, new IntervalWindow(new Instant(1), new Instant(11)), PaneInfo.NO_FIRING)); Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue); assertThat(expected, equalTo(Lists.newArrayList(result))); }
assertThat( multiWindowPushback, containsInAnyOrder(WindowedValue.timestampedValueInGlobalWindow(2, new Instant(-2L)))); assertThat( underlying.inputElems,
@Test public void addElementsPastEndOfTimeThrows() { Instant timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE.plus(Duration.standardMinutes(2)); WindowedValue<Integer> value = WindowedValue.timestampedValueInGlobalWindow(1, timestamp); UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage(timestamp.toString()); bundle.add(value); }
@Test public void addElementsPastEndOfTimeThrows() { Instant timestamp = BoundedWindow.TIMESTAMP_MAX_VALUE.plus(Duration.standardMinutes(2)); WindowedValue<Integer> value = WindowedValue.timestampedValueInGlobalWindow(1, timestamp); UncommittedBundle<Integer> bundle = bundleFactory.createRootBundle(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage(timestamp.toString()); bundle.add(value); }
WindowedValue<Integer> first = WindowedValue.valueInGlobalWindow(1); WindowedValue<Integer> second = WindowedValue.timestampedValueInGlobalWindow(2, new Instant(-1000L)); WindowedValue<Integer> third = WindowedValue.timestampedValueInGlobalWindow(3, new Instant(1234L)); CommittedBundle<Integer> createdBundle = bundleFactory
@SuppressWarnings("unchecked") @Test public void withElementsShouldReturnIndependentBundle() { WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1); WindowedValue<Integer> secondValue = WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L)); CommittedBundle<Integer> committed = afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue)); WindowedValue<Integer> firstReplacement = WindowedValue.of( 9, new Instant(2048L), new IntervalWindow(new Instant(2044L), Instant.now()), PaneInfo.NO_FIRING); WindowedValue<Integer> secondReplacement = WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now()); CommittedBundle<Integer> withed = committed.withElements(ImmutableList.of(firstReplacement, secondReplacement)); assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement)); assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue)); assertThat(withed.getKey(), Matchers.equalTo(committed.getKey())); assertThat(withed.getPCollection(), equalTo(committed.getPCollection())); assertThat( withed.getSynchronizedProcessingOutputWatermark(), equalTo(committed.getSynchronizedProcessingOutputWatermark())); assertThat(withed.getMinimumTimestamp(), equalTo(new Instant(2048L))); }
@SuppressWarnings("unchecked") @Test public void withElementsShouldReturnIndependentBundle() { WindowedValue<Integer> firstValue = WindowedValue.valueInGlobalWindow(1); WindowedValue<Integer> secondValue = WindowedValue.timestampedValueInGlobalWindow(2, new Instant(1000L)); CommittedBundle<Integer> committed = afterCommitGetElementsShouldHaveAddedElements(ImmutableList.of(firstValue, secondValue)); WindowedValue<Integer> firstReplacement = WindowedValue.of( 9, new Instant(2048L), new IntervalWindow(new Instant(2044L), Instant.now()), PaneInfo.NO_FIRING); WindowedValue<Integer> secondReplacement = WindowedValue.timestampedValueInGlobalWindow(-1, Instant.now()); CommittedBundle<Integer> withed = committed.withElements(ImmutableList.of(firstReplacement, secondReplacement)); assertThat(withed.getElements(), containsInAnyOrder(firstReplacement, secondReplacement)); assertThat(committed.getElements(), containsInAnyOrder(firstValue, secondValue)); assertThat(withed.getKey(), equalTo(committed.getKey())); assertThat(withed.getPCollection(), equalTo(committed.getPCollection())); assertThat( withed.getSynchronizedProcessingOutputWatermark(), equalTo(committed.getSynchronizedProcessingOutputWatermark())); assertThat(withed.getMinimumTimestamp(), equalTo(new Instant(2048L))); }
WindowedValue.timestampedValueInGlobalWindow(Duration.ZERO, new Instant(0))); thrown.expect(UserCodeException.class); thrown.expectCause(isA(IllegalArgumentException.class)); WindowedValue.timestampedValueInGlobalWindow(Duration.millis(1L), new Instant(0)));
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testAssignWindowsWithSlidingWindow() { WindowFn slidingWindows = SlidingWindows.of(Duration.millis(10)).every(Duration.millis(5)); WindowAssignTranslator.AssignWindows<String> assignWindows = new WindowAssignTranslator.AssignWindows(slidingWindows); String value = "v1"; Instant timestamp = new Instant(1); WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow(value, timestamp); ArrayList<WindowedValue<String>> expected = new ArrayList<>(); expected.add( WindowedValue.of( value, timestamp, new IntervalWindow(new Instant(0), new Instant(10)), PaneInfo.NO_FIRING)); expected.add( WindowedValue.of( value, timestamp, new IntervalWindow(new Instant(-5), new Instant(5)), PaneInfo.NO_FIRING)); Iterator<WindowedValue<String>> result = assignWindows.flatMap(windowedValue); assertThat(expected, equalTo(Lists.newArrayList(result))); }