private boolean canProduceMultiplePanes(WindowingStrategy<?, ?> strategy) { // The default trigger is Repeatedly.forever(AfterWatermark.pastEndOfWindow()); This fires // for every late-arriving element if allowed lateness is nonzero, and thus we must have // an accumulating mode specified boolean dataCanArriveLate = !(strategy.getWindowFn() instanceof GlobalWindows) && strategy.getAllowedLateness().getMillis() > 0; boolean hasCustomTrigger = !(strategy.getTrigger() instanceof DefaultTrigger); return dataCanArriveLate || hasCustomTrigger; }
private static <T, W extends BoundedWindow> void validateWindowStrategy( WindowingStrategy<T, W> strategy) { if (!strategy.getWindowFn().isNonMerging() && (!strategy.getTrigger().getClass().equals(DefaultTrigger.class) || strategy.getAllowedLateness().isLongerThan(Duration.ZERO))) { throw new UnsupportedOperationException( String.format( "%s does not support merging windowing strategies, except when using the default " + "trigger and zero allowed lateness.", Distinct.class.getSimpleName())); } }
public static <W extends BoundedWindow> WatermarkCallback onGuaranteedFiring( BoundedWindow window, WindowingStrategy<?, W> strategy, Runnable callback) { @SuppressWarnings("unchecked") Instant firingAfter = strategy.getTrigger().getWatermarkThatGuaranteesFiring((W) window); return new WatermarkCallback(firingAfter, callback); }
public static <W extends BoundedWindow> WatermarkCallback onGuaranteedFiring( BoundedWindow window, WindowingStrategy<?, W> strategy, Runnable callback) { @SuppressWarnings("unchecked") Instant firingAfter = strategy.getTrigger().getWatermarkThatGuaranteesFiring((W) window); return new WatermarkCallback(firingAfter, callback); }
private boolean triggersOncePerWindow(WindowingStrategy windowingStrategy) { Trigger trigger = windowingStrategy.getTrigger(); return !(windowingStrategy.getWindowFn() instanceof GlobalWindows) && trigger instanceof DefaultTrigger && ZERO.equals(windowingStrategy.getAllowedLateness()); }
/** * Performs the same check as {@link GroupByKey}, provides more context in exception. * * <p>Verifies that the input PCollection is bounded, or that there is windowing/triggering * being used. Without this, the watermark (at end of global window) will never be reached. * * <p>Throws {@link UnsupportedOperationException} if validation fails. */ private void validateWindowIsSupported(PCollection<Row> upstream) { WindowingStrategy<?, ?> windowingStrategy = upstream.getWindowingStrategy(); if (windowingStrategy.getWindowFn() instanceof GlobalWindows && windowingStrategy.getTrigger() instanceof DefaultTrigger && upstream.isBounded() != BOUNDED) { throw new UnsupportedOperationException( "Please explicitly specify windowing in SQL query using HOP/TUMBLE/SESSION functions " + "(default trigger will be used in this case). " + "Unbounded input with global windowing and default trigger is not supported " + "in Beam SQL aggregations. " + "See GroupByKey section in Beam Programming Guide"); } }
private void verifyInputWindowing(PCollection<KV<KeyT, ValueT>> input) { if (input.isBounded().equals(PCollection.IsBounded.UNBOUNDED)) { checkArgument( !input.getWindowingStrategy().equals(WindowingStrategy.globalDefault()), "Cannot work with %s and GLOBAL %s", PCollection.IsBounded.UNBOUNDED, WindowingStrategy.class.getSimpleName()); checkArgument( input.getWindowingStrategy().getTrigger().getClass().equals(DefaultTrigger.class), "Cannot work with %s trigger. Write works correctly only with %s", input.getWindowingStrategy().getTrigger().getClass().getSimpleName(), DefaultTrigger.class.getSimpleName()); checkArgument( input.getWindowingStrategy().getAllowedLateness().equals(Duration.ZERO), "Write does not allow late data."); } }
public static void applicableTo(PCollection<?> input) { WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy(); // Verify that the input PCollection is bounded, or that there is windowing/triggering being // used. Without this, the watermark (at end of global window) will never be reached. if (windowingStrategy.getWindowFn() instanceof GlobalWindows && windowingStrategy.getTrigger() instanceof DefaultTrigger && input.isBounded() != IsBounded.BOUNDED) { throw new IllegalStateException( "GroupByKey cannot be applied to non-bounded PCollection in " + "the GlobalWindow without a trigger. Use a Window.into or Window.triggering transform " + "prior to GroupByKey."); } // Validate the window merge function. if (windowingStrategy.getWindowFn() instanceof InvalidWindows) { String cause = ((InvalidWindows<?>) windowingStrategy.getWindowFn()).getCause(); throw new IllegalStateException( "GroupByKey must have a valid Window merge function. " + "Invalid because: " + cause); } }
private void applicableTo(PCollection<?> input) { WindowingStrategy<?, ?> outputStrategy = getOutputStrategyInternal(input.getWindowingStrategy()); // Make sure that the windowing strategy is complete & valid. if (outputStrategy.isTriggerSpecified() && !(outputStrategy.getTrigger() instanceof DefaultTrigger) && !(outputStrategy.getWindowFn() instanceof GlobalWindows) && !outputStrategy.isAllowedLatenessSpecified()) { throw new IllegalArgumentException( "Except when using GlobalWindows," + " calling .triggering() to specify a trigger requires that the allowed lateness" + " be specified using .withAllowedLateness() to set the upper bound on how late" + " data can arrive before being dropped. See Javadoc for more details."); } if (!outputStrategy.isModeSpecified() && canProduceMultiplePanes(outputStrategy)) { throw new IllegalArgumentException( "Calling .triggering() to specify a trigger or calling .withAllowedLateness() to" + " specify an allowed lateness greater than zero requires that the accumulation" + " mode be specified using .discardingFiredPanes() or .accumulatingFiredPanes()." + " See Javadoc for more details."); } }
public WindowingStrategy<?, ?> updateWindowingStrategy(WindowingStrategy<?, ?> inputStrategy) { WindowFn<?, ?> inputWindowFn = inputStrategy.getWindowFn(); if (!inputWindowFn.isNonMerging()) { // Prevent merging windows again, without explicit user // involvement, e.g., by Window.into() or Window.remerge(). inputWindowFn = new InvalidWindows<>( "WindowFn has already been consumed by previous GroupByKey", inputWindowFn); } // We also switch to the continuation trigger associated with the current trigger. return inputStrategy .withWindowFn(inputWindowFn) .withTrigger(inputStrategy.getTrigger().getContinuationTrigger()); }
@Override public boolean equals(Object object) { if (!(object instanceof WindowingStrategy)) { return false; } WindowingStrategy<?, ?> other = (WindowingStrategy<?, ?>) object; return isAllowedLatenessSpecified() == other.isAllowedLatenessSpecified() && isModeSpecified() == other.isModeSpecified() && isTimestampCombinerSpecified() == other.isTimestampCombinerSpecified() && getMode().equals(other.getMode()) && getAllowedLateness().equals(other.getAllowedLateness()) && getClosingBehavior().equals(other.getClosingBehavior()) && getOnTimeBehavior().equals(other.getOnTimeBehavior()) && getTrigger().equals(other.getTrigger()) && getTimestampCombiner().equals(other.getTimestampCombiner()) && getWindowFn().equals(other.getWindowFn()); }
@Override public void processElement(WindowedValue<KeyedWorkItem<K, V>> element) throws Exception { KeyedWorkItem<K, V> workItem = element.getValue(); UncommittedBundle<KV<K, Iterable<V>>> bundle = bundleFactory.createKeyedBundle(this.key, outputCollection); outputBundles.add(bundle); RunnerApi.Trigger runnerApiTrigger = TriggerTranslation.toProto(windowingStrategy.getTrigger()); ReduceFnRunner<K, V, Iterable<V>, BoundedWindow> reduceFnRunner = new ReduceFnRunner<>( workItem.key(), windowingStrategy, ExecutableTriggerStateMachine.create( TriggerStateMachines.stateMachineForTrigger(runnerApiTrigger)), stateInternals, timerInternals, new OutputWindowedValueToBundle<>(bundle), null, reduceFn, null); // Drop any elements within expired windows reduceFnRunner.processElements( dropExpiredWindows(workItem.key(), workItem.elementsIterable(), timerInternals)); reduceFnRunner.onTimers(workItem.timersIterable()); reduceFnRunner.persist(); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { KeyedWorkItem<K, InputT> keyedWorkItem = c.element(); K key = keyedWorkItem.key(); StateInternals stateInternals = stateInternalsFactory.stateInternalsForKey(key); TimerInternals timerInternals = timerInternalsFactory.timerInternalsForKey(key); ReduceFnRunner<K, InputT, OutputT, W> reduceFnRunner = new ReduceFnRunner<>( key, windowingStrategy, ExecutableTriggerStateMachine.create( TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(windowingStrategy.getTrigger()))), stateInternals, timerInternals, outputWindowedValue(), sideInputReader, reduceFn, c.getPipelineOptions()); reduceFnRunner.processElements(keyedWorkItem.elementsIterable()); reduceFnRunner.onTimers(keyedWorkItem.timersIterable()); reduceFnRunner.persist(); } }
public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, CombineFnWithContext<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder, PipelineOptions options, SideInputReader sideInputReader) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); // Ensure that the CombineFn can be converted into an AppliedCombineFn AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return combining( strategy, TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(strategy.getTrigger())), combineFn, outputCoder, options, sideInputReader); }
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, creating a {@link * TriggerStateMachine} from its {@link Trigger}. */ public static <W extends BoundedWindow> ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining( WindowingStrategy<?, W> windowingStrategy) throws Exception { return new ReduceFnTester<>( windowingStrategy, TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(windowingStrategy.getTrigger())), SystemReduceFn.buffering(VarIntCoder.of()), IterableCoder.of(VarIntCoder.of()), PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and {@link CombineFn}, * creating a {@link TriggerStateMachine} from the {@link Trigger} in the {@link * WindowingStrategy}. */ public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, CombineFn<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); // Ensure that the CombineFn can be converted into an AppliedCombineFn AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return combining( strategy, TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(strategy.getTrigger())), combineFn, outputCoder); }
/** * Converts a {@link WindowingStrategy} into a {@link RunnerApi.WindowingStrategy}, registering * any components in the provided {@link SdkComponents}. */ public static RunnerApi.WindowingStrategy toProto( WindowingStrategy<?, ?> windowingStrategy, SdkComponents components) throws IOException { SdkFunctionSpec windowFnSpec = toProto(windowingStrategy.getWindowFn(), components); RunnerApi.WindowingStrategy.Builder windowingStrategyProto = RunnerApi.WindowingStrategy.newBuilder() .setOutputTime(toProto(windowingStrategy.getTimestampCombiner())) .setAccumulationMode(toProto(windowingStrategy.getMode())) .setClosingBehavior(toProto(windowingStrategy.getClosingBehavior())) .setAllowedLateness(windowingStrategy.getAllowedLateness().getMillis()) .setTrigger(TriggerTranslation.toProto(windowingStrategy.getTrigger())) .setWindowFn(windowFnSpec) .setAssignsToOneWindow(windowingStrategy.getWindowFn().assignsToOneWindow()) .setOnTimeBehavior(toProto(windowingStrategy.getOnTimeBehavior())) .setWindowCoderId( components.registerCoder(windowingStrategy.getWindowFn().windowCoder())); return windowingStrategyProto.build(); }
@Test public void testWindowIntoSetWindowfn() { WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply(Window.into(FixedWindows.of(Duration.standardMinutes(10)))) .getWindowingStrategy(); assertTrue(strategy.getWindowFn() instanceof FixedWindows); assertTrue(strategy.getTrigger() instanceof DefaultTrigger); assertEquals(AccumulationMode.DISCARDING_FIRED_PANES, strategy.getMode()); }
@Test public void testWindowIntoTriggersAndAccumulating() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); Repeatedly trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(5)); WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply( Window.<String>into(fixed10) .triggering(trigger) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO)) .getWindowingStrategy(); assertEquals(fixed10, strategy.getWindowFn()); assertEquals(trigger, strategy.getTrigger()); assertEquals(AccumulationMode.ACCUMULATING_FIRED_PANES, strategy.getMode()); }
@Test public void testWindowPropagatesEachPart() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); Repeatedly trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(5)); WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply("Mode", Window.<String>configure().accumulatingFiredPanes()) .apply( "Lateness", Window.<String>configure().withAllowedLateness(Duration.standardDays(1))) .apply("Trigger", Window.<String>configure().triggering(trigger)) .apply("Window", Window.into(fixed10)) .getWindowingStrategy(); assertEquals(fixed10, strategy.getWindowFn()); assertEquals(trigger, strategy.getTrigger()); assertEquals(AccumulationMode.ACCUMULATING_FIRED_PANES, strategy.getMode()); assertEquals(Duration.standardDays(1), strategy.getAllowedLateness()); }