/** * Gathers all panes of each window into a single output element. * * <p>This will gather all output panes into a single element, which causes them to be colocated * on a single worker. As a result, this is only suitable for {@link PCollection PCollections} * where all of the output elements for each pane fit in memory, such as in tests. */ public static <T> GatherAllPanes<T> globally() { return new GatherAllPanes<>(); }
@Override public PCollectionView<ActualT> expand(PBegin input) { final Coder<T> coder = actual.getCoder(); return actual .apply("FilterActuals", rewindowActuals.prepareActuals()) .apply("GatherPanes", GatherAllPanes.globally()) .apply("ExtractPane", MapElements.via(extractPane)) .setCoder(IterableCoder.of(actual.getCoder())) .apply(Flatten.iterables()) .apply("RewindowActuals", rewindowActuals.windowActuals()) .apply( ParDo.of( new DoFn<T, T>() { @ProcessElement public void processElement(ProcessContext context) throws CoderException { context.output(CoderUtils.clone(coder, context.element())); } })) .apply(actualView); } }
.apply("GatherAllOutputs", GatherAllPanes.globally()) .apply("RewindowActuals", rewindowingStrategy.windowActuals()) .apply("KeyForDummy", WithKeys.of(combinedKey))
@Test @Category(NeedsRunner.class) public void singlePaneSingleReifiedPane() { PCollection<Iterable<ValueInSingleWindow<Iterable<Long>>>> accumulatedPanes = p.apply(GenerateSequence.from(0).to(20000)) .apply(WithTimestamps.of(input -> new Instant(input * 10))) .apply( Window.<Long>into(FixedWindows.of(Duration.standardMinutes(1))) .triggering(AfterWatermark.pastEndOfWindow()) .withAllowedLateness(Duration.ZERO) .discardingFiredPanes()) .apply(WithKeys.<Void, Long>of((Void) null).withKeyType(new TypeDescriptor<Void>() {})) .apply(GroupByKey.create()) .apply(Values.create()) .apply(GatherAllPanes.globally()); PAssert.that(accumulatedPanes) .satisfies( input -> { for (Iterable<ValueInSingleWindow<Iterable<Long>>> windowedInput : input) { if (Iterables.size(windowedInput) > 1) { fail("Expected all windows to have exactly one pane, got " + windowedInput); return null; } } return null; }); p.run(); }
.apply(GroupByKey.create()) .apply(Values.create()) .apply(GatherAllPanes.globally());