private String paneInfoToString(PaneInfo paneInfo) { String paneString = String.format("pane-%d", paneInfo.getIndex()); if (paneInfo.getTiming() == Timing.LATE) { paneString = String.format("%s-late", paneString); } if (paneInfo.isLast()) { paneString = String.format("%s-last", paneString); } return paneString; }
/** * Set the value of the {@link PCollectionView} in the {@link BoundedWindow} to be based on the * specified values, if the values are part of a later pane than currently exist within the {@link * PCollectionViewWindow}. */ private void updatePCollectionViewWindowValues( PCollectionView<?> view, BoundedWindow window, Collection<WindowedValue<?>> windowValues) { PCollectionViewWindow<?> windowedView = PCollectionViewWindow.of(view, window); AtomicReference<Iterable<? extends WindowedValue<?>>> contents = viewByWindows.getUnchecked(windowedView); if (contents.compareAndSet(null, windowValues)) { // the value had never been set, so we set it and are done. return; } PaneInfo newPane = windowValues.iterator().next().getPane(); Iterable<? extends WindowedValue<?>> existingValues; long existingPane; do { existingValues = contents.get(); existingPane = Iterables.isEmpty(existingValues) ? -1L : existingValues.iterator().next().getPane().getIndex(); } while (newPane.getIndex() > existingPane && !contents.compareAndSet(existingValues, windowValues)); }
res.append("-"); res.append(pane.getIndex());
@ProcessElement public void processElement(@Element Attribution attribution, @Timestamp Instant timestamp, BoundedWindow window, PaneInfo pane, OutputReceiver<String> output) { StringBuilder builder = new StringBuilder(String.format( "%s: %s %s %-7s", Utils.formatWindow(window), attribution, Utils.formatTime(timestamp), pane.getTiming())); if (pane.getTiming() != PaneInfo.Timing.UNKNOWN) { builder.append(String.format(" index=%d", pane.getIndex())); if (pane.getNonSpeculativeIndex() > -1) builder.append(" onTimeIndex=" + pane.getNonSpeculativeIndex()); if (pane.isFirst()) builder.append(" isFirst"); if (pane.isLast()) builder.append(" isLast"); } output.output(builder.toString()); } }
@ProcessElement public void processElement(@Element KV<String, Integer> kv, @Timestamp Instant timestamp, BoundedWindow window, PaneInfo pane, OutputReceiver<String> output) { StringBuilder builder = new StringBuilder(String.format( "%s: %s:%-2d %s %-7s index=%d", Utils.formatWindow(window), kv.getKey(), kv.getValue(), Utils.formatTime(timestamp), pane.getTiming(), pane.getIndex())); if (pane.getNonSpeculativeIndex() > -1) builder.append(" onTimeIndex=" + pane.getNonSpeculativeIndex()); if (pane.isFirst()) builder.append(" isFirst"); if (pane.isLast()) builder.append(" isLast"); output.output(builder.toString()); } }
private PendingJobData startWriteRename( TableDestination finalTableDestination, Iterable<String> tempTableNames, ProcessContext c) throws Exception { WriteDisposition writeDisposition = (c.pane().getIndex() == 0) ? firstPaneWriteDisposition : WriteDisposition.WRITE_APPEND; CreateDisposition createDisposition = (c.pane().getIndex() == 0) ? firstPaneCreateDisposition : CreateDisposition.CREATE_NEVER; List<TableReference> tempTables = StreamSupport.stream(tempTableNames.spliterator(), false) .map(table -> BigQueryHelpers.fromJsonString(table, TableReference.class)) .collect(Collectors.toList()); ; // Make sure each destination table gets a unique job id. String jobIdPrefix = BigQueryHelpers.createJobId( c.sideInput(jobIdToken), finalTableDestination, -1, c.pane().getIndex()); BigQueryHelpers.PendingJob retryJob = startCopy( bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)), bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)), jobIdPrefix, finalTableDestination.getTableReference(), tempTables, writeDisposition, createDisposition); return new PendingJobData(retryJob, finalTableDestination, tempTables); }
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { String filenamePrefix = outputFilePrefix.isDirectory() ? "" : firstNonNull(outputFilePrefix.getFilename(), ""); IntervalWindow interval = (IntervalWindow) window; String windowStr = String.format("%s-%s", interval.start().toString(), interval.end().toString()); String filename = String.format( "%s-%s-%s-of-%s-pane-%s%s%s.avro", filenamePrefix, windowStr, shardNumber, numShards, paneInfo.getIndex(), paneInfo.isLast() ? "-last" : "", outputFileHints.getSuggestedFilenameSuffix()); return outputFilePrefix.getCurrentDirectory().resolve(filename, RESOLVE_FILE); }
/** * We should fire a non-empty ON_TIME pane in the GlobalWindow when the watermark moves to * end-of-time. */ @Test public void fireNonEmptyOnDrainInGlobalWindow() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(new GlobalWindows()) .withTrigger(Repeatedly.forever(AfterPane.elementCountAtLeast(3))) .withMode(AccumulationMode.DISCARDING_FIRED_PANES)); tester.advanceInputWatermark(new Instant(0)); final int n = 20; for (int i = 0; i < n; i++) { tester.injectElements(TimestampedValue.of(i, new Instant(i))); } List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertEquals(n / 3, output.size()); for (int i = 0; i < output.size(); i++) { assertEquals(Timing.EARLY, output.get(i).getPane().getTiming()); assertEquals(i, output.get(i).getPane().getIndex()); assertEquals(3, Iterables.size(output.get(i).getValue())); } tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); output = tester.extractOutput(); assertEquals(1, output.size()); assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming()); assertEquals(n / 3, output.get(0).getPane().getIndex()); assertEquals(n - ((n / 3) * 3), Iterables.size(output.get(0).getValue())); }
for (int i = 0; i < output.size(); i++) { assertEquals(Timing.EARLY, output.get(i).getPane().getTiming()); assertEquals(i, output.get(i).getPane().getIndex()); assertEquals(4, Iterables.size(output.get(i).getValue())); assertEquals(1, output.size()); assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming()); assertEquals((n + 3) / 4, output.get(0).getPane().getIndex()); assertEquals(0, Iterables.size(output.get(0).getValue()));
boolean isFirst = previousPane == null; Timing previousTiming = isFirst ? null : previousPane.getTiming(); long index = isFirst ? 0 : previousPane.getIndex() + 1; long nonSpeculativeIndex = isFirst ? 0 : previousPane.getNonSpeculativeIndex() + 1; Instant outputWM = timerInternals.currentOutputWatermarkTime();