@Test public void testIncludeSameComponentAtDifferentPaths() { final HasDisplayData subComponent1 = new HasDisplayData() { @Override public void populateDisplayData(Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; final HasDisplayData subComponent2 = new HasDisplayData() { @Override public void populateDisplayData(Builder builder) { builder.add(DisplayData.item("foo2", "bar2")); } }; HasDisplayData component = new HasDisplayData() { @Override public void populateDisplayData(Builder builder) { builder.include("p1", subComponent1).include("p2", subComponent2); } }; DisplayData data = DisplayData.from(component); assertThat(data, includesDisplayDataFor("p1", subComponent1)); assertThat(data, includesDisplayDataFor("p2", subComponent2)); }
@Test public void testIncludes() { final HasDisplayData subComponent = new HasDisplayData() { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; DisplayData data = DisplayData.from( new HasDisplayData() { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.include("p", subComponent); } }); assertThat(data, includesDisplayDataFor("p", subComponent)); }
@Test public void testDisplayData() { SerializableBoundedSource boundedSource = new SerializableBoundedSource() { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; SerializableUnboundedSource unboundedSource = new SerializableUnboundedSource() { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; Duration maxReadTime = Duration.standardMinutes(2345); Read.Bounded<String> bounded = Read.from(boundedSource); BoundedReadFromUnboundedSource<String> unbounded = Read.from(unboundedSource).withMaxNumRecords(1234).withMaxReadTime(maxReadTime); DisplayData boundedDisplayData = DisplayData.from(bounded); assertThat(boundedDisplayData, hasDisplayItem("source", boundedSource.getClass())); assertThat(boundedDisplayData, includesDisplayDataFor("source", boundedSource)); DisplayData unboundedDisplayData = DisplayData.from(unbounded); assertThat(unboundedDisplayData, hasDisplayItem("source", unboundedSource.getClass())); assertThat(unboundedDisplayData, includesDisplayDataFor("source", unboundedSource)); assertThat(unboundedDisplayData, hasDisplayItem("maxRecords", 1234)); assertThat(unboundedDisplayData, hasDisplayItem("maxReadTime", maxReadTime)); }
@Test public void testForwardsDisplayData() { TestCountingSource src = new TestCountingSource(1234) { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; BoundedReadFromUnboundedSource<KV<Integer, Integer>> read = Read.from(src).withMaxNumRecords(5); assertThat(DisplayData.from(read), includesDisplayDataFor("source", src)); }
@Test public void testDoFnWithContextDisplayData() { DoFn<String, String> fn = new DoFn<String, String>() { @ProcessElement public void proccessElement(ProcessContext c) {} @Override public void populateDisplayData(Builder builder) { builder.add(DisplayData.item("fnMetadata", "foobar")); } }; SingleOutput<String, String> parDo = ParDo.of(fn); DisplayData displayData = DisplayData.from(parDo); assertThat(displayData, includesDisplayDataFor("fn", fn)); assertThat(displayData, hasDisplayItem("fn", fn.getClass())); }
@Test public void testDoFnDisplayData() { DoFn<String, String> fn = new DoFn<String, String>() { @ProcessElement public void processElement(ProcessContext c) {} @Override public void populateDisplayData(Builder builder) { builder.add(DisplayData.item("doFnMetadata", "bar")); } }; SingleOutput<String, String> parDo = ParDo.of(fn); DisplayData displayData = DisplayData.from(parDo); assertThat( displayData, hasDisplayItem( allOf( hasKey("fn"), hasType(DisplayData.Type.JAVA_CLASS), DisplayDataMatchers.hasValue(fn.getClass().getName())))); assertThat(displayData, includesDisplayDataFor("fn", fn)); }
@Test public void testAssignDisplayDataUnchanged() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); Window<Object> original = Window.into(windowFn); WindowingStrategy<?, ?> updated = WindowingStrategy.globalDefault().withWindowFn(windowFn); DisplayData displayData = DisplayData.from(new Window.Assign<>(original, updated)); assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(displayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(displayData, not(hasDisplayItem("trigger"))); assertThat(displayData, not(hasDisplayItem("accumulationMode"))); assertThat(displayData, not(hasDisplayItem("allowedLateness"))); assertThat(displayData, not(hasDisplayItem("closingBehavior"))); assertThat(displayData, not(hasDisplayItem("timestampCombiner"))); }
@Test public void testComposedCombineDisplayData() { SimpleFunction<String, String> extractFn = new SimpleFunction<String, String>() { @Override public String apply(String input) { return input; } }; DisplayDataCombineFn combineFn1 = new DisplayDataCombineFn("value1"); DisplayDataCombineFn combineFn2 = new DisplayDataCombineFn("value2"); CombineFns.ComposedCombineFn<String> composedCombine = CombineFns.compose() .with(extractFn, combineFn1, new TupleTag<>()) .with(extractFn, combineFn2, new TupleTag<>()); DisplayData displayData = DisplayData.from(composedCombine); assertThat(displayData, hasDisplayItem("combineFn1", combineFn1.getClass())); assertThat(displayData, hasDisplayItem("combineFn2", combineFn2.getClass())); assertThat(displayData, includesDisplayDataFor("combineFn1", combineFn1)); assertThat(displayData, includesDisplayDataFor("combineFn2", combineFn2)); }
@Test public void testWithOutputTagsDisplayData() { DoFn<String, String> fn = new DoFn<String, String>() { @ProcessElement public void proccessElement(ProcessContext c) {} @Override public void populateDisplayData(Builder builder) { builder.add(DisplayData.item("fnMetadata", "foobar")); } }; ParDo.MultiOutput<String, String> parDo = ParDo.of(fn).withOutputTags(new TupleTag<>(), TupleTagList.empty()); DisplayData displayData = DisplayData.from(parDo); assertThat(displayData, includesDisplayDataFor("fn", fn)); assertThat(displayData, hasDisplayItem("fn", fn.getClass())); } }
@Test public void testDisplayData() { UniqueInts combineFn = new UniqueInts() { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("fnMetadata", "foobar")); } }; Combine.Globally<?, ?> combine = Combine.globally(combineFn).withFanout(1234); DisplayData displayData = DisplayData.from(combine); assertThat(displayData, hasDisplayItem("combineFn", combineFn.getClass())); assertThat(displayData, hasDisplayItem("emitDefaultOnEmptyInput", true)); assertThat(displayData, hasDisplayItem("fanout", 1234)); assertThat(displayData, includesDisplayDataFor("combineFn", combineFn)); }
@Test public void testIncludes() { final HasDisplayData subComponent = builder -> builder.add(DisplayData.item("foo", "bar")); HasDisplayData hasSubcomponent = builder -> builder.include("p", subComponent); HasDisplayData wrongPath = builder -> builder.include("q", subComponent); HasDisplayData deeplyNested = builder -> builder.include("p", builder1 -> builder1.include("p", subComponent)); HasDisplayData sameDisplayItemDifferentComponent = builder -> builder.add(DisplayData.item("foo", "bar")); Matcher<DisplayData> matcher = includesDisplayDataFor("p", subComponent); assertFalse( "should not match sub-component at different path", matcher.matches(DisplayData.from(wrongPath))); assertFalse( "should not match deeply nested sub-component", matcher.matches(DisplayData.from(deeplyNested))); assertFalse( "should not match identical display data from different component", matcher.matches(DisplayData.from(sameDisplayItemDifferentComponent))); assertThat(DisplayData.from(hasSubcomponent), matcher); }
@Test public void testDisplayData() { ByteSource inputSource = new ByteSource("foobar.txt", 1) { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; CompressedSource<?> compressedSource = CompressedSource.from(inputSource); CompressedSource<?> gzipSource = compressedSource.withDecompression(CompressionMode.GZIP); DisplayData compressedSourceDisplayData = DisplayData.from(compressedSource); DisplayData gzipDisplayData = DisplayData.from(gzipSource); assertThat(compressedSourceDisplayData, hasDisplayItem("compressionMode")); assertThat(gzipDisplayData, hasDisplayItem("compressionMode", CompressionMode.GZIP.toString())); assertThat(compressedSourceDisplayData, hasDisplayItem("source", inputSource.getClass())); assertThat(compressedSourceDisplayData, includesDisplayDataFor("source", inputSource)); }
@Test public void testDisplayData() { DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant( DefaultFilenamePolicy.fromParams( new Params() .withBaseFilename( getBaseOutputDirectory() .resolve("file", StandardResolveOptions.RESOLVE_FILE)) .withShardTemplate("-SS-of-NN"))); SimpleSink<Void> sink = new SimpleSink<Void>( getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; WriteFiles<String, ?, String> write = WriteFiles.to(sink); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("sink", sink.getClass())); assertThat(displayData, includesDisplayDataFor("sink", sink)); }
@Test public void testShardedDisplayData() { DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant( DefaultFilenamePolicy.fromParams( new Params() .withBaseFilename( getBaseOutputDirectory() .resolve("file", StandardResolveOptions.RESOLVE_FILE)) .withShardTemplate("-SS-of-NN"))); SimpleSink<Void> sink = new SimpleSink<Void>( getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; WriteFiles<String, ?, String> write = WriteFiles.to(sink).withNumShards(1); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("sink", sink.getClass())); assertThat(displayData, includesDisplayDataFor("sink", sink)); assertThat(displayData, hasDisplayItem("numShards", 1)); }
DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("sink", sink.getClass())); assertThat(displayData, includesDisplayDataFor("sink", sink)); assertThat(displayData, hasDisplayItem("spam", "ham"));
@Test public void testDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData displayData = DisplayData.from(window); assertThat(displayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(displayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(displayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( displayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(displayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(displayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat(displayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }
@Test @Category(ValidatesRunner.class) public void testPrimitiveDisplayData() { FixedWindows windowFn = FixedWindows.of(Duration.standardHours(5)); AfterWatermark.FromEndOfWindow triggerBuilder = AfterWatermark.pastEndOfWindow(); Duration allowedLateness = Duration.standardMinutes(10); Window.ClosingBehavior closingBehavior = Window.ClosingBehavior.FIRE_IF_NON_EMPTY; TimestampCombiner timestampCombiner = TimestampCombiner.END_OF_WINDOW; Window<?> window = Window.into(windowFn) .triggering(triggerBuilder) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness, closingBehavior) .withTimestampCombiner(timestampCombiner); DisplayData primitiveDisplayData = Iterables.getOnlyElement( DisplayDataEvaluator.create().displayDataForPrimitiveTransforms(window)); assertThat(primitiveDisplayData, hasDisplayItem("windowFn", windowFn.getClass())); assertThat(primitiveDisplayData, includesDisplayDataFor("windowFn", windowFn)); assertThat(primitiveDisplayData, hasDisplayItem("trigger", triggerBuilder.toString())); assertThat( primitiveDisplayData, hasDisplayItem("accumulationMode", AccumulationMode.ACCUMULATING_FIRED_PANES.toString())); assertThat(primitiveDisplayData, hasDisplayItem("allowedLateness", allowedLateness)); assertThat(primitiveDisplayData, hasDisplayItem("closingBehavior", closingBehavior.toString())); assertThat( primitiveDisplayData, hasDisplayItem("timestampCombiner", timestampCombiner.toString())); }