@Override public Coder<T> getOutputCoder() { return boundedSource.getOutputCoder(); }
@Override public Coder<T> getOutputCoder() { return underlying.getOutputCoder(); } }
/** * Given a reference {@code Source} and a list of {@code Source}s, assert that the union of the * records read from the list of sources is equal to the records read from the reference source. */ public static <T> void assertSourcesEqualReferenceSource( BoundedSource<T> referenceSource, List<? extends BoundedSource<T>> sources, PipelineOptions options) throws Exception { Coder<T> coder = referenceSource.getOutputCoder(); List<T> referenceRecords = readFromSource(referenceSource, options); List<T> bundleRecords = new ArrayList<>(); for (BoundedSource<T> source : sources) { assertThat( "Coder type for source " + source + " is not compatible with Coder type for referenceSource " + referenceSource, source.getOutputCoder(), equalTo(coder)); List<T> elems = readFromSource(source, options); bundleRecords.addAll(elems); } List<ReadableStructuralValue<T>> bundleValues = createStructuralValues(coder, bundleRecords); List<ReadableStructuralValue<T>> referenceValues = createStructuralValues(coder, referenceRecords); assertThat(bundleValues, containsInAnyOrder(referenceValues.toArray())); }
@Override public final PCollection<T> expand(PBegin input) { source.validate(); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, source.getOutputCoder()); }
/** * Assert that a {@code Reader} returns a {@code Source} that, when read from, produces the same * records as the reader. */ public static <T> void assertUnstartedReaderReadsSameAsItsSource( BoundedSource.BoundedReader<T> reader, PipelineOptions options) throws Exception { Coder<T> coder = reader.getCurrentSource().getOutputCoder(); List<T> expected = readFromUnstartedReader(reader); List<T> actual = readFromSource(reader.getCurrentSource(), options); List<ReadableStructuralValue<T>> expectedStructural = createStructuralValues(coder, expected); List<ReadableStructuralValue<T>> actualStructural = createStructuralValues(coder, actual); assertThat(actualStructural, containsInAnyOrder(expectedStructural.toArray())); }
+ "original source: %s, primary: %s, residual: %s", splitFraction, numItemsToReadBeforeSplit, source, primary, residual); Coder<T> coder = primary.getOutputCoder(); List<ReadableStructuralValue<T>> primaryValues = createStructuralValues(coder, primaryItems); List<ReadableStructuralValue<T>> currentValues = createStructuralValues(coder, currentItems);
@Override public PCollection<T> expand(PBegin input) { return input .apply(Impulse.create()) .apply(ParDo.of(new SplitBoundedSourceFn<>(source, DEFAULT_BUNDLE_SIZE))) .setCoder(new BoundedSourceCoder<>()) .apply(Reshuffle.viaRandomKey()) .apply(ParDo.of(new ReadFromBoundedSourceFn<>())) .setCoder(source.getOutputCoder()); } }