@Override public Coder<T> getOutputCoder() { return source.getOutputCoder(); }
@Override public final PCollection<T> expand(PBegin input) { source.validate(); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, source.getOutputCoder()); }
@Override public final PCollection<ValueWithRecordId<T>> expand(PInput input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.UNBOUNDED, ValueWithRecordId.ValueWithRecordIdCoder.of(source.getOutputCoder())); }
@Override public PCollection<T> expand(PBegin input) { Coder<Shard<T>> shardCoder = SerializableCoder.of((Class<Shard<T>>) (Class) Shard.class); PCollection<ValueWithRecordId<T>> read = input .apply( "Create", Create.of( new AutoValue_BoundedReadFromUnboundedSource_Shard.Builder<T>() .setSource(source) .setMaxNumRecords(maxNumRecords) .setMaxReadTime(maxReadTime) .build()) .withCoder(shardCoder)) .apply("Split", ParDo.of(new SplitFn<>())) .setCoder(shardCoder) .apply("Reshuffle", Reshuffle.viaRandomKey()) .apply("Read", ParDo.of(new ReadFn<>())) .setCoder(ValueWithRecordId.ValueWithRecordIdCoder.of(source.getOutputCoder())); if (source.requiresDeduping()) { read = read.apply( Distinct.<ValueWithRecordId<T>, byte[]>withRepresentativeValueFn( ValueWithRecordId::getId) .withRepresentativeType(TypeDescriptor.of(byte[].class))); } return read.apply("StripIds", ParDo.of(new ValueWithRecordId.StripIdsDoFn<>())) .setCoder(source.getOutputCoder()); }
source.getOutputCoder(), GlobalWindow.Coder.INSTANCE); JavaDStream<WindowedValue<T>> readUnboundedStream = mapWithStateDStream