/** * Returns the {@link CombineFn} bound to its coders. * * <p>For internal use. */ public AppliedCombineFn<? super K, ? super InputT, ?, OutputT> getAppliedFn( CoderRegistry registry, Coder<? extends KV<K, ? extends Iterable<InputT>>> inputCoder, WindowingStrategy<?, ?> windowingStrategy) { KvCoder<K, InputT> kvCoder = getKvCoder(inputCoder); return AppliedCombineFn.withInputCoder(fn, registry, kvCoder, sideInputs, windowingStrategy); }
final Coder<K> keyCoder, final AppliedCombineFn<K, InputT, AccumT, OutputT> combineFn) { final StateTag<CombiningState<InputT, AccumT, OutputT>> bufferTag; if (combineFn.getFn() instanceof CombineFnWithContext) { bufferTag = StateTags.makeSystemTagInternal( StateTags.combiningValueWithContext( BUFFER_NAME, combineFn.getAccumulatorCoder(), (CombineFnWithContext<InputT, AccumT, OutputT>) combineFn.getFn())); StateTags.combiningValue( BUFFER_NAME, combineFn.getAccumulatorCoder(), (CombineFn<InputT, AccumT, OutputT>) combineFn.getFn()));
private static <K, InputT, AccumT, OutputT> AppliedCombineFn<K, InputT, AccumT, OutputT> create( GlobalCombineFn<InputT, AccumT, OutputT> fn, Coder<AccumT> accumulatorCoder, Iterable<PCollectionView<?>> sideInputViews, KvCoder<K, InputT> kvCoder, WindowingStrategy<?, ?> windowingStrategy) { return new AppliedCombineFn<>(fn, accumulatorCoder, sideInputViews, kvCoder, windowingStrategy); }
public static <K, InputT, AccumT, OutputT> AppliedCombineFn<K, InputT, AccumT, OutputT> withAccumulatorCoder( GlobalCombineFn<? super InputT, AccumT, OutputT> fn, Coder<AccumT> accumCoder, Iterable<PCollectionView<?>> sideInputViews, KvCoder<K, InputT> kvCoder, WindowingStrategy<?, ?> windowingStrategy) { // Casting down the K and InputT is safe because they're only used as inputs. @SuppressWarnings("unchecked") GlobalCombineFn<InputT, AccumT, OutputT> clonedFn = (GlobalCombineFn<InputT, AccumT, OutputT>) SerializableUtils.clone(fn); return create(clonedFn, accumCoder, sideInputViews, kvCoder, windowingStrategy); }
public static <K, InputT, AccumT, OutputT> AppliedCombineFn<K, InputT, AccumT, OutputT> withInputCoder( GlobalCombineFn<? super InputT, AccumT, OutputT> fn, CoderRegistry registry, KvCoder<K, InputT> kvCoder, Iterable<PCollectionView<?>> sideInputViews, WindowingStrategy<?, ?> windowingStrategy) { // Casting down the K and InputT is safe because they're only used as inputs. @SuppressWarnings("unchecked") GlobalCombineFn<InputT, AccumT, OutputT> clonedFn = (GlobalCombineFn<InputT, AccumT, OutputT>) SerializableUtils.clone(fn); try { Coder<AccumT> accumulatorCoder = clonedFn.getAccumulatorCoder(registry, kvCoder.getValueCoder()); return create(clonedFn, accumulatorCoder, sideInputViews, kvCoder, windowingStrategy); } catch (CannotProvideCoderException e) { throw new IllegalStateException("Could not determine coder for accumulator", e); } }
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder( GlobalCombineFn<InputT, AccumT, ?> combineFn, AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform) throws CannotProvideCoderException { @SuppressWarnings("unchecked") PCollection<KV<K, InputT>> mainInput = (PCollection<KV<K, InputT>>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform)); KvCoder<K, InputT> inputCoder = (KvCoder<K, InputT>) mainInput.getCoder(); return AppliedCombineFn.withInputCoder( combineFn, transform.getPipeline().getCoderRegistry(), inputCoder, transform.getTransform().getSideInputs(), ((PCollection<?>) Iterables.getOnlyElement(transform.getOutputs().values())) .getWindowingStrategy()) .getAccumulatorCoder(); }
context.getInput(primitiveTransform).getWindowingStrategy()); stepContext.addEncodingInput(fn.getAccumulatorCoder());
public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, TriggerStateMachine triggerStateMachine, CombineFnWithContext<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder, PipelineOptions options, SideInputReader sideInputReader) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); AppliedCombineFn<String, Integer, AccumT, OutputT> fn = AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return new ReduceFnTester<>( strategy, triggerStateMachine, SystemReduceFn.combining(StringUtf8Coder.of(), fn), outputCoder, options, sideInputReader); }
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and {@link CombineFn}, * creating a {@link TriggerStateMachine} from the {@link Trigger} in the {@link * WindowingStrategy}. */ public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, CombineFn<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); // Ensure that the CombineFn can be converted into an AppliedCombineFn AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return combining( strategy, TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(strategy.getTrigger())), combineFn, outputCoder); }
public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, CombineFnWithContext<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder, PipelineOptions options, SideInputReader sideInputReader) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); // Ensure that the CombineFn can be converted into an AppliedCombineFn AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return combining( strategy, TriggerStateMachines.stateMachineForTrigger( TriggerTranslation.toProto(strategy.getTrigger())), combineFn, outputCoder, options, sideInputReader); }
/** * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, {@link CombineFn}, * and {@link TriggerStateMachine}, for mocking the interaction between {@link ReduceFnRunner} and * the {@link TriggerStateMachine}. Ignores the {@link Trigger} in the {@link WindowingStrategy}. */ public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining( WindowingStrategy<?, W> strategy, TriggerStateMachine triggerStateMachine, CombineFn<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder) throws Exception { CoderRegistry registry = CoderRegistry.createDefault(); AppliedCombineFn<String, Integer, AccumT, OutputT> fn = AppliedCombineFn.withInputCoder( combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())); return new ReduceFnTester<>( strategy, triggerStateMachine, SystemReduceFn.combining(StringUtf8Coder.of(), fn), outputCoder, PipelineOptionsFactory.create(), NullSideInputReader.empty()); }
SystemReduceFn.combining( inputKvCoder.getKeyCoder(), AppliedCombineFn.withInputCoder( combineFn, input.getPipeline().getCoderRegistry(), inputKvCoder));
SystemReduceFn.combining( VarLongCoder.of(), AppliedCombineFn.withInputCoder( Sum.ofLongs(), CoderRegistry.createDefault(),
SystemReduceFn.combining( inputKvCoder.getKeyCoder(), AppliedCombineFn.withInputCoder( combineFn, input.getPipeline().getCoderRegistry(), inputKvCoder));
SystemReduceFn.combining( VarLongCoder.of(), AppliedCombineFn.withInputCoder( Sum.ofLongs(), CoderRegistry.createDefault(),
AppliedCombineFn.withInputCoder( transform.getFn(), input.getPipeline().getCoderRegistry(), inputKvCoder));