@Override public Object[] addInput(Object[] accumulator, DataT value) { for (int i = 0; i < combineFnCount; ++i) { Object input = extractInputFns.get(i).apply(value); accumulator[i] = combineFns.get(i).addInput(accumulator[i], input); } return accumulator; }
private static <InputT, AccumT, OutputT> List<AccumT> combineInputs( CombineFn<InputT, AccumT, OutputT> fn, Iterable<? extends Iterable<InputT>> shards) { List<AccumT> accumulators = new ArrayList<>(); int maybeCompact = 0; for (Iterable<InputT> shard : shards) { AccumT accumulator = fn.createAccumulator(); for (InputT elem : shard) { accumulator = fn.addInput(accumulator, elem); } if (maybeCompact++ % 2 == 0) { accumulator = fn.compact(accumulator); } accumulators.add(accumulator); } return accumulators; }
@ProcessElement public void processElement(ProcessContext context, BoundedWindow window) { WindowedStructuralKey<K> key = WindowedStructuralKey.create(keyCoder, context.element().getKey(), window); AccumT accumulator = accumulators.get(key); Instant assignedTs = timestampCombiner.assign(window, context.timestamp()); if (accumulator == null) { accumulator = combineFn.createAccumulator(); accumulators.put(key, accumulator); timestamps.put(key, assignedTs); } accumulators.put(key, combineFn.addInput(accumulator, context.element().getValue())); timestamps.put(key, timestampCombiner.combine(assignedTs, timestamps.get(key))); }
@Override public void add(InputT value) { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT current = state.value(); if (current == null) { current = combineFn.createAccumulator(); } current = combineFn.addInput(current, value); state.update(current); } catch (Exception e) { throw new RuntimeException("Error adding to state." , e); } }
@Override public void add(InputT value) { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT current = state.value(); if (current == null) { current = combineFn.createAccumulator(); } current = combineFn.addInput(current, value); state.update(current); } catch (Exception e) { throw new RuntimeException("Error adding to state.", e); } }
@Override public void add(InputT value) { try { org.apache.flink.api.common.state.ValueState<AccumT> state = flinkStateBackend.getPartitionedState( namespace.stringKey(), StringSerializer.INSTANCE, flinkStateDescriptor); AccumT current = state.value(); if (current == null) { current = combineFn.createAccumulator(); } current = combineFn.addInput(current, value); state.update(current); } catch (Exception e) { throw new RuntimeException("Error adding to state.", e); } }
/** * Applies this {@code CombineFn} to a collection of input values to produce a combined output * value. * * <p>Useful when using a {@code CombineFn} separately from a {@code Combine} transform. Does * not invoke the {@link #mergeAccumulators} operation. */ public OutputT apply(Iterable<? extends InputT> inputs) { AccumT accum = createAccumulator(); for (InputT input : inputs) { accum = addInput(accum, input); } return extractOutput(accum); }
@Override public AccumT addInput(AccumT accumulator, InputT value) { return fn.addInput(accumulator, value); }
@Override public AccumT addInput( AccumT accumulator, InputT input, PipelineOptions options, SideInputReader sideInputReader, Collection<? extends BoundedWindow> windows) { return combineFn.addInput(accumulator, input); }
@Override public AccumT addInput(AccumT accumulator, InputT input, Context c) { return combineFn.addInput(accumulator, input); }
@Override public void add(InputT input) { isCleared = false; accum = combineFn.addInput(accum, input); }
@Override public Object addInput(Object accumulator, T input) { T processedInput = getInput(input); return (processedInput == null) ? accumulator : combineFn.addInput(accumulator, getInput(input)); }
@Override public AccumT addInput(AccumT accumulator, InputOrAccum<InputT, AccumT> value) { if (value.accum == null) { return fn.addInput(accumulator, value.input); } else { return fn.mergeAccumulators(ImmutableList.of(accumulator, value.accum)); } }