public FlinkReduceFunction( CombineFnBase.GlobalCombineFn<?, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public FlinkReduceFunction( CombineFnBase.GlobalCombineFn<?, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public ApexReadUnboundedInputOperator( UnboundedSource<OutputT, CheckpointMarkT> source, ApexPipelineOptions options) { this.pipelineOptions = new SerializablePipelineOptions(options); this.source = source; this.isBoundedSource = false; }
public EvaluationContext(JavaSparkContext jsc, Pipeline pipeline, PipelineOptions options) { this.jsc = jsc; this.pipeline = pipeline; this.options = options; this.serializableOptions = new SerializablePipelineOptions(options); }
public SourceInputFormat( String stepName, BoundedSource<T> initialSource, PipelineOptions options) { this.stepName = stepName; this.initialSource = initialSource; this.serializedOptions = new SerializablePipelineOptions(options); }
public FlinkPartialReduceFunction( CombineFnBase.GlobalCombineFn<InputT, AccumT, ?> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public ApexReadUnboundedInputOperator( UnboundedSource<OutputT, CheckpointMarkT> source, boolean isBoundedSource, ApexPipelineOptions options) { this.pipelineOptions = new SerializablePipelineOptions(options); this.source = source; this.isBoundedSource = isBoundedSource; }
public FlinkMergingNonShuffleReduceFunction( CombineFnBase.GlobalCombineFn<InputT, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public SourceInputFormat( String stepName, BoundedSource<T> initialSource, PipelineOptions options) { this.stepName = stepName; this.initialSource = initialSource; this.serializedOptions = new SerializablePipelineOptions(options); }
public FlinkPartialReduceFunction( CombineFnBase.GlobalCombineFn<InputT, AccumT, ?> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public SourceInputFormat( String stepName, BoundedSource<T> initialSource, PipelineOptions options) { this.stepName = stepName; this.initialSource = initialSource; this.serializedOptions = new SerializablePipelineOptions(options); }
public FlinkMergingNonShuffleReduceFunction( CombineFnBase.GlobalCombineFn<InputT, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public FlinkReduceFunction( CombineFnBase.GlobalCombineFn<?, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public FlinkPartialReduceFunction( CombineFnBase.GlobalCombineFn<InputT, AccumT, ?> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
public FlinkMergingNonShuffleReduceFunction( CombineFnBase.GlobalCombineFn<InputT, AccumT, OutputT> combineFn, WindowingStrategy<Object, W> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions) { this.combineFn = combineFn; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); }
GearpumpSource(PipelineOptions options) { this.serializedOptions = new SerializablePipelineOptions(options); }
public FlinkStatefulDoFnFunction( DoFn<KV<K, V>, OutputT> dofn, String stepName, WindowingStrategy<?, ?> windowingStrategy, Map<PCollectionView<?>, WindowingStrategy<?, ?>> sideInputs, PipelineOptions pipelineOptions, Map<TupleTag<?>, Integer> outputMap, TupleTag<OutputT> mainOutputTag) { this.dofn = dofn; this.stepName = stepName; this.windowingStrategy = windowingStrategy; this.sideInputs = sideInputs; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); this.outputMap = outputMap; this.mainOutputTag = mainOutputTag; }
@SuppressWarnings("unchecked") public BoundedSourceWrapper( String stepName, PipelineOptions pipelineOptions, BoundedSource<OutputT> source, int parallelism) throws Exception { this.stepName = stepName; this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); long desiredBundleSize = source.getEstimatedSizeBytes(pipelineOptions) / parallelism; // get the splits early. we assume that the generated splits are stable, // this is necessary so that the mapping of state to source is correct // when restoring splitSources = source.split(desiredBundleSize, pipelineOptions); }
@SuppressWarnings("unchecked") public ApexGroupByKeyOperator( ApexPipelineOptions pipelineOptions, PCollection<KV<K, V>> input, ApexStateBackend stateBackend) { checkNotNull(pipelineOptions); this.serializedOptions = new SerializablePipelineOptions(pipelineOptions); this.windowingStrategy = (WindowingStrategy<V, BoundedWindow>) input.getWindowingStrategy(); this.keyCoder = ((KvCoder<K, V>) input.getCoder()).getKeyCoder(); this.valueCoder = ((KvCoder<K, V>) input.getCoder()).getValueCoder(); this.stateInternalsFactory = stateBackend.newStateInternalsFactory(keyCoder); TimerInternals.TimerDataCoder timerCoder = TimerInternals.TimerDataCoder.of(windowingStrategy.getWindowFn().windowCoder()); this.timerInternals = new ApexTimerInternals<>(timerCoder); }
@Test public void testSerializationAndDeserialization() throws Exception { PipelineOptions options = PipelineOptionsFactory.fromArgs("--foo=testValue", "--ignoredField=overridden") .as(MyOptions.class); SerializablePipelineOptions serializableOptions = new SerializablePipelineOptions(options); assertEquals("testValue", serializableOptions.get().as(MyOptions.class).getFoo()); assertEquals("overridden", serializableOptions.get().as(MyOptions.class).getIgnoredField()); SerializablePipelineOptions copy = SerializableUtils.clone(serializableOptions); assertEquals("testValue", copy.get().as(MyOptions.class).getFoo()); assertEquals("not overridden", copy.get().as(MyOptions.class).getIgnoredField()); }