public <F> F clean(F f) { return environment.clean(f); }
/** * Invokes the {@link org.apache.flink.api.java.ClosureCleaner} * on the given function if closure cleaning is enabled in the {@link ExecutionConfig}. * * @return The cleaned Function */ protected <F> F clean(F f) { return getExecutionEnvironment().clean(f); }
/** * Invokes the {@link org.apache.flink.api.java.ClosureCleaner} * on the given function if closure cleaning is enabled in the {@link ExecutionConfig}. * * @return The cleaned Function */ <F> F clean(F f) { return inputStream.getExecutionEnvironment().clean(f); }
protected <F> F clean(F f) { return getExecutionEnvironment().clean(f); } }
/** * Send late arriving data to the side output identified by the given {@link OutputTag}. Data * is considered late after the watermark has passed the end of the window plus the allowed * lateness set using {@link #allowedLateness(Time)}. * * <p>You can get the stream of late data using * {@link SingleOutputStreamOperator#getSideOutput(OutputTag)} on the * {@link SingleOutputStreamOperator} resulting from the windowed operation * with the same {@link OutputTag}. */ @PublicEvolving public AllWindowedStream<T, W> sideOutputLateData(OutputTag<T> outputTag) { Preconditions.checkNotNull(outputTag, "Side output tag must not be null."); this.lateDataOutputTag = input.getExecutionEnvironment().clean(outputTag); return this; }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Note that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ public <R> SingleOutputStreamOperator<R> apply(WindowFunction<T, R, K, W> function, TypeInformation<R> resultType) { function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableWindowFunction<>(function), resultType, function); }
/** * Applies a reduce function to the window. The window function is called for each evaluation * of the window for each key individually. The output of the reduce function is interpreted * as a regular non-windowed stream. * * <p>This window will try and incrementally aggregate data as much as the window policies * permit. For example, tumbling time windows can aggregate the data, meaning that only one * element per key is stored. Sliding time windows will aggregate on the granularity of the * slide interval, so a few elements are stored per key (one per slide interval). * Custom windows may not be able to incrementally aggregate, or may need to store extra values * in an aggregation tree. * * @param function The reduce function. * @return The data stream that is the result of applying the reduce function to the window. */ @SuppressWarnings("unchecked") public SingleOutputStreamOperator<T> reduce(ReduceFunction<T> function) { if (function instanceof RichFunction) { throw new UnsupportedOperationException("ReduceFunction of reduce can not be a RichFunction. " + "Please use reduce(ReduceFunction, WindowFunction) instead."); } //clean the closure function = input.getExecutionEnvironment().clean(function); return reduce(function, new PassThroughWindowFunction<K, W, T>()); }
/** * Send late arriving data to the side output identified by the given {@link OutputTag}. Data * is considered late after the watermark has passed the end of the window plus the allowed * lateness set using {@link #allowedLateness(Time)}. * * <p>You can get the stream of late data using * {@link SingleOutputStreamOperator#getSideOutput(OutputTag)} on the * {@link SingleOutputStreamOperator} resulting from the windowed operation * with the same {@link OutputTag}. */ @PublicEvolving public WindowedStream<T, K, W> sideOutputLateData(OutputTag<T> outputTag) { Preconditions.checkNotNull(outputTag, "Side output tag must not be null."); this.lateDataOutputTag = input.getExecutionEnvironment().clean(outputTag); return this; }
/** * Applies a reduce function to the window. The window function is called for each evaluation * of the window for each key individually. The output of the reduce function is interpreted * as a regular non-windowed stream. * * <p>This window will try and incrementally aggregate data as much as the window policies permit. * For example, tumbling time windows can aggregate the data, meaning that only one element per * key is stored. Sliding time windows will aggregate on the granularity of the slide interval, * so a few elements are stored per key (one per slide interval). * Custom windows may not be able to incrementally aggregate, or may need to store extra values * in an aggregation tree. * * @param function The reduce function. * @return The data stream that is the result of applying the reduce function to the window. */ @SuppressWarnings("unchecked") public SingleOutputStreamOperator<T> reduce(ReduceFunction<T> function) { if (function instanceof RichFunction) { throw new UnsupportedOperationException("ReduceFunction of reduce can not be a RichFunction. " + "Please use reduce(ReduceFunction, WindowFunction) instead."); } //clean the closure function = input.getExecutionEnvironment().clean(function); String callLocation = Utils.getCallLocationName(); String udfName = "AllWindowedStream." + callLocation; return reduce(function, new PassThroughAllWindowFunction<W, T>()); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ public <R> SingleOutputStreamOperator<R> apply(AllWindowFunction<T, R, W> function, TypeInformation<R> resultType) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableAllWindowFunction<>(function), resultType, callLocation); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Note that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ @Internal public <R> SingleOutputStreamOperator<R> process(ProcessWindowFunction<T, R, K, W> function, TypeInformation<R> resultType) { function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableProcessWindowFunction<>(function), resultType, function); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The process window function. * @return The data stream that is the result of applying the window function to the window. */ @PublicEvolving public <R> SingleOutputStreamOperator<R> process(ProcessAllWindowFunction<T, R, W> function, TypeInformation<R> resultType) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableProcessAllWindowFunction<>(function), resultType, callLocation); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ public <R> SingleOutputStreamOperator<R> apply(AllWindowFunction<T, R, W> function) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); TypeInformation<R> resultType = getAllWindowFunctionReturnType(function, getInputType()); return apply(new InternalIterableAllWindowFunction<>(function), resultType, callLocation); }
clean(function); StreamSource<OUT, ?> sourceOperator; if (function instanceof StoppableFunction) {
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The process window function. * @return The data stream that is the result of applying the window function to the window. */ @PublicEvolving public <R> SingleOutputStreamOperator<R> process(ProcessAllWindowFunction<T, R, W> function) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); TypeInformation<R> resultType = getProcessAllWindowFunctionReturnType(function, getInputType()); return apply(new InternalIterableProcessAllWindowFunction<>(function), resultType, callLocation); }
in.getExecutionEnvironment().clean(func), timeout, bufSize,
/** * Completes the join operation with the user function that is executed * for each combination of elements with the same key in a window. * * <p>Note: This method's return type does not support setting an operator-specific parallelism. * Due to binary backwards compatibility, this cannot be altered. Use the * {@link #with(JoinFunction, TypeInformation)}, method to set an operator-specific parallelism. */ public <T> DataStream<T> apply(JoinFunction<T1, T2, T> function, TypeInformation<T> resultType) { //clean the closure function = input1.getExecutionEnvironment().clean(function); coGroupedWindowedStream = input1.coGroup(input2) .where(keySelector1) .equalTo(keySelector2) .window(windowAssigner) .trigger(trigger) .evictor(evictor) .allowedLateness(allowedLateness); return coGroupedWindowedStream .apply(new JoinCoGroupFunction<>(function), resultType); }
/** * Completes the join operation with the user function that is executed * for each combination of elements with the same key in a window. * * <p>Note: This method's return type does not support setting an operator-specific parallelism. * Due to binary backwards compatibility, this cannot be altered. Use the * {@link #with(JoinFunction, TypeInformation)}, method to set an operator-specific parallelism. */ public <T> DataStream<T> apply(FlatJoinFunction<T1, T2, T> function, TypeInformation<T> resultType) { //clean the closure function = input1.getExecutionEnvironment().clean(function); coGroupedWindowedStream = input1.coGroup(input2) .where(keySelector1) .equalTo(keySelector2) .window(windowAssigner) .trigger(trigger) .evictor(evictor) .allowedLateness(allowedLateness); return coGroupedWindowedStream .apply(new FlatJoinCoGroupFunction<>(function), resultType); }
Preconditions.checkNotNull(outputType); final ProcessJoinFunction<IN1, IN2, OUT> cleanedUdf = left.getExecutionEnvironment().clean(processJoinFunction);
function = input1.getExecutionEnvironment().clean(function);