congrats Icon
New! Announcing Tabnine Chat Beta
Learn More
Tabnine Logo
UserCodeWrapper
Code IndexAdd Tabnine to your IDE (free)

How to use
UserCodeWrapper
in
org.apache.flink.api.common.operators.util

Best Java code snippets using org.apache.flink.api.common.operators.util.UserCodeWrapper (Showing top 20 results out of 315)

origin: apache/flink

  @Override
  protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
    FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
    
    FunctionUtils.setFunctionRuntimeContext(function, ctx);
    FunctionUtils.openFunction(function, this.parameters);
    
    ArrayList<T> result = new ArrayList<T>(inputData.size());
    ListCollector<T> collector = new ListCollector<T>(result);

    for (T element : inputData) {
      function.flatMap(element, collector);
    }
    
    FunctionUtils.closeFunction(function);
    
    return result;
  }
}
origin: apache/flink

private String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
  try {
    if (wrapper.hasObject()) {
      try {
        return wrapper.getUserCodeObject().toString();
      }
      catch (Throwable t) {
        return wrapper.getUserCodeClass().getName();
      }
    }
    else {
      return wrapper.getUserCodeClass().getName();
    }
  }
  catch (Throwable t) {
    return null;
  }
}

origin: apache/flink

if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) &&
    this.estimatedOutputSize >= 0) {
  estimator.addFileInputCost(this.estimatedOutputSize, costs);
    ((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) &&
    this.estimatedOutputSize >= 0) {
origin: apache/flink

/**
 * Marks the group reduce operation as combinable. Combinable operations may pre-reduce the
 * data before the actual group reduce operations. Combinable user-defined functions
 * must implement the interface {@link GroupCombineFunction}.
 * 
 * @param combinable Flag to mark the group reduce operation as combinable.
 */
public void setCombinable(boolean combinable) {
  // sanity check
  if (combinable && !GroupCombineFunction.class.isAssignableFrom(this.userFunction.getUserCodeClass())) {
    throw new IllegalArgumentException("Cannot set a UDF as combinable if it does not implement the interface " +
        GroupCombineFunction.class.getName());
  } else {
    this.combinable = combinable;
  }
}

origin: apache/flink

MapOperatorBase<?, ?, ?> worksetMapper = (MapOperatorBase<?, ?, ?>) worksetSelfJoin.getFirstInput();
assertEquals(IdentityMapper.class, worksetMapper.getUserCodeWrapper().getUserCodeClass());
assertEquals(NextWorksetMapper.class, nextWorksetMapper.getUserCodeWrapper().getUserCodeClass());
if (solutionSetJoin.getUserCodeWrapper().getUserCodeObject() instanceof WrappingFunction) {
  WrappingFunction<?> wf = (WrappingFunction<?>) solutionSetJoin.getUserCodeWrapper().getUserCodeObject();
  assertEquals(SolutionWorksetJoin.class, wf.getWrappedFunction().getClass());
  assertEquals(SolutionWorksetJoin.class, solutionSetJoin.getUserCodeWrapper().getUserCodeClass());
origin: apache/flink

super(pactContract);
if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) {
  throw new IllegalArgumentException("Input format has not been set.");
if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) {
  setParallelism(1);
  this.sequentialInput = true;
                        pactContract.getUserCodeWrapper().getUserCodeClass());
origin: apache/flink

  @Override
  protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
    MapPartitionFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
    
    FunctionUtils.setFunctionRuntimeContext(function, ctx);
    FunctionUtils.openFunction(function, this.parameters);
    
    ArrayList<OUT> result = new ArrayList<OUT>(inputData.size() / 4);

    TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
    TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

    CopyingIterator<IN> source = new CopyingIterator<IN>(inputData.iterator(), inSerializer);
    CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer);

    function.mapPartition(source, resultCollector);

    result.trimToSize();
    FunctionUtils.closeFunction(function);
    return result;
  }
}
origin: org.apache.flink/flink-optimizer_2.11

private String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
  try {
    if (wrapper.hasObject()) {
      try {
        return wrapper.getUserCodeObject().toString();
      }
      catch (Throwable t) {
        return wrapper.getUserCodeClass().getName();
      }
    }
    else {
      return wrapper.getUserCodeClass().getName();
    }
  }
  catch (Throwable t) {
    return null;
  }
}

origin: org.apache.flink/flink-optimizer

if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) &&
    this.estimatedOutputSize >= 0) {
  estimator.addFileInputCost(this.estimatedOutputSize, costs);
    ((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) &&
    this.estimatedOutputSize >= 0) {
origin: apache/flink

private <OUT> List<OUT> executeDataSource(GenericDataSourceBase<?, ?> source, int superStep)
    throws Exception {
  @SuppressWarnings("unchecked")
  GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
  // build the runtime context and compute broadcast variables, if necessary
  TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 1, 0, 1, 0);
  
  RuntimeUDFContext ctx;
  MetricGroup metrics = new UnregisteredMetricsGroup();
  if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
    ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
        new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
  } else {
    ctx = null;
  }
  return typedSource.executeOnCollections(ctx, executionConfig);
}

origin: apache/flink

format = getOperator().getFormatWrapper().getUserCodeObject();
Configuration config = getOperator().getParameters();
format.configure(config);
origin: org.apache.flink/flink-optimizer_2.10

private String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
  try {
    if (wrapper.hasObject()) {
      try {
        return wrapper.getUserCodeObject().toString();
      }
      catch (Throwable t) {
        return wrapper.getUserCodeClass().getName();
      }
    }
    else {
      return wrapper.getUserCodeClass().getName();
    }
  }
  catch (Throwable t) {
    return null;
  }
}

origin: org.apache.flink/flink-optimizer_2.11

if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) &&
    this.estimatedOutputSize >= 0) {
  estimator.addFileInputCost(this.estimatedOutputSize, costs);
    ((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) &&
    this.estimatedOutputSize >= 0) {
origin: apache/flink

private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception {
  Operator<?> inputOp = operator.getInput();
  if (inputOp == null) {
    throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input.");
  }
  
  @SuppressWarnings("unchecked")
  List<IN> inputData = (List<IN>) execute(inputOp, superStep);
  
  @SuppressWarnings("unchecked")
  SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator;
  
  // build the runtime context and compute broadcast variables, if necessary
  TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
  RuntimeUDFContext ctx;
  MetricGroup metrics = new UnregisteredMetricsGroup();
  if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
    ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
        new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
    
    for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
      List<?> bcData = execute(bcInputs.getValue());
      ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
    }
  } else {
    ctx = null;
  }
  return typedOp.executeOnCollections(inputData, ctx, executionConfig);
}

origin: apache/flink

  @Override
  protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
    MapFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
    
    FunctionUtils.setFunctionRuntimeContext(function, ctx);
    FunctionUtils.openFunction(function, this.parameters);
    
    ArrayList<OUT> result = new ArrayList<OUT>(inputData.size());

    TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
    TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

    for (IN element : inputData) {
      IN inCopy = inSerializer.copy(element);
      OUT out = function.map(inCopy);
      result.add(outSerializer.copy(out));
    }

    FunctionUtils.closeFunction(function);
    
    return result;
  }
}
origin: org.apache.flink/flink-optimizer

private String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
  try {
    if (wrapper.hasObject()) {
      try {
        return wrapper.getUserCodeObject().toString();
      }
      catch (Throwable t) {
        return wrapper.getUserCodeClass().getName();
      }
    }
    else {
      return wrapper.getUserCodeClass().getName();
    }
  }
  catch (Throwable t) {
    return null;
  }
}

origin: org.apache.flink/flink-optimizer_2.10

if (FileInputFormat.class.isAssignableFrom(getOperator().getFormatWrapper().getUserCodeClass()) &&
    this.estimatedOutputSize >= 0) {
  estimator.addFileInputCost(this.estimatedOutputSize, costs);
    ((ReplicatingInputFormat<?,?>) getOperator().getFormatWrapper().getUserCodeObject()).getReplicatedInputFormat();
if (FileInputFormat.class.isAssignableFrom(inputFormat.getClass()) &&
    this.estimatedOutputSize >= 0) {
origin: apache/flink

private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
  Operator<?> inputOp = sink.getInput();
  if (inputOp == null) {
    throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
  }
  
  @SuppressWarnings("unchecked")
  List<IN> input = (List<IN>) execute(inputOp);
  
  @SuppressWarnings("unchecked")
  GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;
  // build the runtime context and compute broadcast variables, if necessary
  TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
  RuntimeUDFContext ctx;
  MetricGroup metrics = new UnregisteredMetricsGroup();
    
  if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
    ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
        new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
  } else {
    ctx = null;
  }
  typedSink.executeOnCollections(input, ctx, executionConfig);
}

origin: apache/flink

ReduceFunction<T> function = this.userFunction.getUserCodeObject();
origin: com.alibaba.blink/flink-optimizer

private String getDescriptionForUserCode(UserCodeWrapper<?> wrapper) {
  try {
    if (wrapper.hasObject()) {
      try {
        return wrapper.getUserCodeObject().toString();
      }
      catch (Throwable t) {
        return wrapper.getUserCodeClass().getName();
      }
    }
    else {
      return wrapper.getUserCodeClass().getName();
    }
  }
  catch (Throwable t) {
    return null;
  }
}

org.apache.flink.api.common.operators.utilUserCodeWrapper

Javadoc

UDf operators can have either a class or an object containing the user code, this is the common interface to access them.

Most used methods

  • getUserCodeObject
    Gets the user code object, which may be either a function or an input or output format. The subclass
  • getUserCodeClass
    Gets the class of the user code. If the user code is provided as a class, this class is just returne
  • hasObject
    Checks whether the wrapper already has an object, or whether it needs to instantiate it.

Popular in Java

  • Parsing JSON documents to java classes using gson
  • getSystemService (Context)
  • addToBackStack (FragmentTransaction)
  • startActivity (Activity)
  • BorderLayout (java.awt)
    A border layout lays out a container, arranging and resizing its components to fit in five regions:
  • Socket (java.net)
    Provides a client-side TCP socket.
  • SecureRandom (java.security)
    This class generates cryptographically secure pseudo-random numbers. It is best to invoke SecureRand
  • Timer (java.util)
    Timers schedule one-shot or recurring TimerTask for execution. Prefer java.util.concurrent.Scheduled
  • JarFile (java.util.jar)
    JarFile is used to read jar entries and their associated data from jar files.
  • Option (scala)
  • Top 12 Jupyter Notebook extensions
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now