How to use

FlinkStreamingPipelineTranslator

in

org.apache.beam.runners.flink

@Override
public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
 LOG.info("{} enterCompositeTransform- {}", genSpaces(this.depth), node.getFullName());
 this.depth++;
 PTransform<?, ?> transform = node.getTransform();
 if (transform != null) {
  StreamTransformTranslator<?> translator =
    FlinkStreamingTransformTranslators.getTranslator(transform);
  if (translator != null && applyCanTranslate(transform, node, translator)) {
   applyStreamingTransform(transform, node, translator);
   LOG.info("{} translated- {}", genSpaces(this.depth), node.getFullName());
   return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
  }
 }
 return CompositeBehavior.ENTER_TRANSFORM;
}

private <T extends PTransform<?, ?>> void applyStreamingTransform(
  PTransform<?, ?> transform,
  TransformHierarchy.Node node,
  StreamTransformTranslator<?> translator) {
 @SuppressWarnings("unchecked")
 T typedTransform = (T) transform;
 @SuppressWarnings("unchecked")
 StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 // create the applied PTransform on the streamingContext
 streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
 typedTranslator.translateNode(typedTransform, streamingContext);
}

@Override
public void leaveCompositeTransform(TransformHierarchy.Node node) {
 this.depth--;
 LOG.info("{} leaveCompositeTransform- {}", genSpaces(this.depth), node.getFullName());
}

/**
 * Depending on if the job is a Streaming or a Batch one, this method creates
 * the necessary execution environment and pipeline translator, and translates
 * the {@link org.apache.beam.sdk.values.PCollection} program into
 * a {@link org.apache.flink.api.java.DataSet}
 * or {@link org.apache.flink.streaming.api.datastream.DataStream} one.
 * */
public void translate(FlinkRunner flinkRunner, Pipeline pipeline) {
 this.flinkBatchEnv = null;
 this.flinkStreamEnv = null;
 pipeline.replaceAll(FlinkTransformOverrides.getDefaultOverrides(options.isStreaming()));
 PipelineTranslationOptimizer optimizer =
   new PipelineTranslationOptimizer(TranslationMode.BATCH, options);
 optimizer.translate(pipeline);
 TranslationMode translationMode = optimizer.getTranslationMode();
 FlinkPipelineTranslator translator;
 if (translationMode == TranslationMode.STREAMING) {
  this.flinkStreamEnv = createStreamExecutionEnvironment();
  translator = new FlinkStreamingPipelineTranslator(flinkRunner, flinkStreamEnv, options);
 } else {
  this.flinkBatchEnv = createBatchExecutionEnvironment();
  translator = new FlinkBatchPipelineTranslator(flinkBatchEnv, options);
 }
 translator.translate(pipeline);
}

private <T extends PTransform<?, ?>> boolean applyCanTranslate(
  PTransform<?, ?> transform,
  TransformHierarchy.Node node,
  StreamTransformTranslator<?> translator) {
 @SuppressWarnings("unchecked")
 T typedTransform = (T) transform;
 @SuppressWarnings("unchecked")
 StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
 return typedTranslator.canTranslate(typedTransform, streamingContext);
}

@Override
public void leaveCompositeTransform(TransformHierarchy.Node node) {
 this.depth--;
 LOG.info("{} leaveCompositeTransform- {}", genSpaces(this.depth), node.getFullName());
}

/**
 * Depending on if the job is a Streaming or a Batch one, this method creates the necessary
 * execution environment and pipeline translator, and translates the {@link
 * org.apache.beam.sdk.values.PCollection} program into a {@link
 * org.apache.flink.api.java.DataSet} or {@link
 * org.apache.flink.streaming.api.datastream.DataStream} one.
 */
public void translate(Pipeline pipeline) {
 this.flinkBatchEnv = null;
 this.flinkStreamEnv = null;
 PipelineTranslationModeOptimizer optimizer = new PipelineTranslationModeOptimizer(options);
 optimizer.translate(pipeline);
 // Needs to be done before creating the Flink ExecutionEnvironments
 prepareFilesToStageForRemoteClusterExecution(options);
 FlinkPipelineTranslator translator;
 if (options.isStreaming()) {
  this.flinkStreamEnv =
    FlinkExecutionEnvironments.createStreamExecutionEnvironment(
      options, options.getFilesToStage());
  translator = new FlinkStreamingPipelineTranslator(flinkStreamEnv, options);
 } else {
  this.flinkBatchEnv =
    FlinkExecutionEnvironments.createBatchExecutionEnvironment(
      options, options.getFilesToStage());
  translator = new FlinkBatchPipelineTranslator(flinkBatchEnv, options);
 }
 pipeline.replaceAll(FlinkTransformOverrides.getDefaultOverrides(options));
 translator.translate(pipeline);
}

@Override
public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
 LOG.info("{} enterCompositeTransform- {}", genSpaces(this.depth), node.getFullName());
 this.depth++;
 PTransform<?, ?> transform = node.getTransform();
 if (transform != null) {
  StreamTransformTranslator<?> translator =
    FlinkStreamingTransformTranslators.getTranslator(transform);
  if (translator != null && applyCanTranslate(transform, node, translator)) {
   applyStreamingTransform(transform, node, translator);
   LOG.info("{} translated- {}", genSpaces(this.depth), node.getFullName());
   return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
  }
 }
 return CompositeBehavior.ENTER_TRANSFORM;
}

private <T extends PTransform<?, ?>> void applyStreamingTransform(
  PTransform<?, ?> transform,
  TransformHierarchy.Node node,
  StreamTransformTranslator<?> translator) {
 @SuppressWarnings("unchecked")
 T typedTransform = (T) transform;
 @SuppressWarnings("unchecked")
 StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 // create the applied PTransform on the streamingContext
 streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
 typedTranslator.translateNode(typedTransform, streamingContext);
}

@Override
public void leaveCompositeTransform(TransformHierarchy.Node node) {
 this.depth--;
 LOG.info("{} leaveCompositeTransform- {}", genSpaces(this.depth), node.getFullName());
}

   FlinkExecutionEnvironments.createStreamExecutionEnvironment(
     options, options.getFilesToStage());
 translator = new FlinkStreamingPipelineTranslator(flinkStreamEnv, options);
} else {
 this.flinkBatchEnv =

@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
 LOG.info("{} visitPrimitiveTransform- {}", genSpaces(this.depth), node.getFullName());
 // get the transformation corresponding to hte node we are
 // currently visiting and translate it into its Flink alternative.
 PTransform<?, ?> transform = node.getTransform();
 StreamTransformTranslator<?> translator =
   FlinkStreamingTransformTranslators.getTranslator(transform);
 if (translator == null || !applyCanTranslate(transform, node, translator)) {
  String transformUrn = PTransformTranslation.urnForTransform(transform);
  LOG.info(transformUrn);
  throw new UnsupportedOperationException(
    "The transform " + transformUrn + " is currently not supported.");
 }
 applyStreamingTransform(transform, node, translator);
}

private <T extends PTransform<?, ?>> boolean applyCanTranslate(
  PTransform<?, ?> transform,
  TransformHierarchy.Node node,
  StreamTransformTranslator<?> translator) {
 @SuppressWarnings("unchecked")
 T typedTransform = (T) transform;
 @SuppressWarnings("unchecked")
 StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
 return typedTranslator.canTranslate(typedTransform, streamingContext);
}

@Override
public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
 LOG.info("{} enterCompositeTransform- {}", genSpaces(this.depth), node.getFullName());
 this.depth++;
 PTransform<?, ?> transform = node.getTransform();
 if (transform != null) {
  StreamTransformTranslator<?> translator =
    FlinkStreamingTransformTranslators.getTranslator(transform);
  if (translator != null && applyCanTranslate(transform, node, translator)) {
   applyStreamingTransform(transform, node, translator);
   LOG.info("{} translated- {}", genSpaces(this.depth), node.getFullName());
   return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
  }
 }
 return CompositeBehavior.ENTER_TRANSFORM;
}

private <T extends PTransform<?, ?>> void applyStreamingTransform(
  PTransform<?, ?> transform,
  TransformHierarchy.Node node,
  StreamTransformTranslator<?> translator) {
 @SuppressWarnings("unchecked")
 T typedTransform = (T) transform;
 @SuppressWarnings("unchecked")
 StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 // create the applied PTransform on the streamingContext
 streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
 typedTranslator.translateNode(typedTransform, streamingContext);
}

@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
 LOG.info("{} visitPrimitiveTransform- {}", genSpaces(this.depth), node.getFullName());
 // get the transformation corresponding to hte node we are
 // currently visiting and translate it into its Flink alternative.
 PTransform<?, ?> transform = node.getTransform();
 StreamTransformTranslator<?> translator =
   FlinkStreamingTransformTranslators.getTranslator(transform);
 if (translator == null || !applyCanTranslate(transform, node, translator)) {
  LOG.info(node.getTransform().getClass().toString());
  throw new UnsupportedOperationException(
    "The transform " + transform + " is currently not supported.");
 }
 applyStreamingTransform(transform, node, translator);
}

private <T extends PTransform<?, ?>> boolean applyCanTranslate(
  PTransform<?, ?> transform,
  TransformHierarchy.Node node,
  StreamTransformTranslator<?> translator) {
 @SuppressWarnings("unchecked")
 T typedTransform = (T) transform;
 @SuppressWarnings("unchecked")
 StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator;
 streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline()));
 return typedTranslator.canTranslate(typedTransform, streamingContext);
}

@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
 LOG.info("{} visitPrimitiveTransform- {}", genSpaces(this.depth), node.getFullName());
 // get the transformation corresponding to hte node we are
 // currently visiting and translate it into its Flink alternative.
 PTransform<?, ?> transform = node.getTransform();
 StreamTransformTranslator<?> translator =
   FlinkStreamingTransformTranslators.getTranslator(transform);
 if (translator == null || !applyCanTranslate(transform, node, translator)) {
  String transformUrn = PTransformTranslation.urnForTransform(transform);
  LOG.info(transformUrn);
  throw new UnsupportedOperationException(
    "The transform " + transformUrn + " is currently not supported.");
 }
 applyStreamingTransform(transform, node, translator);
}