How to use

getStreamGraph

method

in

org.apache.flink.streaming.api.environment.StreamExecutionEnvironment

/**
 * Creates the plan with which the system will execute the program, and
 * returns it as a String using a JSON representation of the execution data
 * flow graph. Note that this needs to be called, before the plan is
 * executed.
 *
 * @return The execution plan of the program, as a JSON String.
 */
public String getExecutionPlan() {
  return getStreamGraph().getStreamingPlanAsJSON();
}

private static StreamOperator<?> getOperatorForDataStream(DataStream<?> dataStream) {
  StreamExecutionEnvironment env = dataStream.getExecutionEnvironment();
  StreamGraph streamGraph = env.getStreamGraph();
  return streamGraph.getStreamNode(dataStream.getId()).getOperator();
}

private static StreamOperator<?> getOperatorFromDataStream(DataStream<?> dataStream) {
  StreamExecutionEnvironment env = dataStream.getExecutionEnvironment();
  StreamGraph streamGraph = env.getStreamGraph();
  return streamGraph.getStreamNode(dataStream.getId()).getOperator();
}

AutoCancellableJob(Deadline deadline, final ClusterClient<?> clusterClient, final StreamExecutionEnvironment env) {
  Preconditions.checkNotNull(env);
  this.clusterClient = Preconditions.checkNotNull(clusterClient);
  this.jobGraph = env.getStreamGraph().getJobGraph();
  this.jobId = Preconditions.checkNotNull(jobGraph.getJobID());
  this.deadline = deadline;
}

/**
 * Tests that in a streaming use case where checkpointing is enabled, there is no default strategy set on the
 * client side.
 */
@Test
public void testFallbackStrategyOnClientSideWhenCheckpointingEnabled() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.enableCheckpointing(500);
  env.fromElements(1).print();
  StreamGraph graph = env.getStreamGraph();
  JobGraph jobGraph = graph.getJobGraph();
  RestartStrategies.RestartStrategyConfiguration restartStrategy =
    jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();
  Assert.assertNotNull(restartStrategy);
  Assert.assertTrue(restartStrategy instanceof RestartStrategies.FallbackRestartStrategyConfiguration);
}

public static JobGraph stoppableJob(final StopJobSignal stopJobSignal) {
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.addSource(new InfiniteSourceFunction(stopJobSignal))
    .setParallelism(2)
    .shuffle()
    .addSink(new DiscardingSink<>())
    .setParallelism(2);
  return env.getStreamGraph().getJobGraph();
}

@Test
public void testConsecutiveSplitRejection() {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  DataStreamSource<Integer> src = env.fromElements(0, 0);
  OutputSelector<Integer> outputSelector = new DummyOutputSelector<>();
  src.split(outputSelector).split(outputSelector).addSink(new DiscardingSink<>());
  expectedException.expect(IllegalStateException.class);
  expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs.");
  env.getStreamGraph();
}

private JobGraph createJobGraph(ExecutionMode mode) {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  env.setStateBackend((StateBackend) new MemoryStateBackend());
  switch (mode) {
    case MIGRATE:
      createMigrationJob(env);
      break;
    case RESTORE:
      createRestoredJob(env);
      break;
  }
  return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

@Test
public void testSplitAfterSideOutputRejection() {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  DataStreamSource<Integer> src = env.fromElements(0, 0);
  OutputTag<Integer> outputTag = new OutputTag<Integer>("dummy"){};
  OutputSelector<Integer> outputSelector = new DummyOutputSelector<>();
  src.getSideOutput(outputTag).split(outputSelector).addSink(new DiscardingSink<>());
  expectedException.expect(IllegalStateException.class);
  expectedException.expectMessage("Split after side-outputs are not supported. Splits are deprecated. Please use side-outputs.");
  env.getStreamGraph();
}

/**
 * Tests that a changed operator name does not affect the hash.
 */
@Test
public void testChangedOperatorName() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
  env.addSource(new NoOpSourceFunction(), "A").map(new NoOpMapFunction());
  JobGraph jobGraph = env.getStreamGraph().getJobGraph();
  JobVertexID expected = jobGraph.getVerticesAsArray()[0].getID();
  env = StreamExecutionEnvironment.createLocalEnvironment();
  env.addSource(new NoOpSourceFunction(), "B").map(new NoOpMapFunction());
  jobGraph = env.getStreamGraph().getJobGraph();
  JobVertexID actual = jobGraph.getVerticesAsArray()[0].getID();
  assertEquals(expected, actual);
}

@Test
public void testStreaming() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(1);
  DataStream<Integer> input = env.fromCollection(inputData);
  input
      .flatMap(new NotifyingMapper())
      .writeUsingOutputFormat(new DummyOutputFormat()).disableChaining();
  JobGraph jobGraph = env.getStreamGraph().getJobGraph();
  submitJobAndVerifyResults(jobGraph);
}

@Test
public void testSelectBetweenConsecutiveSplitRejection() {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  DataStreamSource<Integer> src = env.fromElements(0, 0);
  OutputSelector<Integer> outputSelector = new DummyOutputSelector<>();
  src.split(outputSelector).select("dummy").split(outputSelector).addSink(new DiscardingSink<>());
  expectedException.expect(IllegalStateException.class);
  expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs.");
  env.getStreamGraph();
}

/**
 * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of
 * maximum parallelism.
 */
@Test
public void testSetupOfKeyGroupPartitioner() {
  int maxParallelism = 42;
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().setMaxParallelism(maxParallelism);
  DataStream<Integer> source = env.fromElements(1, 2, 3);
  DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap());
  keyedResult.addSink(new DiscardingSink<>());
  StreamGraph graph = env.getStreamGraph();
  StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId());
  StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner();
}

/**
 * Tests that a manual hash at the beginning of a chain is accepted.
 */
@Test
public void testManualHashAssignmentForStartNodeInInChain() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
  env.setParallelism(4);
  env.addSource(new NoOpSourceFunction()).uid("source")
      .map(new NoOpMapFunction())
      .addSink(new NoOpSinkFunction());
  env.getStreamGraph().getJobGraph();
}

/**
 * Tests that a manual hash for an intermediate chain node is accepted.
 */
@Test
public void testManualHashAssignmentForIntermediateNodeInChain() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
  env.setParallelism(4);
  env.addSource(new NoOpSourceFunction())
      // Intermediate chained node
      .map(new NoOpMapFunction()).uid("map")
      .addSink(new NoOpSinkFunction());
  env.getStreamGraph().getJobGraph();
}

@Test
public void testOutputTypeConfigurationWithTwoInputTransformation() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  DataStream<Integer> source1 = env.fromElements(1, 10);
  DataStream<Integer> source2 = env.fromElements(2, 11);
  ConnectedStreams<Integer, Integer> connectedSource = source1.connect(source2);
  OutputTypeConfigurableOperationWithTwoInputs outputTypeConfigurableOperation = new OutputTypeConfigurableOperationWithTwoInputs();
  DataStream<Integer> result = connectedSource.transform(
      "Two input and output type configurable operation",
      BasicTypeInfo.INT_TYPE_INFO,
      outputTypeConfigurableOperation);
  result.addSink(new DiscardingSink<>());
  env.getStreamGraph();
  assertEquals(BasicTypeInfo.INT_TYPE_INFO, outputTypeConfigurableOperation.getTypeInformation());
}

@Test
public void testKeybyBetweenConsecutiveSplitRejection() {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  DataStreamSource<Integer> src = env.fromElements(0, 0);
  OutputSelector<Integer> outputSelector = new DummyOutputSelector<>();
  src.split(outputSelector).select("dummy").keyBy(x -> x).split(outputSelector).addSink(new DiscardingSink<>());
  expectedException.expect(IllegalStateException.class);
  expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs.");
  env.getStreamGraph();
}

@Test
public void testUnionBetweenConsecutiveSplitRejection() {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  DataStreamSource<Integer> src = env.fromElements(0, 0);
  OutputSelector<Integer> outputSelector = new DummyOutputSelector<>();
  src.split(outputSelector).select("dummy").union(src.map(x -> x)).split(outputSelector).addSink(new DiscardingSink<>());
  expectedException.expect(IllegalStateException.class);
  expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs.");
  env.getStreamGraph();
}

/**
 * Tests that a collision on the manual hash throws an Exception.
 */
@Test(expected = IllegalArgumentException.class)
public void testManualHashAssignmentCollisionThrowsException() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
  env.setParallelism(4);
  env.disableOperatorChaining();
  env.addSource(new NoOpSourceFunction()).uid("source")
      .map(new NoOpMapFunction()).uid("source") // Collision
      .addSink(new NoOpSinkFunction());
  // This call is necessary to generate the job graph
  env.getStreamGraph().getJobGraph();
}

@Test
public void testUserProvidedHashingOnChainSupported() {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
  env.addSource(new NoOpSourceFunction(), "src").setUidHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
      .map(new NoOpMapFunction()).setUidHash("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
      .filter(new NoOpFilterFunction()).setUidHash("cccccccccccccccccccccccccccccccc")
      .keyBy(new NoOpKeySelector())
      .reduce(new NoOpReduceFunction()).name("reduce").setUidHash("dddddddddddddddddddddddddddddddd");
  env.getStreamGraph().getJobGraph();
}