How to use

disableSysoutLogging

method

in

org.apache.flink.api.common.ExecutionConfig

public static StreamExecutionEnvironment prepareExecutionEnv(ParameterTool parameterTool)
  throws Exception {
  if (parameterTool.getNumberOfParameters() < 5) {
    System.out.println("Missing parameters!\n" +
      "Usage: Kafka --input-topic <topic> --output-topic <topic> " +
      "--bootstrap.servers <kafka brokers> " +
      "--zookeeper.connect <zk quorum> --group.id <some id>");
    throw new Exception("Missing parameters!\n" +
      "Usage: Kafka --input-topic <topic> --output-topic <topic> " +
      "--bootstrap.servers <kafka brokers> " +
      "--zookeeper.connect <zk quorum> --group.id <some id>");
  }
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
  env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
  env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  return env;
}

public void run() throws Exception {
  LOG.info("Random seed = {}", RANDOM_SEED);
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  for (int parallelism = MAX_PARALLELISM; parallelism > 0; parallelism--) {
    LOG.info("Parallelism = {}", parallelism);
    env.setParallelism(parallelism);
    testReduce(env);
    testGroupedReduce(env);
    testJoin(env);
    testCross(env);
  }
}

@Test
public void testDisjointDataflows() {
  try {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(5);
    env.getConfig().disableSysoutLogging();
    // generate two different flows
    env.generateSequence(1, 10).output(new DiscardingOutputFormat<Long>());
    env.generateSequence(1, 10).output(new DiscardingOutputFormat<Long>());
  }
  catch (Exception e) {
    e.printStackTrace();
    fail(e.getMessage());
  }
}

@Test
public void testProgramWithAutoParallelism() throws Exception {
  ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(ExecutionConfig.PARALLELISM_AUTO_MAX);
  env.getConfig().disableSysoutLogging();
  DataSet<Integer> result = env
      .createInput(new ParallelismDependentInputFormat())
      .rebalance()
      .mapPartition(new ParallelismDependentMapPartition());
  List<Integer> resultCollection = new ArrayList<>();
  result.output(new LocalCollectionOutputFormat<>(resultCollection));
  try {
    env.execute();
    assertEquals(PARALLELISM, resultCollection.size());
  }
  catch (Exception ex) {
    assertTrue(
      ExceptionUtils.findThrowableWithMessage(ex, ExecutionGraphBuilder.PARALLELISM_AUTO_MAX_ERROR_MESSAGE).isPresent());
  }
}

@Test
public void testExecuteAfterGetExecutionPlan() {
  ExecutionEnvironment env = new LocalEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Integer> baseSet = env.fromElements(1, 2);
  DataSet<Integer> result = baseSet.map(new MapFunction<Integer, Integer>() {
    @Override public Integer map(Integer value) throws Exception {
      return value * 2;
    }});
  result.output(new DiscardingOutputFormat<Integer>());
  try {
    env.getExecutionPlan();
    env.execute();
  }
  catch (Exception e) {
    e.printStackTrace();
    fail("Cannot run both #getExecutionPlan and #execute.");
  }
}

@Test
public void testIdentityMapWithMissingTypesAndStringTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Tuple3<Integer, Long, String>> identityMapDs = ds
    .map(new Mapper<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>>())
    .returns(new TypeHint<Tuple3<Integer, Long, String>>(){});
  List<Tuple3<Integer, Long, String>> result = identityMapDs.collect();
  String expectedResult = "(2,2,Hello)\n" +
    "(3,2,Hello world)\n" +
    "(1,1,Hi)\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testFaultyMergeAccumulator() throws Exception {
  ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  // Test Exception forwarding with faulty Accumulator implementation
  env.generateSequence(0, 10000)
    .map(new FaultyMergeAccumulatorUsingMapper())
    .output(new DiscardingOutputFormat<>());
  assertAccumulatorsShouldFail(env.execute());
}

@Test
public void testFlatMapWithClassTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Integer> identityMapDs = ds
    .flatMap(new FlatMapper<Tuple3<Integer, Long, String>, Integer>())
    .returns(Integer.class);
  List<Integer> result = identityMapDs.collect();
  String expectedResult = "2\n" +
    "3\n" +
    "1\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testFaultyAccumulator() throws Exception {
  ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  // Test Exception forwarding with faulty Accumulator implementation
  env.generateSequence(0, 10000)
    .map(new FaultyAccumulatorUsingMapper())
    .output(new DiscardingOutputFormat<>());
  assertAccumulatorsShouldFail(env.execute());
}

/**
 * This verifies that an event time source works when setting stream time characteristic to
 * processing time. In this case, the watermarks should just be swallowed.
 */
@Test
public void testEventTimeSourceWithProcessingTime() throws Exception {
  StreamExecutionEnvironment env =
      StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(2);
  env.getConfig().disableSysoutLogging();
  env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
  DataStream<Integer> source1 = env.addSource(new MyTimestampSource(0, 10));
  source1
    .map(new IdentityMap())
    .transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(false));
  env.execute();
  // verify that we don't get any watermarks, the source is used as watermark source in
  // other tests, so it normally emits watermarks
  Assert.assertTrue(CustomOperator.finalWatermarks[0].size() == 0);
}

@Test
public void testCombineGroupWithTypeInformationTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Integer> resultDs = ds
    .groupBy(0)
    .combineGroup(new GroupCombiner<Tuple3<Integer, Long, String>, Integer>())
    .returns(BasicTypeInfo.INT_TYPE_INFO);
  List<Integer> result = resultDs.collect();
  String expectedResult = "2\n" +
    "3\n" +
    "1\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testUnsortedGroupReduceWithTypeInformationTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Integer> resultDs = ds
    .groupBy(0)
    .reduceGroup(new GroupReducer<Tuple3<Integer, Long, String>, Integer>())
    .returns(BasicTypeInfo.INT_TYPE_INFO);
  List<Integer> result = resultDs.collect();
  String expectedResult = "2\n" +
    "3\n" +
    "1\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testIdentityMapWithMissingTypesAndTypeInformationTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Tuple3<Integer, Long, String>> identityMapDs = ds
    // all following generics get erased during compilation
    .map(new Mapper<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>>())
    .returns(new TupleTypeInfo<Tuple3<Integer, Long, String>>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));
  List<Tuple3<Integer, Long, String>> result = identityMapDs
    .collect();
  String expectedResult = "(2,2,Hello)\n" +
    "(3,2,Hello world)\n" +
    "(1,1,Hi)\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testSortedGroupReduceWithTypeInformationTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Integer> resultDs = ds
    .groupBy(0)
    .sortGroup(0, Order.ASCENDING)
    .reduceGroup(new GroupReducer<Tuple3<Integer, Long, String>, Integer>())
    .returns(BasicTypeInfo.INT_TYPE_INFO);
  List<Integer> result = resultDs.collect();
  String expectedResult = "2\n" +
    "3\n" +
    "1\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testInvalidTypeAccumulator() throws Exception {
  ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  // Test Exception forwarding with faulty Accumulator implementation
  env.generateSequence(0, 10000)
    .map(new IncompatibleAccumulatorTypesMapper())
    .map(new IncompatibleAccumulatorTypesMapper2())
    .output(new DiscardingOutputFormat<>());
  try {
    env.execute();
    fail("Should have failed.");
  } catch (JobExecutionException e) {
    assertTrue("Root cause should be:",
        e.getCause() instanceof Exception);
    assertTrue("Root cause should be:",
        e.getCause().getCause() instanceof UnsupportedOperationException);
  }
}

@Test
public void testFlatJoinWithTypeInformationTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Integer> resultDs = ds1
    .join(ds2)
    .where(0)
    .equalTo(0)
    .with(new FlatJoiner<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Integer>())
    .returns(BasicTypeInfo.INT_TYPE_INFO);
  List<Integer> result = resultDs.collect();
  String expectedResult = "2\n" +
    "3\n" +
    "1\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testJoinWithTypeInformationTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Integer> resultDs = ds1
    .join(ds2)
    .where(0)
    .equalTo(0)
    .with(new Joiner<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Integer>())
    .returns(BasicTypeInfo.INT_TYPE_INFO);
  List<Integer> result = resultDs.collect();
  String expectedResult = "2\n" +
    "3\n" +
    "1\n";
  compareResultAsText(result, expectedResult);
}

@Test
public void testCoGroupWithTypeInformationTypeHint() throws Exception {
  final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
  DataSet<Integer> resultDs = ds1
    .coGroup(ds2)
    .where(0)
    .equalTo(0)
    .with(new CoGrouper<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Integer>())
    .returns(BasicTypeInfo.INT_TYPE_INFO);
  List<Integer> result = resultDs.collect();
  String expectedResult = "2\n" +
    "3\n" +
    "1\n";
  compareResultAsText(result, expectedResult);
}

/**
 * Creates a streaming JobGraph from the StreamEnvironment.
 */
private JobGraph createJobGraph(
  int parallelism,
  int numberOfRetries,
  long restartDelay) {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(parallelism);
  env.disableOperatorChaining();
  env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(numberOfRetries, restartDelay));
  env.getConfig().disableSysoutLogging();
  DataStream<Integer> stream = env
    .addSource(new InfiniteTestSource())
    .shuffle()
    .map(new StatefulCounter());
  stream.addSink(new DiscardingSink<>());
  return env.getStreamGraph().getJobGraph();
}

/**
 * These check whether timestamps are properly ignored when they are disabled.
 */
@Test
public void testDisabledTimestamps() throws Exception {
  final int numElements = 10;
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
  env.setParallelism(PARALLELISM);
  env.getConfig().disableSysoutLogging();
  DataStream<Integer> source1 = env.addSource(new MyNonWatermarkingSource(numElements));
  DataStream<Integer> source2 = env.addSource(new MyNonWatermarkingSource(numElements));
  source1
      .map(new IdentityMap())
      .connect(source2).map(new IdentityCoMap())
      .transform("Custom Operator", BasicTypeInfo.INT_TYPE_INFO, new DisabledTimestampCheckingOperator())
      .addSink(new DiscardingSink<Integer>());
  env.execute();
}