How to use

flatMap

method

in

org.apache.spark.streaming.api.java.JavaDStream

@Override
public <U> SparkCollection<U> flatMap(StageSpec stageSpec, FlatMapFunction<T, U> function) {
 return wrap(stream.flatMap(function));
}

@Override
public JavaPairDStream<String, Integer> process(JavaDStream<String> input) {
  JavaDStream<String> words = input.flatMap(new FlatMapFunction<String, String>() {
    @Override
    public Iterable<String> call(String x) {
      return Arrays.asList(x.split(" "));
    }
  });
  JavaPairDStream<String, Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() {
    @Override
    public Tuple2<String, Integer> call(String s) {
      return new Tuple2<String, Integer>(s, 1);
    }
  }).reduceByKey(new Function2<Integer, Integer, Integer>() {
    @Override
    public Integer call(Integer i1, Integer i2) {
      return i1 + i2;
    }
  });
  return wordCounts;
}

@Test
public void testFlatMap() {
 List<List<String>> inputData = Arrays.asList(
  Arrays.asList("go", "giants"),
  Arrays.asList("boo", "dodgers"),
  Arrays.asList("athletics"));
 List<List<String>> expected = Arrays.asList(
  Arrays.asList("g", "o", "g", "i", "a", "n", "t", "s"),
  Arrays.asList("b", "o", "o", "d", "o", "d", "g", "e", "r", "s"),
  Arrays.asList("a", "t", "h", "l", "e", "t", "i", "c", "s"));
 JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<String> flatMapped = stream.flatMap(
   s -> Arrays.asList(s.split("(?!^)")).iterator());
 JavaTestUtils.attachTestOutputStream(flatMapped);
 List<List<String>> result = JavaTestUtils.runStreams(ssc, 3, 3);
 assertOrderInvariantEquals(expected, result);
}

@Test
public void testFlatMap() {
 List<List<String>> inputData = Arrays.asList(
  Arrays.asList("go", "giants"),
  Arrays.asList("boo", "dodgers"),
  Arrays.asList("athletics"));
 List<List<String>> expected = Arrays.asList(
  Arrays.asList("g", "o", "g", "i", "a", "n", "t", "s"),
  Arrays.asList("b", "o", "o", "d", "o", "d", "g", "e", "r", "s"),
  Arrays.asList("a", "t", "h", "l", "e", "t", "i", "c", "s"));
 JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<String> flatMapped = stream.flatMap(
   s -> Arrays.asList(s.split("(?!^)")).iterator());
 JavaTestUtils.attachTestOutputStream(flatMapped);
 List<List<String>> result = JavaTestUtils.runStreams(ssc, 3, 3);
 assertOrderInvariantEquals(expected, result);
}

static void streamSpansToStorage(
  JavaDStream<byte[]> stream,
  ReadSpans readSpans,
  AdjustAndConsumeSpansSharingTraceId adjustAndConsumeSpansSharingTraceId
) {
 JavaDStream<Span> spans = stream.flatMap(readSpans);
 // TODO: plug in some filter to drop spans regardless of trace ID
 // spans = spans.filter(spanFilter);
 JavaPairDStream<String, Iterable<Span>> tracesById = spans
   .mapToPair(s -> new Tuple2<>(Util.toLowerHex(s.traceIdHigh, s.traceId), s))
   .groupByKey();
 tracesById.foreachRDD(rdd -> {
  rdd.values().foreachPartition(adjustAndConsumeSpansSharingTraceId);
 });
}

@SuppressWarnings("unchecked")
@Test
public void testFlatMap() {
 List<List<String>> inputData = Arrays.asList(
   Arrays.asList("go", "giants"),
   Arrays.asList("boo", "dodgers"),
   Arrays.asList("athletics"));
 List<List<String>> expected = Arrays.asList(
   Arrays.asList("g","o","g","i","a","n","t","s"),
   Arrays.asList("b", "o", "o", "d","o","d","g","e","r","s"),
   Arrays.asList("a","t","h","l","e","t","i","c","s"));
 JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<String> flatMapped =
  stream.flatMap(x -> Arrays.asList(x.split("(?!^)")).iterator());
 JavaTestUtils.attachTestOutputStream(flatMapped);
 List<List<String>> result = JavaTestUtils.runStreams(ssc, 3, 3);
 assertOrderInvariantEquals(expected, result);
}

@SuppressWarnings("unchecked")
@Test
public void testFlatMap() {
 List<List<String>> inputData = Arrays.asList(
   Arrays.asList("go", "giants"),
   Arrays.asList("boo", "dodgers"),
   Arrays.asList("athletics"));
 List<List<String>> expected = Arrays.asList(
   Arrays.asList("g","o","g","i","a","n","t","s"),
   Arrays.asList("b", "o", "o", "d","o","d","g","e","r","s"),
   Arrays.asList("a","t","h","l","e","t","i","c","s"));
 JavaDStream<String> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<String> flatMapped =
  stream.flatMap(x -> Arrays.asList(x.split("(?!^)")).iterator());
 JavaTestUtils.attachTestOutputStream(flatMapped);
 List<List<String>> result = JavaTestUtils.runStreams(ssc, 3, 3);
 assertOrderInvariantEquals(expected, result);
}

JavaDStream<String> words = statuses.flatMap(l -> Arrays.asList(l.split(" ")).iterator());

JavaDStream<String> words = lines.flatMap(new FlatMapFunction <String, String>(){
  @Override
  public Iterable<String> call(String x) {

JavaDStream<String> words = unionStreams.flatMap(new FlatMapFunction<byte[], String>() {
 @Override
 public Iterator<String> call(byte[] line) {

JavaDStream<ApacheAccessLog> accessLogsDStream = lines.flatMap(line -> {
  List<ApacheAccessLog> list = new ArrayList<>();
  try {