How to use

window

method

in

org.apache.spark.streaming.api.java.JavaDStream

public void processAccessLogs(String outDir, JavaDStream<ApacheAccessLog> accessLogsDStream) {
 JavaDStream<ApacheAccessLog> windowDStream = accessLogsDStream.window(
   Flags.getInstance().getWindowLength(),
   Flags.getInstance().getSlideInterval());

@SuppressWarnings("unchecked")
@Test
public void testWindow() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6,1,2,3),
   Arrays.asList(7,8,9,4,5,6),
   Arrays.asList(7,8,9));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 4, 4);
 assertOrderInvariantEquals(expected, result);
}

@SuppressWarnings("unchecked")
@Test
public void testWindowWithSlideDuration() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9),
   Arrays.asList(10,11,12),
   Arrays.asList(13,14,15),
   Arrays.asList(16,17,18));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3,4,5,6),
   Arrays.asList(1,2,3,4,5,6,7,8,9,10,11,12),
   Arrays.asList(7,8,9,10,11,12,13,14,15,16,17,18),
   Arrays.asList(13,14,15,16,17,18));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(4000), new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 8, 4);
 assertOrderInvariantEquals(expected, result);
}

@SuppressWarnings("unchecked")
@Test
public void testWindow() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6,1,2,3),
   Arrays.asList(7,8,9,4,5,6),
   Arrays.asList(7,8,9));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 4, 4);
 assertOrderInvariantEquals(expected, result);
}

@SuppressWarnings("unchecked")
@Test
public void testWindowWithSlideDuration() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9),
   Arrays.asList(10,11,12),
   Arrays.asList(13,14,15),
   Arrays.asList(16,17,18));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3,4,5,6),
   Arrays.asList(1,2,3,4,5,6,7,8,9,10,11,12),
   Arrays.asList(7,8,9,10,11,12,13,14,15,16,17,18),
   Arrays.asList(13,14,15,16,17,18));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(4000), new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 8, 4);
 assertOrderInvariantEquals(expected, result);
}

if (config.hasPath(WINDOW_SLIDE_MILLISECONDS_CONFIG)) {
 int slideDuration = config.getInt(WINDOW_SLIDE_MILLISECONDS_CONFIG);
 dStream = dStream.window(new Duration(windowDuration), new Duration(slideDuration));
} else {
 dStream = dStream.window(new Duration(windowDuration));

JavaDStream<String> threeSecondsEverySecond = streamOfRecords.window(new Duration(3000));
  streamOfRecords.window(new Duration(5000), new Duration(2000));

@Override
public SparkCollection<T> window(StageSpec stageSpec, Windower windower) {
 String stageName = stageSpec.getName();
 return wrap(stream.transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.in", null))
        .window(Durations.seconds(windower.getWidth()), Durations.seconds(windower.getSlideInterval()))
        .transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.out",
                              sec.getDataTracer(stageName))));
}