public void processAccessLogs(String outDir, JavaDStream<ApacheAccessLog> accessLogsDStream) { JavaDStream<ApacheAccessLog> windowDStream = accessLogsDStream.window( Flags.getInstance().getWindowLength(), Flags.getInstance().getSlideInterval());
@SuppressWarnings("unchecked") @Test public void testWindow() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6,1,2,3), Arrays.asList(7,8,9,4,5,6), Arrays.asList(7,8,9)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> windowed = stream.window(new Duration(2000)); JavaTestUtils.attachTestOutputStream(windowed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 4, 4); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testWindowWithSlideDuration() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9), Arrays.asList(10,11,12), Arrays.asList(13,14,15), Arrays.asList(16,17,18)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(1,2,3,4,5,6), Arrays.asList(1,2,3,4,5,6,7,8,9,10,11,12), Arrays.asList(7,8,9,10,11,12,13,14,15,16,17,18), Arrays.asList(13,14,15,16,17,18)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> windowed = stream.window(new Duration(4000), new Duration(2000)); JavaTestUtils.attachTestOutputStream(windowed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 8, 4); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testWindow() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6,1,2,3), Arrays.asList(7,8,9,4,5,6), Arrays.asList(7,8,9)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> windowed = stream.window(new Duration(2000)); JavaTestUtils.attachTestOutputStream(windowed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 4, 4); assertOrderInvariantEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testWindowWithSlideDuration() { List<List<Integer>> inputData = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9), Arrays.asList(10,11,12), Arrays.asList(13,14,15), Arrays.asList(16,17,18)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(1,2,3,4,5,6), Arrays.asList(1,2,3,4,5,6,7,8,9,10,11,12), Arrays.asList(7,8,9,10,11,12,13,14,15,16,17,18), Arrays.asList(13,14,15,16,17,18)); JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1); JavaDStream<Integer> windowed = stream.window(new Duration(4000), new Duration(2000)); JavaTestUtils.attachTestOutputStream(windowed); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 8, 4); assertOrderInvariantEquals(expected, result); }
@Override public SparkCollection<T> window(StageSpec stageSpec, Windower windower) { String stageName = stageSpec.getName(); return wrap(stream.transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.in", null)) .window(Durations.seconds(windower.getWidth()), Durations.seconds(windower.getSlideInterval())) .transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.out", sec.getDataTracer(stageName)))); }