JavaPairDStream<String, Long> ipAddressRequestCount = ip.countByValueAndWindow( Flags.getInstance().getWindowLength(), Flags.getInstance().getSlideInterval()); requestCount.print();
@SuppressWarnings("unchecked") @Test public void testCountByValueAndWindow() { List<List<String>> inputData = Arrays.asList( Arrays.asList("hello", "world"), Arrays.asList("hello", "moon"), Arrays.asList("hello")); List<HashSet<Tuple2<String, Long>>> expected = Arrays.asList( Sets.newHashSet( new Tuple2<>("hello", 1L), new Tuple2<>("world", 1L)), Sets.newHashSet( new Tuple2<>("hello", 2L), new Tuple2<>("world", 1L), new Tuple2<>("moon", 1L)), Sets.newHashSet( new Tuple2<>("hello", 2L), new Tuple2<>("moon", 1L))); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, Long> counted = stream.countByValueAndWindow(new Duration(2000), new Duration(1000)); JavaTestUtils.attachTestOutputStream(counted); List<List<Tuple2<String, Long>>> result = JavaTestUtils.runStreams(ssc, 3, 3); List<Set<Tuple2<String, Long>>> unorderedResult = new ArrayList<>(); for (List<Tuple2<String, Long>> res: result) { unorderedResult.add(Sets.newHashSet(res)); } Assert.assertEquals(expected, unorderedResult); }
@SuppressWarnings("unchecked") @Test public void testCountByValueAndWindow() { List<List<String>> inputData = Arrays.asList( Arrays.asList("hello", "world"), Arrays.asList("hello", "moon"), Arrays.asList("hello")); List<HashSet<Tuple2<String, Long>>> expected = Arrays.asList( Sets.newHashSet( new Tuple2<>("hello", 1L), new Tuple2<>("world", 1L)), Sets.newHashSet( new Tuple2<>("hello", 2L), new Tuple2<>("world", 1L), new Tuple2<>("moon", 1L)), Sets.newHashSet( new Tuple2<>("hello", 2L), new Tuple2<>("moon", 1L))); JavaDStream<String> stream = JavaTestUtils.attachTestInputStream( ssc, inputData, 1); JavaPairDStream<String, Long> counted = stream.countByValueAndWindow(new Duration(2000), new Duration(1000)); JavaTestUtils.attachTestOutputStream(counted); List<List<Tuple2<String, Long>>> result = JavaTestUtils.runStreams(ssc, 3, 3); List<Set<Tuple2<String, Long>>> unorderedResult = new ArrayList<>(); for (List<Tuple2<String, Long>> res: result) { unorderedResult.add(Sets.newHashSet(res)); } Assert.assertEquals(expected, unorderedResult); }