/** * A static factory method to create a {@link StreamingContextJavaFunctions} based on an existing * {@link JavaStreamingContext} instance. */ public static StreamingContextJavaFunctions javaFunctions(JavaStreamingContext streamingContext) { return new StreamingContextJavaFunctions(streamingContext.ssc()); }
/** * A static factory method to create a {@link StreamingContextJavaFunctions} based on an existing * {@link JavaStreamingContext} instance. */ public static StreamingContextJavaFunctions javaFunctions(JavaStreamingContext streamingContext) { return new StreamingContextJavaFunctions(streamingContext.ssc()); }
/** * A static factory method to create a {@link StreamingContextJavaFunctions} based on an existing * {@link JavaStreamingContext} instance. */ public static StreamingContextJavaFunctions javaFunctions(JavaStreamingContext streamingContext) { return new StreamingContextJavaFunctions(streamingContext.ssc()); }
/** * A static factory method to create a {@link StreamingContextJavaFunctions} based on an existing * {@link JavaStreamingContext} instance. */ public static StreamingContextJavaFunctions javaFunctions(JavaStreamingContext streamingContext) { return new StreamingContextJavaFunctions(streamingContext.ssc()); }
/** * A static factory method to create a {@link StreamingContextJavaFunctions} based on an existing * {@link JavaStreamingContext} instance. */ public static StreamingContextJavaFunctions javaFunctions(JavaStreamingContext streamingContext) { return new StreamingContextJavaFunctions(streamingContext.ssc()); }
final long batchDuration = jsc.ssc().graph().batchDuration().milliseconds(); ReceiverStreamListener listener = new ReceiverStreamListener(kafkaConfig, batchDuration);
jsc.awaitTermination(); }catch (Exception ex ) { jsc.ssc().sc().cancelAllJobs(); jsc.stop(true, false); System.exit(-1);
@SuppressWarnings("unchecked") @Test public void testQueueStream() { ssc.stop(); // Create a new JavaStreamingContext without checkpointing SparkConf conf = new SparkConf() .setMaster("local[2]") .setAppName("test") .set("spark.streaming.clock", "org.apache.spark.util.ManualClock"); ssc = new JavaStreamingContext(conf, new Duration(1000)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9)); JavaSparkContext jsc = new JavaSparkContext(ssc.ssc().sc()); JavaRDD<Integer> rdd1 = jsc.parallelize(Arrays.asList(1, 2, 3)); JavaRDD<Integer> rdd2 = jsc.parallelize(Arrays.asList(4, 5, 6)); JavaRDD<Integer> rdd3 = jsc.parallelize(Arrays.asList(7,8,9)); Queue<JavaRDD<Integer>> rdds = new LinkedList<>(); rdds.add(rdd1); rdds.add(rdd2); rdds.add(rdd3); JavaDStream<Integer> stream = ssc.queueStream(rdds); JavaTestUtils.attachTestOutputStream(stream); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 3, 3); Assert.assertEquals(expected, result); }
@SuppressWarnings("unchecked") @Test public void testQueueStream() { ssc.stop(); // Create a new JavaStreamingContext without checkpointing SparkConf conf = new SparkConf() .setMaster("local[2]") .setAppName("test") .set("spark.streaming.clock", "org.apache.spark.util.ManualClock"); ssc = new JavaStreamingContext(conf, new Duration(1000)); List<List<Integer>> expected = Arrays.asList( Arrays.asList(1,2,3), Arrays.asList(4,5,6), Arrays.asList(7,8,9)); JavaSparkContext jsc = new JavaSparkContext(ssc.ssc().sc()); JavaRDD<Integer> rdd1 = jsc.parallelize(Arrays.asList(1, 2, 3)); JavaRDD<Integer> rdd2 = jsc.parallelize(Arrays.asList(4, 5, 6)); JavaRDD<Integer> rdd3 = jsc.parallelize(Arrays.asList(7,8,9)); Queue<JavaRDD<Integer>> rdds = new LinkedList<>(); rdds.add(rdd1); rdds.add(rdd2); rdds.add(rdd3); JavaDStream<Integer> stream = ssc.queueStream(rdds); JavaTestUtils.attachTestOutputStream(stream); List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 3, 3); Assert.assertEquals(expected, result); }
private <K, S, T> void testOperation( List<List<K>> input, StateSpec<K, Integer, S, T> mapWithStateSpec, List<Set<T>> expectedOutputs, List<Set<Tuple2<K, S>>> expectedStateSnapshots) { int numBatches = expectedOutputs.size(); JavaDStream<K> inputStream = JavaTestUtils.attachTestInputStream(ssc, input, 2); JavaMapWithStateDStream<K, Integer, S, T> mapWithStateDStream = JavaPairDStream.fromJavaDStream( inputStream.map(x -> new Tuple2<>(x, 1))).mapWithState(mapWithStateSpec); List<Set<T>> collectedOutputs = Collections.synchronizedList(new ArrayList<Set<T>>()); mapWithStateDStream.foreachRDD(rdd -> collectedOutputs.add(Sets.newHashSet(rdd.collect()))); List<Set<Tuple2<K, S>>> collectedStateSnapshots = Collections.synchronizedList(new ArrayList<Set<Tuple2<K, S>>>()); mapWithStateDStream.stateSnapshots().foreachRDD(rdd -> collectedStateSnapshots.add(Sets.newHashSet(rdd.collect()))); BatchCounter batchCounter = new BatchCounter(ssc.ssc()); ssc.start(); ((ManualClock) ssc.ssc().scheduler().clock()) .advance(ssc.ssc().progressListener().batchDuration() * numBatches + 1); batchCounter.waitUntilBatchesCompleted(numBatches, 10000); Assert.assertEquals(expectedOutputs, collectedOutputs); Assert.assertEquals(expectedStateSnapshots, collectedStateSnapshots); } }
private <K, S, T> void testOperation( List<List<K>> input, StateSpec<K, Integer, S, T> mapWithStateSpec, List<Set<T>> expectedOutputs, List<Set<Tuple2<K, S>>> expectedStateSnapshots) { int numBatches = expectedOutputs.size(); JavaDStream<K> inputStream = JavaTestUtils.attachTestInputStream(ssc, input, 2); JavaMapWithStateDStream<K, Integer, S, T> mapWithStateDStream = JavaPairDStream.fromJavaDStream( inputStream.map(x -> new Tuple2<>(x, 1))).mapWithState(mapWithStateSpec); List<Set<T>> collectedOutputs = Collections.synchronizedList(new ArrayList<Set<T>>()); mapWithStateDStream.foreachRDD(rdd -> collectedOutputs.add(Sets.newHashSet(rdd.collect()))); List<Set<Tuple2<K, S>>> collectedStateSnapshots = Collections.synchronizedList(new ArrayList<Set<Tuple2<K, S>>>()); mapWithStateDStream.stateSnapshots().foreachRDD(rdd -> collectedStateSnapshots.add(Sets.newHashSet(rdd.collect()))); BatchCounter batchCounter = new BatchCounter(ssc.ssc()); ssc.start(); ((ManualClock) ssc.ssc().scheduler().clock()) .advance(ssc.ssc().progressListener().batchDuration() * numBatches + 1); batchCounter.waitUntilBatchesCompleted(numBatches, 10000); Assert.assertEquals(expectedOutputs, collectedOutputs); Assert.assertEquals(expectedStateSnapshots, collectedStateSnapshots); } }
Long maxRecordsPerBatch = options.getMaxRecordsPerBatch(); SourceDStream<T, CheckpointMarkT> sourceDStream = new SourceDStream<>(jssc.ssc(), source, rc, maxRecordsPerBatch);