StreamingContextJavaFunctions(StreamingContext ssc) { super(ssc.sparkContext()); this.ssc = ssc; } }
StreamingContextJavaFunctions(StreamingContext ssc) { super(ssc.sparkContext()); this.ssc = ssc; } }
StreamingContextJavaFunctions(StreamingContext ssc) { super(ssc.sparkContext()); this.ssc = ssc; } }
StreamingContextJavaFunctions(StreamingContext ssc) { super(ssc.sparkContext()); this.ssc = ssc; } }
StreamingContextJavaFunctions(StreamingContext ssc) { super(ssc.sparkContext()); this.ssc = ssc; } }
private RDD<WindowedValue<T>> generateRdd() { return rdds.size() > 0 ? rdds.poll().rdd() : ssc().sparkContext().emptyRDD(JavaSparkContext$.MODULE$.<WindowedValue<T>>fakeClassTag()); }
@Override public scala.Option<RDD<Tuple2<Source<T>, CheckpointMarkT>>> compute(Time validTime) { RDD<Tuple2<Source<T>, CheckpointMarkT>> rdd = new SourceRDD.Unbounded<>( ssc().sparkContext(), options, createMicrobatchSource(), numPartitions); return scala.Option.apply(rdd); }
@Override public void run(DatasetContext datasetContext) throws Exception { PipelineRuntime pipelineRuntime = new SparkPipelineRuntime(sec); SparkExecutionPluginContext sparkPluginContext = new BasicSparkExecutionPluginContext(sec, JavaSparkContext.fromSparkContext(stream.context().sparkContext()), datasetContext, pipelineRuntime, stageSpec); wrappedCompute.initialize(sparkPluginContext); } }, Exception.class);
SourceDStream( StreamingContext ssc, UnboundedSource<T, CheckpointMarkT> unboundedSource, SerializablePipelineOptions options, Long boundMaxRecords) { super(ssc, JavaSparkContext$.MODULE$.fakeClassTag()); this.unboundedSource = unboundedSource; this.options = options; SparkPipelineOptions sparkOptions = options.get().as(SparkPipelineOptions.class); // Reader cache expiration interval. 50% of batch interval is added to accommodate latency. this.readerCacheInterval = 1.5 * sparkOptions.getBatchIntervalMillis(); this.boundReadDuration = boundReadDuration( sparkOptions.getReadTimePercentage(), sparkOptions.getMinReadTimeMillis()); // set initial parallelism once. this.initialParallelism = ssc().sparkContext().defaultParallelism(); checkArgument(this.initialParallelism > 0, "Number of partitions must be greater than zero."); this.boundMaxRecords = boundMaxRecords; try { this.numPartitions = createMicrobatchSource().split(sparkOptions).size(); } catch (Exception e) { throw new RuntimeException(e); } }