Tabnine Logo
JavaDStream.window
Code IndexAdd Tabnine to your IDE (free)

How to use
window
method
in
org.apache.spark.streaming.api.java.JavaDStream

Best Java code snippets using org.apache.spark.streaming.api.java.JavaDStream.window (Showing top 8 results out of 315)

origin: databricks/learning-spark

public void processAccessLogs(String outDir, JavaDStream<ApacheAccessLog> accessLogsDStream) {
 JavaDStream<ApacheAccessLog> windowDStream = accessLogsDStream.window(
   Flags.getInstance().getWindowLength(),
   Flags.getInstance().getSlideInterval());
origin: org.apache.spark/spark-streaming_2.11

@SuppressWarnings("unchecked")
@Test
public void testWindow() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6,1,2,3),
   Arrays.asList(7,8,9,4,5,6),
   Arrays.asList(7,8,9));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 4, 4);
 assertOrderInvariantEquals(expected, result);
}
origin: org.apache.spark/spark-streaming_2.10

@SuppressWarnings("unchecked")
@Test
public void testWindowWithSlideDuration() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9),
   Arrays.asList(10,11,12),
   Arrays.asList(13,14,15),
   Arrays.asList(16,17,18));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3,4,5,6),
   Arrays.asList(1,2,3,4,5,6,7,8,9,10,11,12),
   Arrays.asList(7,8,9,10,11,12,13,14,15,16,17,18),
   Arrays.asList(13,14,15,16,17,18));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(4000), new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 8, 4);
 assertOrderInvariantEquals(expected, result);
}
origin: org.apache.spark/spark-streaming_2.10

@SuppressWarnings("unchecked")
@Test
public void testWindow() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6,1,2,3),
   Arrays.asList(7,8,9,4,5,6),
   Arrays.asList(7,8,9));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 4, 4);
 assertOrderInvariantEquals(expected, result);
}
origin: org.apache.spark/spark-streaming_2.11

@SuppressWarnings("unchecked")
@Test
public void testWindowWithSlideDuration() {
 List<List<Integer>> inputData = Arrays.asList(
   Arrays.asList(1,2,3),
   Arrays.asList(4,5,6),
   Arrays.asList(7,8,9),
   Arrays.asList(10,11,12),
   Arrays.asList(13,14,15),
   Arrays.asList(16,17,18));
 List<List<Integer>> expected = Arrays.asList(
   Arrays.asList(1,2,3,4,5,6),
   Arrays.asList(1,2,3,4,5,6,7,8,9,10,11,12),
   Arrays.asList(7,8,9,10,11,12,13,14,15,16,17,18),
   Arrays.asList(13,14,15,16,17,18));
 JavaDStream<Integer> stream = JavaTestUtils.attachTestInputStream(ssc, inputData, 1);
 JavaDStream<Integer> windowed = stream.window(new Duration(4000), new Duration(2000));
 JavaTestUtils.attachTestOutputStream(windowed);
 List<List<Integer>> result = JavaTestUtils.runStreams(ssc, 8, 4);
 assertOrderInvariantEquals(expected, result);
}
origin: cloudera-labs/envelope

if (config.hasPath(WINDOW_SLIDE_MILLISECONDS_CONFIG)) {
 int slideDuration = config.getInt(WINDOW_SLIDE_MILLISECONDS_CONFIG);
 dStream = dStream.window(new Duration(windowDuration), new Duration(slideDuration));
} else {
 dStream = dStream.window(new Duration(windowDuration));
origin: spirom/learning-spark-with-java

JavaDStream<String> threeSecondsEverySecond = streamOfRecords.window(new Duration(3000));
  streamOfRecords.window(new Duration(5000), new Duration(2000));
origin: co.cask.cdap/hydrator-spark-core2

@Override
public SparkCollection<T> window(StageSpec stageSpec, Windower windower) {
 String stageName = stageSpec.getName();
 return wrap(stream.transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.in", null))
        .window(Durations.seconds(windower.getWidth()), Durations.seconds(windower.getSlideInterval()))
        .transform(new CountingTransformFunction<T>(stageName, sec.getMetrics(), "records.out",
                              sec.getDataTracer(stageName))));
}
org.apache.spark.streaming.api.javaJavaDStreamwindow

Popular methods of JavaDStream

  • foreachRDD
  • map
  • mapToPair
  • union
  • filter
  • flatMap
  • dstream
  • countByValue
  • cache
  • transformToPair
  • count
  • transform
  • count,
  • transform,
  • countByValueAndWindow,
  • flatMapToPair,
  • print,
  • reduceByWindow,
  • repartition,
  • glom,
  • mapPartitions

Popular in Java

  • Creating JSON documents from java classes using gson
  • compareTo (BigDecimal)
  • requestLocationUpdates (LocationManager)
  • getSystemService (Context)
  • SQLException (java.sql)
    An exception that indicates a failed JDBC operation. It provides the following information about pro
  • Vector (java.util)
    Vector is an implementation of List, backed by an array and synchronized. All optional operations in
  • Pattern (java.util.regex)
    Patterns are compiled regular expressions. In many cases, convenience methods such as String#matches
  • HttpServlet (javax.servlet.http)
    Provides an abstract class to be subclassed to create an HTTP servlet suitable for a Web site. A sub
  • BasicDataSource (org.apache.commons.dbcp)
    Basic implementation of javax.sql.DataSource that is configured via JavaBeans properties. This is no
  • FileUtils (org.apache.commons.io)
    General file manipulation utilities. Facilities are provided in the following areas: * writing to a
  • Top plugins for Android Studio
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now