@Test public void testJsonRDDToDataFrame() { // This is a test for the deprecated API in SPARK-15615. JavaRDD<String> rdd = jsc.parallelize(Arrays.asList("{\"a\": 2}")); Dataset<Row> df = spark.read().json(rdd); Assert.assertEquals(1L, df.count()); Assert.assertEquals(2L, df.collectAsList().get(0).getLong(0)); }
@Before public void setUp() throws IOException { spark = SparkSession.builder() .master("local[*]") .appName("testing") .getOrCreate(); path = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "datasource").getCanonicalFile(); if (path.exists()) { path.delete(); } List<String> jsonObjects = new ArrayList<>(10); for (int i = 0; i < 10; i++) { jsonObjects.add("{\"a\":" + i + ", \"b\":\"str" + i + "\"}"); } Dataset<String> ds = spark.createDataset(jsonObjects, Encoders.STRING()); df = spark.read().json(ds); df.createOrReplaceTempView("jsonTable"); }
@Before public void setUp() throws IOException { spark = SparkSession.builder() .master("local[*]") .appName("testing") .getOrCreate(); path = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "datasource").getCanonicalFile(); if (path.exists()) { path.delete(); } List<String> jsonObjects = new ArrayList<>(10); for (int i = 0; i < 10; i++) { jsonObjects.add("{\"a\":" + i + ", \"b\":\"str" + i + "\"}"); } Dataset<String> ds = spark.createDataset(jsonObjects, Encoders.STRING()); df = spark.read().json(ds); df.createOrReplaceTempView("jsonTable"); }
@Before public void setUp() throws IOException { spark = SparkSession.builder() .master("local[*]") .appName("testing") .getOrCreate(); path = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "datasource").getCanonicalFile(); if (path.exists()) { path.delete(); } List<String> jsonObjects = new ArrayList<>(10); for (int i = 0; i < 10; i++) { jsonObjects.add("{\"a\":" + i + ", \"b\":\"str" + i + "\"}"); } Dataset<String> ds = spark.createDataset(jsonObjects, Encoders.STRING()); df = spark.read().json(ds); df.createOrReplaceTempView("jsonTable"); }
@Test public void testJsonRDDToDataFrame() { // This is a test for the deprecated API in SPARK-15615. JavaRDD<String> rdd = jsc.parallelize(Arrays.asList("{\"a\": 2}")); Dataset<Row> df = spark.read().json(rdd); Assert.assertEquals(1L, df.count()); Assert.assertEquals(2L, df.collectAsList().get(0).getLong(0)); }
@Test public void testJsonRDDToDataFrame() { // This is a test for the deprecated API in SPARK-15615. JavaRDD<String> rdd = jsc.parallelize(Arrays.asList("{\"a\": 2}")); Dataset<Row> df = spark.read().json(rdd); Assert.assertEquals(1L, df.count()); Assert.assertEquals(2L, df.collectAsList().get(0).getLong(0)); }
"this is another simple string.")); Dataset<Row> df1 = spark.read().json(jsonDS); StructType actualSchema1 = df1.schema(); Assert.assertEquals(expectedSchema, actualSchema1); Assert.assertEquals(expectedResult, actual1); Dataset<Row> df2 = spark.read().schema(expectedSchema).json(jsonDS); StructType actualSchema2 = df2.schema(); Assert.assertEquals(expectedSchema, actualSchema2);
"this is another simple string.")); Dataset<Row> df1 = spark.read().json(jsonDS); StructType actualSchema1 = df1.schema(); Assert.assertEquals(expectedSchema, actualSchema1); Assert.assertEquals(expectedResult, actual1); Dataset<Row> df2 = spark.read().schema(expectedSchema).json(jsonDS); StructType actualSchema2 = df2.schema(); Assert.assertEquals(expectedSchema, actualSchema2);
"this is another simple string.")); Dataset<Row> df1 = spark.read().json(jsonDS); StructType actualSchema1 = df1.schema(); Assert.assertEquals(expectedSchema, actualSchema1); Assert.assertEquals(expectedResult, actual1); Dataset<Row> df2 = spark.read().schema(expectedSchema).json(jsonDS); StructType actualSchema2 = df2.schema(); Assert.assertEquals(expectedSchema, actualSchema2);
@Override public Dataset<Row> json(final org.apache.spark.sql.Dataset<String> jsonDataset) { final boolean userTriggered = initializeFunction(jsonDataset); final Dataset<Row> result = Dataset.from(super.json(jsonDataset)); this.setIsUserTriggered(userTriggered); return result; }
@Override public Dataset<Row> json(final RDD<String> jsonRDD) { final boolean userTriggered = initializeFunction(jsonRDD); final Dataset<Row> result = Dataset.from(super.json(jsonRDD)); this.setIsUserTriggered(userTriggered); return result; }
@Override public Dataset<Row> json(final scala.collection.Seq<String> paths) { final boolean userTriggered = initializeFunction(paths); final Dataset<Row> result = Dataset.from(super.json(paths)); this.setIsUserTriggered(userTriggered); return result; }
@Override public Dataset<Row> json(final String... paths) { final boolean userTriggered = initializeFunction(paths); final Dataset<Row> result = Dataset.from(super.json(paths)); this.setIsUserTriggered(userTriggered); return result; }
@Before public void setUp() throws IOException { hc = TestHive$.MODULE$; List<String> jsonObjects = new ArrayList<>(10); for (int i = 0; i < 10; i++) { jsonObjects.add("{\"key\":" + i + ", \"value\":\"str" + i + "\"}"); } df = hc.read().json(hc.createDataset(jsonObjects, Encoders.STRING())); df.createOrReplaceTempView("window_table"); }
@Before public void setUp() throws IOException { hc = TestHive$.MODULE$; List<String> jsonObjects = new ArrayList<>(10); for (int i = 0; i < 10; i++) { jsonObjects.add("{\"key\":" + i + ", \"value\":\"str" + i + "\"}"); } df = hc.read().json(hc.createDataset(jsonObjects, Encoders.STRING())); df.createOrReplaceTempView("window_table"); }