@Test public void deserializeStrangeString() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("deserialize"); Writable w = new Text("{\"one\":\"\\a\\v\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("",result.get("one").toString().trim()); } }
@Test public void testTimestampDeSerialize() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(Timestamp.valueOf("2013-03-27 23:18:40.0"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testTimestampDeSerializeWithNanoseconds() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40.123456\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals( Timestamp.valueOf("2013-03-27 23:18:40.123456"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testUTCTimestampDeSerialize() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27T23:18:40Z\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(Timestamp.valueOf("2013-03-27 23:18:40.0"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testTimestampDeSerializeWithPlusHours() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2017-08-17T10:46:04+0300\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals( Timestamp.valueOf("2017-08-17 07:46:04.0"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testNestedCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "col1,col2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,struct<time1:string>"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"col1\":\"forme\",\"col2\":{\"Time\":\"foryou\"}}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("col1")).equals("forme")); StructObjectInspector soi2 = (StructObjectInspector) soi.getStructFieldRef("col2").getFieldObjectInspector(); Object col2 = soi.getStructFieldData(res, soi.getStructFieldRef("col2")); assertTrue(soi2.getStructFieldData(col2, soi2.getStructFieldRef("time1")).equals("foryou")); }
@Test public void testNestedGetJsonObject() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"pagehit\":{ \"kind\": \"youtube#video\" } ,\"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.pagehit"); assertEquals("{\"kind\":\"youtube#video\"}", output.toString()); }
@Test public void testTimestampDeSerializeNumericTimestampWithNanoseconds() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":1367801925.123}"); // JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(getDate("2013-05-05 17:58:45.123"), jstOi.getPrimitiveJavaObject(result.get("five")) ); }
@Test public void testTimestampDeSerializeNumericTimestamp() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":1367801925}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(getDate("2013-05-05 17:58:45.0" ), jstOi.getPrimitiveJavaObject(result.get("five")) ); }
@Test public void testGetJsonObject() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.totalresults"); assertEquals("1", output.toString()); }
@Test public void testTimestampDeSerializeNumericTimestampWithMilliseconds() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":1367801925123}"); // JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(getDate("2013-05-05 17:58:45.123"), jstOi.getPrimitiveJavaObject(result.get("five")) ); }
@Test public void testStringWhenJson() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); String sres = loi.getPrimitiveJavaObject(res); assertEquals("page", sres); }
@Test public void testStringWhenNotJson() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field"); assertNull(output); }
@Test public void testStringWhenFieldIsNotInJson() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field"); assertNull(output); }
/** * Test of deserialize method, but passing an array. */ @Test public void testDeserialize() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("deserialize"); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("poop",result.get("four")); assertTrue(result.get("three") instanceof JSONArray); assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); assertEquals("red", ((JSONArray)result.get("three")).get(0)); }
@Test public void testDeSerializeEmptyArray() throws Exception { // Test that timestamp object can be deserialized Writable[] wa = new Writable[]{ new Text("{\"country\":\"Switzerland\",\"languages\":[\"Italian\"],\"religions\":\"\"}") }; for (Writable w : wa) { JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(), ObjectInspector.Category.LIST); ListObjectInspector loi = (ListObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr); assertEquals(-1, loi.getListLength(val)); } }
@Test public void testDeSerializeEmptyMap() throws Exception { // Test that timestamp object can be deserialized Writable[] wa = new Writable[] { new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\":\"\"}") }; for (Writable w : wa ) { JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(), ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr); assertEquals(-1, moi.getMapSize(val)); } }
@Test public void testDeserializePartialFieldSet() throws Exception { Writable w = new Text("{\"missing\":\"whocares\",\"one\":true,\"three\":[\"red\",\"yellow\",[\"blue\",\"azure\",\"cobalt\",\"teal\"],\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JsonSerDe instance = new JsonSerDe(); initialize(instance); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals(result.get("four"),"poop"); assertTrue( result.get("three") instanceof JSONArray); assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); assertEquals( ((JSONArray)result.get("three")).get(0),"red"); }
@Test public void testDeserialize2() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",[\"blue\",\"azure\",\"cobalt\",\"teal\"],\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("poop", result.get("four")); assertTrue(result.get("three") instanceof JSONArray); assertTrue(((JSONArray) result.get("three")).get(0) instanceof String); assertEquals("red", ((JSONArray) result.get("three")).get(0)); }
@Test public void testDeSerializeMap() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\": { \"f\": \"v\", \"n\":null} }"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(),ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr) ; assertEquals(2, moi.getMapSize(val)); assertEquals("v", moi.getMapValueElement(val, "f")); ObjectInspector voi = moi.getMapValueObjectInspector(); assertTrue(voi instanceof StringObjectInspector); StringObjectInspector svoi = (StringObjectInspector) voi; assertNull( svoi.getPrimitiveJavaObject(moi.getMapValueElement(val,"n"))); }