@Test public void testCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "time1,time2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty("mapping.time2", "time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"Time\":\"forme\",\"time\":\"foryou\"}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time1")).equals("forme")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time2")).equals("foryou")); }
@Test public void testTimestampDeSerialize() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(Timestamp.valueOf("2013-03-27 23:18:40.0"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testTimestampDeSerializeWithNanoseconds() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40.123456\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals( Timestamp.valueOf("2013-03-27 23:18:40.123456"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testTimestampDeSerializeWithPlusHours() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2017-08-17T10:46:04+0300\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals( Timestamp.valueOf("2017-08-17 07:46:04.0"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testUTCTimestampDeSerialize() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27T23:18:40Z\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(Timestamp.valueOf("2013-03-27 23:18:40.0"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
@Test public void testNestedCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "col1,col2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,struct<time1:string>"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"col1\":\"forme\",\"col2\":{\"Time\":\"foryou\"}}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("col1")).equals("forme")); StructObjectInspector soi2 = (StructObjectInspector) soi.getStructFieldRef("col2").getFieldObjectInspector(); Object col2 = soi.getStructFieldData(res, soi.getStructFieldRef("col2")); assertTrue(soi2.getStructFieldData(col2, soi2.getStructFieldRef("time1")).equals("foryou")); }
@Test public void testNestedGetJsonObject() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"pagehit\":{ \"kind\": \"youtube#video\" } ,\"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.pagehit"); assertEquals("{\"kind\":\"youtube#video\"}", output.toString()); }
@Test public void testExplicitNullValueDefault() throws SerDeException, IOException { System.out.println("testExplicitNullValue"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "stringCol,nullCol,missingCol"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string,string"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); // Load json string with one 'null' value and one 'missing' value Object res = serde.deserialize(new Text("{\"stringCol\":\"str\",\"nullCol\":null}")); // Get the serialized json string String jsonStr = serde.serialize(res, soi).toString(); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("stringCol")).equals("str")); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("nullCol"))); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("missingCol"))); assertEquals(jsonStr,"{\"stringCol\":\"str\"}"); }
@Test public void testTimestampDeSerializeNumericTimestampWithNanoseconds() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":1367801925.123}"); // JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(getDate("2013-05-05 17:58:45.123"), jstOi.getPrimitiveJavaObject(result.get("five")) ); }
@Test public void testGetJsonObject() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.totalresults"); assertEquals("1", output.toString()); }
@Test public void testTimestampDeSerializeNumericTimestamp() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":1367801925}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(getDate("2013-05-05 17:58:45.0" ), jstOi.getPrimitiveJavaObject(result.get("five")) ); }
@Test public void testStringWhenJson() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); String sres = loi.getPrimitiveJavaObject(res); assertEquals("page", sres); }
@Test public void testTimestampDeSerializeNumericTimestampWithMilliseconds() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":1367801925123}"); // JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(getDate("2013-05-05 17:58:45.123"), jstOi.getPrimitiveJavaObject(result.get("five")) ); }
@Test public void testStringWhenNotJson() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": \"page\", \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field"); assertNull(output); }
@Test public void testStringWhenFieldIsNotInJson() throws SerDeException, JSONException { Writable w = new Text( "{ \"kind\": \"youtube#videoListResponse\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/mYlS4-ghMGhc1wTFCwoQl3IYDZc\\\"\", \"pageInfo\": { \"totalResults\": 1, \"resultsPerPage\": 1 }, \"items\": [ { \"kind\": \"youtube#video\", \"etag\": \"\\\"79S54kzisD_9SOTfQLu_0TVQSpY/A4foLs-VO317Po_ulY6b5mSimZA\\\"\", \"id\": \"wHkPb68dxEw\", \"statistics\": { \"viewCount\": \"9211\", \"likeCount\": \"79\", \"dislikeCount\": \"11\", \"favoriteCount\": \"0\", \"commentCount\": \"29\" }, \"topicDetails\": { \"topicIds\": [ \"/m/02mjmr\" ], \"relevantTopicIds\": [ \"/m/0cnfvd\", \"/m/01jdpf\" ] } } ] }"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); Object res = soi.getStructFieldData(result, soi.getStructFieldRef("pageinfo")); StringObjectInspector loi = (StringObjectInspector) soi.getStructFieldRef("pageinfo") .getFieldObjectInspector(); UDFJson udfJson = new UDFJson(); Text output = udfJson.evaluate(loi.getPrimitiveJavaObject(res), "$.test_field"); assertNull(output); }
@Test public void testDeSerializeEmptyArray() throws Exception { // Test that timestamp object can be deserialized Writable[] wa = new Writable[]{ new Text("{\"country\":\"Switzerland\",\"languages\":[\"Italian\"],\"religions\":\"\"}") }; for (Writable w : wa) { JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(), ObjectInspector.Category.LIST); ListObjectInspector loi = (ListObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr); assertEquals(-1, loi.getListLength(val)); } }
@Test public void testDeSerializeEmptyMap() throws Exception { // Test that timestamp object can be deserialized Writable[] wa = new Writable[] { new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\":\"\"}") }; for (Writable w : wa ) { JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(), ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr); assertEquals(-1, moi.getMapSize(val)); } }
/** * Test scientific notation */ @Test public void testENotationNumbers() throws SerDeException, JSONException { System.out.println("testNumbers"); JsonSerDe serde = getNumericSerde(); Text line = new Text("{ cfloat:3.1415E02, cdouble:-1.65788E-12}"); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); JSONObject result = (JSONObject) serde.deserialize(line); StructField sf = soi.getStructFieldRef("cfloat"); assertTrue(sf.getFieldObjectInspector() instanceof JavaStringFloatObjectInspector); JavaStringFloatObjectInspector foi = (JavaStringFloatObjectInspector) sf.getFieldObjectInspector(); assertEquals(3.1415E02 , foi.get(result.get("cfloat")),0.001); sf = soi.getStructFieldRef("cdouble"); assertTrue(sf.getFieldObjectInspector() instanceof JavaStringDoubleObjectInspector); JavaStringDoubleObjectInspector doi = (JavaStringDoubleObjectInspector) sf.getFieldObjectInspector(); assertEquals(-1.65788E-12 , doi.get(result.get("cdouble")),0.001); }
@Test public void testDeSerializeMap() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"country\":\"Switzerland\",\"languages\":\"Italian\",\"religions\": { \"f\": \"v\", \"n\":null} }"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); StructField sfr = soi.getStructFieldRef("religions"); assertEquals(sfr.getFieldObjectInspector().getCategory(),ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector) sfr.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sfr) ; assertEquals(2, moi.getMapSize(val)); assertEquals("v", moi.getMapValueElement(val, "f")); ObjectInspector voi = moi.getMapValueObjectInspector(); assertTrue(voi instanceof StringObjectInspector); StringObjectInspector svoi = (StringObjectInspector) voi; assertNull( svoi.getPrimitiveJavaObject(moi.getMapValueElement(val,"n"))); }
@Test public void testDeSerializeArray() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"country\":\"Switzerland\",\"languages\": [ \"Italian\" ],\"religions\": \"christian\" }"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); // this one is an actuall array StructField sflang = soi.getStructFieldRef("languages"); // this one has a scalar, which will be promoted to a one element array StructField sfrel = soi.getStructFieldRef("religions"); assertEquals(sflang.getFieldObjectInspector().getCategory(), ObjectInspector.Category.LIST); assertEquals(sfrel.getFieldObjectInspector().getCategory(), ObjectInspector.Category.LIST); ListObjectInspector loi = (ListObjectInspector) sflang.getFieldObjectInspector(); Object val = soi.getStructFieldData(result, sflang); assertEquals(1, loi.getListLength(val)); assertEquals("Italian", loi.getListElement(val, 0)); loi = (ListObjectInspector) sfrel.getFieldObjectInspector(); val = soi.getStructFieldData(result, sfrel); assertEquals(1, loi.getListLength(val)); assertEquals("christian", loi.getListElement(val, 0)); }