static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,five"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,timestamp"); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,languages,religions"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string,map<string,string>".toLowerCase()); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,stuff"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,uniontype<int,double,array<string>,struct<a:int,b:string>,string>".toLowerCase()); instance.initialize(conf, tbl); }
public JsonSerDe getNumericSerde() throws SerDeException { System.out.println("testMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "cboolean,ctinyint,csmallint,cint,cbigint,cfloat,cdouble"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,tinyint,smallint,int,bigint,float,double"); serde.initialize(conf, tbl); return serde; }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,languages,religions"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,array<string>,array<string>".toLowerCase()); instance.initialize(conf, tbl); }
public JsonSerDe getMappedSerde() throws SerDeException { System.out.println("testMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,ts"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,int"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.ts", "timestamp"); serde.initialize(conf, tbl); return serde; }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "kind,etag,pageInfo,v_items"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ("string,string,"+ "struct<totalResults:INT,resultsPerPage:INT>," + "ARRAY<STRUCT<kind:STRING," + "etag:STRING," + "id:STRING," + "v_statistics:STRUCT<viewCount:INT,likeCount:INT,dislikeCount:INT,favoriteCount:INT,commentCount:INT>," + "topicDetails:STRUCT<topicIds:ARRAY<STRING>,relevantTopicIds:ARRAY<STRING>>" + ">>").toLowerCase()); tbl.setProperty("mapping.v_items" , "items"); tbl.setProperty("mapping.v_statistics" , "statistics"); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "kind,etag,pageInfo,v_items"); tbl.setProperty( serdeConstants.LIST_COLUMN_TYPES, ("string,string," + "string," + "ARRAY<STRUCT<kind:STRING," + "etag:STRING," + "id:STRING," + "v_statistics:STRUCT<viewCount:INT,likeCount:INT,dislikeCount:INT,favoriteCount:INT,commentCount:INT>," + "topicDetails:STRUCT<topicIds:ARRAY<STRING>,relevantTopicIds:ARRAY<STRING>>" + ">>").toLowerCase()); tbl.setProperty("mapping.v_items", "items"); tbl.setProperty("mapping.v_statistics", "statistics"); instance.initialize(conf, tbl); tbl.setProperty("mapping.v_items", "items"); tbl.setProperty("mapping.v_statistics", "statistics"); instance.initialize(conf, tbl); }
@Test public void deserializeStrangeString() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("deserialize"); Writable w = new Text("{\"one\":\"\\a\\v\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("",result.get("one").toString().trim()); } }
/** * Test of getSerializedClass method, of class JsonSerDe. */ @Test public void testGetSerializedClass() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("getSerializedClass"); Class expResult = Text.class; Class result = instance.getSerializedClass(); assertEquals(expResult, result); }
@Test public void testCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "time1,time2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty("mapping.time2", "time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"Time\":\"forme\",\"time\":\"foryou\"}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time1")).equals("forme")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time2")).equals("foryou")); }
@Test public void testNestedCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "col1,col2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,struct<time1:string>"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"col1\":\"forme\",\"col2\":{\"Time\":\"foryou\"}}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("col1")).equals("forme")); StructObjectInspector soi2 = (StructObjectInspector) soi.getStructFieldRef("col2").getFieldObjectInspector(); Object col2 = soi.getStructFieldData(res, soi.getStructFieldRef("col2")); assertTrue(soi2.getStructFieldData(col2, soi2.getStructFieldRef("time1")).equals("foryou")); }
@Test public void testExplicitNullValue() throws SerDeException, IOException { System.out.println("testExplicitNullValue"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "stringCol,nullCol,missingCol"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string,string"); // Set 'explicit.null' to true tbl.setProperty(JsonSerDe.PROP_EXPLICIT_NULL, "true"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); // Load json string with one 'null' value and one 'missing' value Object res = serde.deserialize(new Text("{\"stringCol\":\"str\",\"nullCol\":null}")); // Get the serialized json string String jsonStr = serde.serialize(res, soi).toString(); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("stringCol")).equals("str")); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("nullCol"))); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("missingCol"))); assertEquals(jsonStr,"{\"nullCol\":null,\"stringCol\":\"str\",\"missingCol\":null}"); }
@Test public void testExplicitNullValueDefault() throws SerDeException, IOException { System.out.println("testExplicitNullValue"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "stringCol,nullCol,missingCol"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string,string"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); // Load json string with one 'null' value and one 'missing' value Object res = serde.deserialize(new Text("{\"stringCol\":\"str\",\"nullCol\":null}")); // Get the serialized json string String jsonStr = serde.serialize(res, soi).toString(); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("stringCol")).equals("str")); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("nullCol"))); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("missingCol"))); assertEquals(jsonStr,"{\"stringCol\":\"str\"}"); }
/** * Test of deserialize method, but passing an array. */ @Test public void testDeserialize() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("deserialize"); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("poop",result.get("four")); assertTrue(result.get("three") instanceof JSONArray); assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); assertEquals("red", ((JSONArray)result.get("three")).get(0)); }
@Test public void testNestedExplicitNullValue() throws SerDeException, IOException { System.out.println("testNestedExplicitNullValue"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties();
@Test public void testDeserialize2() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",[\"blue\",\"azure\",\"cobalt\",\"teal\"],\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("poop", result.get("four")); assertTrue(result.get("three") instanceof JSONArray); assertTrue(((JSONArray) result.get("three")).get(0) instanceof String); assertEquals("red", ((JSONArray) result.get("three")).get(0)); }
@Test public void testDeserializePartialFieldSet() throws Exception { Writable w = new Text("{\"missing\":\"whocares\",\"one\":true,\"three\":[\"red\",\"yellow\",[\"blue\",\"azure\",\"cobalt\",\"teal\"],\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JsonSerDe instance = new JsonSerDe(); initialize(instance); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals(result.get("four"),"poop"); assertTrue( result.get("three") instanceof JSONArray); assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); assertEquals( ((JSONArray)result.get("three")).get(0),"red"); }
JsonSerDe instance = new JsonSerDe(); initialize(instance);
@Test public void testDeserialize2Initializations() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",[\"blue\",\"azure\",\"cobalt\",\"teal\"],\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals(result.get("four"), "poop"); assertTrue(result.get("three") instanceof JSONArray); assertTrue(((JSONArray) result.get("three")).get(0) instanceof String); assertEquals(((JSONArray) result.get("three")).get(0), "red"); // second initialization, new column initialize2(instance); result = (JSONObject) instance.deserialize(w); assertEquals(result.get("four"), "poop"); assertTrue(result.get("three") instanceof JSONArray); assertTrue(((JSONArray) result.get("three")).get(0) instanceof String); assertEquals(((JSONArray) result.get("three")).get(0), "red"); }