@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length != 1) throw new UDFArgumentException("Function takes one argument"); inOi = arguments[0]; try { Properties props = new Properties(); props.setProperty(serdeConstants.LIST_COLUMNS,"col1"); props.setProperty(serdeConstants.LIST_COLUMN_TYPES, inOi.getTypeName()); serde.initialize(new Configuration(), props); } catch (SerDeException ex) { throw new UDFArgumentException(ex); } return PrimitiveObjectInspectorFactory.javaStringObjectInspector; }
public void initialize(JsonSerDe instance) throws Exception { System.out.println("initialize"); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string"); instance.initialize(conf, tbl); }
public void initialize(JsonSerDe instance) throws Exception { System.out.println("initialize"); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string"); instance.initialize(conf, tbl); }
public void initialize2(JsonSerDe instance) throws Exception { System.out.println("initialize"); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,five"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,string"); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,five"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,timestamp"); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,languages,religions"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string,map<string,string>".toLowerCase()); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,languages,religions"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,array<string>,array<string>".toLowerCase()); instance.initialize(conf, tbl); }
public JsonSerDe getMappedSerde() throws SerDeException { System.out.println("testMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,ts"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,int"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.ts", "timestamp"); serde.initialize(conf, tbl); return serde; }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,stuff"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,uniontype<int,double,array<string>,struct<a:int,b:string>,string>".toLowerCase()); instance.initialize(conf, tbl); }
public JsonSerDe getNumericSerde() throws SerDeException { System.out.println("testMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "cboolean,ctinyint,csmallint,cint,cbigint,cfloat,cdouble"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,tinyint,smallint,int,bigint,float,double"); serde.initialize(conf, tbl); return serde; }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "kind,etag,pageInfo,v_items"); tbl.setProperty( serdeConstants.LIST_COLUMN_TYPES, ("string,string," + "string," + "ARRAY<STRUCT<kind:STRING," + "etag:STRING," + "id:STRING," + "v_statistics:STRUCT<viewCount:INT,likeCount:INT,dislikeCount:INT,favoriteCount:INT,commentCount:INT>," + "topicDetails:STRUCT<topicIds:ARRAY<STRING>,relevantTopicIds:ARRAY<STRING>>" + ">>").toLowerCase()); tbl.setProperty("mapping.v_items", "items"); tbl.setProperty("mapping.v_statistics", "statistics"); instance.initialize(conf, tbl); tbl.setProperty("mapping.v_items", "items"); tbl.setProperty("mapping.v_statistics", "statistics"); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "kind,etag,pageInfo,v_items"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ("string,string,"+ "struct<totalResults:INT,resultsPerPage:INT>," + "ARRAY<STRUCT<kind:STRING," + "etag:STRING," + "id:STRING," + "v_statistics:STRUCT<viewCount:INT,likeCount:INT,dislikeCount:INT,favoriteCount:INT,commentCount:INT>," + "topicDetails:STRUCT<topicIds:ARRAY<STRING>,relevantTopicIds:ARRAY<STRING>>" + ">>").toLowerCase()); tbl.setProperty("mapping.v_items" , "items"); tbl.setProperty("mapping.v_statistics" , "statistics"); instance.initialize(conf, tbl); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); /* create table json_84( ts string, t int, request struct< path:string, ip:string, headers: struct< useragent:array<string> > > ) ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' WITH SERDEPROPERTIES ("mapping.useragent" = "User-Agent") STORED AS TEXTFILE; */ tbl.setProperty(serdeConstants.LIST_COLUMNS, "ts,t,request"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ("string,int," + "struct<path:string,ip:string,headers:struct<useragent:array<string>>>"). toLowerCase()); tbl.setProperty("mapping.useragent" , "User-Agent"); instance.initialize(conf, tbl); }
@Test public void testCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "time1,time2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty("mapping.time2", "time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"Time\":\"forme\",\"time\":\"foryou\"}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time1")).equals("forme")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time2")).equals("foryou")); }
@Test public void testNestedCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "col1,col2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,struct<time1:string>"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"col1\":\"forme\",\"col2\":{\"Time\":\"foryou\"}}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("col1")).equals("forme")); StructObjectInspector soi2 = (StructObjectInspector) soi.getStructFieldRef("col2").getFieldObjectInspector(); Object col2 = soi.getStructFieldData(res, soi.getStructFieldRef("col2")); assertTrue(soi2.getStructFieldData(col2, soi2.getStructFieldRef("time1")).equals("foryou")); }
@Test public void testExplicitNullValue() throws SerDeException, IOException { System.out.println("testExplicitNullValue"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "stringCol,nullCol,missingCol"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string,string"); // Set 'explicit.null' to true tbl.setProperty(JsonSerDe.PROP_EXPLICIT_NULL, "true"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); // Load json string with one 'null' value and one 'missing' value Object res = serde.deserialize(new Text("{\"stringCol\":\"str\",\"nullCol\":null}")); // Get the serialized json string String jsonStr = serde.serialize(res, soi).toString(); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("stringCol")).equals("str")); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("nullCol"))); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("missingCol"))); assertEquals(jsonStr,"{\"nullCol\":null,\"stringCol\":\"str\",\"missingCol\":null}"); }
@Test public void testExplicitNullValueDefault() throws SerDeException, IOException { System.out.println("testExplicitNullValue"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "stringCol,nullCol,missingCol"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string,string"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); // Load json string with one 'null' value and one 'missing' value Object res = serde.deserialize(new Text("{\"stringCol\":\"str\",\"nullCol\":null}")); // Get the serialized json string String jsonStr = serde.serialize(res, soi).toString(); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("stringCol")).equals("str")); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("nullCol"))); assertNull(soi.getStructFieldData(res, soi.getStructFieldRef("missingCol"))); assertEquals(jsonStr,"{\"stringCol\":\"str\"}"); }
serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"structCol\":{\"name\":\"myName\"},\"structNullCol\":{\"name\":null}}"));
instance.initialize(conf, tbl);
"struct<inner_with_dots:string,resultsPerPage:INT>").toLowerCase()); instance.initialize(conf, tbl);