@Test public void testCaseSensitiveMapping() throws SerDeException, IOException { System.out.println("testCaseSensitiveMapping"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "time1,time2"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,string"); // this means, we call it ts but in data it's 'timestamp' tbl.setProperty("mapping.time1", "Time"); tbl.setProperty("mapping.time2", "time"); tbl.setProperty(JsonSerDe.PROP_CASE_INSENSITIVE, "false"); serde.initialize(conf, tbl); StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector(); Object res = serde.deserialize(new Text("{\"Time\":\"forme\",\"time\":\"foryou\"}")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time1")).equals("forme")); assertTrue(soi.getStructFieldData(res, soi.getStructFieldRef("time2")).equals("foryou")); }
getSerializedFieldName(columnNames, i, sf), serializeField( data, sf.getFieldObjectInspector())); } else if(explicitNull) { result.putNull(getSerializedFieldName(columnNames, i, sf));
options = new JsonStructOIOptions(getMappings(tbl, isCaseInsensitive)); options.setCaseInsensitive(isCaseInsensitive);
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,five"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,timestamp"); instance.initialize(conf, tbl); }
/** * Test of getSerializedClass method, of class JsonSerDe. */ @Test public void testGetSerializedClass() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("getSerializedClass"); Class expResult = Text.class; Class result = instance.getSerializedClass(); assertEquals(expResult, result); }
@Test public void testTimestampDeSerialize() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals(Timestamp.valueOf("2013-03-27 23:18:40.0"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
System.out.println("testSerializeTimestamp"); JsonSerDe serde = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,string,timestamp"); // one timestamp field serde.initialize(conf, tbl); Object obj = serde.serialize(row, soi);
@Test public void deserializeStrangeString() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("deserialize"); Writable w = new Text("{\"one\":\"\\a\\v\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("",result.get("one").toString().trim()); } }
JsonSerDe instance = new JsonSerDe(); initialize(instance); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",\"orange\", null],\"two\":null,\"four\":null}"); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JSONObject result = (JSONObject) instance.deserialize(w); assertTrue(JSONObject.NULL == result.get("four"));
@Override // The evaluate() method. The input is passed in as an array of DeferredObjects, so that // computation is not wasted on deserializing them if they're not actually used public Object evaluate(DeferredObject[] arguments) throws HiveException { Object obj = arguments[0].get(); return serde.serializeField(obj, inOi); }
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (arguments.length != 1) throw new UDFArgumentException("Function takes one argument"); inOi = arguments[0]; try { Properties props = new Properties(); props.setProperty(serdeConstants.LIST_COLUMNS,"col1"); props.setProperty(serdeConstants.LIST_COLUMN_TYPES, inOi.getTypeName()); serde.initialize(new Configuration(), props); } catch (SerDeException ex) { throw new UDFArgumentException(ex); } return PrimitiveObjectInspectorFactory.javaStringObjectInspector; }
onMalformedJson("Row is not a valid JSON Object - JSONException: " + e.getMessage()); try { jObj = new JSONObject("{}"); } catch (JSONException ex) { onMalformedJson("Error parsing empty row. This should never happen.");
Object obj = serde.serialize(row, soi);
@Test public void testTimestampDeSerializeWithNanoseconds() throws Exception { // Test that timestamp object can be deserialized Writable w = new Text("{\"one\":true,\"five\":\"2013-03-27 23:18:40.123456\"}"); JSONObject result = (JSONObject) instance.deserialize(w); StructObjectInspector soi = (StructObjectInspector) instance.getObjectInspector(); JavaStringTimestampObjectInspector jstOi = (JavaStringTimestampObjectInspector) soi.getStructFieldRef("five").getFieldObjectInspector(); assertEquals( Timestamp.valueOf("2013-03-27 23:18:40.123456"), jstOi.getPrimitiveJavaObject(result.get("five"))); }
static public void initialize() throws Exception { instance = new JsonSerDe(); Configuration conf = null; Properties tbl = new Properties(); // from google video API tbl.setProperty(serdeConstants.LIST_COLUMNS, "country,languages,religions"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,array<string>,array<string>".toLowerCase()); instance.initialize(conf, tbl); }
/** * Test of deserialize method, but passing an array. */ @Test public void testDeserialize() throws Exception { JsonSerDe instance = new JsonSerDe(); initialize(instance); System.out.println("deserialize"); Writable w = new Text("{\"one\":true,\"three\":[\"red\",\"yellow\",\"orange\"],\"two\":19.5,\"four\":\"poop\"}"); JSONObject result = (JSONObject) instance.deserialize(w); assertEquals("poop",result.get("four")); assertTrue(result.get("three") instanceof JSONArray); assertTrue( ((JSONArray)result.get("three")).get(0) instanceof String ); assertEquals("red", ((JSONArray)result.get("three")).get(0)); }
JsonSerDe instance = new JsonSerDe(); initialize(instance); Writable w = new Text("[true,19.5, [\"red\",\"yellow\",\"orange\"],\"poop\"]"); Object result = instance.deserialize(w); assertTrue(result instanceof JSONArray); StructObjectInspector soi = (StructObjectInspector)instance.getObjectInspector();
/** * Serializes a Union */ private Object serializeUnion(Object obj, UnionObjectInspector oi) { if(obj == null) return null; return serializeField(obj, oi.getObjectInspectors().get(oi.getTag(obj))); }
public void initialize2(JsonSerDe instance) throws Exception { System.out.println("initialize"); Configuration conf = null; Properties tbl = new Properties(); tbl.setProperty(serdeConstants.LIST_COLUMNS, "one,two,three,four,five"); tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, "boolean,float,array<string>,string,string"); instance.initialize(conf, tbl); }