LazySimpleSerDe testSD = new LazySimpleSerDe(); SerDeUtils.initializeSerDe(testSD, conf, tblProps, null); Writable s3 = testSD.serialize(s, hrsd.getObjectInspector()); LOG.info("THREE: {}", s3); Object o3 = testSD.deserialize(s3); Assert.assertFalse(r.getClass().equals(o3.getClass())); HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector()); LOG.info("FOUR: {}", s4); LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector()); LOG.info("FIVE: {}", s5);
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams.getRowTypeInfo()) .getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null; serializeField(serializeStream, f, foi, serdeParams);
private LazySimpleSerDe getLineSerde() throws SerDeException { if (serde == null) { Logger.getLogger(LazySimpleSerDe.class).setLevel(Level.DEBUG); serde = new LazySimpleSerDe(); Configuration job = new Configuration(); Properties tbl = new Properties(); tbl.setProperty("columns", "key,value"); tbl.setProperty("columns.types", keyType.getTypeName() + "," + valType.getTypeName()); serde.initialize(job, tbl); } return serde; }
StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector(); StructField keyRef = lineOI.getStructFieldRef("key"); StructField valueRef = lineOI.getStructFieldRef("value"); while ((line = reader.readLine()) != null) { Text lineText = new Text(line); Object lineObj = serde.deserialize(lineText); List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj); Object f0 = fields.get(0);
@Override public LazySimpleSerDe createSerde() throws SerializationError { try { Properties tableProps = table.getMetadata(); tableProps.setProperty(serdeConstants.LIST_COLUMNS, Joiner.on(",").join(inputColumns)); tableProps.setProperty(serdeConstants.LIST_COLUMN_TYPES, Joiner.on(":").join(inputTypes)); tableProps.setProperty(serdeConstants.FIELD_DELIM, String.valueOf(fieldDelimiter)); tableProps.setProperty(serdeConstants.COLLECTION_DELIM, String.valueOf(collectionDelimiter)); tableProps.setProperty(serdeConstants.MAPKEY_DELIM, String.valueOf(mapKeyDelimiter)); LazySimpleSerDe serde = new LazySimpleSerDe(); SerDeUtils.initializeSerDe(serde, conf, tableProps, null); this.serde = serde; return serde; } catch (SerDeException e) { throw new SerializationError("Error initializing serde", e); } } }
protected void serializeField(ByteStream.Output out, Object obj, ObjectInspector objInspector, LazySerDeParameters serdeParams) throws SerDeException { try { serialize(out, obj, objInspector, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape()); } catch (IOException e) { throw new SerDeException(e); } }
this.recordObjInspector = (LazySimpleStructObjectInspector) serde.getObjectInspector(); this.bucketObjInspectors = getObjectInspectorsForBucketedCols(bucketIds, recordObjInspector); } catch (SerDeException e) {
@Override protected void serializeField(ByteStream.Output out, Object obj, ObjectInspector objInspector, LazySerDeParameters serdeParams) throws SerDeException { if (!objInspector.getCategory().equals(Category.PRIMITIVE) || (objInspector.getTypeName().equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME))) { //do this for all complex types and binary try { serialize(out, SerDeUtils.getJSONString(obj, objInspector, serdeParams.getNullSequence().toString()), PrimitiveObjectInspectorFactory.javaStringObjectInspector, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape()); } catch (IOException e) { throw new SerDeException(e); } } else { //primitives except binary super.serializeField(out, obj, objInspector, serdeParams); } } }
@Override public Object encode(byte[] record) throws SerializationError { try { BytesWritable blob = new BytesWritable(); blob.set(record, 0, record.length); return serde.deserialize(blob); } catch (SerDeException e) { throw new SerializationError("Unable to convert byte[] record into Object", e); } }
Hive1SerDeParametersShim(Configuration configuration, Properties properties, String serDeName) throws SerDeException { this.realSerDeParameters = LazySimpleSerDe.initSerdeParams(configuration, properties, serDeName); }
StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector(); StructField keyRef = lineOI.getStructFieldRef("key"); StructField valueRef = lineOI.getStructFieldRef("value"); while((line = reader.readLine()) != null) { Text lineText = new Text(line); Object lineObj = serde.deserialize(lineText); List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj); Object f0 = fields.get(0);
/** * Creates LazySimpleSerde * @return * @throws SerializationError if serde could not be initialized * @param tbl */ protected static LazySimpleSerDe createSerde(Table tbl, HiveConf conf, char serdeSeparator) throws SerializationError { try { Properties tableProps = MetaStoreUtils.getTableMetadata(tbl); tableProps.setProperty("field.delim", String.valueOf(serdeSeparator)); LazySimpleSerDe serde = new LazySimpleSerDe(); SerDeUtils.initializeSerDe(serde, conf, tableProps, null); return serde; } catch (SerDeException e) { throw new SerializationError("Error initializing serde", e); } }
out.write(separator); serialize(out, list.get(i), eoi, separators, level + 1, nullSequence, escaped, escapeChar, needsEscape); out.write(separator); serialize(out, entry.getKey(), koi, separators, level + 2, nullSequence, escaped, escapeChar, needsEscape); out.write(keyValueSeparator); serialize(out, entry.getValue(), voi, separators, level + 2, nullSequence, escaped, escapeChar, needsEscape); out.write(separator); serialize(out, list.get(i), fields.get(i).getFieldObjectInspector(), separators, level + 1, nullSequence, escaped, escapeChar, needsEscape); escaped, escapeChar, needsEscape); out.write(separator); serialize(out, uoi.getField(obj), ois.get(uoi.getTag(obj)), separators, level + 1, nullSequence, escaped, escapeChar, needsEscape);
@Override protected void serializeField(ByteStream.Output out, Object obj, ObjectInspector objInspector, LazySerDeParameters serdeParams) throws SerDeException { if (!objInspector.getCategory().equals(Category.PRIMITIVE) || (objInspector.getTypeName().equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME))) { //do this for all complex types and binary try { serialize(out, SerDeUtils.getJSONString(obj, objInspector, serdeParams.getNullSequence().toString()), PrimitiveObjectInspectorFactory.javaStringObjectInspector, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape()); } catch (IOException e) { throw new SerDeException(e); } } else { //primitives except binary super.serializeField(out, obj, objInspector, serdeParams); } } }
@Override public Object encode(byte[] record) throws SerializationError { try { BytesWritable blob = new BytesWritable(); blob.set(record, 0, record.length); return serde.deserialize(blob); } catch (SerDeException e) { throw new SerializationError("Unable to convert byte[] record into Object", e); } }
Hive1SerDeParametersShim(Configuration configuration, Properties properties, String serDeName) throws SerDeException { this.realSerDeParameters = LazySimpleSerDe.initSerdeParams(configuration, properties, serDeName); }
StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector(); StructField keyRef = lineOI.getStructFieldRef("key"); StructField valueRef = lineOI.getStructFieldRef("value"); while ((line = reader.readLine()) != null) { Text lineText = new Text(line); Object lineObj = serde.deserialize(lineText); List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj); Object f0 = fields.get(0);
public static LazySimpleSerDe getKeyValueLineSerde(@Nonnull final PrimitiveObjectInspector keyOI, @Nonnull final PrimitiveObjectInspector valueOI) throws SerDeException { LazySimpleSerDe serde = new LazySimpleSerDe(); Configuration conf = new Configuration(); Properties tbl = new Properties(); tbl.setProperty("columns", "key,value"); tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName()); serde.initialize(conf, tbl); return serde; }
String types = typesSb.toString(); serde = new LazySimpleSerDe(); Properties props = new Properties(); if (names.length() > 0) {
List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.rowTypeInfo != null && ((StructTypeInfo) serdeParams.rowTypeInfo) .getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null; serializeField(serializeStream, f, foi, serdeParams);