public static ArrayList<TypeInfo> typeInfosFromStructObjectInspector( StructObjectInspector structObjectInspector) { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); ArrayList<TypeInfo> typeInfoList = new ArrayList<TypeInfo>(fields.size()); for(StructField field : fields) { TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString( field.getFieldObjectInspector().getTypeName()); typeInfoList.add(typeInfo); } return typeInfoList; }
@Override public Object get(int fieldNum) { try { StructField fref = soi.getAllStructFieldRefs().get(fieldNum); return HCatRecordSerDe.serializeField( soi.getStructFieldData(wrappedObject, fref), fref.getFieldObjectInspector()); } catch (SerDeException e) { throw new IllegalStateException("SerDe Exception deserializing",e); } }
private static void unpackStructObject(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj cStatsObj) throws UnsupportedDoubleException { if (oi.getCategory() != ObjectInspector.Category.STRUCT) { throw new RuntimeException("Invalid object datatype : " + oi.getCategory().toString()); } StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); List<Object> list = soi.getStructFieldsDataAsList(o); for (int i = 0; i < fields.size(); i++) { // Get the field objectInspector, fieldName and the field object. ObjectInspector foi = fields.get(i).getFieldObjectInspector(); Object f = (list == null ? null : list.get(i)); String fieldName = fields.get(i).getFieldName(); if (foi.getCategory() == ObjectInspector.Category.PRIMITIVE) { unpackPrimitiveObject(foi, f, fieldName, cStatsObj); } else { unpackStructObject(foi, f, fieldName, cStatsObj); } } } }
private StructField getStructField(StructObjectInspector oi, String name) { if (hiveColIndexParsing) { int colIndex = getColIndex(name); if (colIndex >= 0) { return oi.getAllStructFieldRefs().get(colIndex); } } // FIXME: linear scan inside the below method...get a map here or something.. return oi.getStructFieldRef(name); }
@Override public void setKeyValue(Writable key, Writable val) throws SerDeException { Object keyObj = keySerDe.deserialize(key), valObj = valSerDe.deserialize(val); List<? extends StructField> keyFields = keySoi.getAllStructFieldRefs(), valFields = valSoi.getAllStructFieldRefs(); for (int i = 0; i < keyFields.size(); ++i) { keyObjs[i] = keySoi.getStructFieldData(keyObj, keyFields.get(i)); } for (int i = 0; i < valFields.size(); ++i) { valObjs[i] = valSoi.getStructFieldData(valObj, valFields.get(i)); } }
soi = (StructObjectInspector) parameters[0]; countTruesField = soi.getStructFieldRef("counttrues"); countTruesFieldOI = (WritableLongObjectInspector) countTruesField.getFieldObjectInspector(); countFalsesField = soi.getStructFieldRef("countfalses"); countFalsesFieldOI = (WritableLongObjectInspector) countFalsesField.getFieldObjectInspector(); countNullsField = soi.getStructFieldRef("countnulls"); countNullsFieldOI = (WritableLongObjectInspector) countNullsField.getFieldObjectInspector(); partialResult[0] = new Text(); partialResult[1] = new LongWritable(0); partialResult[2] = new LongWritable(0); result[0] = new Text(); result[1] = new LongWritable(0); result[2] = new LongWritable(0); result[3] = new LongWritable(0); return ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
switch (objIns.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns); switch (poi.getPrimitiveCategory()) { case VOID: return 0; return Murmur3.hash32(text.getBytes(), text.getLength()); return Murmur3.hash32(text.getBytes(), text.getLength()); + poi.getPrimitiveCategory()); int r = 0; ListObjectInspector listOI = (ListObjectInspector)objIns; ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI, byteBuffer); int r = 0; StructObjectInspector structOI = (StructObjectInspector)objIns; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCodeMurmur(structOI.getStructFieldData(o, field), field.getFieldObjectInspector(), byteBuffer); throw new RuntimeException("Unknown type: "+ objIns.getTypeName());
List<? extends StructField> structFields = soi.getAllStructFieldRefs(); assert (columnNames.size() == structFields.size()); if (obj == null) { buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)), structFields.get(i).getFieldObjectInspector()); throw new SerDeException(e); return new Text(sb.toString());
throws IOException, SerDeException { if (obj == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); return; switch (objInspector.getCategory()) { case PRIMITIVE: PrimitiveObjectInspector oi = (PrimitiveObjectInspector) objInspector; if (oi.getPrimitiveCategory() == PrimitiveCategory.BINARY) { BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(obj); byte[] toWrite = new byte[bw.getLength()]; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); separator = (char) separators[level]; StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); list = soi.getStructFieldsDataAsList(obj); out.write(separator); serializeNoEncode(out, list.get(i), fields.get(i).getFieldObjectInspector(), separators, level + 1, nullSequence, escaped, escapeChar, needsEscape); throw new RuntimeException("Unknown category type: "+ objInspector.getCategory());
switch (oi.getCategory()) { case PRIMITIVE: { return oi.getClass().getSimpleName(); ListObjectInspector loi = (ListObjectInspector) oi; return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(loi.getListElementObjectInspector()) + ">"; + getObjectInspectorName(moi.getMapKeyObjectInspector()) + "," + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">"; result.append(oi.getClass().getSimpleName() + "<"); StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { result.append(fields.get(i).getFieldName()); result.append(":"); result.append(getObjectInspectorName(fields.get(i) .getFieldObjectInspector())); if (i == fields.size() - 1) { result.append(">");
switch (outputOI.getCategory()) { case PRIMITIVE: primOutputOI.getTypeInfo()); case STRUCT: StructObjectInspector structOutputOI = (StructObjectInspector) outputOI; List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs(); List<String> structFieldNames = new ArrayList<String>(listFields.size()); List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>( structFieldNames.add(listField.getFieldName()); structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(), listField.getFieldObjectInspector(), oiSettableProperties, false)); return ObjectInspectorFactory.getStandardStructObjectInspector( structFieldNames, structFieldObjectInspectors); ListObjectInspector listOutputOI = (ListObjectInspector) outputOI; + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet.");
switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector loi = (PrimitiveObjectInspector) oi; if (objectInspectorOption == ObjectInspectorCopyOption.DEFAULT) { objectInspectorOption = loi.preferWritable() ? ObjectInspectorCopyOption.WRITABLE : ObjectInspectorCopyOption.JAVA; result = loi.getPrimitiveJavaObject(o); if (loi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) { result = PrimitiveObjectInspectorFactory.javaTimestampObjectInspector.copyObject(result); } else if (loi.getPrimitiveCategory() == int length = loi.getListLength(o); ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { list.add(copyToStandardObject(loi.getListElement(o, i), loi .getListElementObjectInspector(), objectInspectorOption)); List<? extends StructField> fields = soi.getAllStructFieldRefs(); ArrayList<Object> struct = new ArrayList<Object>(fields.size()); for (StructField f : fields) { struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f .getFieldObjectInspector(), objectInspectorOption));
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); } StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { sb.append(separator); } Object column = soi.getStructFieldData(obj, fields.get(i)); if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) { // For primitive object, serialize to plain string sb.append(column == null ? nullString : column.toString()); } else { // For complex object, serialize to JSON format sb.append(SerDeUtils.getJSONString(column, fields.get(i) .getFieldObjectInspector())); } } serializeCache.set(sb.toString()); return serializeCache; }
ObjectInspector.Category oiCategory = oi.getCategory(); switch (oiCategory) { case PRIMITIVE: case LIST: ListObjectInspector loi = (ListObjectInspector) oi; int listSize = loi.getListLength(val); ObjectInspector listElementOI = loi.getListElementObjectInspector(); for (int idx = 0; idx < listSize; ++idx) { convertedList.add(convertValue(loi.getListElement(val, idx), listElementOI)); case MAP: MapObjectInspector moi = (MapObjectInspector) oi; int mapSize = moi.getMapSize(val); ObjectInspector mapKeyOI = moi.getMapKeyObjectInspector(); ObjectInspector mapValOI = moi.getMapValueObjectInspector(); Map<?, ?> mapCol = moi.getMap(val); for (Object mapKey : mapCol.keySet()) { StructObjectInspector soi = (StructObjectInspector) oi; List<Object> convertedRow = new ArrayList<Object>(); for (StructField structField : soi.getAllStructFieldRefs()) { Object convertedFieldValue = convertValue( soi.getStructFieldData(val, structField), structField.getFieldObjectInspector()); convertedRow.add(convertedFieldValue);
throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); List<? extends StructField> fields = soi.getAllStructFieldRefs(); List<Object> list = soi.getStructFieldsDataAsList(obj); List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams.getRowTypeInfo()) .getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null; ObjectInspector foi = fields.get(i).getFieldObjectInspector(); Object f = (list == null ? null : list.get(i)); + "tableType=" + serdeParams.getRowTypeInfo().toString() + "\n" + "dataType=" + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector)); .set(serializeStream.getData(), 0, serializeStream.getLength()); serializedSize = serializeStream.getLength(); lastOperationSerialize = true;
StructField sf = soi.getStructFieldRef(names[i]); if (sf == null) { throw new MetaException("Invalid Field " + names[i]); } else { oi = sf.getFieldObjectInspector(); if (oi.getCategory() != Category.STRUCT) { str_fields.add(new FieldSchema(last_name, oi.getTypeName(), FROM_SERIALIZER)); } else { List<? extends StructField> fields = ((StructObjectInspector) oi) .getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { StructField structField = fields.get(i); String fieldName = structField.getFieldName(); String fieldTypeName = structField.getFieldObjectInspector().getTypeName(); String fieldComment = determineFieldComment(structField.getFieldComment());
private ObjectInspector convertStruct(ObjectInspector inspector, AtomicBoolean foundUnion) { StructObjectInspector structOI = (StructObjectInspector) inspector; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); List<String> names = new ArrayList<>(fields.size()); List<ObjectInspector> inspectors = new ArrayList<>(fields.size()); for (StructField field : fields) { names.add(field.getFieldName()); inspectors.add(convert(field.getFieldObjectInspector(), foundUnion)); } return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); }
switch (oi.getCategory()) { case PRIMITIVE: Writable writable = (Writable) ((PrimitiveObjectInspector) oi).getPrimitiveWritableObject(data); return writableWriter.write(writable, generator); generator.writeBeginArray(); for (Map.Entry<?, ?> entry : moi.getMap(data).entrySet()) { Result result = write(entry.getValue(), moi.getMapValueObjectInspector(), generator); if (!result.isSuccesful()) { return result; StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> refs = soi.getAllStructFieldRefs(); String actualFieldName = alias.toES(structField.getFieldName()); if (shouldKeep(generator.getParentPath(), actualFieldName)) { generator.writeFieldName(actualFieldName); Result result = write(soi.getStructFieldData(data, structField), structField.getFieldObjectInspector(), generator); if (!result.isSuccesful()) { return result;
private boolean serialize(Object obj, ObjectInspector objInspector, int level, ByteStream.Output ss) throws IOException { switch (objInspector.getCategory()) { case PRIMITIVE: LazyUtils.writePrimitiveUTF8(ss, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape); char keyValueSeparator = (char) separators[level + 1]; MapObjectInspector moi = (MapObjectInspector) objInspector; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map<?, ?> map = moi.getMap(obj); if (map == null) { return false; sep = (char) separators[level]; StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); list = soi.getStructFieldsDataAsList(obj); if (list == null) { return false; serialize(currentItem, fields.get(i).getFieldObjectInspector(), level + 1, ss); throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
switch (oi.getCategory()) { case PRIMITIVE: { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; result = poi.getTypeInfo(); break; ListObjectInspector loi = (ListObjectInspector) oi; result = TypeInfoFactory .getListTypeInfo(getTypeInfoFromObjectInspector(loi .getListElementObjectInspector())); break; MapObjectInspector moi = (MapObjectInspector) oi; result = TypeInfoFactory.getMapTypeInfo( getTypeInfoFromObjectInspector(moi.getMapKeyObjectInspector()), getTypeInfoFromObjectInspector(moi.getMapValueObjectInspector())); break; List<? extends StructField> fields = soi.getAllStructFieldRefs(); List<String> fieldNames = new ArrayList<String>(fields.size()); List<TypeInfo> fieldTypeInfos = new ArrayList<TypeInfo>(fields.size()); for (StructField f : fields) { fieldNames.add(f.getFieldName()); fieldTypeInfos.add(getTypeInfoFromObjectInspector(f .getFieldObjectInspector()));