public static String[] columnNamesFromStructObjectInspector( StructObjectInspector structObjectInspector) throws HiveException { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); String[] result = new String[fields.size()]; int i = 0; for(StructField field : fields) { result[i++] = field.getFieldName(); } return result; }
private static StructField[] initializeFieldsFromOi(ObjectInspector inspector) { if (inspector instanceof StructObjectInspector) { List<? extends StructField> fieldList = ((StructObjectInspector) inspector).getAllStructFieldRefs(); StructField[] fields = new StructField[fieldList.size()]; fieldList.toArray(fields); return fields; } else { return null; } }
protected static ObjectInspector[] getObjectInspectorsForBucketedCols(List<Integer> bucketIds , StructObjectInspector recordObjInspector) { ObjectInspector[] result = new ObjectInspector[bucketIds.size()]; for (int i = 0; i < bucketIds.size(); i++) { int bucketId = bucketIds.get(i); result[i] = recordObjInspector.getAllStructFieldRefs().get(bucketId).getFieldObjectInspector(); } return result; }
void init(List<StructObjectInspector> unionObjectInspectors) { this.unionObjectInspectors = unionObjectInspectors; int totalSize = 0; for (int i = 0; i < unionObjectInspectors.size(); i++) { totalSize += unionObjectInspectors.get(i).getAllStructFieldRefs().size(); } fields = new ArrayList<MyField>(totalSize); for (int i = 0; i < unionObjectInspectors.size(); i++) { StructObjectInspector oi = unionObjectInspectors.get(i); for (StructField sf : oi.getAllStructFieldRefs()) { fields.add(new MyField(i, sf)); } } }
/** * Compiles the appropriate vector expression writers based on a struct object * inspector. */ public static VectorExpressionWriter[] genVectorStructExpressionWritables( StructObjectInspector oi) throws HiveException { VectorExpressionWriter[] writers = new VectorExpressionWriter[oi.getAllStructFieldRefs().size()]; final List<? extends StructField> fields = oi.getAllStructFieldRefs(); int i = 0; for(StructField field : fields) { writers[i++] = genVectorExpressionWritable(field.getFieldObjectInspector()); } return writers; }
public static int getStructSize(ObjectInspector oi) throws SerDeException { if (oi.getCategory() != Category.STRUCT) { throw new SerDeException("Unexpected category " + oi.getCategory()); } return ((StructObjectInspector)oi).getAllStructFieldRefs().size(); }
/** * Note that all column indexes are with respect to your record structure, not the Hive table structure. */ public RecordInspectorImpl(ObjectInspector objectInspector, int recordIdColumn) { if (!(objectInspector instanceof StructObjectInspector)) { throw new IllegalArgumentException("Serious problem, expected a StructObjectInspector, " + "but got a " + objectInspector.getClass().getName()); } structObjectInspector = (StructObjectInspector) objectInspector; List<? extends StructField> structFields = structObjectInspector.getAllStructFieldRefs(); recordIdentifierField = structFields.get(recordIdColumn); }
/** * It writes all the fields contained inside a group to the RecordConsumer. * * @param value The list of values contained in the group. * @param inspector The object inspector used to get the correct value type. * @param type Type that contains information about the group schema. */ private void writeGroupFields(final Object value, final StructObjectInspector inspector, final GroupType type) { if (value != null) { List<? extends StructField> fields = inspector.getAllStructFieldRefs(); List<Object> fieldValuesList = inspector.getStructFieldsDataAsList(value); for (int i = 0; i < type.getFieldCount(); i++) { Type fieldType = type.getType(i); String fieldName = fieldType.getName(); Object fieldValue = fieldValuesList.get(i); if (fieldValue != null) { ObjectInspector fieldInspector = fields.get(i).getFieldObjectInspector(); recordConsumer.startField(fieldName, i); writeValue(fieldValue, fieldInspector, fieldType); recordConsumer.endField(fieldName, i); } } } }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
/** * Copy fields in the input row to the output array of standard objects. * * @param result * output list of standard objects. * @param row * input row. * @param soi * Object inspector for the to-be-copied columns. * @param objectInspectorOption */ public static void copyToStandardObject(List<Object> result, Object row, StructObjectInspector soi, ObjectInspectorCopyOption objectInspectorOption) { List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (StructField f : fields) { result.add(copyToStandardObject(soi.getStructFieldData(row, f), f.getFieldObjectInspector(), objectInspectorOption)); } }
return new DeserializerOrcWriter(serDe, sourceOi, allocSize); for (StructField sf : sourceOi.getAllStructFieldRefs()) { Category c = sf.getFieldObjectInspector().getCategory(); if (c != Category.PRIMITIVE) { LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter: " + c + " is not supported");
public static boolean isSupportedKey(ObjectInspector keyOi) { List<? extends StructField> keyFields = ((StructObjectInspector)keyOi).getAllStructFieldRefs(); for (StructField field : keyFields) { if (!MapJoinKey.isSupportedField(field.getFieldObjectInspector())) { return false; } } return true; }
private List<Object> nextOutputRow() throws HiveException { List<Object> oRow = new ArrayList<Object>(); Object iRow = rollingPart.nextOutputRow(); int i = 0; for (; i < fnOutputs.length; i++) { oRow.add(fnOutputs[i].remove(0)); } for (StructField f : rollingPart.getOutputOI().getAllStructFieldRefs()) { oRow.add(rollingPart.getOutputOI().getStructFieldData(iRow, f)); } if ( rnkLimit != null ) { rnkLimit.updateRank(oRow); } return oRow; }
private ArrayList<FieldNode> prunedInputList(List<FieldNode> prunedCols, PTFInputDef tDef) { ArrayList<FieldNode> prunedInputCols = new ArrayList<>(); StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); FieldNode fn = lookupColumn(prunedCols, fName); if (fn != null) { prunedInputCols.add(fn); } } return prunedInputCols; } }
protected static ObjectInspector[] getObjectInspectorsForBucketedCols(List<Integer> bucketIds , StructObjectInspector recordObjInspector) throws SerializationError { ObjectInspector[] result = new ObjectInspector[bucketIds.size()]; for (int i = 0; i < bucketIds.size(); i++) { int bucketId = bucketIds.get(i); result[i] = recordObjInspector.getAllStructFieldRefs().get( bucketId ).getFieldObjectInspector(); } return result; }
public static ArrayList<TypeInfo> typeInfosFromStructObjectInspector( StructObjectInspector structObjectInspector) { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); ArrayList<TypeInfo> typeInfoList = new ArrayList<TypeInfo>(fields.size()); for(StructField field : fields) { TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString( field.getFieldObjectInspector().getTypeName()); typeInfoList.add(typeInfo); } return typeInfoList; }
@Override public Writable serialize(final Object obj, final ObjectInspector objInspector) throws SerDeException { if (!objInspector.getCategory().equals(Category.STRUCT)) { throw new SerDeException("Cannot serialize " + objInspector.getCategory() + ". Can only serialize a struct"); } serializedSize = ((StructObjectInspector)objInspector).getAllStructFieldRefs().size(); status = LAST_OPERATION.SERIALIZE; parquetRow.value = obj; parquetRow.inspector= (StructObjectInspector)objInspector; return parquetRow; }
public static String[] columnNamesFromStructObjectInspector( StructObjectInspector structObjectInspector) throws HiveException { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); String[] result = new String[fields.size()]; int i = 0; for(StructField field : fields) { result[i++] = field.getFieldName(); } return result; }
private StructField getStructField(StructObjectInspector oi, String name) { if (hiveColIndexParsing) { int colIndex = getColIndex(name); if (colIndex >= 0) { return oi.getAllStructFieldRefs().get(colIndex); } } // FIXME: linear scan inside the below method...get a map here or something.. return oi.getStructFieldRef(name); }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }