public static String[] columnNamesFromStructObjectInspector( StructObjectInspector structObjectInspector) throws HiveException { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); String[] result = new String[fields.size()]; int i = 0; for(StructField field : fields) { result[i++] = field.getFieldName(); } return result; }
protected void init(List<StructField> fields) { this.fields = new ArrayList<MyField>(fields.size()); this.originalColumnNames = new ArrayList<String>(fields.size()); for (int i = 0; i < fields.size(); i++) { this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields .get(i).getFieldObjectInspector())); this.originalColumnNames.add(fields.get(i).getFieldName()); } }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
private ObjectInspector convertStruct(ObjectInspector inspector, AtomicBoolean foundUnion) { StructObjectInspector structOI = (StructObjectInspector) inspector; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); List<String> names = new ArrayList<>(fields.size()); List<ObjectInspector> inspectors = new ArrayList<>(fields.size()); for (StructField field : fields) { names.add(field.getFieldName()); inspectors.add(convert(field.getFieldObjectInspector(), foundUnion)); } return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); }
public static StructField getStandardStructFieldRef(String fieldName, List<? extends StructField> fields) { fieldName = fieldName.toLowerCase(); for (int i = 0; i < fields.size(); i++) { if (fields.get(i).getFieldName().equals(fieldName)) { return fields.get(i); } } // For backward compatibility: fieldNames can also be integer Strings. try { int i = Integer.parseInt(fieldName); if (i >= 0 && i < fields.size()) { return fields.get(i); } } catch (NumberFormatException e) { // ignore } throw new RuntimeException("cannot find field " + fieldName + " from " + fields); // return null; }
public MyField(int fieldID, StructField field) { this.fieldID = fieldID; this.fieldName = field.getFieldName().toLowerCase(); this.fieldObjectInspector = field.getFieldObjectInspector(); this.fieldComment = field.getFieldComment(); }
private ArrayList<FieldNode> prunedInputList(List<FieldNode> prunedCols, PTFInputDef tDef) { ArrayList<FieldNode> prunedInputCols = new ArrayList<>(); StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); FieldNode fn = lookupColumn(prunedCols, fName); if (fn != null) { prunedInputCols.add(fn); } } return prunedInputCols; } }
public static String getStandardStructTypeName(StructObjectInspector soi) { StringBuilder sb = new StringBuilder(); sb.append("struct<"); List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { sb.append(","); } sb.append(fields.get(i).getFieldName()); sb.append(":"); sb.append(fields.get(i).getFieldObjectInspector().getTypeName()); } sb.append(">"); return sb.toString(); }
@Override public String getTypeName() { StringBuilder buffer = new StringBuilder(); buffer.append("struct<"); for(int i=0; i < fields.size(); ++i) { StructField field = fields.get(i); if (i != 0) { buffer.append(","); } buffer.append(field.getFieldName()); buffer.append(":"); buffer.append(field.getFieldObjectInspector().getTypeName()); } buffer.append(">"); return buffer.toString(); }
public static String[] columnNamesFromStructObjectInspector( StructObjectInspector structObjectInspector) throws HiveException { List<? extends StructField> fields = structObjectInspector.getAllStructFieldRefs(); String[] result = new String[fields.size()]; int i = 0; for(StructField field : fields) { result[i++] = field.getFieldName(); } return result; }
@Override public void write(Object value) { for (int i = 0; i < structFields.size(); i++) { StructField field = structFields.get(i); Object fieldValue = inspector.getStructFieldData(value, field); if (fieldValue != null) { String fieldName = field.getFieldName(); DataWriter writer = structWriters[i]; recordConsumer.startField(fieldName, i); writer.write(fieldValue); recordConsumer.endField(fieldName, i); } } } }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
/** * Create a new struct object inspector for the list of struct fields, first removing the * prefix from the field name. * @param fields * @param prefixToRemove * @return */ private static ObjectInspector createStructFromFields(List<StructField> fields, String prefixToRemove) { int prefixLength = prefixToRemove.length() + 1; // also remove the '.' after the prefix ArrayList<String> fieldNames = new ArrayList<String>(); ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>(); for (StructField field : fields) { fieldNames.add(field.getFieldName().substring(prefixLength)); fieldOIs.add(field.getFieldObjectInspector()); } return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs); }
private ArrayList<FieldNode> prunedInputList(List<FieldNode> prunedCols, PTFInputDef tDef) { ArrayList<FieldNode> prunedInputCols = new ArrayList<>(); StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); FieldNode fn = lookupColumn(prunedCols, fName); if (fn != null) { prunedInputCols.add(fn); } } return prunedInputCols; } }
private ObjectInspector convertStruct(ObjectInspector inspector, AtomicBoolean foundUnion) { StructObjectInspector structOI = (StructObjectInspector) inspector; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); List<String> names = new ArrayList<>(fields.size()); List<ObjectInspector> inspectors = new ArrayList<>(fields.size()); for (StructField field : fields) { names.add(field.getFieldName()); inspectors.add(convert(field.getFieldObjectInspector(), foundUnion)); } return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors); }
@Override public boolean equals(Object o) { if (o == null || o.getClass() != getClass()) { return false; } else if (o == this) { return true; } else { List<StructField> other = ((OrcStructInspector) o).fields; if (other.size() != fields.size()) { return false; } for(int i = 0; i < fields.size(); ++i) { StructField left = other.get(i); StructField right = fields.get(i); if (!(left.getFieldName().equalsIgnoreCase(right.getFieldName()) && left.getFieldObjectInspector().equals (right.getFieldObjectInspector()))) { return false; } } return true; } } }
/** * Get the list of field names as csv from a StructObjectInspector. */ public static String getFieldNames(StructObjectInspector soi) { List<? extends StructField> fields = soi.getAllStructFieldRefs(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < fields.size(); i++) { if (i > 0) { sb.append(","); } sb.append(fields.get(i).getFieldName()); } return sb.toString(); }
@Override public void write(Object value) { for (int i = 0; i < structFields.size(); i++) { StructField field = structFields.get(i); Object fieldValue = inspector.getStructFieldData(value, field); if (fieldValue != null) { String fieldName = field.getFieldName(); DataWriter writer = structWriters[i]; recordConsumer.startField(fieldName, i); writer.write(fieldValue); recordConsumer.endField(fieldName, i); } } } }
private static void addInputColumnsToList(ShapeDetails shape, ArrayList<String> fieldNames, ArrayList<ObjectInspector> fieldOIs) { StructObjectInspector OI = shape.getOI(); for (StructField f : OI.getAllStructFieldRefs()) { fieldNames.add(f.getFieldName()); fieldOIs.add(f.getFieldObjectInspector()); } }
public String getTypeName() { StringBuilder sb = new StringBuilder("struct<"); boolean first = true; for (StructField structField : getAllStructFieldRefs()) { if (first) { first = false; } else { sb.append(","); } sb.append(structField.getFieldName()).append(":") .append(structField.getFieldObjectInspector().getTypeName()); } sb.append(">"); return sb.toString(); }