int length = loi.getListLength(o); ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { list.add(copyToStandardObject(loi.getListElement(o, i), loi .getListElementObjectInspector(), objectInspectorOption)); MapObjectInspector moi = (MapObjectInspector) oi; Map<Object, Object> map = new LinkedHashMap<Object, Object>(); Map<? extends Object, ? extends Object> omap = moi.getMap(o); for (Map.Entry<? extends Object, ? extends Object> entry : omap .entrySet()) { map.put(copyToStandardObject(entry.getKey(), moi .getMapKeyObjectInspector(), objectInspectorOption), copyToStandardObject(entry.getValue(), moi .getMapValueObjectInspector(), objectInspectorOption)); List<? extends StructField> fields = soi.getAllStructFieldRefs(); ArrayList<Object> struct = new ArrayList<Object>(fields.size()); for (StructField f : fields) { struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f .getFieldObjectInspector(), objectInspectorOption)); UnionObjectInspector uoi = (UnionObjectInspector)oi; List<ObjectInspector> objectInspectors = uoi.getObjectInspectors(); Object object = copyToStandardObject( uoi.getField(o), objectInspectors.get(uoi.getTag(o)),
ObjectInspector writableOI = getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE); Object writableValue = value == null ? value : ObjectInspectorConverters.getConverter(oi, writableOI).convert(value); switch (writableOI.getCategory()) { case PRIMITIVE: PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; return PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector( poi.getTypeInfo(), writableValue); case LIST: ListObjectInspector loi = (ListObjectInspector) oi; return ObjectInspectorFactory.getStandardConstantListObjectInspector( getStandardObjectInspector( loi.getListElementObjectInspector(), ObjectInspectorCopyOption.WRITABLE ), MapObjectInspector moi = (MapObjectInspector) oi; return ObjectInspectorFactory.getStandardConstantMapObjectInspector( getStandardObjectInspector( moi.getMapKeyObjectInspector(), ObjectInspectorCopyOption.WRITABLE ), getStandardObjectInspector( moi.getMapValueObjectInspector(), ObjectInspectorCopyOption.WRITABLE ), case STRUCT: StructObjectInspector soi = (StructObjectInspector) oi;
@Override public int compare(Object object1, Object object2) { int result = 0; /**If multiple fields are mentioned for sorting a record then inside the loop we do will do sorting for each field*/ for (int i = 0; i < noOfInputFields; i++) { Object o1 = structObjectInspector.getStructFieldData(object1, fields[i]); Object o2 = structObjectInspector.getStructFieldData(object2, fields[i]); result = ObjectInspectorUtils.compare(o1, fields[i].getFieldObjectInspector(), o2, fields[i].getFieldObjectInspector()); if (result != 0) { /**Ordering*/ if (sortOrder == SORT_ORDER_TYPE.DESC) { result *= -1; } return result; } } return result; } });
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial != null) { MaxAgg myagg = (MaxAgg) agg; int r = ObjectInspectorUtils.compare(myagg.o, outputOI, partial, inputOI); if (myagg.o == null || r < 0) { myagg.o = ObjectInspectorUtils.copyToStandardObject(partial, inputOI, ObjectInspectorCopyOption.JAVA); } } }
+ poi.getPrimitiveCategory()); r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI, byteBuffer); int r = 0; MapObjectInspector mapOI = (MapObjectInspector)objIns; ObjectInspector keyOI = mapOI.getMapKeyObjectInspector(); ObjectInspector valueOI = mapOI.getMapValueObjectInspector(); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCodeMurmur(entry.getKey(), keyOI, byteBuffer) ^ hashCode(entry.getValue(), valueOI); int r = 0; StructObjectInspector structOI = (StructObjectInspector)objIns; List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCodeMurmur(structOI.getStructFieldData(o, field), field.getFieldObjectInspector(), byteBuffer); UnionObjectInspector uOI = (UnionObjectInspector)objIns; byte tag = uOI.getTag(o); return hashCodeMurmur(uOI.getField(o), uOI.getObjectInspectors().get(tag), byteBuffer); throw new RuntimeException("Unknown type: "+ objIns.getTypeName());
return o1.getTypeName().equals(o2.getTypeName()); ((ListObjectInspector) o1).getListElementObjectInspector(); ObjectInspector child2 = ((ListObjectInspector) o2).getListElementObjectInspector(); return compareTypes(child1, child2); MapObjectInspector mapOI2 = (MapObjectInspector) o2; if (compareTypes(childKey1, childKey2)) { ObjectInspector childVal1 = mapOI1.getMapValueObjectInspector(); ObjectInspector childVal2 = mapOI2.getMapValueObjectInspector(); if (compareTypes(childVal1, childVal2)) { return true; StructField field2 = it2.next(); if (!compareTypes(field1.getFieldObjectInspector(), field2.getFieldObjectInspector())) { return false; Iterator<? extends ObjectInspector> it2 = ois2.iterator(); while (it1.hasNext()) { if (!compareTypes(it1.next(), it2.next())) { return false;
/** * Copy fields in the input row to the output array of standard objects. * * @param result * output list of standard objects. * @param row * input row. * @param soi * Object inspector for the to-be-copied columns. * @param objectInspectorOption */ public static void copyToStandardObject(List<Object> result, Object row, StructObjectInspector soi, ObjectInspectorCopyOption objectInspectorOption) { List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (StructField f : fields) { result.add(copyToStandardObject(soi.getStructFieldData(row, f), f.getFieldObjectInspector(), objectInspectorOption)); } }
SourceInfo(TableDesc table, ExprNodeDesc partKey, String columnName, String columnType, JobConf jobConf) throws SerDeException { this.partKey = partKey; this.columnName = columnName; this.columnType = columnType; deserializer = ReflectionUtils.newInstance(table.getDeserializerClass(), null); deserializer.initialize(jobConf, table.getProperties()); ObjectInspector inspector = deserializer.getObjectInspector(); if (LOG.isDebugEnabled()) { LOG.debug("Type of obj insp: " + inspector.getTypeName()); } soi = (StructObjectInspector) inspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); assert(fields.size() > 1) : "expecting single field in input"; field = fields.get(0); fieldInspector = ObjectInspectorUtils.getStandardObjectInspector(field.getFieldObjectInspector()); } }
switch (oi.getCategory()) { case PRIMITIVE: return true; case LIST: ListObjectInspector loi = (ListObjectInspector) oi; return compareSupported(loi.getListElementObjectInspector()); case STRUCT: StructObjectInspector soi = (StructObjectInspector) oi; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int f = 0; f < fields.size(); f++) { if (!compareSupported(fields.get(f).getFieldObjectInspector())) { return false; UnionObjectInspector uoi = (UnionObjectInspector) oi; for (ObjectInspector eoi : uoi.getObjectInspectors()) { if (!compareSupported(eoi)) { return false;
if (!arguments[ARRAY_IDX].getCategory().equals(Category.LIST)) { throw new UDFArgumentTypeException(ARRAY_IDX, "\"" + org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "\" " + "expected at function ARRAY_CONTAINS, but " + "\"" + arguments[ARRAY_IDX].getTypeName() + "\" " + "is found"); arrayElementOI = arrayOI.getListElementObjectInspector(); if (!ObjectInspectorUtils.compareTypes(arrayElementOI, valueOI)) { throw new UDFArgumentTypeException(VALUE_IDX, "\"" + arrayElementOI.getTypeName() + "\"" + " expected at function ARRAY_CONTAINS, but " + "\"" + valueOI.getTypeName() + "\"" if (!ObjectInspectorUtils.compareSupported(valueOI)) { throw new UDFArgumentException("The function " + FUNC_NAME + " does not support comparison for "
bw.set(false); if (arguments[0].get() == null) { return null; prepareInSet(arguments); if (constantInSet.contains(((ListObjectInspector) compareOI).getList(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); return bw; if (constantInSet.contains(((MapObjectInspector) compareOI).getMap(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); if (constantInSet.contains(((StructObjectInspector) compareOI).getStructFieldsDataAsList(conversionHelper .convertIfNecessary(arguments[0].get(), argumentOIs[0])))) { bw.set(true); + compareOI.getCategory()); if (ObjectInspectorUtils.compare( conversionHelper.convertIfNecessary( arguments[0].get(), argumentOIs[0]), compareOI,
@Override public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { if (!(arguments.length >= 1 && arguments.length <= 3)) { throw new UDFArgumentTypeException(arguments.length - 1, "Incorrect invocation of " + _getFnName() + ": _FUNC_(expr, amt, default)"); } amt = 1; if (arguments.length > 1) { ObjectInspector amtOI = arguments[1]; if (!ObjectInspectorUtils.isConstantObjectInspector(amtOI) || (amtOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || ((PrimitiveObjectInspector) amtOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.INT) { throw new UDFArgumentTypeException(1, _getFnName() + " amount must be a integer value " + amtOI.getTypeName() + " was passed as parameter 1."); } Object o = ((ConstantObjectInspector) amtOI).getWritableConstantValue(); amt = ((IntWritable) o).get(); if (amt < 0) { throw new UDFArgumentTypeException(1, " amount can not be nagative. Specified: " + amt); } } if (arguments.length == 3) { defaultValueConverter = ObjectInspectorConverters.getConverter(arguments[2], arguments[0]); } firstArgOI = arguments[0]; return ObjectInspectorUtils.getStandardObjectInspector(firstArgOI, ObjectInspectorCopyOption.WRITABLE); }
private void validateAndSetupPatternStr(MatchPath evaluator, List<PTFExpressionDef> args) throws SemanticException { PTFExpressionDef symboPatternArg = args.get(0); ObjectInspector symbolPatternArgOI = symboPatternArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) || (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) { throwErrorWithSignature("Currently the symbol Pattern must be a Constant String."); } evaluator.patternStr = ((ConstantObjectInspector)symbolPatternArgOI). getWritableConstantValue().toString(); }
public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { super.init(m, parameters); inputOI = (MapObjectInspector) parameters[0]; if (inputOI.getMapKeyObjectInspector().getCategory() != Category.PRIMITIVE) { throw new HiveException("Vector map keys must be a primitive."); } if (inputOI.getMapValueObjectInspector().getCategory() != Category.PRIMITIVE || !NumericUtil.isNumericCategory( ((PrimitiveObjectInspector) inputOI.getMapValueObjectInspector()).getPrimitiveCategory())) { throw new HiveException("Vector values must be numeric."); } stdMapOI = ObjectInspectorFactory. getStandardMapObjectInspector( ObjectInspectorUtils.getStandardObjectInspector(inputOI.getMapKeyObjectInspector(), ObjectInspectorUtils.ObjectInspectorCopyOption.JAVA), PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); //// XXX make return type numeric type of input, //// not doubles... return stdMapOI; }
public static List<Object> getComplexFieldsAsList(LazyBinaryStruct lazyBinaryStruct, ArrayList<Object> objectArrayBuffer, LazyBinaryStructObjectInspector lazyBinaryStructObjectInspector) { List<? extends StructField> fields = lazyBinaryStructObjectInspector.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { StructField field = fields.get(i); ObjectInspector objectInspector = field.getFieldObjectInspector(); Category category = objectInspector.getCategory(); Object object = lazyBinaryStruct.getField(i); if (category == Category.PRIMITIVE) { objectArrayBuffer.set(i, object); } else { objectArrayBuffer.set(i, ObjectInspectorUtils.copyToStandardObject( object, objectInspector, ObjectInspectorCopyOption.WRITABLE)); } } return objectArrayBuffer; }
@Override public ObjectInspector initialize(ObjectInspector[] arg0) throws UDFArgumentException { if (arg0.length < 2) { throw new UDFArgumentException(" Expecting at least two arrays as arguments "); } ObjectInspector first = arg0[0]; listInspectorArr = new ListObjectInspector[arg0.length]; if (first.getCategory() == Category.LIST) { listInspectorArr[0] = (ListObjectInspector) first; } else { throw new UDFArgumentException(" Expecting an array as first argument "); } for (int i = 1; i < arg0.length; ++i) { if (arg0[i].getCategory() != Category.LIST) { throw new UDFArgumentException(" Expecting arrays arguments "); } ListObjectInspector checkInspector = (ListObjectInspector) arg0[i]; if (!ObjectInspectorUtils.compareTypes(listInspectorArr[0].getListElementObjectInspector(), checkInspector.getListElementObjectInspector())) { throw new UDFArgumentException(" Array types must match " + listInspectorArr[0].getTypeName() + " != " + checkInspector.getTypeName()); } listInspectorArr[i] = checkInspector; } retValInspector = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(first); return retValInspector; }
@Override public Object evaluate(DeferredObject[] arg0) throws HiveException { Map hiveMap = mapInspector.getMap(arg0[0].get()); List keyValues = inspectList(keyListInspector.getList(arg0[1].get())); /// Convert all the keys to standard keys Map stdKeys = stdKeys(hiveMap); Map retVal = (Map) retValInspector.create(); for (Object keyObj : keyValues) { if (stdKeys.containsKey(keyObj)) { Object hiveKey = stdKeys.get(keyObj); Object hiveVal = hiveMap.get(hiveKey); Object keyStd = ObjectInspectorUtils.copyToStandardObject(hiveKey, mapInspector.getMapKeyObjectInspector()); Object valStd = ObjectInspectorUtils.copyToStandardObject(hiveVal, mapInspector.getMapValueObjectInspector()); retVal.put(keyStd, valStd); } } return retVal; }
/** * Convert a Object to a standard Java object in compliance with JDBC 3.0 (see JDBC 3.0 * Specification, Table B-3: Mapping from JDBC Types to Java Object Types). * * This method is kept consistent with HiveResultSetMetaData#hiveTypeToSqlType . */ public static Object toThriftPayload(Object val, ObjectInspector valOI, int version) { if (valOI.getCategory() == ObjectInspector.Category.PRIMITIVE) { if (val == null) { return null; } Object obj = ObjectInspectorUtils.copyToStandardObject(val, valOI, ObjectInspectorUtils.ObjectInspectorCopyOption.JAVA); // uses string type for binary before HIVE_CLI_SERVICE_PROTOCOL_V6 if (version < 5 && ((PrimitiveObjectInspector)valOI).getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.BINARY) { // todo HIVE-5269 return new String((byte[])obj); } return obj; } // for now, expose non-primitive as a string // TODO: expose non-primitive as a structured object while maintaining JDBC compliance return SerDeUtils.getJSONString(val, valOI); }
@SuppressWarnings("unchecked") private List<Object> mergeJoinComputeKeys(Object row, Byte alias) throws HiveException { if ((joinKeysObjectInspectors != null) && (joinKeysObjectInspectors[alias] != null)) { return JoinUtil.computeKeys(row, joinKeys[alias], joinKeysObjectInspectors[alias]); } else { row = ObjectInspectorUtils.copyToStandardObject(row, inputObjInspectors[alias], ObjectInspectorCopyOption.WRITABLE); StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[alias]; StructField sf = soi.getStructFieldRef(Utilities.ReduceField.KEY.toString()); return (List<Object>) soi.getStructFieldData(row, sf); } }
@Override public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { super.init(m, parameters); // init output object inspectors // The output of a partial aggregation is a list if (m == Mode.PARTIAL1) { inputOI = parameters[0]; return ObjectInspectorFactory.getStandardListObjectInspector( ObjectInspectorUtils.getStandardObjectInspector(inputOI)); } else { if (!(parameters[0] instanceof ListObjectInspector)) { //no map aggregation. inputOI = ObjectInspectorUtils.getStandardObjectInspector(parameters[0]); return ObjectInspectorFactory.getStandardListObjectInspector(inputOI); } else { internalMergeOI = (ListObjectInspector) parameters[0]; inputOI = internalMergeOI.getListElementObjectInspector(); loi = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI); return loi; } } }