/** * Computes the hash code for the given bucketed fields * @param bucketFields * @param bucketFieldInspectors * @return */ @Deprecated public static int getBucketHashCodeOld(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) { int hashCode = 0; for (int i = 0; i < bucketFields.length; i++) { int fieldHash = ObjectInspectorUtils.hashCode(bucketFields[i], bucketFieldInspectors[i]); hashCode = 31 * hashCode + fieldHash; } return hashCode; }
for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCodeMurmur(entry.getKey(), keyOI, byteBuffer) ^ hashCode(entry.getValue(), valueOI);
@Override public int hashCode() { ///LOG.info( "Object is " + o + " :: " + o.getClass().getCanonicalName() + " Inspector = " + oi + " ::" + oi.getCategory() + " type =" + oi.getTypeName()); return ObjectInspectorUtils.hashCode(o, oi); }
for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCodeMurmur(entry.getKey(), keyOI, byteBuffer) ^ hashCode(entry.getValue(), valueOI);
ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI); List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCode(structOI.getStructFieldData(o, field), field.getFieldObjectInspector()); UnionObjectInspector uOI = (UnionObjectInspector)objIns; byte tag = uOI.getTag(o); return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
@Override public int hashCode() { return ObjectInspectorUtils.hashCode(o, oi); }
@Override public int hashCode() { return ObjectInspectorUtils.hashCode(o, oi); }
@Override public int hashCode() { return ObjectInspectorUtils.hashCode(o, oi); }
/** * Computes the hash code for the given bucketed fields * @param bucketFields * @param bucketFieldInspectors * @return */ @Deprecated public static int getBucketHashCodeOld(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) { int hashCode = 0; for (int i = 0; i < bucketFields.length; i++) { int fieldHash = ObjectInspectorUtils.hashCode(bucketFields[i], bucketFieldInspectors[i]); hashCode = 31 * hashCode + fieldHash; } return hashCode; }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { // See // http://java.sun.com/j2se/1.5.0/docs/api/java/util/List.html#hashCode() int r = 0; for (int i = 0; i < arguments.length; i++) { r = r * 31 + ObjectInspectorUtils.hashCode(arguments[i].get(), argumentOIs[i]); } result.set(r); return result; }
@Override public Object evaluate(DeferredObject[] arguments) throws HiveException { // See // http://java.sun.com/j2se/1.5.0/docs/api/java/util/List.html#hashCode() int r = 0; for (int i = 0; i < arguments.length; i++) { r = r * 31 + ObjectInspectorUtils.hashCode(arguments[i].get(), argumentOIs[i]); } result.set(r); return result; }
private int findWriterOffset(Object row) throws HiveException { if (!multiFileSpray) { return 0; } else { int keyHashCode = 0; for (int i = 0; i < partitionEval.length; i++) { Object o = partitionEval[i].evaluate(row); keyHashCode = keyHashCode * 31 + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); } key.setHashCode(keyHashCode); int bucketNum = prtner.getBucket(key, null, totalFiles); return bucketMap.get(bucketNum); } }
private int computeHashCode(Object row, int buckNum) throws HiveException { // Evaluate the HashCode int keyHashCode = 0; if (partitionEval.length == 0) { // If no partition cols and not doing an update or delete, just distribute the data uniformly // to provide better load balance. If the requirement is to have a single reducer, we should // set the number of reducers to 1. Use a constant seed to make the code deterministic. // For acid operations make sure to send all records with the same key to the same // FileSinkOperator, as the RecordUpdater interface can't manage multiple writers for a file. if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) { if (random == null) { random = new Random(12345); } keyHashCode = random.nextInt(); } else { keyHashCode = 1; } } else { for (int i = 0; i < partitionEval.length; i++) { Object o = partitionEval[i].evaluate(row); keyHashCode = keyHashCode * 31 + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]); } } int hashCode = buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum; if (isLogTraceEnabled) { LOG.trace("Going to return hash code " + hashCode); } return hashCode; }
Object o = partitionEval[i].evaluate(row); keyHashCode = keyHashCode * 31 + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]);
Object o = partitionEval[i].evaluate(row); keyHashCode = keyHashCode * 31 + ObjectInspectorUtils.hashCode(o, partitionObjectInspectors[i]);
private int computeBucketNumber(Object row, int numBuckets) throws HiveException { int buckNum = 0; if (conf.getWriteType() == AcidUtils.Operation.UPDATE || conf.getWriteType() == AcidUtils.Operation.DELETE) { // We don't need to evalute the hash code. Instead read the bucket number directly from // the row. I don't need to evaluate any expressions as I know I am reading the ROW__ID // column directly. Object recIdValue = acidRowInspector.getStructFieldData(row, recIdField); buckNum = bucketInspector.get(recIdInspector.getStructFieldData(recIdValue, bucketField)); if (isLogTraceEnabled) { LOG.trace("Acid choosing bucket number " + buckNum); } } else { for (int i = 0; i < bucketEval.length; i++) { Object o = bucketEval[i].evaluate(row); buckNum = buckNum * 31 + ObjectInspectorUtils.hashCode(o, bucketObjectInspectors[i]); } } // similar to hive's default partitioner, refer DefaultHivePartitioner return (buckNum & Integer.MAX_VALUE) % numBuckets; }
ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI); List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCode(structOI.getStructFieldData(o, field), field.getFieldObjectInspector()); UnionObjectInspector uOI = (UnionObjectInspector)objIns; byte tag = uOI.getTag(o); return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI); List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCode(structOI.getStructFieldData(o, field), field.getFieldObjectInspector()); UnionObjectInspector uOI = (UnionObjectInspector)objIns; byte tag = uOI.getTag(o); return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI); List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCode(structOI.getStructFieldData(o, field), field.getFieldObjectInspector()); UnionObjectInspector uOI = (UnionObjectInspector)objIns; byte tag = uOI.getTag(o); return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
ObjectInspector elemOI = listOI.getListElementObjectInspector(); for (int ii = 0; ii < listOI.getListLength(o); ++ii) { r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI); Map<?, ?> map = mapOI.getMap(o); for (Map.Entry<?,?> entry : map.entrySet()) { r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI); List<? extends StructField> fields = structOI.getAllStructFieldRefs(); for (StructField field : fields) { r = 31 * r + hashCode(structOI.getStructFieldData(o, field), field.getFieldObjectInspector()); UnionObjectInspector uOI = (UnionObjectInspector)objIns; byte tag = uOI.getTag(o); return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));