@Override public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { NumericStatsAgg myagg = (NumericStatsAgg) agg; if (myagg.numDV == null) { int numVectors = parameters[1] == null ? 0 : PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI); if (numVectors > MAX_BIT_VECTORS) { throw new HiveException("The maximum allowed value for number of bit vectors " + " is " + MAX_BIT_VECTORS + ", but was passed " + numVectors + " bit vectors"); } myagg.initNDVEstimator(numVectors); } //Update null counter if a null value is seen if (parameters[0] == null) { myagg.countNulls++; } else { try { myagg.update(parameters[0], inputOI); } catch (NumberFormatException e) { if (!warned) { warned = true; LOG.warn(getClass().getSimpleName() + " " + StringUtils.stringifyException(e)); LOG.warn(getClass().getSimpleName() + " ignoring similar exceptions."); } } } }
@Override public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { NumericStatsAgg myagg = (NumericStatsAgg) agg; if (myagg.numDV == null) { int numVectors = parameters[1] == null ? 0 : PrimitiveObjectInspectorUtils.getInt(parameters[1], numVectorsOI); if (numVectors > MAX_BIT_VECTORS) { throw new HiveException("The maximum allowed value for number of bit vectors " + " is " + MAX_BIT_VECTORS + ", but was passed " + numVectors + " bit vectors"); } myagg.initNDVEstimator(numVectors); } //Update null counter if a null value is seen if (parameters[0] == null) { myagg.countNulls++; } else { try { myagg.update(parameters[0], inputOI); } catch (NumberFormatException e) { if (!warned) { warned = true; LOG.warn(getClass().getSimpleName() + " " + StringUtils.stringifyException(e)); LOG.warn(getClass().getSimpleName() + " ignoring similar exceptions."); } } } }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial != null) { NumericStatsAgg myagg = (NumericStatsAgg) agg; if (myagg.numDV == null) { Object partialValue = soi.getStructFieldData(partial, numBitVectorsField); int numVectors = numBitVectorsFieldOI.get(partialValue); if (numVectors <= 0) { return; } myagg.initNDVEstimator(numVectors); } // Update min if min is lesser than the smallest value seen so far Object minValue = soi.getStructFieldData(partial, minField); myagg.updateMin(minValue, minFieldOI); // Update max if max is greater than the largest value seen so far Object maxValue = soi.getStructFieldData(partial, maxField); myagg.updateMax(maxValue, maxFieldOI); // Update the null counter Object countNull = soi.getStructFieldData(partial, countNullsField); myagg.countNulls += countNullsFieldOI.get(countNull); // Merge numDistinctValue Estimators Object numDistinct = soi.getStructFieldData(partial, ndvField); String v = ndvFieldOI.getPrimitiveJavaObject(numDistinct); NumDistinctValueEstimator o = new NumDistinctValueEstimator(v, myagg.numDV.getnumBitVectors()); myagg.numDV.mergeEstimators(o); } } }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial != null) { NumericStatsAgg myagg = (NumericStatsAgg) agg; if (myagg.numDV == null) { Object partialValue = soi.getStructFieldData(partial, numBitVectorsField); int numVectors = numBitVectorsFieldOI.get(partialValue); if (numVectors <= 0) { return; } myagg.initNDVEstimator(numVectors); } // Update min if min is lesser than the smallest value seen so far Object minValue = soi.getStructFieldData(partial, minField); myagg.updateMin(minValue, minFieldOI); // Update max if max is greater than the largest value seen so far Object maxValue = soi.getStructFieldData(partial, maxField); myagg.updateMax(maxValue, maxFieldOI); // Update the null counter Object countNull = soi.getStructFieldData(partial, countNullsField); myagg.countNulls += countNullsFieldOI.get(countNull); // Merge numDistinctValue Estimators Object numDistinct = soi.getStructFieldData(partial, ndvField); String v = ndvFieldOI.getPrimitiveJavaObject(numDistinct); NumDistinctValueEstimator o = new NumDistinctValueEstimator(v, myagg.numDV.getnumBitVectors()); myagg.numDV.mergeEstimators(o); } } }
@Override public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { NumericStatsAgg myagg = (NumericStatsAgg) agg; if (myagg.numDV == null) { String func = null; int numVectors = 0; // func may be null when GBY op is closing. // see mvn test -Dtest=TestMiniTezCliDriver -Dqfile=explainuser_3.q // original behavior is to create FMSketch func = parameters[1] == null ? "fm" : PrimitiveObjectInspectorUtils.getString( parameters[1], funcOI); if (parameters.length == 3) { numVectors = parameters[2] == null ? 0 : PrimitiveObjectInspectorUtils.getInt( parameters[2], numVectorsOI); if (numVectors > MAX_BIT_VECTORS) { throw new HiveException("The maximum allowed value for number of bit vectors " + " is " + MAX_BIT_VECTORS + ", but was passed " + numVectors + " bit vectors"); } } myagg.initNDVEstimator(func, numVectors); } // Update null counter if a null value is seen if (parameters[0] == null) { myagg.countNulls++; } else { myagg.update(parameters[0], inputOI); } }