/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. * @param parameters * The row, can be inspected by the OIs passed in init(). */ public void aggregate(AggregationBuffer agg, Object[] parameters) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { iterate(agg, parameters); } else { assert (parameters.length == 1); merge(agg, parameters[0]); } }
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. * @param parameters * The row, can be inspected by the OIs passed in init(). */ public void aggregate(AggregationBuffer agg, Object[] parameters) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { iterate(agg, parameters); } else { assert (parameters.length == 1); merge(agg, parameters[0]); } }
private List<Object> runPartial2(List<Object> values) throws Exception { List<Object> ret = new ArrayList<>(); int batchSize = 1; Iterator<Object> iter = values.iterator(); do { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.PARTIAL2, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (int i = 0; i < batchSize - 1 && iter.hasNext(); i++) { eval.merge(buf, iter.next()); } batchSize <<= 1; ret.add(eval.terminatePartial(buf)); // back-check to force at least 1 output; and this should have a partial which is empty } while (iter.hasNext()); return ret; } }
private Object runPartialFinal(List<Object[]> values) throws Exception { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.FINAL, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (Object partialResult : runPartial1(values)) { eval.merge(buf, partialResult); } return eval.terminate(buf); }
private Object runPartial2Final(List<Object[]> values) throws Exception { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.FINAL, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (Object partialResult : runPartial2(runPartial1(values))) { eval.merge(buf, partialResult); } return eval.terminate(buf); }
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. * @param parameters * The row, can be inspected by the OIs passed in init(). */ public void aggregate(AggregationBuffer agg, Object[] parameters) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { iterate(agg, parameters); } else { assert (parameters.length == 1); merge(agg, parameters[0]); } }
/** * This function will be called by GroupByOperator when it sees a new input * row. * * @param agg * The object to store the aggregation result. * @param parameters * The row, can be inspected by the OIs passed in init(). */ public void aggregate(AggregationBuffer agg, Object[] parameters) throws HiveException { if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) { iterate(agg, parameters); } else { assert (parameters.length == 1); merge(agg, parameters[0]); } }
@Override public Object exec(Tuple input) throws IOException { try { if (!inited) { schemaAndEvaluatorInfo.init(getInputSchema(), instantiateUDAF(funcName), Mode.FINAL, constantsInfo); schemaAndEvaluatorInfo.evaluator.configure(instantiateMapredContext()); inited = true; } DataBag b = (DataBag)input.get(0); AggregationBuffer agg = schemaAndEvaluatorInfo.evaluator.getNewAggregationBuffer(); for (Iterator<Tuple> it = b.iterator(); it.hasNext();) { Tuple t = it.next(); schemaAndEvaluatorInfo.evaluator.merge(agg, t.get(0)); } Object returnValue = schemaAndEvaluatorInfo.evaluator.terminate(agg); Object result = HiveUtils.convertHiveToPig(returnValue, schemaAndEvaluatorInfo.outputObjectInspector, null); return result; } catch (Exception e) { throw new IOException(e); } } }
@Override public Tuple exec(Tuple input) throws IOException { try { if (!inited) { schemaAndEvaluatorInfo.init(getInputSchema(), instantiateUDAF(funcName), Mode.PARTIAL2, constantsInfo); inited = true; } DataBag b = (DataBag)input.get(0); AggregationBuffer agg = schemaAndEvaluatorInfo.evaluator.getNewAggregationBuffer(); for (Iterator<Tuple> it = b.iterator(); it.hasNext();) { Tuple t = it.next(); schemaAndEvaluatorInfo.evaluator.merge(agg, t.get(0)); } Object returnValue = schemaAndEvaluatorInfo.evaluator.terminatePartial(agg); Tuple result = tf.newTuple(); result.append(HiveUtils.convertHiveToPig(returnValue, schemaAndEvaluatorInfo.intermediateOutputObjectInspector, null)); return result; } catch (Exception e) { throw new IOException(e); } } }