public GenericUDAFEvaluator getGenericWindowingEvaluator(String functionName, List<ObjectInspector> argumentOIs, boolean isDistinct, boolean isAllColumns) throws SemanticException { functionName = functionName.toLowerCase(); WindowFunctionInfo info = getWindowFunctionInfo(functionName); if (info == null) { return null; } if (!functionName.equals(FunctionRegistry.LEAD_FUNC_NAME) && !functionName.equals(FunctionRegistry.LAG_FUNC_NAME)) { return getGenericUDAFEvaluator(functionName, argumentOIs, true, isDistinct, isAllColumns); } // this must be lead/lag UDAF ObjectInspector args[] = new ObjectInspector[argumentOIs.size()]; GenericUDAFResolver udafResolver = info.getGenericUDAFResolver(); GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( argumentOIs.toArray(args), true, isDistinct, isAllColumns); return ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); }
public GenericUDAFEvaluator getGenericWindowingEvaluator(String functionName, List<ObjectInspector> argumentOIs, boolean isDistinct, boolean isAllColumns) throws SemanticException { functionName = functionName.toLowerCase(); WindowFunctionInfo info = getWindowFunctionInfo(functionName); if (info == null) { return null; } if (!functionName.equals(FunctionRegistry.LEAD_FUNC_NAME) && !functionName.equals(FunctionRegistry.LAG_FUNC_NAME)) { return getGenericUDAFEvaluator(functionName, argumentOIs, true, isDistinct, isAllColumns); } // this must be lead/lag UDAF ObjectInspector args[] = new ObjectInspector[argumentOIs.size()]; GenericUDAFResolver udafResolver = info.getGenericUDAFResolver(); GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( argumentOIs.toArray(args), true, isDistinct, isAllColumns); return ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); }
if (udafResolver instanceof GenericUDAFResolver2) { udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); } else { udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters());
if (udafResolver instanceof GenericUDAFResolver2) { udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); } else { udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters());
if (udafResolver instanceof GenericUDAFResolver2) { udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); } else { udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters());
if (udafResolver instanceof GenericUDAFResolver2) { udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); } else { udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters());
public GenericUDAFExecutor(GenericUDAFResolver2 evaluatorFactory, GenericUDAFParameterInfo info) throws Exception { this.evaluatorFactory = evaluatorFactory; this.info = info; GenericUDAFEvaluator eval0 = evaluatorFactory.getEvaluator(info); partialOIs = new ObjectInspector[] { eval0.init(GenericUDAFEvaluator.Mode.PARTIAL1, info.getParameterObjectInspectors()) }; }
private List<Object> runPartial2(List<Object> values) throws Exception { List<Object> ret = new ArrayList<>(); int batchSize = 1; Iterator<Object> iter = values.iterator(); do { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.PARTIAL2, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (int i = 0; i < batchSize - 1 && iter.hasNext(); i++) { eval.merge(buf, iter.next()); } batchSize <<= 1; ret.add(eval.terminatePartial(buf)); // back-check to force at least 1 output; and this should have a partial which is empty } while (iter.hasNext()); return ret; } }
private List<Object> runPartial1(List<Object[]> values) throws Exception { List<Object> ret = new ArrayList<>(); int batchSize = 1; Iterator<Object[]> iter = values.iterator(); do { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.PARTIAL1, info.getParameterObjectInspectors()); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (int i = 0; i < batchSize - 1 && iter.hasNext(); i++) { eval.iterate(buf, iter.next()); } batchSize <<= 1; ret.add(eval.terminatePartial(buf)); // back-check to force at least 1 output; and this should have a partial which is empty } while (iter.hasNext()); return ret; }
private Object runComplete(List<Object[]> values) throws SemanticException, HiveException { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.COMPLETE, info.getParameterObjectInspectors()); AggregationBuffer agg = eval.getNewAggregationBuffer(); for (Object[] parameters : values) { eval.iterate(agg, parameters); } return eval.terminate(agg); }
private Object runPartialFinal(List<Object[]> values) throws Exception { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.FINAL, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (Object partialResult : runPartial1(values)) { eval.merge(buf, partialResult); } return eval.terminate(buf); }
private Object runPartial2Final(List<Object[]> values) throws Exception { GenericUDAFEvaluator eval = evaluatorFactory.getEvaluator(info); eval.init(GenericUDAFEvaluator.Mode.FINAL, partialOIs); AggregationBuffer buf = eval.getNewAggregationBuffer(); for (Object partialResult : runPartial2(runPartial1(values))) { eval.merge(buf, partialResult); } return eval.terminate(buf); }
public GenericUDAFEvaluator getGenericWindowingEvaluator(String functionName, List<ObjectInspector> argumentOIs, boolean isDistinct, boolean isAllColumns) throws SemanticException { functionName = functionName.toLowerCase(); WindowFunctionInfo info = getWindowFunctionInfo(functionName); if (info == null) { return null; } if (!functionName.equals(FunctionRegistry.LEAD_FUNC_NAME) && !functionName.equals(FunctionRegistry.LAG_FUNC_NAME)) { return getGenericUDAFEvaluator(functionName, argumentOIs, isDistinct, isAllColumns); } // this must be lead/lag UDAF ObjectInspector args[] = new ObjectInspector[argumentOIs.size()]; GenericUDAFResolver udafResolver = info.getGenericUDAFResolver(); GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( argumentOIs.toArray(args), isDistinct, isAllColumns); return ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); }
private GenericUDAFEvaluator getPartial1Evaluator() throws HiveException { if (partial1EvaluatorByVoid) { partial1Evaluator = null; } if (null == partial1Evaluator) { if (null == inputInspectors) { SimpleGenericUDAFParameterInfo paramInfo = getLazyVoidOneParam(); partial1Evaluator = resolver.getEvaluator(paramInfo); inputInspectors = paramInfo.getParameterObjectInspectors(); partial1EvaluatorByVoid = true; } else { SimpleGenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( inputInspectors, false, false, false); partial1Evaluator = resolver.getEvaluator(paramInfo); partial1EvaluatorByVoid = false; } partialResultInspector = partial1Evaluator.init( GenericUDAFEvaluator.Mode.PARTIAL1, inputInspectors); } return this.partial1Evaluator; }
private GenericUDAFEvaluator getFinalEvaluator() throws HiveException { if (finalEvaluatorByVoid) { // If it is not from the real finalEvaluator finalEvaluator = null; } if (null == finalEvaluator) { // If real params are null, use the one Void params. if (null == inputInspectors) { SimpleGenericUDAFParameterInfo paramInfo = getLazyVoidOneParam(); finalEvaluator = resolver.getEvaluator(paramInfo); inputInspectors = paramInfo.getParameterObjectInspectors(); finalEvaluatorByVoid = true; } else { SimpleGenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( inputInspectors, false, false, false); finalEvaluator = resolver.getEvaluator(paramInfo); finalEvaluatorByVoid = false; } returnInspector = finalEvaluator.init( GenericUDAFEvaluator.Mode.FINAL, inputInspectors); } return this.finalEvaluator; }
private GenericUDAFEvaluator getPartial2Evaluator() throws HiveException { if (partial2EvaluatorByVoid) { partial2Evaluator = null; } if (null == partial2Evaluator) { if (null == partialResultInspector) { SimpleGenericUDAFParameterInfo parameterInfo = getLazyVoidOneParam(); partial2Evaluator = resolver.getEvaluator(parameterInfo); partialResultInspector = parameterInfo.getParameterObjectInspectors()[0]; partial2EvaluatorByVoid = true; } else { SimpleGenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( inputInspectors, false, false, false); partial2Evaluator = resolver.getEvaluator(paramInfo); partial2EvaluatorByVoid = false; } // The input of Partial 2 is th Output of the Partial 1. ObjectInspector[] partial1Types = new ObjectInspector[1]; partial1Types[0] = partialResultInspector; partial2Evaluator.init( GenericUDAFEvaluator.Mode.PARTIAL2, partial1Types); } return this.partial2Evaluator; }
private static GenericUDAFEvaluator getEvaluator(TypeInfo inputTypeInfo, GenericUDAFResolver udaf, ConstantObjectInspectInfo constantsInfo) throws IOException { try { GenericUDAFEvaluator evaluator; ObjectInspector[] arguments = getInputObjectInspectorAsArray(inputTypeInfo, constantsInfo); if (udaf instanceof GenericUDAFResolver2) { GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( arguments, false, false); evaluator = ((GenericUDAFResolver2)udaf).getEvaluator(paramInfo); } else { TypeInfo[] params = ((StructTypeInfo)inputTypeInfo) .getAllStructFieldTypeInfos().toArray(new TypeInfo[0]); evaluator = udaf.getEvaluator(params); } return evaluator; } catch (Exception e) { throw new IOException(e); } }
parameters, isDistinct, isAllColumns); udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); } else { udafEvaluator = udafResolver.getEvaluator(parameters);
if (udafResolver instanceof GenericUDAFResolver2) { udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); } else { udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters());
if (udafResolver instanceof GenericUDAFResolver2) { udafEvaluator = ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); } else { udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters());