private static int[] projectVariablesImpl(IOperatorSchema opSchema, Iterable<LogicalVariable> variables, int variableCount) { int[] projectionList = new int[variableCount]; int k = 0; for (LogicalVariable v : variables) { projectionList[k++] = opSchema.findVariable(v); } return projectionList; } }
public static int[] variablesToFieldIndexes(Collection<LogicalVariable> varLogical, IOperatorSchema opSchema) { int[] tuplePos = new int[varLogical.size()]; int i = 0; for (LogicalVariable var : varLogical) { tuplePos[i] = opSchema.findVariable(var); i++; } return tuplePos; }
private int[] createFilterIndexes(List<LogicalVariable> filterVars, IOperatorSchema opSchema) { if (filterVars != null && !filterVars.isEmpty()) { final int size = filterVars.size(); int[] result = new int[size]; for (int i = 0; i < size; ++i) { result[i] = opSchema.findVariable(filterVars.get(i)); } return result; } return null; }
protected int[] getKeyIndexes(List<LogicalVariable> keyVarList, IOperatorSchema[] inputSchemas) { if (keyVarList == null) { return null; } int[] keyIndexes = new int[keyVarList.size()]; for (int i = 0; i < keyVarList.size(); i++) { keyIndexes[i] = inputSchemas[0].findVariable(keyVarList.get(i)); } return keyIndexes; }
protected int[] getKeyIndexes(List<LogicalVariable> keyVarList, IOperatorSchema[] inputSchemas) { if (keyVarList == null) { return null; } int[] keyIndexes = new int[keyVarList.size()]; for (int i = 0; i < keyVarList.size(); i++) { keyIndexes[i] = inputSchemas[0].findVariable(keyVarList.get(i)); } return keyIndexes; }
@Override public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) { for (IOperatorSchema s : sources) { for (LogicalVariable v : s) { if (target.findVariable(v) < 0) { target.addVariable(v); } } } } };
protected int[] getKeysAndDecs(IOperatorSchema inputSchema) { int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchema); int sz = inputSchema.getSize(); int fdSz = sz - columnList.size(); int[] fdColumns = new int[fdSz]; int j = 0; for (LogicalVariable v : inputSchema) { if (!columnList.contains(v)) { fdColumns[j++] = inputSchema.findVariable(v); } } int[] keysAndDecs = new int[keys.length + fdColumns.length]; for (int i = 0; i < keys.length; i++) { keysAndDecs[i] = keys[i]; } for (int i = 0; i < fdColumns.length; i++) { keysAndDecs[i + keys.length] = fdColumns[i]; } return keysAndDecs; }
private IScalarEvaluatorFactory createVariableEvaluatorFactory(VariableReferenceExpression expr, IOperatorSchema[] inputSchemas) throws AlgebricksException { LogicalVariable variable = expr.getVariableReference(); for (IOperatorSchema scm : inputSchemas) { int pos = scm.findVariable(variable); if (pos >= 0) { return new ColumnAccessEvalFactory(pos); } } throw new AlgebricksException("Variable " + variable + " could not be found in any input schema."); }
@Override public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) throws AlgebricksException { /** make sure distinct key vars laid-out first */ for (LogicalVariable keyVar : getDistinctByVarList()) { target.addVariable(keyVar); } /** add other source vars */ for (IOperatorSchema srcSchema : sources) { for (LogicalVariable srcVar : srcSchema) if (target.findVariable(srcVar) < 0) { target.addVariable(srcVar); } } } };
@Override public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { int[] keys = new int[hashFields.size()]; IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[hashFields.size()]; int i = 0; IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider(); IVariableTypeEnvironment env = context.getTypeEnvironment(op); for (LogicalVariable v : hashFields) { keys[i] = opSchema.findVariable(v); hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v)); ++i; } ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories); IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf); return new Pair<>(conn, null); }
protected int[] getFdColumns(GroupByOperator gby, IOperatorSchema inputSchema) throws AlgebricksException { int numFds = gby.getDecorList().size(); int fdColumns[] = new int[numFds]; int j = 0; for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) { ILogicalExpression expr = p.second.getValue(); if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) { throw new AlgebricksException("pre-sorted group-by expects variable references."); } VariableReferenceExpression v = (VariableReferenceExpression) expr; LogicalVariable decor = v.getVariableReference(); fdColumns[j++] = inputSchema.findVariable(decor); } return fdColumns; }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { AggregateOperator aggOp = (AggregateOperator) op; List<LogicalVariable> variables = aggOp.getVariables(); List<Mutable<ILogicalExpression>> expressions = aggOp.getExpressions(); int[] outColumns = new int[variables.size()]; for (int i = 0; i < outColumns.length; i++) { outColumns[i] = opSchema.findVariable(variables.get(i)); } IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[expressions.size()]; IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); for (int i = 0; i < aggFactories.length; i++) { AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expressions.get(i).getValue(); aggFactories[i] = expressionRuntimeProvider.createAggregateFunctionFactory(aggFun, context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context); } AggregateRuntimeFactory runtime = new AggregateRuntimeFactory(aggFactories); runtime.setSourceLocation(aggOp.getSourceLocation()); // contribute one Asterix framewriter RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); builder.contributeMicroOperator(aggOp, runtime, recDesc); // and contribute one edge from its child ILogicalOperator src = aggOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, aggOp, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { ProjectOperator project = (ProjectOperator) op; int[] projectionList = new int[project.getVariables().size()]; int i = 0; for (LogicalVariable v : project.getVariables()) { int pos = inputSchemas[0].findVariable(v); if (pos < 0) { throw new AlgebricksException("Could not find variable " + v + "."); } projectionList[i++] = pos; } StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList, flushFramesRapidly); runtime.setSourceLocation(project.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(project, runtime, recDesc); ILogicalOperator src = project.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, project, 0); }
@Override public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { int n = partitioningFields.size(); int[] sortFields = new int[n]; IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n]; IVariableTypeEnvironment env = context.getTypeEnvironment(op); int i = 0; for (OrderColumn oc : partitioningFields) { LogicalVariable var = oc.getColumn(); sortFields[i] = opSchema.findVariable(var); Object type = env.getVarType(var); IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider(); comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC); i++; } FieldRangePartitionComputerFactory partitionerFactory; if (rangeMapIsComputedAtRunTime) { partitionerFactory = new DynamicFieldRangePartitionComputerFactory(sortFields, comps, rangeMapKeyInContext, op.getSourceLocation()); } else { partitionerFactory = new StaticFieldRangePartitionComputerFactory(sortFields, comps, rangeMap); } IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, partitionerFactory); return new Pair<>(conn, null); }
@Override public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { int n = partitioningFields.size(); int[] sortFields = new int[n]; IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n]; INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider(); INormalizedKeyComputerFactory nkcf = null; IVariableTypeEnvironment env = context.getTypeEnvironment(op); int i = 0; for (OrderColumn oc : partitioningFields) { LogicalVariable var = oc.getColumn(); sortFields[i] = opSchema.findVariable(var); Object type = env.getVarType(var); OrderKind order = oc.getOrder(); if (i == 0 && nkcfProvider != null && type != null) { nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC); } IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider(); comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC); i++; } ITuplePartitionComputerFactory tpcf = new StaticFieldRangePartitionComputerFactory(sortFields, comps, rangeMap); IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf); return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null); }
columns[i++] = inputSchemas[0].findVariable(v);
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { SelectOperator select = (SelectOperator) op; IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory( select.getCondition().getValue(), context.getTypeEnvironment(op), inputSchemas, context); StreamSelectRuntimeFactory runtime = new StreamSelectRuntimeFactory(cond, null, context.getBinaryBooleanInspectorFactory(), select.getRetainMissing(), inputSchemas[0].findVariable(select.getMissingPlaceholderVariable()), context.getMissingWriterFactory()); runtime.setSourceLocation(select.getSourceLocation()); // contribute one Asterix framewriter RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); builder.contributeMicroOperator(select, runtime, recDesc); // and contribute one edge from its child ILogicalOperator src = select.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, select, 0); }
for (OrderColumn oc : sortColumns) { LogicalVariable var = oc.getColumn(); sortFields[i] = opSchema.findVariable(var); Object type = env.getVarType(var); OrderKind order = oc.getOrder();
@Override public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { int n = sortColumns.length; int[] sortFields = new int[n]; IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n]; IBinaryHashFunctionFactory[] hashFuns = new IBinaryHashFunctionFactory[n]; IVariableTypeEnvironment env = context.getTypeEnvironment(op); INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider(); INormalizedKeyComputerFactory nkcf = null; for (int i = 0; i < n; i++) { sortFields[i] = opSchema.findVariable(sortColumns[i].getColumn()); Object type = env.getVarType(sortColumns[i].getColumn()); IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider(); comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder() == OrderKind.ASC); IBinaryHashFunctionFactoryProvider bhffp = context.getBinaryHashFunctionFactoryProvider(); hashFuns[i] = bhffp.getBinaryHashFunctionFactory(type); if (i == 0 && nkcfProvider != null && type != null) { nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, sortColumns[i].getOrder() == OrderKind.ASC); } } ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(sortFields, hashFuns); IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf); return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE); }
int tupleFieldIndex = inputSchemas[0].findVariable(vrExpr.getVariableReference()); return new TupleFieldEvaluatorFactory(tupleFieldIndex);