@Override public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { int[] keys = new int[hashFields.size()]; IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[hashFields.size()]; int i = 0; IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider(); IVariableTypeEnvironment env = context.getTypeEnvironment(op); for (LogicalVariable v : hashFields) { keys[i] = opSchema.findVariable(v); hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v)); ++i; } ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories); IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf); return new Pair<>(conn, null); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); MicroUnionAllRuntimeFactory runtime = new MicroUnionAllRuntimeFactory(op.getInputs().size()); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recordDescriptor); super.contributeRuntimeOperator(builder, context, op, opSchema, inputSchemas, outerPlanSchema); } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { SubplanOperator subplan = (SubplanOperator) op; if (subplan.getNestedPlans().size() != 1) { throw new NotImplementedException("Subplan currently works only for one nested plan with one root."); } List<List<AlgebricksPipeline>> subplans = compileSubplansImpl(inputSchemas[0], subplan, opSchema, context); assert subplans.size() == 1; List<AlgebricksPipeline> np = subplans.get(0); RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); IMissingWriterFactory[] missingWriterFactories = new IMissingWriterFactory[np.get(0).getOutputWidth()]; for (int i = 0; i < missingWriterFactories.length; i++) { missingWriterFactories[i] = context.getMissingWriterFactory(); } RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); SubplanRuntimeFactory runtime = new SubplanRuntimeFactory(np, missingWriterFactories, inputRecordDesc, recDesc, null); runtime.setSourceLocation(subplan.getSourceLocation()); builder.contributeMicroOperator(subplan, runtime, recDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { propagatedSchema.addAllVariables(outerPlanSchema); NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory(); runtime.setSourceLocation(op.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(op, runtime, recDesc); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); UnionAllOperatorDescriptor opDesc = new UnionAllOperatorDescriptor(builder.getJobSpec(), op.getInputs().size(), recordDescriptor); opDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc); super.contributeRuntimeOperator(builder, context, op, opSchema, inputSchemas, outerPlanSchema); } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); MaterializingOperatorDescriptor materializationOpDesc = new MaterializingOperatorDescriptor(builder.getJobSpec(), recDescriptor, isSingleActivity); materializationOpDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, materializationOpDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op; assert insertDeleteOp.getOperation() == Kind.INSERT; assert insertDeleteOp.isBulkload(); IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); JobSpecification spec = builder.getJobSpec(); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, primaryKeys, payload, additionalFilteringKeys, additionalNonFilterVars, inputDesc, context, spec, true); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(insertDeleteOp.getSourceLocation()); builder.contributeHyracksOperator(insertDeleteOp, opDesc); builder.contributeAlgebricksPartitionConstraint(opDesc, runtimeAndConstraints.second); ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, insertDeleteOp, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { IndexInsertDeleteUpsertOperator indexInsertDeleteOp = (IndexInsertDeleteUpsertOperator) op; assert indexInsertDeleteOp.getOperation() == Kind.INSERT; assert indexInsertDeleteOp.isBulkload(); IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); JobSpecification spec = builder.getJobSpec(); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec, true); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(indexInsertDeleteOp.getSourceLocation()); builder.contributeHyracksOperator(indexInsertDeleteOp, opDesc); builder.contributeAlgebricksPartitionConstraint(opDesc, runtimeAndConstraints.second); ILogicalOperator src = indexInsertDeleteOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, indexInsertDeleteOp, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); int[] primaryKeyFields = JobGenHelper.variablesToFieldIndexes(primaryKeyLogicalVars, inputSchemas[0]); //get dataset splits IPushRuntimeFactory runtime = dataset.getCommitRuntimeFactory(metadataProvider, primaryKeyFields, isSink); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
/** * The output record descriptor of forward operator is the same as the output record descriptor of the data source * which is located at index 0. * @param builder Hyracks job builder * @param context job generation context * @param op {@see {@link ForwardOperator}} * @param propagatedSchema not used * @param inputSchemas schemas of all inputs * @param outerPlanSchema not used * @throws AlgebricksException */ @Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { ForwardOperator forwardOp = (ForwardOperator) op; RecordDescriptor dataInputDescriptor = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(forwardOp.getInputs().get(0).getValue()), inputSchemas[0], context); ForwardOperatorDescriptor forwardDescriptor = new ForwardOperatorDescriptor(builder.getJobSpec(), forwardOp.getRangeMapKey(), dataInputDescriptor); builder.contributeHyracksOperator(forwardOp, forwardDescriptor); ILogicalOperator dataSource = forwardOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(dataSource, 0, forwardOp, 0); ILogicalOperator rangemapSource = forwardOp.getInputs().get(1).getValue(); builder.contributeGraphEdge(rangemapSource, 0, forwardOp, 1); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { IOperatorDescriptorRegistry spec = builder.getJobSpec(); int[] keysAndDecs = getKeysAndDecs(inputSchemas[0]); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper .variablesToAscBinaryComparatorFactories(columnList, context.getTypeEnvironment(op), context); IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[] {}; AbstractAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aggFactories, keysAndDecs); aggregatorFactory.setSourceLocation(op.getSourceLocation()); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); /* make fd columns part of the key but the comparator only compares the distinct key columns */ PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keysAndDecs, comparatorFactories, aggregatorFactory, recordDescriptor); opDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { TokenizeOperator tokenizeOp = (TokenizeOperator) op; if (tokenizeOp.getOperation() != Kind.INSERT || !tokenizeOp.isBulkload()) { throw new AlgebricksException("Tokenize Operator only works when bulk-loading data."); } IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); JobSpecification spec = builder.getJobSpec(); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getTokenizerRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, null, inputDesc, context, spec, true); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(tokenizeOp.getSourceLocation()); builder.contributeHyracksOperator(tokenizeOp, opDesc); builder.contributeAlgebricksPartitionConstraint(opDesc, runtimeAndConstraints.second); ILogicalOperator src = tokenizeOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, tokenizeOp, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { ProjectOperator project = (ProjectOperator) op; int[] projectionList = new int[project.getVariables().size()]; int i = 0; for (LogicalVariable v : project.getVariables()) { int pos = inputSchemas[0].findVariable(v); if (pos < 0) { throw new AlgebricksException("Could not find variable " + v + "."); } projectionList[i++] = pos; } StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList, flushFramesRapidly); runtime.setSourceLocation(project.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(project, runtime, recDesc); ILogicalOperator src = project.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, project, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { LimitOperator limit = (LimitOperator) op; IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); IVariableTypeEnvironment env = context.getTypeEnvironment(op); IScalarEvaluatorFactory maxObjectsFact = expressionRuntimeProvider .createEvaluatorFactory(limit.getMaxObjects().getValue(), env, inputSchemas, context); ILogicalExpression offsetExpr = limit.getOffset().getValue(); IScalarEvaluatorFactory offsetFact = (offsetExpr == null) ? null : expressionRuntimeProvider.createEvaluatorFactory(offsetExpr, env, inputSchemas, context); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); StreamLimitRuntimeFactory runtime = new StreamLimitRuntimeFactory(maxObjectsFact, offsetFact, null, context.getBinaryIntegerInspectorFactory()); runtime.setSourceLocation(limit.getSourceLocation()); builder.contributeMicroOperator(limit, runtime, recDesc); // and contribute one edge from its child ILogicalOperator src = limit.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, limit, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { SelectOperator select = (SelectOperator) op; IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory( select.getCondition().getValue(), context.getTypeEnvironment(op), inputSchemas, context); StreamSelectRuntimeFactory runtime = new StreamSelectRuntimeFactory(cond, null, context.getBinaryBooleanInspectorFactory(), select.getRetainMissing(), inputSchemas[0].findVariable(select.getMissingPlaceholderVariable()), context.getMissingWriterFactory()); runtime.setSourceLocation(select.getSourceLocation()); // contribute one Asterix framewriter RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); builder.contributeMicroOperator(select, runtime, recDesc); // and contribute one edge from its child ILogicalOperator src = select.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, select, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { IOperatorDescriptorRegistry spec = builder.getJobSpec(); RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); ReplicateOperator rop = (ReplicateOperator) op; int outputArity = rop.getOutputArity(); boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags(); ReplicateOperatorDescriptor ropDesc = new ReplicateOperatorDescriptor(spec, recDescriptor, outputArity, outputMaterializationFlags); ropDesc.setSourceLocation(rop.getSourceLocation()); contributeOpDesc(builder, rop, ropDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { SplitOperator sop = (SplitOperator) op; int outputArity = sop.getOutputArity(); int defaultBranch = sop.getDefaultBranch(); boolean propageToAllBranchAsDefault = sop.getPropageToAllBranchAsDefault(); IOperatorDescriptorRegistry spec = builder.getJobSpec(); RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); IScalarEvaluatorFactory brachingExprEvalFactory = expressionRuntimeProvider.createEvaluatorFactory( sop.getBranchingExpression().getValue(), context.getTypeEnvironment(op), inputSchemas, context); IBinaryIntegerInspectorFactory intInsepctorFactory = context.getBinaryIntegerInspectorFactory(); SplitOperatorDescriptor sopDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity, brachingExprEvalFactory, intInsepctorFactory, defaultBranch, propageToAllBranchAsDefault); sopDesc.setSourceLocation(sop.getSourceLocation()); contributeOpDesc(builder, sop, sopDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { ScriptOperator scriptOp = (ScriptOperator) op; IScriptDescription scriptDesc = scriptOp.getScriptDescription(); if (scriptDesc.getKind() != ScriptKind.STRING_STREAMING) { throw new IllegalStateException(); } StringStreamingScriptDescription sssd = (StringStreamingScriptDescription) scriptDesc; StringStreamingRuntimeFactory runtime = new StringStreamingRuntimeFactory(sssd.getCommand(), sssd.getPrinterFactories(), sssd.getFieldDelimiter(), sssd.getParserFactory()); runtime.setSourceLocation(scriptOp.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(scriptOp, runtime, recDesc); // and contribute one edge from its child ILogicalOperator src = scriptOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, scriptOp, 0); }
DataSourceScanOperator scan = (DataSourceScanOperator) op; IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); List<LogicalVariable> vars = scan.getVariables(); List<LogicalVariable> projectVars = scan.getProjectVariables();
@Override public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { int n = sortColumns.length; int[] sortFields = new int[n]; IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n]; IBinaryHashFunctionFactory[] hashFuns = new IBinaryHashFunctionFactory[n]; IVariableTypeEnvironment env = context.getTypeEnvironment(op); INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider(); INormalizedKeyComputerFactory nkcf = null; for (int i = 0; i < n; i++) { sortFields[i] = opSchema.findVariable(sortColumns[i].getColumn()); Object type = env.getVarType(sortColumns[i].getColumn()); IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider(); comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder() == OrderKind.ASC); IBinaryHashFunctionFactoryProvider bhffp = context.getBinaryHashFunctionFactoryProvider(); hashFuns[i] = bhffp.getBinaryHashFunctionFactory(type); if (i == 0 && nkcfProvider != null && type != null) { nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, sortColumns[i].getOrder() == OrderKind.ASC); } } ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(sortFields, hashFuns); IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf); return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE); }