public JavaPlan translateToPlan(List<DataSink<?>> sinks, String jobName) { List<GenericDataSinkBase<?>> planSinks = new ArrayList<GenericDataSinkBase<?>>(); for (DataSink<?> sink : sinks) { planSinks.add(translate(sink)); } JavaPlan p = new JavaPlan(planSinks); p.setJobName(jobName); return p; }
private void translateBcVariables(DataSet<?> setOrOp, Operator<?> dataFlowOp) { // check if this is actually an operator that could have broadcast variables if (setOrOp instanceof UdfOperator) { if (!(dataFlowOp instanceof AbstractUdfOperator<?, ?>)) { throw new RuntimeException("Error while creating the data flow plan for the program: A UDF operation was not translated to a UDF operator."); } UdfOperator<?> udfOp = (UdfOperator<?>) setOrOp; AbstractUdfOperator<?, ?> udfDataFlowOp = (AbstractUdfOperator<?, ?>) dataFlowOp; for (Map.Entry<String, DataSet<?>> bcVariable : udfOp.getBroadcastSets().entrySet()) { Operator<?> bcInput = translate(bcVariable.getValue()); udfDataFlowOp.setBroadcastVariable(bcVariable.getKey(), bcInput); } } } }
private <T> GenericDataSinkBase<T> translate(DataSink<T> sink) { // translate the input recursively Operator<T> input = translate(sink.getDataSet()); // translate the sink itself and connect it to the input GenericDataSinkBase<T> translatedSink = sink.translateToDataFlow(input); return translatedSink; }
private <T> BulkIterationBase<T> translateBulkIteration(BulkIterationResultSet<?> untypedIterationEnd) { @SuppressWarnings("unchecked") BulkIterationResultSet<T> iterationEnd = (BulkIterationResultSet<T>) untypedIterationEnd; BulkIterationBase<T> iterationOperator = new BulkIterationBase<T>(new UnaryOperatorInformation<T, T>(iterationEnd.getType(), iterationEnd.getType()), "Bulk Iteration"); IterativeDataSet<T> iterationHead = iterationEnd.getIterationHead(); translated.put(iterationHead, iterationOperator.getPartialSolution()); Operator<T> translatedBody = translate(iterationEnd.getNextPartialSolution()); iterationOperator.setNextPartialSolution(translatedBody); iterationOperator.setMaximumNumberOfIterations(iterationHead.getMaxIterations()); iterationOperator.setInput(translate(iterationHead.getInput())); iterationOperator.getAggregators().addAll(iterationHead.getAggregators()); if(iterationEnd.getTerminationCriterion() != null) { iterationOperator.setTerminationCriterion(translate(iterationEnd.getTerminationCriterion())); } return iterationOperator; }
translated.put(worksetPlaceHolder, iterationOperator.getWorkset()); Operator<D> translatedSolutionSet = translate(iterationEnd.getNextSolutionSet()); Operator<W> translatedWorkset = translate(iterationEnd.getNextWorkset()); iterationOperator.setInitialSolutionSet(translate(iterationHead.getInitialSolutionSet())); iterationOperator.setInitialWorkset(translate(iterationHead.getInitialWorkset()));
private <I, O> eu.stratosphere.api.common.operators.SingleInputOperator<?, O, ?> translateSingleInputOperator(SingleInputOperator<?, ?, ?> op) { @SuppressWarnings("unchecked") SingleInputOperator<I, O, ?> typedOp = (SingleInputOperator<I, O, ?>) op; @SuppressWarnings("unchecked") DataSet<I> typedInput = (DataSet<I>) op.getInput(); Operator<I> input = translate(typedInput); eu.stratosphere.api.common.operators.SingleInputOperator<?, O, ?> dataFlowOp = typedOp.translateToDataFlow(input); if (op instanceof UdfOperator<?> ) { @SuppressWarnings("unchecked") SingleInputUdfOperator<I, O, ?> udfOp = (SingleInputUdfOperator<I, O, ?>) op; // set configuration parameters Configuration opParams = udfOp.getParameters(); if (opParams != null) { dataFlowOp.getParameters().addAll(opParams); } // set the semantic properties dataFlowOp.setSemanticProperties(udfOp.getSematicProperties()); } return dataFlowOp; }