private List<?> execute(Operator<?> operator) throws Exception { return execute(operator, 0); }
@Override public JobExecutionResult execute(String jobName) throws Exception { Plan p = createProgramPlan(jobName); // We need to reverse here. Object-Reuse enabled, means safe mode is disabled. CollectionExecutor exec = new CollectionExecutor(getConfig()); this.lastJobExecutionResult = exec.execute(p); return this.lastJobExecutionResult; }
result = executeBulkIteration((BulkIterationBase<?>) operator); result = executeDeltaIteration((DeltaIterationBase<?, ?>) operator); result = executeUnaryOperator((SingleInputOperator<?, ?, ?>) operator, superStep); result = executeBinaryOperator((DualInputOperator<?, ?, ?, ?>) operator, superStep); result = executeDataSource((GenericDataSourceBase<?, ?>) operator, superStep); executeDataSink((GenericDataSinkBase<?>) operator, superStep); result = Collections.emptyList();
public JobExecutionResult execute(Plan program) throws Exception { long startTime = System.currentTimeMillis(); initCache(program.getCachedFiles()); Collection<? extends GenericDataSinkBase<?>> sinks = program.getDataSinks(); for (Operator<?> sink : sinks) { execute(sink); } long endTime = System.currentTimeMillis(); Map<String, OptionalFailure<Object>> accumulatorResults = AccumulatorHelper.toResultMap(accumulators); return new JobExecutionResult(null, endTime - startTime, accumulatorResults); }
public JobExecutionResult execute(Plan program) throws Exception { long startTime = System.currentTimeMillis(); initCache(program.getCachedFiles()); Collection<? extends GenericDataSinkBase<?>> sinks = program.getDataSinks(); for (Operator<?> sink : sinks) { execute(sink); } long endTime = System.currentTimeMillis(); Map<String, OptionalFailure<Object>> accumulatorResults = AccumulatorHelper.toResultMap(accumulators); return new JobExecutionResult(null, endTime - startTime, accumulatorResults); }
result = executeBulkIteration((BulkIterationBase<?>) operator); result = executeDeltaIteration((DeltaIterationBase<?, ?>) operator); result = executeUnaryOperator((SingleInputOperator<?, ?, ?>) operator, superStep); result = executeBinaryOperator((DualInputOperator<?, ?, ?, ?>) operator, superStep); result = executeDataSource((GenericDataSourceBase<?, ?>) operator, superStep); executeDataSink((GenericDataSinkBase<?>) operator, superStep); result = Collections.emptyList();
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception { Operator<?> inputOp = operator.getInput(); if (inputOp == null) { throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep); @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) { List<?> bcData = execute(bcInputs.getValue()); ctx.setBroadcastVariable(bcInputs.getKey(), bcData); } } else { ctx = null; } return typedOp.executeOnCollections(inputData, ctx, executionConfig); }
@Override public JobExecutionResult execute(String jobName) throws Exception { Plan p = createProgramPlan(jobName); // We need to reverse here. Object-Reuse enabled, means safe mode is disabled. CollectionExecutor exec = new CollectionExecutor(getConfig()); this.lastJobExecutionResult = exec.execute(p); return this.lastJobExecutionResult; }
public JobExecutionResult execute(Plan program) throws Exception { long startTime = System.currentTimeMillis(); initCache(program.getCachedFiles()); Collection<? extends GenericDataSinkBase<?>> sinks = program.getDataSinks(); for (Operator<?> sink : sinks) { execute(sink); } long endTime = System.currentTimeMillis(); Map<String, OptionalFailure<Object>> accumulatorResults = AccumulatorHelper.toResultMap(accumulatorRegistry.getAccumulators()); return new JobExecutionResult(null, endTime - startTime, accumulatorResults); }
result = executeBulkIteration((BulkIterationBase<?>) operator); result = executeDeltaIteration((DeltaIterationBase<?, ?>) operator); result = executeUnaryOperator((SingleInputOperator<?, ?, ?>) operator, superStep); result = executeBinaryOperator((DualInputOperator<?, ?, ?, ?>) operator, superStep); result = executeDataSource((GenericDataSourceBase<?, ?>) operator, superStep); executeDataSink((GenericDataSinkBase<?>) operator, superStep); result = Collections.emptyList();
@Override public JobSubmissionResult executeInternal(String jobName, boolean detached) throws Exception { Plan p = createProgramPlan(jobName); // We need to reverse here. Object-Reuse enabled, means safe mode is disabled. CollectionExecutor exec = new CollectionExecutor(getConfig()); this.lastJobExecutionResult = exec.execute(p); return this.lastJobExecutionResult; }
List<T> solutionInputData = (List<T>) execute(solutionInput); List<T> worksetInputData = (List<T>) execute(worksetInput); List<T> solutionSetDelta = (List<T>) execute(iteration.getSolutionSetDelta(), superstep); this.intermediateResults.put(iteration.getSolutionSetDelta(), solutionSetDelta); currentWorkset = execute(iteration.getNextWorkset(), superstep);
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception { Operator<?> inputOp = sink.getInput(); if (inputOp == null) { throw new InvalidProgramException("The data sink " + sink.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> input = (List<IN>) execute(inputOp); @SuppressWarnings("unchecked") GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); } else { ctx = null; } typedSink.executeOnCollections(input, ctx, executionConfig); }
private List<?> execute(Operator<?> operator) throws Exception { return execute(operator, 0); }
private List<?> execute(Operator<?> operator) throws Exception { return execute(operator, 0); }
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception { Operator<?> inputOp = operator.getInput(); if (inputOp == null) { throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep); @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) { List<?> bcData = execute(bcInputs.getValue()); ctx.setBroadcastVariable(bcInputs.getKey(), bcData); } } else { ctx = null; } return typedOp.executeOnCollections(inputData, ctx, executionConfig); }
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception { Operator<?> inputOp = operator.getInput(); if (inputOp == null) { throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep); @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulatorRegistry, metrics) : new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulatorRegistry, metrics); for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) { List<?> bcData = execute(bcInputs.getValue()); ctx.setBroadcastVariable(bcInputs.getKey(), bcData); } } else { ctx = null; } return typedOp.executeOnCollections(inputData, ctx, executionConfig); }