public GraphChiExecutor(GraphChiPlatform platform, Job job) { super(job.getCrossPlatformExecutor()); this.job = job; this.platform = platform; this.configuration = job.getConfiguration(); }
/** * Execute a plan. * * @param jobName name of the {@link Job} or {@code null} * @param rheemPlan the plan to execute * @param udfJars JARs that declare the code for the UDFs * @see ReflectionUtils#getDeclaringJar(Class) */ public void execute(String jobName, Monitor monitor, RheemPlan rheemPlan, String... udfJars) { this.createJob(jobName, monitor, rheemPlan, udfJars).execute(); }
/** * Create a new {@link Job} that should execute the given {@link RheemPlan} eventually. * * @param experiment {@link Experiment} for that profiling entries will be created * @see ReflectionUtils#getDeclaringJar(Class) */ public Job createJob(String jobName, RheemPlan rheemPlan, Experiment experiment, String... udfJars) { return new Job(this, jobName, null, rheemPlan, experiment, udfJars); }
final Configuration configuration = job.getConfiguration(); if (this.sparkContextReference != null && !this.sparkContextReference.isDisposed()) { final JavaSparkContext sparkContext = this.sparkContextReference.get(); ); if (job.getName() != null) { sparkConf.set("spark.app.name", job.getName()); this.sparkContextReference = new SparkContextReference(job.getCrossPlatformExecutor(), new JavaSparkContext(sparkConf)); this.registerJarIfNotNull(ReflectionUtils.getDeclaringJar(RheemBasic.class)); // rheem-basic this.registerJarIfNotNull(ReflectionUtils.getDeclaringJar(RheemContext.class)); // rheem-core final Set<String> udfJarPaths = job.getUdfJarPaths(); if (udfJarPaths.isEmpty()) { this.logger.warn("Non-local SparkContext but not UDF JARs have been declared.");
public Configuration getConfiguration() { return this.job.getConfiguration(); }
this.prepareRheemPlan(); this.estimateKeyFigures(); ExecutionPlan executionPlan = this.createInitialExecutionPlan(); this.optimizationRound.stop(); if (this.experiment != null) { while (!this.execute(executionPlan, executionId)) { this.optimizationRound.start(); if (this.postProcess(executionPlan, executionId)) { executionId++; if (this.experiment != null) { this.logExecution(); this.stopWatch.stopAll(); this.stopWatch.start("Post-processing", "Release Resources"); this.releaseResources(); this.stopWatch.stop("Post-processing"); this.logger.info("StopWatch results:\n{}", this.stopWatch.toPrettyString());
@Override public void warmUp(Configuration configuration) { super.warmUp(configuration); // Run a most simple Spark job. this.logger.info("Running warm-up Spark job..."); long startTime = System.currentTimeMillis(); final RheemContext rheemCtx = new RheemContext(configuration); SparkCollectionSource<Integer> source = new SparkCollectionSource<>( Collections.singleton(0), DataSetType.createDefault(Integer.class) ); SparkLocalCallbackSink<Integer> sink = new SparkLocalCallbackSink<>( dq -> { }, DataSetType.createDefault(Integer.class) ); source.connectTo(0, sink, 0); final Job job = rheemCtx.createJob("Warm up", new RheemPlan(sink)); // Make sure not to have the warm-up jobs bloat the execution logs. job.getConfiguration().setProperty("rheem.core.log.enabled", "false"); job.execute(); long stopTime = System.currentTimeMillis(); this.logger.info("Spark warm-up finished in {}.", Formats.formatDuration(stopTime - startTime, true)); }
public PushExecutorTemplate(Job job) { super(job == null ? null : job.getCrossPlatformExecutor()); this.job = job; }
final Collection<ChannelInstance> producedChannelInstances; this.job.reportProgress(task.getOperator().getName(), 50); long startTime = System.currentTimeMillis(); try { long executionDuration = endTime - startTime; this.job.reportProgress(task.getOperator().getName(), 100);
/** * Build an execution plan. * * @param rheemPlan the plan to translate * @param udfJars JARs that declare the code for the UDFs * @see ReflectionUtils#getDeclaringJar(Class) */ public ExecutionPlan buildInitialExecutionPlan(String jobName, RheemPlan rheemPlan, String... udfJars) { return this.createJob(jobName, null, rheemPlan, udfJars).buildInitialExecutionPlan(); }
this.rheemPlan = rheemPlan; for (String udfJar : udfJars) { this.addUdfJar(udfJar);
public Configuration getConfiguration() { return this.job.getConfiguration(); }
final Collection<ChannelInstance> producedChannelInstances; this.job.reportProgress(task.getOperator().getName(), 50); this.job.reportProgress(task.getOperator().getName(), 100);
public JavaExecutor(JavaPlatform javaPlatform, Job job) { super(job); this.platform = javaPlatform; this.compiler = new FunctionCompiler(job.getConfiguration()); }
/** * Execute a plan. * * @param jobName name of the {@link Job} or {@code null} * @param rheemPlan the plan to execute * @param experiment {@link Experiment} for that profiling entries will be created * @param udfJars JARs that declare the code for the UDFs * @see ReflectionUtils#getDeclaringJar(Class) */ public void execute(String jobName, RheemPlan rheemPlan, Experiment experiment, String... udfJars) { this.createJob(jobName, rheemPlan, experiment, udfJars).execute(); }
/** * Create a new {@link Job} that should execute the given {@link RheemPlan} eventually. * * @see ReflectionUtils#getDeclaringJar(Class) */ public Job createJob(String jobName, Monitor monitor, RheemPlan rheemPlan, String... udfJars) { return new Job(this, jobName, monitor, rheemPlan, new Experiment("unknown", new Subject("unknown", "unknown")), udfJars); }
/** * Create a new, plain instance. */ public OptimizationContext(Job job) { this( job, null, null, -1, new ChannelConversionGraph(job.getConfiguration()), initializePruningStrategies(job.getConfiguration()) ); }
/** * Creates a new instance. Useful for testing. * * @param operator the single {@link Operator} of this instance */ public OptimizationContext(Job job, Operator operator) { this( job, null, null, -1, new ChannelConversionGraph(job.getConfiguration()), initializePruningStrategies(job.getConfiguration()) ); this.addOneTimeOperator(operator); }