protected Platform(String name, String configName) { this.name = name; this.configName = configName; this.configureDefaults(Configuration.getDefaultConfiguration()); }
/** * <i>Shortcut.</i> Creates an {@link Executor} using the {@link #getExecutorFactory()}. * * @return the {@link Executor} */ public Executor createExecutor(Job job) { return this.getExecutorFactory().create(job); }
/** * Calculates the overall {@link TimeEstimate} of this instance. * * @return the overall {@link TimeEstimate} */ public TimeEstimate getOverallTimeEstimate(Configuration configuration) { final long platformInitializationTime = this.initializedPlatforms.stream() .mapToLong(platform -> platform.getInitializeMillis(configuration)) .sum(); final TimeEstimate executionTime = this.atomicExecutionGroups.stream() .map(group -> group.estimateExecutionTime()) .reduce(TimeEstimate.ZERO, TimeEstimate::plus); return executionTime.plus(platformInitializationTime); }
(platform, requestee) -> platform.getInitializeMillis(requestee.getConfiguration()), configuration ); new FunctionalKeyValueProvider<>( fallbackProvider, (platform, requestee) -> platform.createLoadProfileToTimeConverter( requestee.getConfiguration() new FunctionalKeyValueProvider<>( fallbackProvider, (platform, requestee) -> platform.createTimeToCostConverter( requestee.getConfiguration()
@Override public String toString() { return String.format("Platform[%s]", this.getName()); }
/** * Determines whether the {@code task} should reuse the {@link PlatformExecution} of the {@code producer}. * * @param producer whose {@link PlatformExecution} might be reused * @param inputChannel connects the {@code producer} to the {@code task} * @param task might reuse the {@link PlatformExecution} of {@code producer} * @return whether to reuse */ private boolean checkIfShouldReusePlatformExecution(ExecutionTask producer, Channel inputChannel, ExecutionTask task) { final Platform producerPlatform = producer.getPlatform(); return producerPlatform.equals(task.getPlatform()) && producerPlatform.isSinglePlatformExecutionPossible(producer, inputChannel, task); }
@Override public void warmUp(Configuration configuration) { super.warmUp(configuration); // Run a most simple Spark job. this.logger.info("Running warm-up Spark job..."); long startTime = System.currentTimeMillis(); final RheemContext rheemCtx = new RheemContext(configuration); SparkCollectionSource<Integer> source = new SparkCollectionSource<>( Collections.singleton(0), DataSetType.createDefault(Integer.class) ); SparkLocalCallbackSink<Integer> sink = new SparkLocalCallbackSink<>( dq -> { }, DataSetType.createDefault(Integer.class) ); source.connectTo(0, sink, 0); final Job job = rheemCtx.createJob("Warm up", new RheemPlan(sink)); // Make sure not to have the warm-up jobs bloat the execution logs. job.getConfiguration().setProperty("rheem.core.log.enabled", "false"); job.execute(); long stopTime = System.currentTimeMillis(); this.logger.info("Spark warm-up finished in {}.", Formats.formatDuration(stopTime - startTime, true)); }
@SuppressWarnings("unused") public String toNameString() { return String.format("%s[%s-%d:%d-%6x]", this.getClass().getSimpleName(), this.platformExecution.getPlatform().getName(), this.platformExecution.getSequenceNumber(), this.sequenceNumber, this.hashCode()); }
public Map toJsonMap() { HashMap<String, Object> jsonMap = new HashMap<>(); ArrayList<Map> operators = new ArrayList<>(); jsonMap.put("platform", this.getPlatformExecution().getPlatform().getName()); jsonMap.put("operators", operators); Set<ExecutionTask> seenTasks = new HashSet<>(); for (ExecutionTask startTask : this.startTasks) { this.toJsonMapAux(startTask, seenTasks, operators); } return jsonMap; }
private Executor getOrCreateExecutorFor(ExecutionStage stage) { return this.executors.computeIfAbsent( stage.getPlatformExecution().getPlatform(), platform -> { // It is important to register the Executor. This way, we ensure that it will also not be disposed // among disconnected PlatformExecutions. The downside is, that we only remove it, once the // execution is done. final Executor executor = platform.getExecutorFactory().create(this.job); this.registerGlobal(executor); return executor; } ); }
operator.getClass().getCanonicalName(), operator.getName(), operator.getPlatform().getName() ); instance.operators.add(operatorNode);
.provideFor(consideredPlatform); this.experiment.getSubject().addConfiguration( String.format("Costs per ms (%s)", consideredPlatform.getName()), timeToCostConverter.getCostsPerMillisecond() ); this.experiment.getSubject().addConfiguration( String.format("Fix costs (%s)", consideredPlatform.getName()), timeToCostConverter.getFixCosts() );