/** * Create a {@link PartialExecution} according to the given parameters. * * @param executionLineageNodes {@link ExecutionLineageNode}s reflecting what has been executed * @param executionDuration the measured execution duration in milliseconds * @return the {@link PartialExecution} or {@code null} if nothing has been executed */ protected PartialExecution createPartialExecution( Collection<ExecutionLineageNode> executionLineageNodes, long executionDuration) { if (executionLineageNodes.isEmpty()) return null; final PartialExecution partialExecution = PartialExecution.createFromMeasurement( executionDuration, executionLineageNodes, this.getConfiguration() ); return partialExecution; }
@Override public void add(PartialExecution partialExecution) { this.partialExecutions.add(partialExecution); if (this.logger.isInfoEnabled()) { this.logger.info( "Executed {} items in {} (estimated {}).", partialExecution.getAtomicExecutionGroups().size(), Formats.formatDuration(partialExecution.getMeasuredExecutionTime()), partialExecution.getOverallTimeEstimate(this.getConfiguration()) ); } }
/** * Creates a new instance. * * @param id the ID of the new instance * @param partialExecution provides data for the new instance * @param configuration required to calculate the estimated execution time */ public PartialExecutionMeasurement(String id, PartialExecution partialExecution, Configuration configuration) { super(id); // TODO: Capture what has been executed? this.executionMillis = partialExecution.getMeasuredExecutionTime(); this.estimatedExecutionMillis = partialExecution.getOverallTimeEstimate(configuration); }
for (PartialExecution partialExecution : partialExecutions) { if (this.logger.isDebugEnabled()) { for (AtomicExecutionGroup atomicExecutionGroup : partialExecution.getAtomicExecutionGroups()) { if (!(atomicExecutionGroup.getEstimationContext() instanceof OptimizationContext.OperatorContext)) { continue; .flatMap(partialExecution -> partialExecution.getInvolvedPlatforms().stream()) .map(platform -> this.configuration.getTimeToCostConverterProvider().provideFor(platform).getFixCosts()) .reduce(0d, (a, b) -> a + b);
@Override public PartialExecution deserialize(JSONObject json, Class<? extends PartialExecution> cls) { final long measuredExecutionTime = json.getLong("millis"); final double lowerCost = json.optDouble("lowerCost", -1); final double uppserCost = json.optDouble("upperCost", -1); final Collection<AtomicExecutionGroup> atomicExecutionGroups = JsonSerializables.deserializeAllAsList( json.getJSONArray("execGroups"), new AtomicExecutionGroup.Serializer(this.configuration), AtomicExecutionGroup.class ); final Collection<Platform> initializedPlatforms = JsonSerializables.deserializeAllAsList(json.optJSONArray("initPlatforms"), Platform.jsonSerializer); final PartialExecution partialExecution = new PartialExecution( atomicExecutionGroups, measuredExecutionTime, lowerCost, uppserCost ); partialExecution.initializedPlatforms.addAll(initializedPlatforms); return partialExecution; } }
if (this.numActions == 0) partialExecution.addInitializedPlatform(SparkPlatform.getInstance()); this.numActions++;
/** * Creates a new instance according to the measurement data. * * @param measuredExecutionTime the measured execution time * @param executionLineageNodes the {@link ExecutionLineageNode}s reflecting what has been executed * @param configuration the execution {@link Configuration} * @return the new instance */ public static PartialExecution createFromMeasurement( long measuredExecutionTime, Collection<ExecutionLineageNode> executionLineageNodes, Configuration configuration) { // Calculate possible costs. double lowerCost = Double.POSITIVE_INFINITY, upperCost = Double.NEGATIVE_INFINITY; final Set<Platform> platforms = executionLineageNodes.stream() .map(node -> ((ExecutionOperator) node.getOperatorContext().getOperator()).getPlatform()) .collect(Collectors.toSet()); for (Platform platform : platforms) { final TimeToCostConverter timeToCostConverter = configuration.getTimeToCostConverterProvider().provideFor(platform); final ProbabilisticDoubleInterval costs = timeToCostConverter.convertWithoutFixCosts(TimeEstimate.ZERO.plus(measuredExecutionTime)); lowerCost = Math.min(lowerCost, costs.getLowerEstimate()); upperCost = Math.max(upperCost, costs.getUpperEstimate()); } return new PartialExecution(measuredExecutionTime, lowerCost, upperCost, executionLineageNodes, configuration); }