@Override public void set(final String key, final Object value) { checkKeyValue(key, value); if (this.inExecute) throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key); else this.sparkMemory.get(key).setValue(new ObjectWritable<>(value)); }
floatAccum.setValue(5.0f); assertEquals((Float) 5.0f, floatAccum.value());
floatAccum.setValue(5.0f); assertEquals((Float) 5.0f, floatAccum.value());
floatAccum.setValue(5.0f); assertEquals((Float) 5.0f, floatAccum.value());
@Override public void set(final String key, final Object value) { checkKeyValue(key, value); if (this.inExecute) throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key); else this.sparkMemory.get(key).setValue(new ObjectWritable<>(value)); }
@Override public void set(final String key, final Object value) { checkKeyValue(key, value); if (this.inExecute) { throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key); } else { this.sparkMemory.get(key).setValue(new ObjectWritable<>(value)); } }
/** Init metrics accumulator if it has not been initiated. This method is idempotent. */ public static void init(SparkPipelineOptions opts, JavaSparkContext jsc) { if (instance == null) { synchronized (MetricsAccumulator.class) { if (instance == null) { Optional<CheckpointDir> maybeCheckpointDir = opts.isStreaming() ? Optional.of(new CheckpointDir(opts.getCheckpointDir())) : Optional.absent(); Accumulator<MetricsContainerStepMap> accumulator = jsc.sc() .accumulator( new SparkMetricsContainerStepMap(), ACCUMULATOR_NAME, new MetricsAccumulatorParam()); if (maybeCheckpointDir.isPresent()) { Optional<MetricsContainerStepMap> maybeRecoveredValue = recoverValueFromCheckpoint(jsc, maybeCheckpointDir.get()); if (maybeRecoveredValue.isPresent()) { accumulator.setValue(maybeRecoveredValue.get()); } } instance = accumulator; } } LOG.info("Instantiated metrics accumulator: " + instance.value()); } }
/** Init aggregators accumulator if it has not been initiated. This method is idempotent. */ public static void init(SparkPipelineOptions opts, JavaSparkContext jsc) { if (instance == null) { synchronized (AggregatorsAccumulator.class) { if (instance == null) { Optional<CheckpointDir> maybeCheckpointDir = opts.isStreaming() ? Optional.of(new CheckpointDir(opts.getCheckpointDir())) : Optional.absent(); Accumulator<NamedAggregators> accumulator = jsc.sc().accumulator(new NamedAggregators(), ACCUMULATOR_NAME, new AggAccumParam()); if (maybeCheckpointDir.isPresent()) { Optional<NamedAggregators> maybeRecoveredValue = recoverValueFromCheckpoint(jsc, maybeCheckpointDir.get()); if (maybeRecoveredValue.isPresent()) { accumulator.setValue(maybeRecoveredValue.get()); } } instance = accumulator; } } LOG.info("Instantiated aggregators accumulator: " + instance.value()); } }