public long getValue() { if (accumulator != null) { return accumulator.value(); } else { return accumValue; } }
SparkCounter snapshot() { return new SparkCounter(name, displayName, accumulator.value()); }
System.out.println("Lines with 'KK6JKQ': " + count.value()); }}); callSigns.saveAsTextFile(outputDir + "/callsigns"); System.out.println("Blank lines: "+ blankLines.value()); if (invalidSignCount.value() < 0.1 * validSignCount.value()) { contactCounts.saveAsTextFile(outputDir + "/contactCount"); } else { System.out.println("Too many errors " + invalidSignCount.value() + " for " + validSignCount.value()); System.exit(1);
@Override public Set<String> keys() { if (this.inExecute) return this.broadcast.getValue().keySet(); else { final Set<String> trueKeys = new HashSet<>(); this.sparkMemory.forEach((key, value) -> { if (!value.value().isEmpty()) trueKeys.add(key); }); return Collections.unmodifiableSet(trueKeys); } }
protected void broadcastMemory(final JavaSparkContext sparkContext) { this.broadcast.destroy(true); // do we need to block? final Map<String, Object> toBroadcast = new HashMap<>(); this.sparkMemory.forEach((key, object) -> { if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast()) toBroadcast.put(key, object.value()); }); this.broadcast = sparkContext.broadcast(toBroadcast); }
@Override public <R> R get(final String key) throws IllegalArgumentException { if (!this.memoryComputeKeys.containsKey(key)) throw Memory.Exceptions.memoryDoesNotExist(key); if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast()) throw Memory.Exceptions.memoryDoesNotExist(key); final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value()); if (null == r || r.isEmpty()) throw Memory.Exceptions.memoryDoesNotExist(key); else return r.get(); }
public T getValue() { if (accumulator != null) { return accumulator.value(); } else { return null; } }
public long getValue() { if (accumulator != null) { return accumulator.value(); } else { return accumValue; } }
private static void checkpoint() throws IOException { if (checkpointFilePath != null) { Checkpoint.writeObject(fileSystem, checkpointFilePath, instance.value()); } }
private static void checkpoint() throws IOException { if (checkpointFilePath != null) { Checkpoint.writeObject(fileSystem, checkpointFilePath, instance.value()); } }
protected void broadcastMemory(final JavaSparkContext sparkContext) { this.broadcast.destroy(true); // do we need to block? final Map<String, Object> toBroadcast = new HashMap<>(); this.sparkMemory.forEach((key, object) -> { if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast()) { toBroadcast.put(key, object.value()); } }); this.broadcast = sparkContext.broadcast(toBroadcast); }
protected void broadcastMemory(final JavaSparkContext sparkContext) { this.broadcast.destroy(true); // do we need to block? final Map<String, Object> toBroadcast = new HashMap<>(); this.sparkMemory.forEach((key, object) -> { if (!object.value().isEmpty() && this.memoryComputeKeys.get(key).isBroadcast()) toBroadcast.put(key, object.value()); }); this.broadcast = sparkContext.broadcast(toBroadcast); }
private Counters getCounters() { Counters c = new Counters(); Map<String, Map<String, Long>> values = counters.value(); for (Map.Entry<String, Map<String, Long>> e : values.entrySet()) { CounterGroup cg = c.getGroup(e.getKey()); for (Map.Entry<String, Long> f : e.getValue().entrySet()) { cg.findCounter(f.getKey()).setValue(f.getValue()); } } return c; }
@Override public OptionalLong getMeasuredCardinality() { if (this.accumulator != null) { this.setMeasuredCardinality(this.accumulator.value()); } return super.getMeasuredCardinality(); }
@Override public MetricResults metrics() { return asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value()); }
@Override protected void doDispose() { if (this.accumulator != null) { this.setMeasuredCardinality(this.accumulator.value()); this.accumulator = null; } if (this.isRddCached() && this.rdd != null) { Actions.doSafe(this.rdd::unpersist); logger.debug("Unpersisted {}.", this.rdd); this.rdd = null; } }
@Override public <R> R get(final String key) throws IllegalArgumentException { if (!this.memoryComputeKeys.containsKey(key)) throw Memory.Exceptions.memoryDoesNotExist(key); if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast()) throw Memory.Exceptions.memoryDoesNotExist(key); final ObjectWritable<R> r = (ObjectWritable<R>) (this.inExecute ? this.broadcast.value().get(key) : this.sparkMemory.get(key).value()); if (null == r || r.isEmpty()) throw Memory.Exceptions.memoryDoesNotExist(key); else return r.get(); }