@Override public double getDouble(String name, double defaultValue) { return encapsulated.getDouble(name, defaultValue); }
/** * Extracts the percent of Regions for a table to have been reported to enable quota violation * state change. * * @param conf The configuration object. * @return The percent of regions reported to use. */ static Double getRegionReportPercent(Configuration conf) { return conf.getDouble(QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY, QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT); }
@Override public double getDouble(String name, double defaultValue) { return conf.getDouble(name, defaultValue); }
/** * Extracts the percent of Regions for a table to have been reported to enable quota violation * state change. * * @param conf The configuration object. * @return The percent of regions reported to use. */ static Double getRegionReportPercent(Configuration conf) { return conf.getDouble(POLICY_REFRESHER_CHORE_REPORT_PERCENT_KEY, POLICY_REFRESHER_CHORE_REPORT_PERCENT_DEFAULT); } }
public static double getRegionReplicaStoreFileRefreshMultiplier(Configuration conf) { return conf.getDouble(REGION_REPLICA_STORE_FILE_REFRESH_MEMSTORE_MULTIPLIER, DEFAULT_REGION_REPLICA_STORE_FILE_REFRESH_MEMSTORE_MULTIPLIER); }
public AdaptiveMemStoreCompactionStrategy(Configuration conf, String cfName) { super(conf, cfName); compactionThreshold = conf.getDouble(ADAPTIVE_COMPACTION_THRESHOLD_KEY, ADAPTIVE_COMPACTION_THRESHOLD_DEFAULT); initialCompactionProbability = conf.getDouble(ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_KEY, ADAPTIVE_INITIAL_COMPACTION_PROBABILITY_DEFAULT); resetStats(); }
/** * Get the variable as a double * @param conf configuration to retrieve it from * @param var variable to retrieve * @return value, or default value if value not in config file */ public static double getDoubleVar(Configuration conf, ConfVars var) { assert var.defaultVal.getClass() == Double.class; String val = conf.get(var.varname); return val == null ? conf.getDouble(var.hiveName, (Double)var.defaultVal) : Double.valueOf(val); }
private static double parseDecayFactor(String ns, Configuration conf) { double factor = conf.getDouble(ns + "." + IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY, 0.0); if (factor == 0.0) { factor = conf.getDouble(ns + "." + IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY, IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_DEFAULT); } else if ((factor > 0.0) && (factor < 1)) { LOG.warn(IPC_FCQ_DECAYSCHEDULER_FACTOR_KEY + " is deprecated. Please use " + IPC_SCHEDULER_DECAYSCHEDULER_FACTOR_KEY + "."); } if (factor <= 0 || factor >= 1) { throw new IllegalArgumentException("Decay Factor " + "must be between 0 and 1"); } return factor; }
public LossyCounting() { Configuration conf = HBaseConfiguration.create(); this.errorRate = conf.getDouble(HConstants.DEFAULT_LOSSY_COUNTING_ERROR_RATE, 0.02); this.bucketSize = (long) Math.ceil(1.0 / errorRate); this.currentTerm = 1; this.totalDataCount = 0; this.data = new ConcurrentHashMap<>(); calculateCurrentTerm(); }
public void onConfigurationChange(Configuration conf) { // update CoDel Scheduler tunables int codelTargetDelay = conf.getInt(CALL_QUEUE_CODEL_TARGET_DELAY, CALL_QUEUE_CODEL_DEFAULT_TARGET_DELAY); int codelInterval = conf.getInt(CALL_QUEUE_CODEL_INTERVAL, CALL_QUEUE_CODEL_DEFAULT_INTERVAL); double codelLifoThreshold = conf.getDouble(CALL_QUEUE_CODEL_LIFO_THRESHOLD, CALL_QUEUE_CODEL_DEFAULT_LIFO_THRESHOLD); for (BlockingQueue<CallRunner> queue : queues) { if (queue instanceof AdaptiveLifoCoDelCallQueue) { ((AdaptiveLifoCoDelCallQueue) queue).updateTunables(codelTargetDelay, codelInterval, codelLifoThreshold); } } } }
/** * Start up our handlers. */ protected void startHandlers(final String nameSuffix, final int numHandlers, final List<BlockingQueue<CallRunner>> callQueues, final int qindex, final int qsize, final int port, final AtomicInteger activeHandlerCount) { final String threadPrefix = name + Strings.nullToEmpty(nameSuffix); double handlerFailureThreshhold = conf == null ? 1.0 : conf.getDouble( HConstants.REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT, HConstants.DEFAULT_REGION_SERVER_HANDLER_ABORT_ON_ERROR_PERCENT); for (int i = 0; i < numHandlers; i++) { final int index = qindex + (i % qsize); String name = "RpcServer." + threadPrefix + ".handler=" + handlers.size() + ",queue=" + index + ",port=" + port; Handler handler = getHandler(name, handlerFailureThreshhold, callQueues.get(index), activeHandlerCount); handler.start(); handlers.add(handler); } LOG.debug("Started handlerCount={} with threadPrefix={}, numCallQueues={}, port={}", handlers.size(), threadPrefix, qsize, port); }
@Override public boolean isAllWorkerContainersSucceeded() { if (containerId2Status.isEmpty()) { return false; } Double failedNum = 0.0; for (Entry<XLearningContainerId, XLearningContainerStatus> e : containerId2Status.entrySet()) { if (!e.getValue().equals(XLearningContainerStatus.SUCCEEDED)) { failedNum += 1; if (this.getConfig().getBoolean(XLearningConfiguration.XLEARNING_TF_EVALUATOR, XLearningConfiguration.DEFAULT_XLEARNING_TF_EVALUATOR) && e.getKey().toString().equals(applicationContext.getTfEvaluatorId())) { failedNum -= 1; } } } if (!this.getConfig().getBoolean(XLearningConfiguration.XLEARNING_MODE_SINGLE, XLearningConfiguration.DEFAULT_XLEARNING_MODE_SINGLE)) { if (failedNum > 0) { return false; } } else { Double jobFailedNum = containerId2Status.size() * this.getConfig().getDouble(XLearningConfiguration.XLEARNING_CONTAINER_MAX_FAILURES_RATE, XLearningConfiguration.DEFAULT_XLEARNING_CONTAINER_FAILURES_RATE); if (failedNum >= jobFailedNum) { return false; } } return true; }
@Override protected void configureForRegion(HRegion region) { super.configureForRegion(region); Configuration conf = getConf(); TableDescriptor desc = region.getTableDescriptor(); if (desc != null) { this.desiredMaxFileSize = desc.getMaxFileSize(); } if (this.desiredMaxFileSize <= 0) { this.desiredMaxFileSize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); } double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D); this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter; long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate); // make sure the long value won't overflow with jitter if (this.jitterRate > 0 && jitterValue > (Long.MAX_VALUE - this.desiredMaxFileSize)) { this.desiredMaxFileSize = Long.MAX_VALUE; } else { this.desiredMaxFileSize += jitterValue; } }
private void initInmemoryFlushSize(Configuration conf) { double factor = 0; long memstoreFlushSize = getRegionServices().getMemStoreFlushSize(); int numStores = getRegionServices().getNumStores(); if (numStores <= 1) { // Family number might also be zero in some of our unit test case numStores = 1; } factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.0); if(factor != 0.0) { // multiply by a factor (the same factor for all index types) inmemoryFlushSize = (long) (factor * memstoreFlushSize) / numStores; } else { inmemoryFlushSize = IN_MEMORY_FLUSH_MULTIPLIER * conf.getLong(MemStoreLAB.CHUNK_SIZE_KEY, MemStoreLAB.CHUNK_SIZE_DEFAULT); inmemoryFlushSize -= ChunkCreator.SIZEOF_CHUNK_HEADER; } }
CALL_QUEUE_CODEL_DEFAULT_TARGET_DELAY); int codelInterval = conf.getInt(CALL_QUEUE_CODEL_INTERVAL, CALL_QUEUE_CODEL_DEFAULT_INTERVAL); double codelLifoThreshold = conf.getDouble(CALL_QUEUE_CODEL_LIFO_THRESHOLD, CALL_QUEUE_CODEL_DEFAULT_LIFO_THRESHOLD); queueInitArgs = new Object[] { maxQueueLength, codelTargetDelay, codelInterval,
Double jobFailedNum = containerId2Status.size() * this.getConfig().getDouble(XLearningConfiguration.XLEARNING_CONTAINER_MAX_FAILURES_RATE, XLearningConfiguration.DEFAULT_XLEARNING_CONTAINER_FAILURES_RATE); if (failedNum >= jobFailedNum) { return true;
this.maxRitPercent = conf.getDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT, HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);
int maxMem = Integer.valueOf(envs.get(XLearningConstants.Environment.XLEARNING_CONTAINER_MAX_MEMORY.toString())); LOG.info("maxMem : " + maxMem); workerMemory = workerMemory + (applicationAttemptID.getAttemptId() - 1) * (int) Math.ceil(workerMemory * conf.getDouble(XLearningConfiguration.XLEARNING_WORKER_MEM_AUTO_SCALE, XLearningConfiguration.DEFAULT_XLEARNING_WORKER_MEM_AUTO_SCALE)); LOG.info("Auto Scale the Worker Memory from " + conf.getInt(XLearningConfiguration.XLEARNING_WORKER_MEMORY, XLearningConfiguration.DEFAULT_XLEARNING_WORKER_MEMORY) + " to " + workerMemory); if (workerMemory > maxMem) { psMemory = psMemory + (applicationAttemptID.getAttemptId() - 1) * (int) Math.ceil(psMemory * conf.getDouble(XLearningConfiguration.XLEARNING_PS_MEM_AUTO_SCALE, XLearningConfiguration.DEFAULT_XLEARNING_PS_MEM_AUTO_SCALE)); LOG.info("Auto Scale the Ps Memory from " + conf.getInt(XLearningConfiguration.XLEARNING_PS_MEMORY, XLearningConfiguration.DEFAULT_XLEARNING_PS_MEMORY) + " to " + psMemory); if (psMemory > maxMem) {
if (cpuStatistics.size() != 0) { Double cpuMemUsagedMax = cpuStatistics.get("CPUMEM").get(1); if (cpuMemUsagedMax * 1024.0 / workerMemory < conf.getDouble(XLearningConfiguration.XLEARNING_CONTAINER_MEM_USAGE_WARN_FRACTION, XLearningConfiguration.DEFAULT_XLEARNING_CONTAINER_MEM_USAGE_WARN_FRACTION)) { containerMessage.put(AMParams.CONTAINER_CPU_USAGE_WARN_MEM, "true"); } else { if (cpuStatistics.size() != 0) { Double cpuMemUsagedMax = cpuStatistics.get("CPUMEM").get(1); if (cpuMemUsagedMax * 1024.0 / workerMemory < conf.getDouble(XLearningConfiguration.XLEARNING_CONTAINER_MEM_USAGE_WARN_FRACTION, XLearningConfiguration.DEFAULT_XLEARNING_CONTAINER_MEM_USAGE_WARN_FRACTION)) { containerMessage.put(AMParams.CONTAINER_CPU_USAGE_WARN_MEM, "true"); } else {
ReencryptionUpdater(final FSDirectory fsd, final CompletionService<ReencryptionTask> service, final ReencryptionHandler rh, final Configuration conf) { dir = fsd; batchService = service; handler = rh; this.throttleLimitRatio = conf.getDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_KEY, DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_DEFAULT); Preconditions.checkArgument(throttleLimitRatio > 0.0f, DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_KEY + " is not positive."); }