public static void setFloatVar(Configuration conf, ConfVars var, float val) { assert (var.valClass == Float.class) : var.varname; conf.setFloat(var.varname, val); }
/** * Enable a basic on-heap cache for these jobs. Any BlockCache implementation based on * direct memory will likely cause the map tasks to OOM when opening the region. This * is done here instead of in TableSnapshotRegionRecordReader in case an advanced user * wants to override this behavior in their job. */ public static void resetCacheConfig(Configuration conf) { conf.setFloat( HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); conf.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0f); conf.unset(HConstants.BUCKET_CACHE_IOENGINE_KEY); }
public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); // no need for a block cache conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0); int ret = ToolRunner.run(conf, new HFilePrettyPrinter(), args); System.exit(ret); } }
/** Use this benchmark with default options */ public EncodedSeekPerformanceTest() { configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f); randomizer = new Random(42L); numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS; }
public TestMiniClusterLoadSequential(boolean isMultiPut, DataBlockEncoding dataBlockEncoding) { this.isMultiPut = isMultiPut; this.dataBlockEncoding = dataBlockEncoding; conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024); // We don't want any region reassignments by the load balancer during the test. conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, 10.0f); }
private void setupQuickStart(TezEdgeProperty edgeProp, Vertex v) throws IOException { if (!edgeProp.isSlowStart()) { Configuration pluginConf = new Configuration(false); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName()); pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0); pluginConf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, 0); UserPayload payload = TezUtils.createUserPayloadFromConf(pluginConf); desc.setUserPayload(payload); v.setVertexManagerPlugin(desc); } }
@Before public void before() { conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 1.0f); conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L); conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0); conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f); loadBalancer.setConf(conf); }
/** * setup config values necessary for store */ protected void config() { this.conf = TEST_UTIL.getConfiguration(); this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, minFiles); this.conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, maxFiles); this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY, minSize); this.conf.setLong(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, maxSize); this.conf.setFloat(CompactionConfiguration.HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.0F); }
@Before public void setUpConf() { conf = HBaseConfiguration.create(); conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, DEFAULT_MEMSTORE_SIZE); }
@Test public void testScanQueueWithZeroScanRatio() throws Exception { Configuration schedConf = HBaseConfiguration.create(); schedConf.setFloat(RpcExecutor.CALL_QUEUE_HANDLER_FACTOR_CONF_KEY, 1.0f); schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_READ_SHARE_CONF_KEY, 0.5f); schedConf.setFloat(RWQueueRpcExecutor.CALL_QUEUE_SCAN_SHARE_CONF_KEY, 0f); PriorityFunction priority = mock(PriorityFunction.class); when(priority.getPriority(any(), any(), any())).thenReturn(HConstants.NORMAL_QOS); RpcScheduler scheduler = new SimpleRpcScheduler(schedConf, 2, 1, 1, priority, HConstants.QOS_THRESHOLD); assertNotEquals(null, scheduler); }
private void setUp(boolean useBucketCache) throws IOException { test_util = HBaseTestingUtility.createLocalHTU(); conf = test_util.getConfiguration(); if (useBucketCache) { conf.setInt("hbase.bucketcache.size", 400); conf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); conf.setInt("hbase.bucketcache.writer.threads", 10); conf.setFloat("hfile.block.cache.size", 0.2f); conf.setFloat("hbase.regionserver.global.memstore.size", 0.1f); } tableName = TableName.valueOf(name.getMethodName()); }
@BeforeClass public static void beforeAllTests() throws Exception { conf = HBaseConfiguration.create(); conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class); conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f); conf.setFloat("hbase.regions.slop", 0.0f); conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0); loadBalancer = new StochasticLoadBalancer(); loadBalancer.setConf(conf); }
private static StripeCompactionPolicy createPolicy(Configuration conf, long splitSize, float splitCount, int initialCount, boolean hasTtl) throws Exception { conf.setLong(StripeStoreConfig.SIZE_TO_SPLIT_KEY, splitSize); conf.setFloat(StripeStoreConfig.SPLIT_PARTS_KEY, splitCount); conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, initialCount); StoreConfigInformation sci = mock(StoreConfigInformation.class); when(sci.getStoreFileTtl()).thenReturn(hasTtl ? defaultTtl : Long.MAX_VALUE); StripeStoreConfig ssc = new StripeStoreConfig(conf, sci); return new StripeCompactionPolicy(conf, sci, ssc); }
@Test public void testL2CacheWithInvalidBucketSize() { Configuration c = new Configuration(this.conf); c.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); c.set(BlockCacheFactory.BUCKET_CACHE_BUCKETS_KEY, "256,512,1024,2048,4000,4096"); c.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 1024); try { BlockCacheFactory.createBlockCache(c); fail("Should throw IllegalArgumentException when passing illegal value for bucket size"); } catch (IllegalArgumentException e) { } } }
@BeforeClass public static void setUpBeforeClass() throws Exception { conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true); conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f); chunkPoolDisabledBeforeTest = ChunkCreator.chunkPoolDisabled; ChunkCreator.chunkPoolDisabled = false; long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage() .getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false)); chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); assertTrue(chunkCreator != null); }
@Before public void setup() throws IOException { CONF = TEST_UTIL.getConfiguration(); // Disable block cache. CONF.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); dir = TEST_UTIL.getDataTestDir("TestHRegion").toString(); tableName = TableName.valueOf(name.getMethodName()); }
@Test public void testCompoundBloomFilter() throws IOException { conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); for (int t = 0; t < NUM_TESTS; ++t) { conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) TARGET_ERROR_RATES[t]); testIdMsg = "in test #" + t + ":"; Random generationRand = new Random(GENERATION_SEED); List<KeyValue> kvs = createSortedKeyValues(generationRand, NUM_KV[t]); BloomType bt = BLOOM_TYPES[t]; Path sfPath = writeStoreFile(t, bt, kvs); readStoreFile(t, bt, kvs, sfPath); } }
@Before public void setup() throws IOException { TEST_UTIL = HBaseTestingUtility.createLocalHTU(); CONF = TEST_UTIL.getConfiguration(); // Disable block cache. CONF.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); dir = TEST_UTIL.getDataTestDir("TestHRegion").toString(); tableName = TableName.valueOf(name.getMethodName()); }
@Test public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.02f); conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.03f); RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); assertFalse(manager.isTunerOn()); }
@Test public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception { Configuration conf = HBaseConfiguration.create(); conf.setFloat(MemorySizeUtil.MEMSTORE_SIZE_KEY, 0.02f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.03f); RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf); HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), new MemstoreFlusherStub(0), new RegionServerStub(conf), regionServerAccounting); assertFalse(manager.isTunerOn()); }