private int getValueSize() { if (config.isUseVariableDataSize()) { return vvRandom.nextInt( Math.abs(config.getDataSizeUpperBound() - config.getDataSizeLowerBound())) + config.getDataSizeLowerBound(); } return config.getDataSize(); }
private void startWritesInternal(LoadPattern loadPattern, int windowSize, long windowDurationInSec, int bulkSize) { logger.info("Starting NdBenchDriver writes..."); NdBenchOperation operation; operation = new WriteOperation(clientRef.get()); KeyGeneratorFactory keyGeneratorFactory = new KeyGeneratorFactory(); KeyGenerator<String> keyGenerator = keyGeneratorFactory.getKeyGenerator(loadPattern, config.getNumKeys(), windowSize, windowDurationInSec, config.isPreloadKeys(), config.getZipfExponent()); keyGeneratorWriteRef.set(keyGenerator); startOperation(config.isWriteEnabled(), config.getNumWriters(), writeWorkers, tpWriteRef, writeLimiter, operation, keyGenerator, config.isAutoTuneEnabled(), bulkSize); writesStarted.set(true); }
public TunableConfig(IConfiguration configuration) { this.readRateLimit = configuration.getReadRateLimit(); this.writeRateLimit = configuration.getWriteRateLimit(); }
private void startReadsInternal(LoadPattern loadPattern, int windowSize, long windowDurationInSec, int bulkSize) { logger.info("Starting NdBenchDriver reads..."); NdBenchOperation operation; operation = new ReadOperation(clientRef.get()); KeyGeneratorFactory keyGeneratorFactory = new KeyGeneratorFactory(); KeyGenerator<String> keyGenerator = keyGeneratorFactory.getKeyGenerator(loadPattern, config.getNumKeys(), windowSize, windowDurationInSec, config.isPreloadKeys(), config.getZipfExponent()); keyGeneratorReadRef.set(keyGenerator); startOperation( config.isReadEnabled(), config.getNumReaders(), readWorkers, tpReadRef, readLimiter, operation, keyGenerator, config.isAutoTuneEnabled(), bulkSize); readsStarted.set(true); }
esConfig.getBulkWriteBatchSize(), esConfig.isRandomizeStrings() ? dataGenerator : new FakeWordDictionaryBasedDataGenerator(dataGenerator, coreConfig.getDataSize())); if (coreConfig.isAutoTuneEnabled()) { this.autoTuner = new EsAutoTuner( coreConfig.getAutoTuneRampPeriodMillisecs(), coreConfig.getAutoTuneIncrementIntervalMillisecs(), coreConfig.getWriteRateLimit(), coreConfig.getAutoTuneFinalWriteRate(), coreConfig.getAutoTuneWriteFailureRatioThreshold()); } else {
void updateRPS() { int secondsFreq = config.getStatsUpdateFreqSeconds(); String bottleneckMsg = "If this occurs consistently the benchmark client could be the bottleneck."; if (this.config.isReadEnabled() && readsStarted.get() && readRps < expectedReadRate) { logger.warn("Observed Read RPS ({}) less than expected read rate + ({}).\n{}", readRps, expectedReadRate, bottleneckMsg); if (this.config.isWriteEnabled() && writesStarted.get() && writeRps < expectedwriteRate) { logger.warn("Observed Write RPS ({}) less than expected write rate + ({}).\n{}", writeRps, expectedwriteRate, bottleneckMsg);
@Override public String getConnectionInfo() { int bytesPerCol=coreConfig.getDataSize(); int numColsPerRow=config.getColsPerRow(); int numRowsPerPartition=config.getRowsPerPartition(); int numPartitions= coreConfig.getNumKeys(); int RF = 3; Long numNodes = cluster.getMetadata().getAllHosts() .stream() .collect(groupingBy(Host::getDatacenter,counting())) .values().stream().findFirst().get(); int partitionSizeInBytes = bytesPerCol * numColsPerRow * numRowsPerPartition; long totalSizeInBytes = (long) partitionSizeInBytes * numPartitions * RF; long totalSizeInBytesPerNode = totalSizeInBytes / numNodes; return String.format("Cluster Name - %s : Keyspace Name - %s : CF Name - %s ::: ReadCL - %s : WriteCL - %s ::: " + "DataSize per Node: ~[%s], Total DataSize on Cluster: ~[%s], Num nodes in C* DC: %s, PartitionSize: %s", clusterName, keyspaceName, tableName, config.getReadConsistencyLevel(), config.getWriteConsistencyLevel(), humanReadableByteCount(totalSizeInBytesPerNode), humanReadableByteCount(totalSizeInBytes), numNodes, humanReadableByteCount(partitionSizeInBytes)); } }
config.getBackfillKeySlots(), config.getNumKeys());
logger.info("\n\nWorker threads: " + numWorkersConfig + ", Num Keys: " + config.getNumKeys() + "\n\n");
public void onWriteRateLimitChange() { checkAndInitRateLimit(writeLimiter, config.getWriteRateLimit(), "writeLimiter"); }
private void checkAndInitTimer() { /** CODE TO PERIODICALLY LOG RPS */ ExecutorService timer = timerRef.get(); if (timer == null) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("ndbench-updaterps-pool-%d") .setDaemon(false).build(); timer = Executors.newFixedThreadPool(1, threadFactory); timer.submit(() -> { while (!Thread.currentThread().isInterrupted()) { rpsCount.updateRPS(); Thread.sleep(config.getStatsUpdateFreqSeconds() * 1000); } return null; }); timerRef.set(timer); } }
private void startReadsInternal(LoadPattern loadPattern, int windowSize, long windowDurationInSec, int bulkSize) { logger.info("Starting NdBenchDriver reads..."); NdBenchOperation operation; operation = new ReadOperation(clientRef.get()); KeyGeneratorFactory keyGeneratorFactory = new KeyGeneratorFactory(); KeyGenerator<String> keyGenerator = keyGeneratorFactory.getKeyGenerator(loadPattern, config.getNumKeys(), windowSize, windowDurationInSec, config.isPreloadKeys(), config.getZipfExponent()); keyGeneratorReadRef.set(keyGenerator); startOperation( config.isReadEnabled(), config.getNumReaders(), readWorkers, tpReadRef, readLimiter, operation, keyGenerator, config.isAutoTuneEnabled(), bulkSize); readsStarted.set(true); }
void updateRPS() { int secondsFreq = config.getStatsUpdateFreqSeconds(); String bottleneckMsg = "If this occurs consistently the benchmark client could be the bottleneck."; if (this.config.isReadEnabled() && readsStarted.get() && readRps < expectedReadRate) { logger.warn("Observed Read RPS ({}) less than expected read rate + ({}).\n{}", readRps, expectedReadRate, bottleneckMsg); if (this.config.isWriteEnabled() && writesStarted.get() && writeRps < expectedwriteRate) { logger.warn("Observed Write RPS ({}) less than expected write rate + ({}).\n{}", writeRps, expectedwriteRate, bottleneckMsg);
@Override public String getConnectionInfo() { int bytesPerCol=coreConfig.getDataSize(); int numColsPerRow=config.getColsPerRow(); int numRowsPerPartition=config.getRowsPerPartition(); int numPartitions= coreConfig.getNumKeys(); int RF = 3; Long numNodes = cluster.getMetadata().getAllHosts() .stream() .collect(groupingBy(Host::getDatacenter,counting())) .values().stream().findFirst().get(); int partitionSizeInBytes = bytesPerCol * numColsPerRow * numRowsPerPartition; long totalSizeInBytes = (long) partitionSizeInBytes * numPartitions * RF; long totalSizeInBytesPerNode = totalSizeInBytes / numNodes; return String.format("Cluster Name - %s : Keyspace Name - %s : CF Name - %s ::: ReadCL - %s : WriteCL - %s ::: " + "DataSize per Node: ~[%s], Total DataSize on Cluster: ~[%s], Num nodes in C* DC: %s, PartitionSize: %s", clusterName, keyspaceName, tableName, config.getReadConsistencyLevel(), config.getWriteConsistencyLevel(), humanReadableByteCount(totalSizeInBytesPerNode), humanReadableByteCount(totalSizeInBytes), numNodes, humanReadableByteCount(partitionSizeInBytes)); } }
config.getBackfillKeySlots(), config.getNumKeys());
logger.info("\n\nWorker threads: " + numWorkersConfig + ", Num Keys: " + config.getNumKeys() + "\n\n");
public void onWriteRateLimitChange() { checkAndInitRateLimit(writeLimiter, config.getWriteRateLimit(), "writeLimiter"); }
private void checkAndInitTimer() { /** CODE TO PERIODICALLY LOG RPS */ ExecutorService timer = timerRef.get(); if (timer == null) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setNameFormat("ndbench-updaterps-pool-%d") .setDaemon(false).build(); timer = Executors.newFixedThreadPool(1, threadFactory); timer.submit(() -> { while (!Thread.currentThread().isInterrupted()) { rpsCount.updateRPS(); Thread.sleep(config.getStatsUpdateFreqSeconds() * 1000); } return null; }); timerRef.set(timer); } }
private void startWritesInternal(LoadPattern loadPattern, int windowSize, long windowDurationInSec, int bulkSize) { logger.info("Starting NdBenchDriver writes..."); NdBenchOperation operation; operation = new WriteOperation(clientRef.get()); KeyGeneratorFactory keyGeneratorFactory = new KeyGeneratorFactory(); KeyGenerator<String> keyGenerator = keyGeneratorFactory.getKeyGenerator(loadPattern, config.getNumKeys(), windowSize, windowDurationInSec, config.isPreloadKeys(), config.getZipfExponent()); keyGeneratorWriteRef.set(keyGenerator); startOperation(config.isWriteEnabled(), config.getNumWriters(), writeWorkers, tpWriteRef, writeLimiter, operation, keyGenerator, config.isAutoTuneEnabled(), bulkSize); writesStarted.set(true); }
when(config.getStatsUpdateFreqSeconds()).thenReturn(10); when(config.isReadEnabled()).thenReturn(true); when(config.isWriteEnabled()).thenReturn(true);