float rollTsSec = getMillisFromLastRoll() / 1000.0f; LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)", StringUtils.humanSize(totalSynced.get()), StringUtils.humanSize(totalSynced.get() / rollTsSec))); if (LOG.isTraceEnabled() && (syncWaitMs > 10 || slotIndex < syncMaxSlot)) { LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)", StringUtils.humanTimeDiff(syncWaitMs), slotIndex, StringUtils.humanSize(totalSyncedToStore), StringUtils.humanSize(syncedPerSec)));
@VisibleForTesting public void setOwner(String owner) { this.owner = StringUtils.isEmpty(owner) ? null : owner; }
@Override public boolean visit(final Result r) throws IOException { if (r != null && !r.isEmpty()) { long st = 0; if (LOG.isTraceEnabled()) { st = System.currentTimeMillis(); } visitMetaEntry(visitor, r); if (LOG.isTraceEnabled()) { long et = System.currentTimeMillis(); LOG.trace("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st)); } } else if (isDebugEnabled) { LOG.debug("NULL result from meta - ignoring but this is strange."); } return true; } });
public String dumpReplicationSummary() { StringBuilder sb = new StringBuilder(); if (!deletedQueues.isEmpty()) { sb.append("Found " + deletedQueues.size() + " deleted queues" + ", run hbck -fixReplication in order to remove the deleted replication queues\n"); for (String deletedQueue : deletedQueues) { sb.append(" " + deletedQueue + "\n"); } } if (!deadRegionServers.isEmpty()) { sb.append("Found " + deadRegionServers.size() + " dead regionservers" + ", restart one regionserver to transfer the queues of dead regionservers\n"); for (String deadRs : deadRegionServers) { sb.append(" " + deadRs + "\n"); } } if (!peersQueueSize.isEmpty()) { sb.append("Dumping all peers's number of WALs in replication queue\n"); for (Map.Entry<String, Long> entry : peersQueueSize.asMap().entrySet()) { sb.append(" PeerId: " + entry.getKey() + " , sizeOfLogQueue: " + entry.getValue() + "\n"); } } sb.append(" Total size of WALs on HDFS: " + StringUtils.humanSize(totalSizeOfWALs) + "\n"); if (numWalsNotFound > 0) { sb.append(" ERROR: There are " + numWalsNotFound + " WALs not found!!!\n"); } return sb.toString(); }
private void acceptPlan(final HashMap<RegionInfo, RegionStateNode> regions, final Map<ServerName, List<RegionInfo>> plan) throws HBaseIOException { final ProcedureEvent<?>[] events = new ProcedureEvent[regions.size()]; final long st = System.currentTimeMillis(); if (plan == null) { throw new HBaseIOException("unable to compute plans for regions=" + regions.size()); } if (plan.isEmpty()) return; int evcount = 0; for (Map.Entry<ServerName, List<RegionInfo>> entry: plan.entrySet()) { final ServerName server = entry.getKey(); for (RegionInfo hri: entry.getValue()) { final RegionStateNode regionNode = regions.get(hri); regionNode.setRegionLocation(server); events[evcount++] = regionNode.getProcedureEvent(); } } ProcedureEvent.wakeEvents(getProcedureScheduler(), events); final long et = System.currentTimeMillis(); if (LOG.isTraceEnabled()) { LOG.trace("ASSIGN ACCEPT " + events.length + " -> " + StringUtils.humanTimeDiff(et - st)); } }
protected long syncSlots(final FSDataOutputStream stream, final ByteSlot[] slots, final int offset, final int count) throws IOException { long totalSynced = 0; for (int i = 0; i < count; ++i) { final ByteSlot data = slots[offset + i]; data.writeTo(stream); totalSynced += data.size(); } syncStream(stream); sendPostSyncSignal(); if (LOG.isTraceEnabled()) { LOG.trace("Sync slots=" + count + '/' + syncMaxSlot + ", flushed=" + StringUtils.humanSize(totalSynced)); } return totalSynced; }
@Test public void testPerfSingleThread() { final TableName TABLE_NAME = TableName.valueOf("testPerf"); final int NRUNS = 1 * 1000000; // 1M final RegionStates stateMap = new RegionStates(); long st = System.currentTimeMillis(); for (int i = 0; i < NRUNS; ++i) { stateMap.createRegionStateNode(createRegionInfo(TABLE_NAME, i)); } long et = System.currentTimeMillis(); LOG.info(String.format("PERF SingleThread: %s %s/sec", StringUtils.humanTimeDiff(et - st), StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f)))); } }
et = System.nanoTime(); LOG.info("Recovered {} lease in {}", store.getClass().getSimpleName(), StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st))); et = System.nanoTime(); LOG.info("Loaded {} in {}", store.getClass().getSimpleName(), StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));
private String formatQueue(ServerName regionserver, ReplicationQueueStorage queueStorage, ReplicationQueueInfo queueInfo, String queueId, List<String> wals, boolean isDeleted, boolean hdfs) throws Exception { StringBuilder sb = new StringBuilder(); List<ServerName> deadServers; sb.append("Dumping replication queue info for RegionServer: [" + regionserver + "]" + "\n"); sb.append(" Queue znode: " + queueId + "\n"); sb.append(" PeerID: " + queueInfo.getPeerId() + "\n"); sb.append(" Recovered: " + queueInfo.isQueueRecovered() + "\n"); deadServers = queueInfo.getDeadRegionServers(); if (deadServers.isEmpty()) { sb.append(" No dead RegionServers found in this queue." + "\n"); } else { sb.append(" Dead RegionServers: " + deadServers + "\n"); } sb.append(" Was deleted: " + isDeleted + "\n"); sb.append(" Number of WALs in replication queue: " + wals.size() + "\n"); peersQueueSize.addAndGet(queueInfo.getPeerId(), wals.size()); for (String wal : wals) { long position = queueStorage.getWALPosition(regionserver, queueInfo.getPeerId(), wal); sb.append(" Replication position for " + wal + ": " + (position > 0 ? position : "0" + " (not started or nothing to replicate)") + "\n"); } if (hdfs) { FileSystem fs = FileSystem.get(getConf()); sb.append(" Total size of WALs on HDFS for this queue: " + StringUtils.humanSize(getTotalWALSize(fs, wals, regionserver)) + "\n"); } return sb.toString(); }
private void assertMandatoryParameterNotEmpty(String paramValue, String fieldName) { if (StringUtils.isEmpty(paramValue)) { throw new IllegalArgumentException(fieldName + " should be provided."); } }
long et = System.currentTimeMillis(); LOG.info(String.format("PERF STATEMAP INSERT: %s %s/sec", StringUtils.humanTimeDiff(et - st), StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f)))); et = System.currentTimeMillis(); LOG.info(String.format("PERF STATEMAP GET: %s %s/sec", StringUtils.humanTimeDiff(et - st), StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f))));
public void joinCluster() throws IOException { long startTime = System.nanoTime(); LOG.debug("Joining cluster..."); // Scan hbase:meta to build list of existing regions, servers, and assignment. // hbase:meta is online now or will be. Inside loadMeta, we keep trying. Can't make progress // w/o meta. loadMeta(); while (master.getServerManager().countOfRegionServers() < 1) { LOG.info("Waiting for RegionServers to join; current count={}", master.getServerManager().countOfRegionServers()); Threads.sleep(250); } LOG.info("Number of RegionServers={}", master.getServerManager().countOfRegionServers()); // Start the RIT chore master.getMasterProcedureExecutor().addChore(this.ritChore); long costMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); LOG.info("Joined the cluster in {}", StringUtils.humanTimeDiff(costMs)); }
@Override protected boolean shouldSplit() { boolean force = region.shouldForceSplit(); boolean foundABigStore = false; // Get count of regions that have the same common table as this.region int tableRegionsCount = getCountOfCommonTableRegions(); // Get size to check long sizeToCheck = getSizeToCheck(tableRegionsCount); for (HStore store : region.getStores()) { // If any of the stores is unable to split (eg they contain reference files) // then don't split if (!store.canSplit()) { return false; } // Mark if any store is big enough long size = store.getSize(); if (size > sizeToCheck) { LOG.debug("ShouldSplit because " + store.getColumnFamilyName() + " size=" + StringUtils.humanSize(size) + ", sizeToCheck=" + StringUtils.humanSize(sizeToCheck) + ", regionsWithCommonTable=" + tableRegionsCount); foundABigStore = true; } } return foundABigStore || force; }
@VisibleForTesting public void setOwner(String owner) { this.owner = StringUtils.isEmpty(owner) ? null : owner; }
final float addBackTimeSec = addBackTime / 1000.0f; System.out.println("******************************************"); System.out.println("Time - addBack : " + StringUtils.humanTimeDiff(addBackTime)); System.out.println("Ops/sec - addBack : " + StringUtils.humanSize(numOps / addBackTimeSec)); System.out.println("Time - poll : " + StringUtils.humanTimeDiff(pollTime)); System.out.println("Ops/sec - poll : " + StringUtils.humanSize(numOps / pollTimeSec)); System.out.println("Num Operations : " + numOps); System.out.println();
/** * Called by the ProcedureExecutor when the timeout set by setTimeout() is expired. * <p/> * Another usage for this method is to implement retrying. A procedure can set the state to * {@code WAITING_TIMEOUT} by calling {@code setState} method, and throw a * {@link ProcedureSuspendedException} to halt the execution of the procedure, and do not forget a * call {@link #setTimeout(int)} method to set the timeout. And you should also override this * method to wake up the procedure, and also return false to tell the ProcedureExecutor that the * timeout event has been handled. * @return true to let the framework handle the timeout as abort, false in case the procedure * handled the timeout itself. */ protected synchronized boolean setTimeoutFailure(TEnvironment env) { if (state == ProcedureState.WAITING_TIMEOUT) { long timeDiff = EnvironmentEdgeManager.currentTime() - lastUpdate; setFailure("ProcedureExecutor", new TimeoutIOException("Operation timed out after " + StringUtils.humanTimeDiff(timeDiff))); return true; } return false; }
org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(StringUtils.humanSize(entry.getValue().getFileSize())), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(StringUtils.humanSize(entry.getValue().getCurrentPosition())), jamonWriter);
@VisibleForTesting @InterfaceAudience.Private public void setOwner(final String owner) { this.owner = StringUtils.isEmpty(owner) ? null : owner; }
float rollTsSec = getMillisFromLastRoll() / 1000.0f; LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)", StringUtils.humanSize(totalSynced.get()), StringUtils.humanSize(totalSynced.get() / rollTsSec))); if (LOG.isTraceEnabled() && (syncWaitMs > 10 || slotIndex < syncMaxSlot)) { LOG.trace(String.format("Sync wait %s, slotIndex=%s , totalSynced=%s (%s/sec)", StringUtils.humanTimeDiff(syncWaitMs), slotIndex, StringUtils.humanSize(totalSyncedToStore), StringUtils.humanSize(syncedPerSec)));
org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(StringUtils.humanTimeDiff(entry.getValue().getAgeOfLastShippedOp())), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(entry.getValue().getReplicationDelay() == Long.MAX_VALUE ? "UNKNOWN" : StringUtils.humanTimeDiff(entry.getValue().getReplicationDelay())), jamonWriter);