private FSDataOutputStream createFileWithRetries(final FileSystem fs, final Path hbckLockFilePath, final FsPermission defaultPerms) throws IOException { IOException exception = null; do { try { return FSUtils.create(fs, hbckLockFilePath, defaultPerms, false); } catch (IOException ioe) { LOG.info("Failed to create lock file " + hbckLockFilePath.getName() + ", try=" + (retryCounter.getAttemptTimes() + 1) + " of " + retryCounter.getMaxAttempts()); LOG.debug("Failed to create lock file " + hbckLockFilePath.getName(), ioe); try { exception = ioe; retryCounter.sleepUntilNextRetry(); } catch (InterruptedException ie) { throw (InterruptedIOException) new InterruptedIOException( "Can't create lock file " + hbckLockFilePath.getName()) .initCause(ie); } } } while (retryCounter.shouldRetry()); throw exception; } }
int numRetries = getConfiguration().getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); RetryCounter retrier = new RetryCounter(numRetries+1, (int)pause, TimeUnit.MICROSECONDS); while(retrier.shouldRetry()) { int index = getMiniHBaseCluster().getServerWith(firstRegionName); if (index != -1) { retrier.sleepUntilNextRetry();
/** * Sleep for a back off time as supplied by the backoff policy, and increases the attempts */ public void sleepUntilNextRetry() throws InterruptedException { int attempts = getAttemptTimes(); long sleepTime = getBackoffTime(); LOG.trace("Sleeping {} ms before retry #{}...", sleepTime, attempts); retryConfig.getTimeUnit().sleep(sleepTime); useRetry(); }
public long getBackoffTimeAndIncrementAttempts() { long backoffTime = getBackoffTime(); useRetry(); return backoffTime; } }
private void retryOrThrow(RetryCounter retryCounter, KeeperException e, String opName) throws KeeperException { if (!retryCounter.shouldRetry()) { LOG.error("ZooKeeper " + opName + " failed after " + retryCounter.getMaxAttempts() + " attempts"); throw e; } LOG.debug("Retry, connectivity issue (JVM Pause?); quorum=" + quorumServers + "," + "exception=" + e); }
/** * exists is an idempotent operation. Retry before throwing exception * @return A Stat instance */ public Stat exists(String path, Watcher watcher) throws KeeperException, InterruptedException { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { return zk.exists(path, watcher); } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: case SESSIONEXPIRED: case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "exists"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); retryCounter.useRetry(); } }
private Pair<Integer, String> execWithRetries(String hostname, ServiceType service, String... cmd) throws IOException { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { return exec(hostname, service, cmd); } catch (IOException e) { retryOrThrow(retryCounter, e, hostname, cmd); } try { retryCounter.sleepUntilNextRetry(); } catch (InterruptedException ex) { // ignore LOG.warn("Sleep Interrupted:" + ex); } } }
@Test public void testBasics() throws InterruptedException { int maxAttempts = 10; RetryCounterFactory factory = new RetryCounterFactory(maxAttempts, 10, 1000); RetryCounter retryCounter = factory.create(); while (retryCounter.shouldRetry()) { LOG.info("Attempt={}, backoffTime={}", retryCounter.getAttemptTimes(), retryCounter.getBackoffTime()); retryCounter.sleepUntilNextRetry(); } assertTrue(retryCounter.getAttemptTimes() == maxAttempts); } }
private <E extends Exception> void retryOrThrow(RetryCounter retryCounter, E ex, String hostname, String[] cmd) throws E { if (retryCounter.shouldRetry()) { LOG.warn("Remote command: " + StringUtils.join(cmd, " ") + " , hostname:" + hostname + " failed at attempt " + retryCounter.getAttemptTimes() + ". Retrying until maxAttempts: " + retryCounter.getMaxAttempts() + ". Exception: " + ex.getMessage()); return; } throw ex; }
if (!counter.shouldRetry()) { throw new IOException("Cannot cause primary to flush or drop a wal marker after " + "retries. Failing opening of this region replica " counter.sleepUntilNextRetry(); } catch (InterruptedException e) { throw new InterruptedIOException(e.getMessage());
/** * Sleep for a back off time as supplied by the backoff policy, and increases the attempts * @throws InterruptedException */ public void sleepUntilNextRetry() throws InterruptedException { int attempts = getAttemptTimes(); long sleepTime = retryConfig.backoffPolicy.getBackoffTime(retryConfig, attempts); if (LOG.isTraceEnabled()) { LOG.trace("Sleeping " + sleepTime + "ms before retry #" + attempts + "..."); } retryConfig.getTimeUnit().sleep(sleepTime); useRetry(); }
private void retryOrThrow(RetryCounter retryCounter, KeeperException e, String opName) throws KeeperException { LOG.warn("Possibly transient ZooKeeper exception: " + e); if (!retryCounter.shouldRetry()) { LOG.error("ZooKeeper " + opName + " failed after " + retryCounter.getMaxRetries() + " retries"); throw e; } }
public long getBackoffTime() { return this.retryConfig.backoffPolicy.getBackoffTime(this.retryConfig, getAttemptTimes()); }
public RetryCounter create() { return new RetryCounter(retryConfig); } }
/** * getChildren is an idempotent operation. Retry before throwing exception * @return List of children znodes */ public List<String> getChildren(String path, Watcher watcher) throws KeeperException, InterruptedException { RetryCounter retryCounter = retryCounterFactory.create(); while (true) { try { return zk.getChildren(path, watcher); } catch (KeeperException e) { switch (e.code()) { case CONNECTIONLOSS: case SESSIONEXPIRED: case OPERATIONTIMEOUT: retryOrThrow(retryCounter, e, "getChildren"); break; default: throw e; } } retryCounter.sleepUntilNextRetry(); retryCounter.useRetry(); } }
retryCounter.sleepUntilNextRetry(); isRetry = true;
@Test public void testBasics() throws InterruptedException { int maxAttempts = 10; RetryCounterFactory factory = new RetryCounterFactory(maxAttempts, 10, 1000); RetryCounter retryCounter = factory.create(); while (retryCounter.shouldRetry()) { LOG.info("Attempt={}, backoffTime={}", retryCounter.getAttemptTimes(), retryCounter.getBackoffTime()); retryCounter.sleepUntilNextRetry(); } assertTrue(retryCounter.getAttemptTimes() == maxAttempts); } }
private <E extends Exception> void retryOrThrow(RetryCounter retryCounter, E ex, String hostname, String[] cmd) throws E { if (retryCounter.shouldRetry()) { LOG.warn("Remote command: " + StringUtils.join(cmd, " ") + " , hostname:" + hostname + " failed at attempt " + retryCounter.getAttemptTimes() + ". Retrying until maxAttempts: " + retryCounter.getMaxAttempts() + ". Exception: " + ex.getMessage()); return; } throw ex; }
private void retryOrThrow(RetryCounter retryCounter, KeeperException e, String opName) throws KeeperException { LOG.warn("Possibly transient ZooKeeper, quorum=" + quorumServers + ", exception=" + e); if (!retryCounter.shouldRetry()) { LOG.error("ZooKeeper " + opName + " failed after " + retryCounter.getMaxAttempts() + " attempts"); throw e; } }
if (!counter.shouldRetry()) { throw new IOException("Cannot cause primary to flush or drop a wal marker after " + "retries. Failing opening of this region replica " counter.sleepUntilNextRetry(); } catch (InterruptedException e) { throw new InterruptedIOException(e.getMessage());