/** * Writes to filesystem serialized form of segment descriptor if an existing file exists it will try to replace it. * * @param outputFS filesystem. * @param segment DataSegment object. * @param descriptorPath path. * * @throws IOException in case any IO issues occur. */ public static void writeSegmentDescriptor(final FileSystem outputFS, final DataSegment segment, final Path descriptorPath) throws IOException { final DataPusher descriptorPusher = (DataPusher) RetryProxy.create(DataPusher.class, () -> { if (outputFS.exists(descriptorPath)) { if (!outputFS.delete(descriptorPath, false)) { throw new IOException(String.format("Failed to delete descriptor at [%s]", descriptorPath)); } } try (final OutputStream descriptorOut = outputFS.create(descriptorPath, true, DEFAULT_FS_BUFFER_SIZE)) { JSON_MAPPER.writeValue(descriptorOut, segment); descriptorOut.flush(); } }, RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS)); descriptorPusher.push(); }
effectiveRetryPolicy = RetryPolicies.exponentialBackoffRetry( NUM_RETRIES, SECONDS_BETWEEN_RETRIES,
RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS) ); descriptorPusher.push();
RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS) ); zipPusher.push();
rp = RetryPolicies.exponentialBackoffRetry(Long.SIZE - 2, kerberosMinSecondsBeforeRelogin, TimeUnit.MILLISECONDS);
private static NamenodeProtocol createNNProxyWithNamenodeProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries) throws IOException { NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy( address, conf, ugi, NamenodeProtocolPB.class, 0); if (withRetries) { // create the proxy with retries RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200, TimeUnit.MILLISECONDS); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("getBlocks", timeoutPolicy); methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy); NamenodeProtocol translatorProxy = new NamenodeProtocolTranslatorPB(proxy); return (NamenodeProtocol) RetryProxy.create( NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap); } else { return new NamenodeProtocolTranslatorPB(proxy); } }
protected RetryPolicy createThrottleRetryPolicy(final Configuration conf) { return exponentialBackoffRetry( conf.getInt(S3GUARD_DDB_MAX_RETRIES, S3GUARD_DDB_MAX_RETRIES_DEFAULT), conf.getTimeDuration(S3GUARD_DDB_THROTTLE_RETRY_INTERVAL, S3GUARD_DDB_THROTTLE_RETRY_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); } }
/** * Create the throttling policy. * This will be called from the S3ARetryPolicy constructor, so * subclasses must assume they are not initialized. * @param conf configuration to use. * @return the retry policy for throttling events. */ protected RetryPolicy createThrottleRetryPolicy(final Configuration conf) { return exponentialBackoffRetry( conf.getInt(RETRY_THROTTLE_LIMIT, RETRY_THROTTLE_LIMIT_DEFAULT), conf.getTimeDuration(RETRY_THROTTLE_INTERVAL, RETRY_THROTTLE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); }
private static NamenodeProtocol createNNProxyWithNamenodeProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries) throws IOException { NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy( address, conf, ugi, NamenodeProtocolPB.class); if (withRetries) { // create the proxy with retries RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200, TimeUnit.MILLISECONDS); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("getBlocks", timeoutPolicy); methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy); NamenodeProtocol translatorProxy = new NamenodeProtocolTranslatorPB(proxy); return (NamenodeProtocol) RetryProxy.create( NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap); } else { return new NamenodeProtocolTranslatorPB(proxy); } }
private static NamenodeProtocol createNNProxyWithNamenodeProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries) throws IOException { NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy( address, conf, ugi, NamenodeProtocolPB.class); if (withRetries) { // create the proxy with retries RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200, TimeUnit.MILLISECONDS); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("getBlocks", timeoutPolicy); methodNameToPolicyMap.put("getAccessKeys", timeoutPolicy); NamenodeProtocol translatorProxy = new NamenodeProtocolTranslatorPB(proxy); return (NamenodeProtocol) RetryProxy.create( NamenodeProtocol.class, translatorProxy, methodNameToPolicyMap); } else { return new NamenodeProtocolTranslatorPB(proxy); } }
RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS) ); descriptorPusher.push();
/** * Set retry policy. This is driven by the value of * {@link Constants#S3GUARD_DDB_MAX_RETRIES} with an exponential backoff * between each attempt of {@link Constants#S3GUARD_DDB_THROTTLE_RETRY_INTERVAL} * milliseconds. * @param config configuration for data access */ private void initDataAccessRetries(Configuration config) { batchWriteRetryPolicy = RetryPolicies .exponentialBackoffRetry( config.getInt(S3GUARD_DDB_MAX_RETRIES, S3GUARD_DDB_MAX_RETRIES_DEFAULT), conf.getTimeDuration(S3GUARD_DDB_THROTTLE_RETRY_INTERVAL, S3GUARD_DDB_THROTTLE_RETRY_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); final RetryPolicy throttledRetryRetryPolicy = new S3GuardDataAccessRetryPolicy(config); readOp = new Invoker(throttledRetryRetryPolicy, this::readRetryEvent); writeOp = new Invoker(throttledRetryRetryPolicy, this::writeRetryEvent); }
private static NamenodeProtocol createNamenode(InetSocketAddress nameNodeAddr, Configuration conf) throws IOException { RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry( 5, 200, TimeUnit.MILLISECONDS); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); RetryPolicy methodPolicy = RetryPolicies.retryByException( timeoutPolicy, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("getBlocks", methodPolicy); UserGroupInformation ugi; try { ugi = UnixUserGroupInformation.login(conf); } catch (javax.security.auth.login.LoginException e) { throw new IOException(StringUtils.stringifyException(e)); } return (NamenodeProtocol) RetryProxy.create( NamenodeProtocol.class, RPC.getProxy(NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, ugi, conf, NetUtils.getDefaultSocketFactory(conf)), methodNameToPolicyMap); }
RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS) ); descriptorPusher.push();
private static NamenodeProtocol createNamenode(Configuration conf) throws IOException { InetSocketAddress nameNodeAddr = DataNode.createSocketAddr(FileSystem.getDefaultUri(conf).getAuthority()); RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry( 5, 200, TimeUnit.MILLISECONDS); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); RetryPolicy methodPolicy = RetryPolicies.retryByException( timeoutPolicy, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("getBlocks", methodPolicy); UserGroupInformation ugi; try { ugi = UnixUserGroupInformation.login(conf); } catch (javax.security.auth.login.LoginException e) { throw new IOException(StringUtils.stringifyException(e)); } return (NamenodeProtocol) RetryProxy.create( NamenodeProtocol.class, RPC.getProxy(NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, ugi, conf, NetUtils.getDefaultSocketFactory(conf)), methodNameToPolicyMap); }
RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS) ); zipPusher.push();
public void testExponentialRetry() throws UnreliableException { UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(UnreliableInterface.class, unreliableImpl, exponentialBackoffRetry(5, 1L, TimeUnit.NANOSECONDS)); unreliable.alwaysSucceeds(); unreliable.failsOnceThenSucceeds(); try { unreliable.failsTenTimesThenSucceeds(); fail("Should fail"); } catch (UnreliableException e) { // expected } }
RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS) ); zipPusher.push();
@Test public void testExponentialRetry() throws UnreliableException { UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(UnreliableInterface.class, unreliableImpl, exponentialBackoffRetry(5, 1L, TimeUnit.NANOSECONDS)); unreliable.alwaysSucceeds(); unreliable.failsOnceThenSucceeds(); try { unreliable.failsTenTimesThenSucceeds(); fail("Should fail"); } catch (UnreliableException e) { // expected } }
@Test public void testExponentialRetry() throws UnreliableException { UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(UnreliableInterface.class, unreliableImpl, exponentialBackoffRetry(5, 1L, TimeUnit.NANOSECONDS)); unreliable.alwaysSucceeds(); unreliable.failsOnceThenSucceeds(); try { unreliable.failsTenTimesThenSucceeds(); fail("Should fail"); } catch (UnreliableException e) { // expected } }