@Override public String getNewestOffset(SystemStreamPartition ssp, Integer maxRetries) { LOG.info("Fetching newest offset for: {}", ssp); ExponentialSleepStrategy strategy = new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS); Function1<ExponentialSleepStrategy.RetryLoop, String> fetchNewestOffset = new AbstractFunction1<ExponentialSleepStrategy.RetryLoop, String>() { @Override public String apply(ExponentialSleepStrategy.RetryLoop loop) { String result = fetchNewestOffset(ssp); loop.done(); return result; } }; String offset = strategy.run(fetchNewestOffset, new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() { @Override public BoxedUnit apply(Exception exception, ExponentialSleepStrategy.RetryLoop loop) { if (loop.sleepCount() < maxRetries) { LOG.warn(String.format("Fetching newest offset for: %s threw an exception. Retrying.", ssp), exception); } else { LOG.error(String.format("Fetching newest offset for: %s threw an exception.", ssp), exception); loop.done(); throw new SamzaException("Exception while trying to get newest offset", exception); } return null; } }).get(); return offset; }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return getSystemStreamMetadata(streamNames, new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS)); }
retryBackoff.run(fetchMetadataOperation, onExceptionRetryOperation).getOrElse(fallbackOperation); return result;
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return getSystemStreamMetadata(streamNames, new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS)); }
retryBackoff.run(fetchMetadataOperation, onExceptionRetryOperation).getOrElse(fallbackOperation); return result;
@Override public String getNewestOffset(SystemStreamPartition ssp, Integer maxRetries) { LOG.info("Fetching newest offset for: {}", ssp); ExponentialSleepStrategy strategy = new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS); Function1<ExponentialSleepStrategy.RetryLoop, String> fetchNewestOffset = new AbstractFunction1<ExponentialSleepStrategy.RetryLoop, String>() { @Override public String apply(ExponentialSleepStrategy.RetryLoop loop) { String result = fetchNewestOffset(ssp); loop.done(); return result; } }; String offset = strategy.run(fetchNewestOffset, new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() { @Override public BoxedUnit apply(Exception exception, ExponentialSleepStrategy.RetryLoop loop) { if (loop.sleepCount() < maxRetries) { LOG.warn(String.format("Fetching newest offset for: %s threw an exception. Retrying.", ssp), exception); } else { LOG.error(String.format("Fetching newest offset for: %s threw an exception.", ssp), exception); loop.done(); throw new SamzaException("Exception while trying to get newest offset", exception); } return null; } }).get(); return offset; }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return getSystemStreamMetadata(streamNames, new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS)); }
retryBackoff.run(fetchMetadataOperation, onExceptionRetryOperation).getOrElse(fallbackOperation); return result;
ExponentialSleepStrategy strategy = new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS); Map<String, SystemStreamMetadata> result = strategy.run(fetchMetadataOperation, new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() { @Override
/** * get the config for the AM or containers based on the containers' names. * * @return Config the config of this container */ protected Config getConfig() { Config config; try { if (isApplicationMaster) { config = JobModelManager.currentJobModelManager().jobModel().getConfig(); } else { String url = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String response = HttpUtil.read(new URL(url), 30000, new ExponentialSleepStrategy()); config = SamzaObjectMapper.getObjectMapper().readValue(response, JobModel.class).getConfig(); } } catch (IOException e) { throw new SamzaException("can not read the config", e); } // Make system producer drop producer errors for StreamAppender config = new MapConfig(config, ImmutableMap.of(TaskConfig.DROP_PRODUCER_ERRORS(), "true")); return config; }
ExponentialSleepStrategy strategy = new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS); Map<String, SystemStreamMetadata> result = strategy.run(fetchMetadataOperation, new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() { @Override
/** * get the config for the AM or containers based on the containers' names. * * @return Config the config of this container */ protected Config getConfig() { Config config; try { if (isApplicationMaster) { config = JobModelManager.currentJobModelManager().jobModel().getConfig(); } else { String url = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL()); String response = HttpUtil.read(new URL(url), 30000, new ExponentialSleepStrategy()); config = SamzaObjectMapper.getObjectMapper().readValue(response, JobModel.class).getConfig(); } } catch (IOException e) { throw new SamzaException("can not read the config", e); } // Make system producer drop producer errors for StreamAppender config = new MapConfig(config, ImmutableMap.of(TaskConfig.DROP_PRODUCER_ERRORS(), "true")); return config; }
ExponentialSleepStrategy strategy = new ExponentialSleepStrategy(DEFAULT_EXPONENTIAL_SLEEP_BACK_OFF_MULTIPLIER, DEFAULT_EXPONENTIAL_SLEEP_INITIAL_DELAY_MS, DEFAULT_EXPONENTIAL_SLEEP_MAX_DELAY_MS); Map<String, SystemStreamMetadata> result = strategy.run(fetchMetadataOperation, new AbstractFunction2<Exception, ExponentialSleepStrategy.RetryLoop, BoxedUnit>() { @Override
@Test public void testInstantiateProducer() { KafkaSystemProducer ksp = new KafkaSystemProducer("SysName", new ExponentialSleepStrategy(2.0, 200, 10000), new AbstractFunction0<Producer<byte[], byte[]>>() { @Override public Producer<byte[], byte[]> apply() { return new KafkaProducer<>(new HashMap<String, Object>()); } }, new KafkaSystemProducerMetrics("SysName", new MetricsRegistryMap()), new AbstractFunction0<Object>() { @Override public Object apply() { return System.currentTimeMillis(); } }, false); long now = System.currentTimeMillis(); assertTrue((Long)ksp.clock().apply() >= now); } }