public BulkIndexer(EsClient client, IndexType indexType, Size size, IndexingListener indexingListener) { this.client = client; this.indexType = indexType; this.sizeHandler = size.createHandler(Runtime2.INSTANCE); this.indexingListener = indexingListener; BulkProcessorListener bulkProcessorListener = new BulkProcessorListener(); this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener) .setBackoffPolicy(BackoffPolicy.exponentialBackoff()) .setBulkSize(FLUSH_BYTE_SIZE) .setBulkActions(FLUSH_ACTIONS) .setConcurrentRequests(sizeHandler.getConcurrentRequests()) .build(); }
RetryHandler(BackoffPolicy backoffPolicy, BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, ActionListener<BulkResponse> listener, Scheduler scheduler) { this.backoff = backoffPolicy.iterator(); this.consumer = consumer; this.listener = listener; this.scheduler = scheduler; // in contrast to System.currentTimeMillis(), nanoTime() uses a monotonic clock under the hood this.startTimestampNanos = System.nanoTime(); }
/** * Creates an new constant backoff policy with the provided configuration. * * @param delay The delay defines how long to wait between retry attempts. Must not be null. * Must be <= <code>Integer.MAX_VALUE</code> ms. * @param maxNumberOfRetries The maximum number of retries. Must be a non-negative number. * @return A backoff policy with a constant wait time between retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy constantBackoff(TimeValue delay, int maxNumberOfRetries) { return new ConstantBackoff(checkDelay(delay), maxNumberOfRetries); }
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
public static BackoffPolicyBuilder constantBackoff(TimeValue delay) { return new BackoffPolicyBuilder(BackoffPolicy.constantBackoff(delay, Integer.MAX_VALUE)); }
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); } }
.setConcurrentRequests(10) .setBackoffPolicy( BackoffPolicy.exponentialBackoff( TimeValue.timeValueMillis(100), 3)) .build();
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); } }
@Override public Iterator<TimeValue> iterator() { return new WrappedBackoffIterator(delegate.iterator(), onBackoff); } }
/** * Creates an new exponential backoff policy with the provided configuration. * * @param initialDelay The initial delay defines how long to wait for the first retry attempt. Must not be null. * Must be <= <code>Integer.MAX_VALUE</code> ms. * @param maxNumberOfRetries The maximum number of retries. Must be a non-negative number. * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff(TimeValue initialDelay, int maxNumberOfRetries) { return new ExponentialBackoff((int) checkDelay(initialDelay).millis(), maxNumberOfRetries); }
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); } }
private void doStoreResult(TaskResult taskResult, ActionListener<Void> listener) { IndexRequestBuilder index = client.prepareIndex(TASK_INDEX, TASK_TYPE, taskResult.getTask().getTaskId().toString()); try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) { taskResult.toXContent(builder, ToXContent.EMPTY_PARAMS); index.setSource(builder); } catch (IOException e) { throw new ElasticsearchException("Couldn't convert task result to XContent for [{}]", e, taskResult.getTask()); } doStoreResult(STORE_BACKOFF_POLICY.iterator(), index, listener); }
/** * Creates an new constant backoff policy with the provided configuration. * * @param delay The delay defines how long to wait between retry attempts. Must not be null. * Must be <= <code>Integer.MAX_VALUE</code> ms. * @param maxNumberOfRetries The maximum number of retries. Must be a non-negative number. * @return A backoff policy with a constant wait time between retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy constantBackoff(TimeValue delay, int maxNumberOfRetries) { return new ConstantBackoff(checkDelay(delay), maxNumberOfRetries); }
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
RetryHandler(BackoffPolicy backoffPolicy, BiConsumer<BulkRequest, ActionListener<BulkResponse>> consumer, ActionListener<BulkResponse> listener, Scheduler scheduler) { this.backoff = backoffPolicy.iterator(); this.consumer = consumer; this.listener = listener; this.scheduler = scheduler; // in contrast to System.currentTimeMillis(), nanoTime() uses a monotonic clock under the hood this.startTimestampNanos = System.nanoTime(); }
/** * Creates an new constant backoff policy with the provided configuration. * * @param delay The delay defines how long to wait between retry attempts. Must not be null. * Must be <= <code>Integer.MAX_VALUE</code> ms. * @param maxNumberOfRetries The maximum number of retries. Must be a non-negative number. * @return A backoff policy with a constant wait time between retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy constantBackoff(TimeValue delay, int maxNumberOfRetries) { return new ConstantBackoff(checkDelay(delay), maxNumberOfRetries); }
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
@Override public Iterator<TimeValue> iterator() { return new WrappedBackoffIterator(delegate.iterator(), onBackoff); } }
/** * Creates an new constant backoff policy with the provided configuration. * * @param delay The delay defines how long to wait between retry attempts. Must not be null. * Must be <= <code>Integer.MAX_VALUE</code> ms. * @param maxNumberOfRetries The maximum number of retries. Must be a non-negative number. * @return A backoff policy with a constant wait time between retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy constantBackoff(TimeValue delay, int maxNumberOfRetries) { return new ConstantBackoff(checkDelay(delay), maxNumberOfRetries); }