@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); }
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); } }
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); } }
public BulkIndexer(EsClient client, IndexType indexType, Size size, IndexingListener indexingListener) { this.client = client; this.indexType = indexType; this.sizeHandler = size.createHandler(Runtime2.INSTANCE); this.indexingListener = indexingListener; BulkProcessorListener bulkProcessorListener = new BulkProcessorListener(); this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener) .setBackoffPolicy(BackoffPolicy.exponentialBackoff()) .setBulkSize(FLUSH_BYTE_SIZE) .setBulkActions(FLUSH_ACTIONS) .setConcurrentRequests(sizeHandler.getConcurrentRequests()) .build(); }
.setConcurrentRequests(10) .setBackoffPolicy( BackoffPolicy.exponentialBackoff( TimeValue.timeValueMillis(100), 3)) .build();
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
/** * Creates an new exponential backoff policy with a default configuration of 50 ms initial wait period and 8 retries taking * roughly 5.1 seconds in total. * * @return A backoff policy with an exponential increase in wait time for retries. The returned instance is thread safe but each * iterator created from it should only be used by a single thread. */ public static BackoffPolicy exponentialBackoff() { return exponentialBackoff(TimeValue.timeValueMillis(50), 8); }
/** * Get the backoff policy for use with retries. */ BackoffPolicy buildBackoffPolicy() { return exponentialBackoff(mainRequest.getRetryBackoffInitialTime(), mainRequest.getMaxRetries()); }
/** * Build the backoff policy for use with retries. */ BackoffPolicy backoffPolicy() { return exponentialBackoff(mainRequest.getRetryBackoffInitialTime(), mainRequest.getMaxRetries()); }
/** * Build the backoff policy for use with retries. */ BackoffPolicy backoffPolicy() { return exponentialBackoff(mainRequest.getRetryBackoffInitialTime(), mainRequest.getMaxRetries()); }
private BulkProcessor build(final RestHighLevelClient client) { logger.trace("Bulk processor name: [{}] bulkActions: [{}], bulkSize: [{}], flush interval time: [{}]," + " concurrent Request: [{}], backoffPolicyTimeInterval: [{}], backoffPolicyRetries: [{}] ", new Object[]{bulkProcessorName, bulkActions, bulkSize, flushIntervalTime, concurrentRequest, backoffPolicyTimeInterval, backoffPolicyRetries}); BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer = (request, bulkListener) -> client .bulkAsync(request, RequestOptions.DEFAULT, bulkListener); return BulkProcessor.builder(bulkConsumer, getListener()) .setBulkActions(bulkActions) .setBulkSize(bulkSize) .setFlushInterval(flushIntervalTime) .setConcurrentRequests(concurrentRequest) .setBackoffPolicy(BackoffPolicy.exponentialBackoff( Util.getTimeValue(backoffPolicyTimeInterval, DEFAULT_ES_BACKOFF_POLICY_START_DELAY), backoffPolicyRetries)) .build(); }
@Override public void configureBulkProcessorBackoff( BulkProcessor.Builder builder, @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { BackoffPolicy backoffPolicy; if (flushBackoffPolicy != null) { switch (flushBackoffPolicy.getBackoffType()) { case CONSTANT: backoffPolicy = BackoffPolicy.constantBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); break; case EXPONENTIAL: default: backoffPolicy = BackoffPolicy.exponentialBackoff( new TimeValue(flushBackoffPolicy.getDelayMillis()), flushBackoffPolicy.getMaxRetryCount()); } } else { backoffPolicy = BackoffPolicy.noBackoff(); } builder.setBackoffPolicy(backoffPolicy); } }
@Bean public BulkProcessor bulkProcessor() throws UnknownHostException { Settings settings = Settings.EMPTY; if(!EmptyUtils.isEmpty(clusterName)){ settings = Settings.builder() .put("cluster.name", clusterName) .build(); } TransportClient transportClient = new PreBuiltTransportClient(settings) .addTransportAddress(new TransportAddress(InetAddress.getByName(host), port)); return BulkProcessor.builder(transportClient, new BulkProcessor.Listener() { @Override public void beforeBulk(long l, BulkRequest bulkRequest) { } @Override public void afterBulk(long l, BulkRequest bulkRequest, BulkResponse bulkResponse) { } @Override public void afterBulk(long l, BulkRequest bulkRequest, Throwable throwable) { logger.error("{} data bulk failed,reason :{}", bulkRequest.numberOfActions(), throwable); } }).setBulkActions(1000)//分批,每10000条请求当成一批请求。默认值为1000 .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB))//每次5MB,刷新一次bulk。默认为5m .setFlushInterval(TimeValue.timeValueSeconds(5))//每5秒一定执行,不管已经队列积累了多少。默认不设置这个值 .setConcurrentRequests(1)//设置并发请求数,如果是0,那表示只有一个请求就可以被执行,如果为1,则可以积累并被执行。默认为1. .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 3))//这里有个backoff策略,最初等待100ms,然后按照指数增长并重试3次。每当一个或者多个bulk请求失败,并出现EsRejectedExecutionException异常时.就会尝试重试。这个异常表示用于处理请求的可用计算资源太少。如果要禁用这个backoff策略,需要用backoff.nobackoff()。 .build(); }
public BulkIndexer(EsClient client, IndexType indexType, Size size, IndexingListener indexingListener) { this.client = client; this.indexType = indexType; this.sizeHandler = size.createHandler(Runtime2.INSTANCE); this.indexingListener = indexingListener; BulkProcessorListener bulkProcessorListener = new BulkProcessorListener(); this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener) .setBackoffPolicy(BackoffPolicy.exponentialBackoff()) .setBulkSize(FLUSH_BYTE_SIZE) .setBulkActions(FLUSH_ACTIONS) .setConcurrentRequests(sizeHandler.getConcurrentRequests()) .build(); }
@Override public void init(ComputationContext context) { super.init(context); // note that we don't use setFlushInterval because this is done by our timer continueOnFailure = context.getPolicy().continueOnFailure(); long backoffDelayMs = context.getPolicy().getRetryPolicy().getDelay().toMillis(); int retries = context.getPolicy().getRetryPolicy().getMaxRetries(); bulkProcessor = getESClient().bulkProcessorBuilder(this) .setConcurrentRequests(0) .setBulkSize(new ByteSizeValue(esBulkSize, ByteSizeUnit.BYTES)) .setBulkActions(esBulkActions) .setBackoffPolicy(BackoffPolicy.exponentialBackoff( TimeValue.timeValueMillis(backoffDelayMs), retries)) .build(); codec = BulkCodecs.getDataBucketCodec(); context.setTimer("flush", System.currentTimeMillis() + flushIntervalMs); }