append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData); if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) { commitBulkRequest(bulkRequestBuilderTmp); bulkRequestBuilderTmp = transportClient.prepareBulk(); if (bulkRequestBuilder.numberOfActions() > 0) { commitBulkRequest(bulkRequestBuilderTmp);
if (bulkRequestBuilder.numberOfActions() % mapping.getCommitBatch() == 0 && bulkRequestBuilder.numberOfActions() > 0) { long esBatchBegin = System.currentTimeMillis(); BulkResponse rp = bulkRequestBuilder.execute().actionGet(); (System.currentTimeMillis() - batchBegin), (System.currentTimeMillis() - esBatchBegin), bulkRequestBuilder.numberOfActions(), mapping.get_index()); if (bulkRequestBuilder.numberOfActions() > 0) { long esBatchBegin = System.currentTimeMillis(); BulkResponse rp = bulkRequestBuilder.execute().actionGet(); (System.currentTimeMillis() - batchBegin), (System.currentTimeMillis() - esBatchBegin), bulkRequestBuilder.numberOfActions(), mapping.get_index());
/** * 批量提交 * * @param bulkRequestBuilder * @return */ private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { if (bulkRequestBuilder.numberOfActions() > 0) { BulkResponse response = bulkRequestBuilder.execute().actionGet(); if (response.hasFailures()) { for (BulkItemResponse itemResponse : response.getItems()) { if (!itemResponse.isFailed()) { continue; } if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.warn(itemResponse.getFailureMessage()); } else { logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); } } } return !response.hasFailures(); } return true; }
if ( bulkRequest.numberOfActions() == 0 ) { return;
public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { Object[] rowData = getRow(); if ( rowData == null ) { if ( currentRequest != null && currentRequest.numberOfActions() > 0 ) { // didn't fill a whole batch processBatch( false ); } setOutputDone(); return false; } if ( first ) { first = false; setupData(); currentRequest = client.prepareBulk(); requestsBuffer = new ArrayList<IndexRequestBuilder>( this.batchSize ); initFieldIndexes(); } try { data.inputRowBuffer[data.nextBufferRowIdx++] = rowData; return indexRow( data.inputRowMeta, rowData ) || !stopOnError; } catch ( KettleStepException e ) { throw e; } catch ( Exception e ) { rejectAllRows( e.getLocalizedMessage() ); String msg = BaseMessages.getString( PKG, "ElasticSearchBulk.Log.Exception", e.getLocalizedMessage() ); logError( msg ); throw new KettleStepException( msg, e ); } }
requestsBuffer.add( requestBuilder ); if ( currentRequest.numberOfActions() >= batchSize ) { return processBatch( true ); } else {
requestsBuffer.clear(); } else { // have to assume all failed numberOfErrors += currentRequest.numberOfActions(); setErrors( numberOfErrors );
if (bulk.numberOfActions() > 0) { final BulkResponse response = bulk.execute().actionGet(); if (response.hasFailures()) {
int linesOK = currentRequest.numberOfActions() - errorsInBatch;
public int size() { if (builder == null) { return 0; } return builder.numberOfActions(); }
public int numOfPercolateRules() { return this.bulkBuilder.numberOfActions(); }
private void sendBulkRequestIfMoreThan(int size) { totalBulkItems.incrementAndGet(); if (currentRequest.numberOfActions() > size) { long startTime = System.currentTimeMillis(); BulkResponse response = currentRequest.execute().actionGet(); totalBulkTime.addAndGet(System.currentTimeMillis() - startTime); if (randgen.nextDouble() < 0.1) { LOG.info("Indexed [" + totalBulkItems.get() + "] in [" + (totalBulkTime.get()/1000) + "s] of indexing"+"[" + ((System.currentTimeMillis() - runStartTime)/1000) + "s] of wall clock"+" for ["+ (float)(1000.0*totalBulkItems.get())/(System.currentTimeMillis() - runStartTime) + "rec/s]"); } currentRequest = client.prepareBulk(); } } }
private static void test() { for (int i = 0; i < 10; i++) { Map<String, Object> json = new HashMap<String, Object>(); json.put("field", "test"); addUpdateBuilderToBulk(client.prepareUpdate(Config.indexName, Config.typeName, String.valueOf(i)).setUpsert(json)); } System.out.println(bulkRequestBuilder.numberOfActions()); }
private boolean saveToEs(BulkRequestBuilder bulkRequestBuilder){ LOGGER.debug("Inserting {} keys in ES[{}]", bulkRequestBuilder.numberOfActions(), this.indexName); try{ bulkRequestBuilder.execute().addListener(new Runnable() { public void run() { LOGGER.debug("Processing Done!!"); } }); } catch(Exception e){ LOGGER.warn("{} had an Exception in persisting: {}", this.indexName, e); } return false; }
@Override public void end() throws IOException { if (bulkRequest.numberOfActions() > 0) { BulkResponse response = bulkRequest.execute().actionGet(); if (response.hasFailures()) { throw new IOException(response.buildFailureMessage()); } } } }
/** * 判断缓存池是否已满,批量提交 * * @param threshold */ private static void bulkRequest(int threshold) { if (bulkRequestBuilder.numberOfActions() > threshold) { BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet(); if (!bulkResponse.hasFailures()) { bulkRequestBuilder = client.prepareBulk(); } } }
BulkRequestBuilder br = null; /// some looping construct while (rs.next()) { if (br != null && br.numberOfActions() >= batchSize) { processResponse(br.execute().get()); br = null; } if (br == null) { br = client.prepareBulk(); } // do whatever to create IndexRequest br.add(ir); } if (br != null) { processResponse(br.execute().get()); }
protected void executeBulkRequest(BulkRequestBuilder bulkRequest) { if (bulkRequest.numberOfActions() == 0) return; BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (!bulkResponse.hasFailures()) return; for (BulkItemResponse response : bulkResponse) { if (!response.isFailed()) continue; LOG.warning(String.format("Unable to save Entity %s in %s/%s, cause: %s", response.getId(), response.getIndex(), response.getType(), response.getFailureMessage())); } }
private void queueMessage(String message){ try { if(logger.isDebugEnabled()) logger.debug("About to add a message..."); byte[] data = message.getBytes(); currentRequest.add(data, 0, data.length, false); if(logger.isDebugEnabled()) logger.debug("Current size" + currentRequest.numberOfActions()); processBulkIfNeeded(false); } catch (Exception e){ logger.error("Unable to build request"); } }
private boolean saveToEs(BulkRequestBuilder bulk) { logger.info("Inserting {} keys in ES", bulk.numberOfActions()); try { bulk.execute().addListener(new Runnable() { @Override public void run() { logger.info("Processing done!"); } }); } catch (Exception e) { logger.error("failed to execute bulk", e); return false; } return true; } }