private BulkProcessor getBulkProcessor(RestClient.BulkActionResponse... responses) { return new BulkProcessor(mockClientResponses(responses), resource, testSettings); }
private void doWriteToIndex(BytesRef payload) { bulkProcessor.add(payload); payload.reset(); }
@Test public void testBulk00_Empty() throws Exception { BulkProcessor processor = getBulkProcessor(); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(0, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(0, stats.docsAccepted); }
/** * Attempts a flush operation, handling failed documents based on configured error listeners. * @throws EsHadoopException in the event that the bulk operation fails, is aborted, or its errors could not be handled. */ public void flush() { BulkResponse bulk = tryFlush(); if (!bulk.getDocumentErrors().isEmpty()) { int maxErrors = 5; String header = String.format("Could not write all entries for bulk operation [%s/%s]. Error " + "sample (first [%s] error messages):\n", bulk.getDocumentErrors().size(), bulk.getTotalDocs(), maxErrors); StringBuilder message = new StringBuilder(header); int i = 0; for (BulkResponse.BulkError errors : bulk.getDocumentErrors()) { if (i >=maxErrors ) { break; } message.append("\t"); appendError(message, errors.getError()); message.append("\n"); message.append("\t") .append(errors.getDocument().toString()) .append("\n"); i++; } message.append("Bailing out..."); throw new EsHadoopException(message.toString()); } }
@Override public void close() { if (log.isDebugEnabled()) { log.debug("Closing repository and connection to Elasticsearch ..."); } // bail out if closed before if (client == null) { return; } try { if (bulkProcessor != null) { bulkProcessor.close(); // Aggregate stats before discarding them. stats.aggregate(bulkProcessor.stats()); bulkProcessor = null; } if (bulkEntryWriter != null) { bulkEntryWriter.close(); bulkEntryWriter = null; } } finally { client.close(); // Aggregate stats before discarding them. stats.aggregate(client.stats()); client = null; } }
BulkResponse bulkResult = null; boolean trackingArrayExpanded = false; String bulkLoggingID = createDebugTxnID(); initFlushOperation(bulkLoggingID, retryOperation, retries.size(), waitTime); debugLog(bulkLoggingID, "Submitting request"); RestClient.BulkActionResponse bar = restClient.bulk(resource, data); debugLog(bulkLoggingID, "Response received"); totalAttempts++; totalTime += bar.getTimeSpent(); BytesRef newEntry = validateEditedEntry(retryDataBuffer); data.remove(trackingBytesPosition); data.copyFrom(newEntry); debugLog(bulkLoggingID, "Completed. [%d] Original Entries. [%d] Attempts. [%d/%d] Docs Sent. [%d/%d] Docs Skipped. [%d/%d] Docs Aborted.", totalDocs, totalAttempts, debugLog(bulkLoggingID, "Failed. %s", ex.getMessage()); hadWriteErrors = true; throw ex;
public BulkResponse tryFlush() { Assert.isTrue(writeInitialized, "Cannot flush non-initialized write operation"); return bulkProcessor.tryFlush(); }
public void flush() { Assert.isTrue(writeInitialized, "Cannot flush non-initialized write operation"); bulkProcessor.flush(); }
@Override public Stats stats() { Stats copy = new Stats(stats); if (client != null) { // Aggregate stats if it's not already discarded copy.aggregate(client.stats()); } if (bulkProcessor != null) { // Aggregate stats if it's not already discarded copy.aggregate(bulkProcessor.stats()); } return copy; }
private void appendError(StringBuilder message, Throwable exception) { if(exception != null) { message.append(exception); if(exception.getCause() != null) { message.append(';'); appendError(message, exception.getCause()); } } }
/** * Logs flushing messages and performs backoff waiting if there is a wait time for retry. */ private void initFlushOperation(String bulkLoggingID, boolean retryOperation, long retriedDocs, long waitTime) { if (retryOperation) { if (waitTime > 0L) { debugLog(bulkLoggingID, "Retrying [%d] entries after backing off for [%s] ms", retriedDocs, TimeValue.timeValueMillis(waitTime)); try { Thread.sleep(waitTime); } catch (InterruptedException e) { debugLog(bulkLoggingID, "Thread interrupted - giving up on retrying..."); throw new EsHadoopException("Thread interrupted - giving up on retrying...", e); } } else { debugLog(bulkLoggingID, "Retrying [%d] entries immediately (without backoff)", retriedDocs); } } else { debugLog(bulkLoggingID, "Sending batch of [%d] bytes/[%s] entries", data.length(), dataEntries); } }
BulkResponse bulkResult = null; boolean trackingArrayExpanded = false; String bulkLoggingID = createDebugTxnID(); initFlushOperation(bulkLoggingID, retryOperation, retries.size(), waitTime); debugLog(bulkLoggingID, "Submitting request"); RestClient.BulkActionResponse bar = restClient.bulk(resource, data); debugLog(bulkLoggingID, "Response received"); totalAttempts++; totalTime += bar.getTimeSpent(); BytesRef newEntry = validateEditedEntry(retryDataBuffer); data.remove(trackingBytesPosition); data.copyFrom(newEntry); debugLog(bulkLoggingID, "Completed. [%d] Original Entries. [%d] Attempts. [%d/%d] Docs Sent. [%d/%d] Docs Skipped. [%d/%d] Docs Aborted.", totalDocs, totalAttempts, debugLog(bulkLoggingID, "Failed. %s", ex.getMessage()); hadWriteErrors = true; throw ex;
@Override public void close() { if (log.isDebugEnabled()) { log.debug("Closing repository and connection to Elasticsearch ..."); } // bail out if closed before if (client == null) { return; } try { if (bulkProcessor != null) { bulkProcessor.close(); // Aggregate stats before discarding them. stats.aggregate(bulkProcessor.stats()); bulkProcessor = null; } if (bulkEntryWriter != null) { bulkEntryWriter.close(); bulkEntryWriter = null; } } finally { client.close(); // Aggregate stats before discarding them. stats.aggregate(client.stats()); client = null; } }
/** * Attempts a flush operation, handling failed documents based on configured error listeners. * @throws EsHadoopException in the event that the bulk operation fails, is aborted, or its errors could not be handled. */ public void flush() { BulkResponse bulk = tryFlush(); if (!bulk.getDocumentErrors().isEmpty()) { int maxErrors = 5; String header = String.format("Could not write all entries for bulk operation [%s/%s]. Error " + "sample (first [%s] error messages):\n", bulk.getDocumentErrors().size(), bulk.getTotalDocs(), maxErrors); StringBuilder message = new StringBuilder(header); int i = 0; for (BulkResponse.BulkError errors : bulk.getDocumentErrors()) { if (i >=maxErrors ) { break; } message.append("\t"); appendError(message, errors.getError()); message.append("\n"); message.append("\t") .append(errors.getDocument().toString()) .append("\n"); i++; } message.append("Bailing out..."); throw new EsHadoopException(message.toString()); } }
@Test(expected = EsHadoopException.class) public void testBulk09_HandlerStillReturnsGarbage() throws Exception { testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLERS, "garbage"); testSettings.setProperty(BulkWriteHandlerLoader.ES_WRITE_REST_ERROR_HANDLER + ".garbage", StillGarbageHandler.class.getName()); BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .addFailure("index", 401, "invalid", "some failure") .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .addSuccess("index", 201) .generate() ); processData(processor); processor.tryFlush(); fail("This should fail since the retry handler returned garbage"); }
/** * Flushes and closes the bulk processor to further writes. */ @Override public void close() { try { if (!hadWriteErrors) { flush(); } else { if (LOG.isDebugEnabled()) { LOG.debug("Dirty close; ignoring last existing write batch..."); } } if (requiresRefreshAfterBulk && executedBulkWrite) { // refresh batch restClient.refresh(resource); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Refreshing index [%s]", resource)); } } } finally { for (IBulkWriteErrorHandler handler : documentBulkErrorHandlers) { handler.close(); } } }
@Override public Stats stats() { Stats copy = new Stats(stats); if (client != null) { // Aggregate stats if it's not already discarded copy.aggregate(client.stats()); } if (bulkProcessor != null) { // Aggregate stats if it's not already discarded copy.aggregate(bulkProcessor.stats()); } return copy; }
private void appendError(StringBuilder message, Throwable exception) { if(exception != null) { message.append(exception); if(exception.getCause() != null) { message.append(';'); appendError(message, exception.getCause()); } } }
/** * Logs flushing messages and performs backoff waiting if there is a wait time for retry. */ private void initFlushOperation(String bulkLoggingID, boolean retryOperation, long retriedDocs, long waitTime) { if (retryOperation) { if (waitTime > 0L) { debugLog(bulkLoggingID, "Retrying [%d] entries after backing off for [%s] ms", retriedDocs, TimeValue.timeValueMillis(waitTime)); try { Thread.sleep(waitTime); } catch (InterruptedException e) { debugLog(bulkLoggingID, "Thread interrupted - giving up on retrying..."); throw new EsHadoopException("Thread interrupted - giving up on retrying...", e); } } else { debugLog(bulkLoggingID, "Retrying [%d] entries immediately (without backoff)", retriedDocs); } } else { debugLog(bulkLoggingID, "Sending batch of [%d] bytes/[%s] entries", data.length(), dataEntries); } }
@Test public void testBulk00_SuccessFromEmptyResponse() throws Exception { BulkProcessor processor = getBulkProcessor( generator.setInfo(resource, 56) .generate() ); processData(processor); BulkResponse bulkResponse = processor.tryFlush(); assertEquals(5, bulkResponse.getDocsSent()); assertEquals(0, bulkResponse.getDocsSkipped()); assertEquals(0, bulkResponse.getDocsAborted()); processor.close(); Stats stats = processor.stats(); assertEquals(0, stats.bulkRetries); assertEquals(0, stats.docsRetried); assertEquals(5, stats.docsAccepted); }