public void error(String sensorType, BulkWriterResponse errors, MessageGetStrategy messageGetStrategy) { Map<Throwable, Collection<Tuple>> errorMap = errors.getErrors(); for(Map.Entry<Throwable, Collection<Tuple>> entry : errorMap.entrySet()) { error(sensorType, entry.getKey(), entry.getValue(), messageGetStrategy); } }
error(sensorType, response, messageGetStrategy); } else if (response.hasErrors()) { throw new IllegalStateException("Unhandled bulk errors in response: " + response.getErrors()); Set<Tuple> tuplesToAck = new HashSet<>(tupleList); tuplesToAck.removeAll(response.getSuccesses()); response.getErrors().values().forEach(tuplesToAck::removeAll); commit(tuplesToAck);
@Test public void shouldHandleWriteFailure() { // create a tuple and a message associated with that tuple List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); Exception cause = new Exception(); // create a document writer which will fail all writes BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // the writer response should only contain failures assertEquals(0, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); Collection<Tuple> errors = response.getErrors().get(cause); assertTrue(errors.contains(tuples.get(0))); }
@Test public void shouldHandlePartialFailures() { // create a few tuples and the messages associated with the tuples int count = 2; List<Tuple> tuples = createTuples(count); List<JSONObject> messages = createMessages(count); Exception cause = new Exception(); // create a document writer that will fail one and succeed the other BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); results.addSuccess(createDocument(messages.get(1), tuples.get(1))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should contain some successes and some failures assertEquals(1, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); assertTrue(response.getErrors().get(cause).contains(tuples.get(0))); assertTrue(response.getSuccesses().contains(tuples.get(1))); }
@Test public void shouldHandleManyWriteFailures() { // create a few tuples and the messages associated with the tuples int count = 3; List<Tuple> tuples = createTuples(count); List<JSONObject> messages = createMessages(count); Exception cause = new Exception(); // create a document writer which will fail all writes BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); results.addFailure(createDocument(messages.get(1), tuples.get(1)), cause, "error"); results.addFailure(createDocument(messages.get(2), tuples.get(2)), cause, "error"); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // the writer response should only contain failures assertEquals(0, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); Collection<Tuple> errors = response.getErrors().get(cause); assertTrue(errors.contains(tuples.get(0))); assertTrue(errors.contains(tuples.get(1))); assertTrue(errors.contains(tuples.get(2))); }