public void commit(BulkWriterResponse response) { commit(response.getSuccesses()); }
tuplesToAck.removeAll(response.getSuccesses()); response.getErrors().values().forEach(tuplesToAck::removeAll); commit(tuplesToAck);
@Test public void shouldWriteManySuccessfully() { // create a few tuples and the messages associated with the tuples List<Tuple> tuples = createTuples(3); List<JSONObject> messages = createMessages(3); // create a document writer which will successfully write all BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(createDocument(messages.get(0), tuples.get(0))); results.addSuccess(createDocument(messages.get(1), tuples.get(1))); results.addSuccess(createDocument(messages.get(2), tuples.get(2))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); assertTrue(response.getSuccesses().contains(tuples.get(1))); assertTrue(response.getSuccesses().contains(tuples.get(2))); }
@Test public void shouldHandlePartialFailures() { // create a few tuples and the messages associated with the tuples int count = 2; List<Tuple> tuples = createTuples(count); List<JSONObject> messages = createMessages(count); Exception cause = new Exception(); // create a document writer that will fail one and succeed the other BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); results.addSuccess(createDocument(messages.get(1), tuples.get(1))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should contain some successes and some failures assertEquals(1, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); assertTrue(response.getErrors().get(cause).contains(tuples.get(0))); assertTrue(response.getSuccesses().contains(tuples.get(1))); }
@Test public void shouldWriteSuccessfullyWhenMessageTimestampIsString() { List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); // the timestamp is a String, rather than a Long messages.get(0).put(Constants.Fields.TIMESTAMP.getName(), new Long(System.currentTimeMillis()).toString()); // create the document JSONObject message = messages.get(0); String timestamp = (String) message.get(Constants.Fields.TIMESTAMP.getName()); String guid = (String) message.get(Constants.GUID); String sensorType = (String) message.get(Constants.SENSOR_TYPE); TupleBasedDocument document = new TupleBasedDocument(message, guid, sensorType, Long.parseLong(timestamp), tuples.get(0)); // create a document writer which will successfully write that document BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(document); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); }
@Test public void shouldHandleWriteFailure() { // create a tuple and a message associated with that tuple List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); Exception cause = new Exception(); // create a document writer which will fail all writes BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // the writer response should only contain failures assertEquals(0, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); Collection<Tuple> errors = response.getErrors().get(cause); assertTrue(errors.contains(tuples.get(0))); }
@Test public void shouldHandleManyWriteFailures() { // create a few tuples and the messages associated with the tuples int count = 3; List<Tuple> tuples = createTuples(count); List<JSONObject> messages = createMessages(count); Exception cause = new Exception(); // create a document writer which will fail all writes BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addFailure(createDocument(messages.get(0), tuples.get(0)), cause, "error"); results.addFailure(createDocument(messages.get(1), tuples.get(1)), cause, "error"); results.addFailure(createDocument(messages.get(2), tuples.get(2)), cause, "error"); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // the writer response should only contain failures assertEquals(0, response.getSuccesses().size()); assertEquals(1, response.getErrors().size()); Collection<Tuple> errors = response.getErrors().get(cause); assertTrue(errors.contains(tuples.get(0))); assertTrue(errors.contains(tuples.get(1))); assertTrue(errors.contains(tuples.get(2))); }
@Test public void shouldWriteSuccessfully() { // create a tuple and a message associated with that tuple List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); // create a document writer which will successfully write all BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(createDocument(messages.get(0), tuples.get(0))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); }
@Test public void shouldWriteSuccessfullyWhenMissingGUID() { // create a tuple and a message associated with that tuple List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); // remove the GUID from the message assertNotNull(messages.get(0).remove(Constants.GUID)); // create a document writer which will successfully write all BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(createDocument(messages.get(0), tuples.get(0))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); }