private List<JSONObject> doParse(List<String> messages) { // initialize HashSet<String> sensorTypes = new HashSet<>(); sensorTypes.add(sensorType); ParserRunnerImpl runner = new ParserRunnerImpl(sensorTypes); runner.init(() -> parserConfigurations, context); // parse each message List<ParserRunnerResults<JSONObject>> results = messages .stream() .map(str -> str.getBytes()) .map(bytes -> DEFAULT.get(emptyMap(), bytes, false, emptyMap())) .map(msg -> runner.execute(sensorType, msg, parserConfigurations)) .collect(Collectors.toList()); // aggregate both successes and errors into a list that can be returned List<JSONObject> successes = results .stream() .flatMap(result -> result.getMessages().stream()) .collect(Collectors.toList()); successCount += successes.size(); List<JSONObject> errors = results .stream() .flatMap(result -> result.getErrors().stream()) .map(err -> err.getJSONObject()) .collect(Collectors.toList()); errorCount += errors.size(); // return a list of both successes and errors successes.addAll(errors); return successes; }
public static void handleError(OutputCollector collector, MetronError error) { collector.emit(Constants.ERROR_STREAM, new Values(error.getJSONObject())); Optional<Throwable> throwable = error.getThrowable(); if (throwable.isPresent()) { collector.reportError(throwable.get()); } }
public void error(String sensorType, Throwable e, Iterable<Tuple> tuples, MessageGetStrategy messageGetStrategy) { LOG.error(format("Failing %d tuple(s); sensorType=%s", Iterables.size(tuples), sensorType), e); tuples.forEach(t -> { MetronError error = new MetronError() .withSensorType(Collections.singleton(sensorType)) .withErrorType(Constants.ErrorType.INDEXING_ERROR) .withThrowable(e) .addRawMessage(messageGetStrategy.get(t)); collector.emit(Constants.ERROR_STREAM, new Values(error.getJSONObject())); }); if (handleCommit) { commit(tuples); } // there is only one error to report for all of the failed tuples collector.reportError(e); }
@Test public void getJSONObjectShouldIncludeErrorFields() { JSONObject message = new JSONObject(); message.put("field1", "value1"); message.put("field2", "value2"); MetronError error = new MetronError().addRawMessage(message).withErrorFields(Sets.newHashSet("field1", "field2")); JSONObject errorJSON = error.getJSONObject(); assertEquals(Sets.newHashSet("field1", "field2"), Sets.newHashSet(((String) errorJSON.get(Constants.ErrorFields.ERROR_FIELDS.getName())).split(","))); assertEquals("04a2629c39e098c3944be85f35c75876598f2b44b8e5e3f52c59fa1ac182817c", errorJSON.get(Constants.ErrorFields.ERROR_HASH.getName())); } }
@Test public void handleErrorShouldEmitAndReportError() throws Exception { Throwable e = new Exception("error"); MetronError error = new MetronError().withMessage("error message").withThrowable(e); OutputCollector collector = mock(OutputCollector.class); ErrorUtils.handleError(collector, error); verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(collector, times(1)).reportError(any()); } }
@Test public void getJSONObjectShouldHandleThrowable() { Throwable e = new Exception("test exception"); MetronError error = new MetronError().withThrowable(e); JSONObject errorJSON = error.getJSONObject(); assertEquals("java.lang.Exception: test exception", errorJSON.get(Constants.ErrorFields.EXCEPTION.getName())); assertTrue(((String) errorJSON.get(Constants.ErrorFields.STACK.getName())).startsWith("java.lang.Exception: test exception")); assertEquals(e.getMessage(), errorJSON.get(Constants.ErrorFields.MESSAGE.getName())); }
@Test public void testNonBatchErrorPathErrorInWrite() throws Exception { ParserConfigurations configurations = getConfigurations(1); String sensorType = "test"; Tuple t = mock(Tuple.class); when(t.toString()).thenReturn("tuple"); when(t.getValueByField(eq("message"))).thenReturn(new JSONObject()); WriterBolt bolt = new WriterBolt(new WriterHandler(writer), configurations, sensorType); bolt.prepare(new HashMap(), topologyContext, outputCollector); doThrow(new Exception("write error")).when(writer).write(any(), any(), any(), any()); verify(writer, times(1)).init(); bolt.execute(t); verify(outputCollector, times(1)).ack(t); verify(writer, times(1)).write(eq(sensorType), any(), any(), any()); verify(outputCollector, times(1)).reportError(any()); verify(outputCollector, times(0)).fail(any()); MetronError error = new MetronError() .withErrorType(Constants.ErrorType.DEFAULT_ERROR) .withThrowable(new IllegalStateException("Unhandled bulk errors in response: {java.lang.Exception: write error=[tuple]}")) .withSensorType(Collections.singleton(sensorType)) .addRawMessage(new JSONObject()); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); }
@SuppressWarnings("unchecked") @Test public void testExecuteShouldReportError() throws ExecutionException { joinBolt.withMaxCacheSize(100); joinBolt.withMaxTimeRetain(10000); joinBolt.prepare(new HashMap(), topologyContext, outputCollector); when(tuple.getValueByField("key")).thenReturn(key); when(tuple.getValueByField("message")).thenReturn(new JSONObject()); joinBolt.cache = mock(LoadingCache.class); when(joinBolt.cache.get(any())).thenThrow(new RuntimeException(new Exception("join exception"))); joinBolt.execute(tuple); RuntimeException expectedExecutionException = new RuntimeException(new Exception("join exception")); MetronError error = new MetronError() .withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) .withMessage("Joining problem: {}") .withThrowable(expectedExecutionException) .addRawMessage(new JSONObject()); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(outputCollector, times(1)).reportError(any(ExecutionException.class)); verify(outputCollector, times(1)).ack(eq(tuple)); verifyNoMoreInteractions(outputCollector); } }
@Test public void getJSONObjectShouldIncludeRawMessages() { JSONObject message1 = new JSONObject(); JSONObject message2 = new JSONObject(); message1.put("value", "message1"); message2.put("value", "message2"); MetronError error = new MetronError().withRawMessages(Arrays.asList(message1, message2)); JSONObject errorJSON = error.getJSONObject(); assertEquals("{\"value\":\"message1\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName() + "_0")); assertEquals("{\"value\":\"message2\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName() + "_1")); error = new MetronError().addRawMessage("raw message".getBytes()); errorJSON = error.getJSONObject(); assertEquals("raw message", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName())); // It's unclear if we need a rawMessageBytes field so commenting out for now //assertEquals(Bytes.asList("raw message".getBytes()), errorJSON.get(Constants.ErrorFields.RAW_MESSAGE_BYTES.getName())); assertEquals("3b02cb29676bc448c69da1ec5eef7c89f4d6dc6a5a7ce0296ea25b207eea36be", errorJSON.get(Constants.ErrorFields.ERROR_HASH.getName())); error = new MetronError().addRawMessage(message1); errorJSON = error.getJSONObject(); assertEquals("{\"value\":\"message1\"}", errorJSON.get(Constants.ErrorFields.RAW_MESSAGE.getName())); assertEquals("e8aaf87c8494d345aac2d612ffd94fcf0b98c975fe6c4b991e2f8280a3a0bd10", errorJSON.get(Constants.ErrorFields.ERROR_HASH.getName())); }
@Test public void getJSONObjectShouldReturnBasicInformation() { MetronError error = new MetronError() .withMessage("test message") .withErrorType(Constants.ErrorType.PARSER_ERROR) .withSensorType(Collections.singleton("sensorType")); JSONObject errorJSON = error.getJSONObject(); assertEquals("test message", errorJSON.get(Constants.ErrorFields.MESSAGE.getName())); assertEquals(Constants.ErrorType.PARSER_ERROR.getType(), errorJSON.get(Constants.ErrorFields.ERROR_TYPE.getName())); assertEquals("error", errorJSON.get(Constants.SENSOR_TYPE)); assertEquals("sensorType", errorJSON.get(Constants.ErrorFields.FAILED_SENSOR_TYPE.getName())); String hostName = null; try { hostName = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException uhe) { // unable to get the hostname on this machine, don't test it } if (!StringUtils.isEmpty(hostName)) { assertTrue(((String) errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0); assertEquals(hostName, (String) errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())); } assertTrue(((long) errorJSON.get(Constants.ErrorFields.TIMESTAMP.getName())) > 0); }
argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(outputCollector, times(1)).reportError(any(IllegalStateException.class)); verify(outputCollector, times(1)).ack(t1);
argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(outputCollector, times(1)).reportError(any(IllegalStateException.class)); verify(outputCollector, times(1)).ack(t1);
argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); verify(outputCollector, times(1)).ack(t1);
.withErrorType(Constants.ErrorType.ENRICHMENT_ERROR) .withThrowable(new Exception("Could not parse binary stream to JSON")); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject()))); when(tuple.getStringByField("key")).thenReturn(key); when(tuple.getValueByField("message")).thenReturn(originalMessage); }}) .withThrowable(new Exception("[Metron] Could not enrich string: value1")); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));