public static String getJsonWithUpdatedResources(String jsonConf, Map<String, Double> resourceUpdates) { try { JSONParser parser = new JSONParser(); Object obj = parser.parse(jsonConf); JSONObject jsonObject = (JSONObject) obj; Map<String, Double> componentResourceMap = (Map<String, Double>) jsonObject.getOrDefault( Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap<String, Double>() ); for (Map.Entry<String, Double> resourceUpdateEntry : resourceUpdates.entrySet()) { if (NormalizedResources.RESOURCE_NAME_NORMALIZER.getResourceNameMapping().containsValue(resourceUpdateEntry.getKey())) { // if there will be legacy values they will be in the outer conf jsonObject.remove(getCorrespondingLegacyResourceName(resourceUpdateEntry.getKey())); componentResourceMap.remove(getCorrespondingLegacyResourceName(resourceUpdateEntry.getKey())); } componentResourceMap.put(resourceUpdateEntry.getKey(), resourceUpdateEntry.getValue()); } jsonObject.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, componentResourceMap); return jsonObject.toJSONString(); } catch (ParseException ex) { throw new RuntimeException("Failed to parse component resources with json: " + jsonConf); } }
@Override public boolean matches(Object o) { Values values = (Values) o; JSONObject actual = (JSONObject) values.get(0); actual.remove("timestamp"); expected.remove("timestamp"); actual.remove("stack"); expected.remove("stack"); actual.remove("guid"); expected.remove("guid"); return actual.equals(expected); } }
public void removeTimingFields(JSONObject message) { ImmutableSet keys = ImmutableSet.copyOf(message.keySet()); for (Object key : keys) { if (key.toString().endsWith(".ts")) { message.remove(key); } } } }
private JSONObject mutate(JSONObject json, String oldKey, String newKey) { if (json.containsKey(oldKey)) { json.put(newKey, json.remove(oldKey)); } return json; }
@SuppressWarnings("unchecked") private JSONObject mutate(JSONObject json, String oldKey, String newKey) { if (json.containsKey(oldKey)) { json.put(newKey, json.remove(oldKey)); } return json; }
private boolean replaceKey(JSONObject payload, String toKey, String[] fromKeys) { for (String fromKey : fromKeys) { if (payload.containsKey(fromKey)) { Object value = payload.remove(fromKey); payload.put(toKey, value); _LOG.trace("[Metron] Added {} to {}", toKey, payload); return true; } } return false; }
public void transformAndUpdate(JSONObject message, Context context, Map<String, Object>... sensorConfig) { Map<String, Object> currentValue = transform(message, context, sensorConfig); if(currentValue != null) { for(Map.Entry<String, Object> kv : currentValue.entrySet()) { if(kv.getValue() == null) { message.remove(kv.getKey()); } else { message.put(kv.getKey(), kv.getValue()); } } } }
@SuppressWarnings("unchecked") private void parseLogoutMessage(JSONObject json) { json.put("event_subtype", "logout"); String message = (String) json.get("message"); if (message.matches(".*'.*'.*'.*'.*")) { String parts[] = message.split("'"); String ip_src_addr = parts[0]; if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) { ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1); ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]")); json.put("ip_src_addr", ip_src_addr); } json.put("username", parts[1]); json.put("security_domain", parts[3]); json.remove("message"); } }
public HostFromJSONListAdapter(String jsonList) { JSONArray jsonArray = (JSONArray) JSONValue.parse(jsonList); Iterator jsonArrayIterator = jsonArray.iterator(); while(jsonArrayIterator.hasNext()) { JSONObject jsonObject = (JSONObject) jsonArrayIterator.next(); String host = (String) jsonObject.remove("ip"); _known_hosts.put(host, jsonObject); } }
@SuppressWarnings("unchecked") private void parseLoginMessage(JSONObject json) { json.put("event_subtype", "login"); String message = (String) json.get("message"); if (message.contains(":")) { String[] parts = message.split(":"); String user = parts[0]; String ip_src_addr = parts[1]; if (user.contains("user(") && user.contains(")")) { user = user.substring(user.indexOf("user(") + "user(".length()); user = user.substring(0, user.indexOf(")")); json.put("username", user); } if (ip_src_addr.contains("[") && ip_src_addr.contains("]")) { ip_src_addr = ip_src_addr.substring(ip_src_addr.indexOf("[") + 1); ip_src_addr = ip_src_addr.substring(0, ip_src_addr.indexOf("]")); json.put("ip_src_addr", ip_src_addr); } json.remove("message"); } }
private static JSONObject join(JSONObject left, JSONObject right) { JSONObject message = new JSONObject(); message.putAll(left); message.putAll(right); List<Object> emptyKeys = new ArrayList<>(); for(Object key : message.keySet()) { Object value = message.get(key); if(value == null || value.toString().length() == 0) { emptyKeys.add(key); } } for(Object o : emptyKeys) { message.remove(o); } return message; }
@Override public JSONObject joinMessages(Map<String, Tuple> streamMessageMap, MessageGetStrategy messageGetStrategy) { JSONObject message = new JSONObject(); for (String key : streamMessageMap.keySet()) { Tuple tuple = streamMessageMap.get(key); JSONObject obj = (JSONObject) messageGetStrategy.get(tuple); message.putAll(obj); } List<Object> emptyKeys = new ArrayList<>(); for(Object key : message.keySet()) { Object value = message.get(key); if(value == null || value.toString().length() == 0) { emptyKeys.add(key); } } for(Object o : emptyKeys) { message.remove(o); } message.put(getClass().getSimpleName().toLowerCase() + ".joiner.ts", "" + System.currentTimeMillis()); return message; }
private boolean replaceKeyArray(JSONObject payload, String toKey, String[] fromKeys) { for (String fromKey : fromKeys) { if (payload.containsKey(fromKey)) { JSONArray value = (JSONArray) payload.remove(fromKey); if (value != null && !value.isEmpty()) { payload.put(toKey, value.get(0)); _LOG.trace("[Metron] Added {} to {}", toKey, payload); return true; } } } return false; }
@Override public void removeTimingFields(JSONObject message) { ImmutableSet keys = ImmutableSet.copyOf(message.keySet()); for(Object key: keys) { if (key.toString().contains("splitter.begin.ts")) { message.remove(key); } } } }
@Override protected void postParse(JSONObject message) { removeEmptyFields(message); message.remove("timestamp_string"); if (message.containsKey("message")) { String messageValue = (String) message.get("message"); if (messageValue.contains("logged into")) { parseLoginMessage(message); } else if (messageValue.contains("logged out")) { parseLogoutMessage(message); } else if (messageValue.contains("rbm(")) { parseRBMMessage(message); } else { parseOtherMessage(message); } } }
@Test public void testInitializeAdapter() throws Exception { Map<String, JSONObject> mapKnownHosts = new HashMap<>(); HostFromPropertiesFileAdapter hfa = new HostFromPropertiesFileAdapter(mapKnownHosts); Assert.assertFalse(hfa.initializeAdapter(null)); JSONArray jsonArray = (JSONArray) JSONValue.parse(expectedKnownHostsString); Iterator jsonArrayIterator = jsonArray.iterator(); while(jsonArrayIterator.hasNext()) { JSONObject jsonObject = (JSONObject) jsonArrayIterator.next(); String host = (String) jsonObject.remove("ip"); mapKnownHosts.put(host, jsonObject); } hfa = new HostFromPropertiesFileAdapter(mapKnownHosts); Assert.assertTrue(hfa.initializeAdapter(null)); }
@Test public void testSourceTypeMissing() throws Exception { // setup the bolt BulkMessageWriterBolt<IndexingConfigurations> bulkMessageWriterBolt = new BulkMessageWriterBolt<IndexingConfigurations>( "zookeeperUrl", "INDEXING") .withBulkMessageWriter(bulkMessageWriter) .withMessageGetter(MessageGetters.JSON_FROM_FIELD.name()) .withMessageGetterField("message"); bulkMessageWriterBolt.setCuratorFramework(client); bulkMessageWriterBolt.setZKCache(cache); bulkMessageWriterBolt.getConfigurations().updateSensorIndexingConfig(sensorType, new FileInputStream(sampleSensorIndexingConfigPath)); // initialize the bolt bulkMessageWriterBolt.declareOutputFields(declarer); Map stormConf = new HashMap(); bulkMessageWriterBolt.prepare(stormConf, topologyContext, outputCollector); // create a message with no source type JSONObject message = (JSONObject) new JSONParser().parse(sampleMessageString); message.remove("source.type"); when(tuple.getValueByField("message")).thenReturn(message); // the tuple should be handled as an error and ack'd bulkMessageWriterBolt.execute(tuple); verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM), any()); verify(outputCollector, times(1)).ack(tuple); }
@Test public void testEnrich() throws Exception { Map<String, JSONObject> mapKnownHosts = new HashMap<>(); JSONArray jsonArray = (JSONArray) JSONValue.parse(expectedKnownHostsString); Iterator jsonArrayIterator = jsonArray.iterator(); while(jsonArrayIterator.hasNext()) { JSONObject jsonObject = (JSONObject) jsonArrayIterator.next(); String host = (String) jsonObject.remove("ip"); mapKnownHosts.put(host, jsonObject); } HostFromPropertiesFileAdapter hfa = new HostFromPropertiesFileAdapter(mapKnownHosts); JSONObject actualMessage = hfa.enrich(new CacheKey("dummy", ip, null)); Assert.assertNotNull(actualMessage); Assert.assertEquals(expectedMessage, actualMessage); actualMessage = hfa.enrich(new CacheKey("dummy", ip1, null)); JSONObject emptyJson = new JSONObject(); Assert.assertEquals(emptyJson, actualMessage); }
@Test public void shouldWriteSuccessfullyWhenMissingGUID() { // create a tuple and a message associated with that tuple List<Tuple> tuples = createTuples(1); List<JSONObject> messages = createMessages(1); // remove the GUID from the message assertNotNull(messages.get(0).remove(Constants.GUID)); // create a document writer which will successfully write all BulkDocumentWriterResults<TupleBasedDocument> results = new BulkDocumentWriterResults<>(); results.addSuccess(createDocument(messages.get(0), tuples.get(0))); BulkDocumentWriter<TupleBasedDocument> docWriter = mock(BulkDocumentWriter.class); when(docWriter.write()).thenReturn(results); // attempt to write ElasticsearchWriter esWriter = new ElasticsearchWriter(); esWriter.setDocumentWriter(docWriter); esWriter.init(stormConf, topologyContext, writerConfiguration); BulkWriterResponse response = esWriter.write("bro", writerConfiguration, tuples, messages); // response should only contain successes assertFalse(response.hasErrors()); assertTrue(response.getSuccesses().contains(tuples.get(0))); }