/** * Build the Document representing a meta alert to be created. * @param alerts The Elasticsearch results for the meta alerts child documents * @param groups The groups used to create this meta alert * @return A Document representing the new meta alert */ protected Document buildCreateDocument(Iterable<Document> alerts, List<String> groups, String alertField) { // Need to create a Document from the multiget. Scores will be calculated later Map<String, Object> metaSource = new HashMap<>(); List<Map<String, Object>> alertList = new ArrayList<>(); for (Document alert : alerts) { alertList.add(alert.getDocument()); } metaSource.put(alertField, alertList); // Add any meta fields String guid = UUID.randomUUID().toString(); metaSource.put(GUID, guid); metaSource.put(Constants.Fields.TIMESTAMP.getName(), System.currentTimeMillis()); metaSource.put(MetaAlertConstants.GROUPS_FIELD, groups); metaSource.put(MetaAlertConstants.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString()); return new Document(metaSource, guid, MetaAlertConstants.METAALERT_TYPE, System.currentTimeMillis()); }
/** * Copy constructor * @param other The document to be copied. */ public Document(Document other) { this(new HashMap<>(other.getDocument()), other.getGuid(), other.getSensorType(), other.getTimestamp(), other.getDocumentID().orElse(null)); }
public Document(Map<String, Object> document, String guid, String sensorType, Long timestamp, String documentID) { setDocument(document); setGuid(guid); setTimestamp(timestamp); setSensorType(sensorType); setDocumentID(documentID); }
public static Document fromJSON(Map<String, Object> json) { String guid = getGUID(json); Long timestamp = getTimestamp(json).orElse(0L); String sensorType = getSensorType(json); return new Document(json, guid, sensorType, timestamp); }
/** * Adds the provided alerts to a given metaalert. * @param metaAlert The metaalert to be given new children. * @param alerts The alerts to be added as children * @return True if metaalert is modified, false otherwise. */ protected boolean addAlertsToMetaAlert(Document metaAlert, Iterable<Document> alerts) { boolean alertAdded = false; @SuppressWarnings("unchecked") List<Map<String, Object>> currentAlerts = (List<Map<String, Object>>) metaAlert.getDocument() .get(MetaAlertConstants.ALERT_FIELD); if (currentAlerts == null) { currentAlerts = new ArrayList<>(); metaAlert.getDocument().put(MetaAlertConstants.ALERT_FIELD, currentAlerts); } Set<String> currentAlertGuids = currentAlerts.stream().map(currentAlert -> (String) currentAlert.get(GUID)).collect(Collectors.toSet()); for (Document alert : alerts) { String alertGuid = alert.getGuid(); // Only add an alert if it isn't already in the meta alert if (!currentAlertGuids.contains(alertGuid)) { currentAlerts.add(alert.getDocument()); alertAdded = true; } } return alertAdded; }
private Document document(JSONObject message) { String guid = UUID.randomUUID().toString(); String sensorType = "bro"; Long timestamp = System.currentTimeMillis(); return new Document(message, guid, sensorType, timestamp); }
@Test public void testWrite() throws Exception { // create some documents to write List<Document> documents = new ArrayList<>(); for(int i=0; i<10; i++) { Document document = Document.fromJSON(createMessage()); documents.add(document); } // write the documents for(Document doc: documents) { writer.addDocument(doc, "bro_index"); } writer.write(); // ensure the documents were written for(Document expected: documents) { Document actual = retrieveDao.getLatest(expected.getGuid(), expected.getSensorType()); assertNotNull("No document found", actual); assertEquals(expected.getGuid(), actual.getGuid()); assertEquals(expected.getSensorType(), actual.getSensorType()); assertEquals(expected.getDocument(), actual.getDocument()); assertTrue(actual.getDocumentID().isPresent()); // the document ID and GUID should not be the same, since the document ID was auto-generated assertNotEquals(actual.getDocument(), actual.getGuid()); } }
/** * Removes a metaalert link from a given alert. An nonexistent link performs no change. * @param metaAlertGuid The metaalert GUID to link. * @param alert The alert to be linked to. * @return True if the alert changed, false otherwise. */ default boolean removeMetaAlertFromAlert(String metaAlertGuid, Document alert) { List<String> metaAlertField = new ArrayList<>(); @SuppressWarnings("unchecked") List<String> alertField = (List<String>) alert.getDocument() .get(MetaAlertConstants.METAALERT_FIELD); if (alertField != null) { metaAlertField.addAll(alertField); } boolean metaAlertRemoved = metaAlertField.remove(metaAlertGuid); if (metaAlertRemoved) { alert.getDocument().put(MetaAlertConstants.METAALERT_FIELD, metaAlertField); } return metaAlertRemoved; }
@Override public Document update(Document update, Optional<String> index) throws IOException { for (Map.Entry<String, List<String>> kv : BACKING_STORE.entrySet()) { if (kv.getKey().startsWith(update.getSensorType())) { for (Iterator<String> it = kv.getValue().iterator(); it.hasNext(); ) { String doc = it.next(); Map<String, Object> docParsed = parse(doc); if (docParsed.getOrDefault(Constants.GUID, "").equals(update.getGuid())) { it.remove(); } } kv.getValue().add(JSONUtils.INSTANCE.toJSON(update.getDocument(), true)); } } return update; }
protected String getIndexName(Document update, String indexPostFix) throws IOException { return findIndexNameByGUID(update.getGuid(), update.getSensorType()) .orElse(ElasticsearchUtils.getIndexName(update.getSensorType(), indexPostFix, null)); }
@Override public Document update(Document update, Optional<String> rawIndex) throws IOException { Document newVersion = update; // Handle any case where we're given comments in Map form, instead of raw String Object commentsObj = update.getDocument().get(COMMENTS_FIELD); if ( commentsObj instanceof List && ((List<Object>) commentsObj).size() > 0 && ((List<Object>) commentsObj).get(0) instanceof Map) { newVersion = new Document(update); convertCommentsToRaw(newVersion.getDocument()); } try { SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(newVersion); Optional<String> index = SolrUtilities .getIndex(config.getIndexSupplier(), newVersion.getSensorType(), rawIndex); if (index.isPresent()) { this.client.add(index.get(), solrInputDocument); this.client.commit(index.get()); } else { throw new IllegalStateException("Index must be specified or inferred."); } } catch (SolrServerException e) { throw new IOException(e); } return newVersion; }
if (MetaAlertConstants.METAALERT_TYPE.equals(update.getSensorType())) { searchResponse = metaAlertSearchDao.getAllMetaAlertsForAlert(update.getGuid()); } catch (InvalidSearchException e) { throw new IOException("Unable to retrieve metaalerts for alert", e); Document doc = new Document(searchResult.getSource(), searchResult.getId(), MetaAlertConstants.METAALERT_TYPE, 0L); metaAlerts.add(doc);
default Document getPatchedDocument(RetrieveLatestDao retrieveLatestDao, PatchRequest request, Optional<Long> optionalTimestamp ) throws OriginalNotFoundException, IOException { String guid = request.getGuid(); String sensorType = request.getSensorType(); String documentID = null; Long timestamp = optionalTimestamp.orElse(System.currentTimeMillis()); Map<String, Object> originalSource = request.getSource(); if (originalSource == null) { // no document source provided, lookup the latest Document toPatch = retrieveLatestDao.getLatest(guid, sensorType); if(toPatch != null && toPatch.getDocument() != null) { originalSource = toPatch.getDocument(); documentID = toPatch.getDocumentID().orElse(null); } else { String error = format("Document does not exist, but is required; guid=%s, sensorType=%s", guid, sensorType); throw new OriginalNotFoundException(error); } } Map<String, Object> patchedSource = JSONUtils.INSTANCE.applyPatch(request.getPatch(), originalSource); return new Document(patchedSource, guid, sensorType, timestamp, documentID); } }
@Override public Document update(Document update, Optional<String> index) throws IOException { if (MetaAlertConstants.METAALERT_TYPE.equals(update.getSensorType())) { SearchResponse response = getMetaAlertsForAlert(update.getGuid()); Collection<Document> metaAlerts = response .getResults() .stream() .map(result -> toDocument(result, update.getTimestamp())) .collect(Collectors.toList());
@Test public void testPatchDocumentThatHasComment() throws Exception { Document document = createAndIndexDocument("testPatchDocumentWithComment"); // comment on the document String commentText = "New Comment"; String commentUser = "test_user"; long commentTimestamp = 152630493050L; Document withComment = addAlertComment(document.getGuid(), commentText, commentUser, commentTimestamp); // create a patch List<Map<String, Object>> patches = new ArrayList<>(); Map<String, Object> patch = new HashMap<>(); patch.put("op", "add"); patch.put("path", "/project"); patch.put("value", "metron"); patches.add(patch); PatchRequest pr = new PatchRequest(); pr.setGuid(withComment.getGuid()); pr.setIndex(SENSOR_NAME); pr.setSensorType(SENSOR_NAME); pr.setPatch(patches); // patch the document that has been commented on Document patched = getDao().patch(getDao(), pr, Optional.of(withComment.getTimestamp())); Assert.assertEquals("metron", patched.getDocument().get("project")); // ensure the patch was made on the indexed document Document indexed = findUpdatedDoc(patched.getDocument(), patched.getGuid(), SENSOR_NAME); Assert.assertEquals("metron", indexed.getDocument().get("project")); }
private Document toDocument(SearchResult result, Long timestamp) { Document document = Document.fromJSON(result.getSource()); document.setTimestamp(timestamp); document.setDocumentID(result.getId()); return document; }
@Test public void shouldGetLatestWithInvalidTimestamp() throws Exception { // Load alert Document alert = buildAlerts(1).get(0); hbaseDao.update(alert, Optional.empty()); Document actualDocument = hbaseDao.getLatest("message_0", SENSOR_TYPE); Assert.assertEquals(alert, actualDocument); alert.getDocument().put("field", "value"); alert.setTimestamp(0L); hbaseDao.update(alert, Optional.empty()); actualDocument = hbaseDao.getLatest("message_0", SENSOR_TYPE); Assert.assertEquals(alert.getDocument(), actualDocument.getDocument()); }
protected Entry<Document, Optional<String>> findMetaEntry( Map<Document, Optional<String>> expected) { for (Entry<Document, Optional<String>> entry : expected.entrySet()) { if (entry.getKey().getSensorType().equals(METAALERT_TYPE)) { return entry; } } return null; }
protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) { boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert, Collections.singleton(alert.getGuid())); if (metaAlertUpdated) { addAlertsToMetaAlert(metaAlert, Collections.singleton(alert)); } return metaAlertUpdated; }
private Optional<Document> toDocument(SearchHit hit) { Document document = Document.fromJSON(hit.getSource()); document.setDocumentID(hit.getId()); return Optional.of(document); } }