private List<AlertStreamEvent> checkDedup(AlertStreamEvent event, EventUniq key, String stateFiledValue) { if (StringUtils.isBlank(stateFiledValue)) { // without state field, we cannot determine whether it is duplicated // without custom filed values, we cannot determine whether it is duplicated synchronized (withoutStatesCache) { if (withoutStatesCache != null && withoutStatesCache.getIfPresent(key) != null) { if (LOG.isDebugEnabled()) { LOG.debug("Alert event {} with key {} is skipped since it is duplicated", event, key); } return null; } else if (withoutStatesCache != null) { withoutStatesCache.put(key, ""); } } return Arrays.asList(event); } return dedupCache.dedup(event, key, dedupStateField, stateFiledValue, dedupStateCloseValue); }
public synchronized DedupValue[] addOrUpdate(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { Map<EventUniq, ConcurrentLinkedDeque<DedupValue>> events = this.getEvents(); if (!events.containsKey(eventEniq) || (events.containsKey(eventEniq) && events.get(eventEniq).size() > 0 && !StringUtils.equalsIgnoreCase(stateFieldValue, events.get(eventEniq).getLast().getStateFieldValue()))) { DedupValue[] dedupValues = this.add(eventEniq, event, stateFieldValue, stateCloseValue); return dedupValues; } else { // update count this.updateCount(eventEniq); return null; } }
public List<AlertStreamEvent> dedup(AlertStreamEvent event, EventUniq eventEniq, String dedupStateField, String stateFieldValue, String stateCloseValue) { DedupValue[] dedupValues = this.addOrUpdate(eventEniq, event, stateFieldValue, stateCloseValue); if (dedupValues != null) { // any of dedupValues won't be null if (dedupValues.length == 2) { // emit last event which includes count of dedup events & new state event return Arrays.asList( this.mergeEventWithDedupValue(event, dedupValues[0], dedupStateField), this.mergeEventWithDedupValue(event, dedupValues[1], dedupStateField)); } else if (dedupValues.length == 1) { //populate firstOccurrenceTime & count return Arrays.asList(this.mergeEventWithDedupValue(event, dedupValues[0], dedupStateField)); } } // duplicated, will be ignored return null; }
private DedupValue[] add(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { DedupValue dedupValue = null; if (!events.containsKey(eventEniq)) { dedupValue = createDedupValue(eventEniq, event, stateFieldValue); ConcurrentLinkedDeque<DedupValue> dedupValues = new ConcurrentLinkedDeque<>(); dedupValues.add(dedupValue); // skip the event which put failed due to concurrency events.put(eventEniq, dedupValues); LOG.info("{} Add new dedup key {}, and value {}", this.publishName, eventEniq, dedupValues); } else if (!StringUtils.equalsIgnoreCase(stateFieldValue, events.get(eventEniq).getLast().getStateFieldValue())) { // existing a de-dup value, try update or reset DedupValue lastDedupValue = events.get(eventEniq).getLast(); dedupValue = updateDedupValue(lastDedupValue, eventEniq, event, stateFieldValue, stateCloseValue); LOG.info("{} Update dedup key {}, and value {}", this.publishName, eventEniq, dedupValue); } if (dedupValue == null) { return null; } return new DedupValue[] {dedupValue}; }
@Test public void testNormal() throws Exception { Config config = ConfigFactory.load(); DedupCache dedupCache = new DedupCache(config, "testPublishment"); StreamDefinition stream = createStream(); PolicyDefinition policy = createPolicy(stream.getStreamId(), "testPolicy"); String[] states = new String[] {"OPEN", "WARN", "CLOSE"}; Random random = new Random(); for (int i = 0; i < 20; i++) { AlertStreamEvent event = createEvent(stream, policy, new Object[] { System.currentTimeMillis(), "host1", "testPolicy-host1-01", states[random.nextInt(3)], 0, 0 }); HashMap<String, String> dedupFieldValues = new HashMap<String, String>(); dedupFieldValues.put("alertKey", (String) event.getData()[event.getSchema().getColumnIndex("alertKey")]); List<AlertStreamEvent> result = dedupCache.dedup(event, new EventUniq(event.getStreamId(), event.getPolicyId(), event.getCreatedTime(), dedupFieldValues), "state", (String) event.getData()[event.getSchema().getColumnIndex("state")], "closed"); System.out.println((i + 1) + " >>>> " + ToStringBuilder.reflectionToString(result)); } Assert.assertTrue(true); }
@SuppressWarnings("rawtypes") @Override public void init(Config config, Publishment publishment, Map conf) throws Exception { DedupCache dedupCache = new DedupCache(config, publishment.getName()); OverrideDeduplicatorSpec spec = publishment.getOverrideDeduplicator(); if (spec != null && StringUtils.isNotBlank(spec.getClassName())) {
public void removeEvent(EventUniq eventEniq) { if (this.contains(eventEniq)) { this.events.remove(eventEniq); } }
public boolean contains(EventUniq eventEniq) { return this.getEvents().containsKey(eventEniq); }
private DedupValue updateDedupValue(DedupValue lastDedupValue, EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { if (lastDedupValue.getFirstOccurrence() >= eventEniq.timestamp) { // if dedup value happens later then event, dedup state changes. return null; } if (lastDedupValue.getStateFieldValue().equals(stateCloseValue) && eventEniq.timestamp < lastDedupValue.getCloseTime()) { DedupValue dv = createDedupValue(eventEniq, event, stateFieldValue); lastDedupValue.resetTo(dv); } else { // update lastDedupValue, set closeTime when close lastDedupValue.setStateFieldValue(stateFieldValue); if (stateFieldValue.equals(stateCloseValue)) { lastDedupValue.setCloseTime(eventEniq.timestamp); // when close an event, set closeTime for further check } } return lastDedupValue; }
DedupCache dedupCache = new DedupCache(config, "testPublishment"); DefaultDeduplicator deduplicator = new DefaultDeduplicator( "PT1M", Arrays.asList(new String[] {"alertKey"}), "state", "close", dedupCache);
DedupCache dedupCache = new DedupCache(config, "testPublishment"); DefaultDeduplicator deduplicator = new DefaultDeduplicator( "PT10S", Arrays.asList(new String[] {"alertKey"}), null, null, dedupCache);
LOG.info("dedup field: " + this.getDedupStateField()); LOG.info("dedup value: " + stateFiledValue); List<AlertStreamEvent> result = this.getDedupCache().dedup(event, eventkey, this.getDedupStateField(), stateFiledValue, "closed"); return result;