private DedupValue updateDedupValue(DedupValue lastDedupValue, EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { if (lastDedupValue.getFirstOccurrence() >= eventEniq.timestamp) { // if dedup value happens later then event, dedup state changes. return null; } if (lastDedupValue.getStateFieldValue().equals(stateCloseValue) && eventEniq.timestamp < lastDedupValue.getCloseTime()) { DedupValue dv = createDedupValue(eventEniq, event, stateFieldValue); lastDedupValue.resetTo(dv); } else { // update lastDedupValue, set closeTime when close lastDedupValue.setStateFieldValue(stateFieldValue); if (stateFieldValue.equals(stateCloseValue)) { lastDedupValue.setCloseTime(eventEniq.timestamp); // when close an event, set closeTime for further check } } return lastDedupValue; }
@Override public boolean equals(Object dedupValue) { return Objects.equal(this.getStateFieldValue(), ((DedupValue) dedupValue).getStateFieldValue()); }
public void removeEvent(EventUniq eventEniq) { if (this.contains(eventEniq)) { this.events.remove(eventEniq); } }
private DedupValue createDedupValue(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue) { DedupValue dedupValue; dedupValue = new DedupValue(); dedupValue.setFirstOccurrence(eventEniq.timestamp); int idx = event.getSchema().getColumnIndex(DOC_ID); if (idx >= 0) { dedupValue.setDocId(event.getData()[idx].toString()); } else { dedupValue.setDocId(""); } dedupValue.setCount(1); dedupValue.setCloseTime(0); dedupValue.setStateFieldValue(stateFieldValue); return dedupValue; }
@Override public void run() { List<AlertStreamEvent> result = deduplicator.dedup(e2); if (result != null) { allResults.addAll(result); } System.out.println("2 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();
public synchronized DedupValue[] addOrUpdate(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { Map<EventUniq, ConcurrentLinkedDeque<DedupValue>> events = this.getEvents(); if (!events.containsKey(eventEniq) || (events.containsKey(eventEniq) && events.get(eventEniq).size() > 0 && !StringUtils.equalsIgnoreCase(stateFieldValue, events.get(eventEniq).getLast().getStateFieldValue()))) { DedupValue[] dedupValues = this.add(eventEniq, event, stateFieldValue, stateCloseValue); return dedupValues; } else { // update count this.updateCount(eventEniq); return null; } }
private DedupValue[] add(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { DedupValue dedupValue = null; if (!events.containsKey(eventEniq)) { dedupValue = createDedupValue(eventEniq, event, stateFieldValue); ConcurrentLinkedDeque<DedupValue> dedupValues = new ConcurrentLinkedDeque<>(); dedupValues.add(dedupValue); // skip the event which put failed due to concurrency events.put(eventEniq, dedupValues); LOG.info("{} Add new dedup key {}, and value {}", this.publishName, eventEniq, dedupValues); } else if (!StringUtils.equalsIgnoreCase(stateFieldValue, events.get(eventEniq).getLast().getStateFieldValue())) { // existing a de-dup value, try update or reset DedupValue lastDedupValue = events.get(eventEniq).getLast(); dedupValue = updateDedupValue(lastDedupValue, eventEniq, event, stateFieldValue, stateCloseValue); LOG.info("{} Update dedup key {}, and value {}", this.publishName, eventEniq, dedupValue); } if (dedupValue == null) { return null; } return new DedupValue[] {dedupValue}; }
@Test public void test() { Map<DedupKey, Integer> testMap = new HashMap<>(); DedupKey key1 = new DedupKey("policy1", "stream1"); update(testMap, key1); update(testMap, key1); DedupKey key2 = new DedupKey("policy2", "stream2"); update(testMap, key2); Assert.assertTrue(testMap.get(key1) == 1); Assert.assertTrue(testMap.get(key2) == 0); DedupKey key3 = new DedupKey("policy1", "stream1"); update(testMap, key3); Assert.assertTrue(testMap.get(key3) == 2); }
@Override public boolean equals(Object obj) { if (obj == this) { return true; } if (obj instanceof DedupKey) { DedupKey au = (DedupKey) obj; return Objects.equals(au.getOutputStreamId(), this.outputStreamId) && Objects.equals(au.getPolicyId(), this.policyId); } return false; }
public List<AlertStreamEvent> dedup(AlertStreamEvent event, EventUniq eventEniq, String dedupStateField, String stateFieldValue, String stateCloseValue) { DedupValue[] dedupValues = this.addOrUpdate(eventEniq, event, stateFieldValue, stateCloseValue); if (dedupValues != null) { // any of dedupValues won't be null if (dedupValues.length == 2) { // emit last event which includes count of dedup events & new state event return Arrays.asList( this.mergeEventWithDedupValue(event, dedupValues[0], dedupStateField), this.mergeEventWithDedupValue(event, dedupValues[1], dedupStateField)); } else if (dedupValues.length == 1) { //populate firstOccurrenceTime & count return Arrays.asList(this.mergeEventWithDedupValue(event, dedupValues[0], dedupStateField)); } } // duplicated, will be ignored return null; }
public boolean contains(EventUniq eventEniq) { return this.getEvents().containsKey(eventEniq); }
public ConcurrentLinkedDeque<DedupValue> getDedupValuesInConcurrentLinkedDeque() { ConcurrentLinkedDeque<DedupValue> result = new ConcurrentLinkedDeque<DedupValue>(); result.addAll(this.getDedupValues()); return result; }
public DefaultDeduplicator(String intervalMin, List<String> customDedupFields, String dedupStateField, String dedupStateCloseValue, DedupCache dedupCache) { setDedupIntervalMin(intervalMin); if (customDedupFields != null) { this.customDedupFields = customDedupFields; } if (StringUtils.isNotBlank(dedupStateField)) { this.dedupStateField = dedupStateField; } if (StringUtils.isNotBlank(dedupStateCloseValue)) { this.dedupStateCloseValue = dedupStateCloseValue; } this.dedupCache = dedupCache; withoutStatesCache = CacheBuilder.newBuilder().expireAfterWrite( this.dedupIntervalSec, TimeUnit.SECONDS).build(); LOG.info("initialize DefaultDeduplicator with dedupIntervalSec={}, customDedupFields={}", dedupIntervalSec, customDedupFields); }
@Override public void run() { List<AlertStreamEvent> result = deduplicator.dedup(e6); if (result != null) { allResults.addAll(result); } System.out.println("6 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();
@Override public void run() { List<AlertStreamEvent> result = deduplicator.dedup(e4); if (result != null) { allResults.addAll(result); } System.out.println("4 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();
@Override public void run() { List<AlertStreamEvent> result = deduplicator.dedup(e1); if (result != null) { allResults.addAll(result); } System.out.println("1 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();
@Override public void run() { List<AlertStreamEvent> result = deduplicator.dedup(e3); if (result != null) { allResults.addAll(result); } System.out.println("3 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();
@Override public void run() { List<AlertStreamEvent> result = deduplicator.dedup(e7); if (result != null) { allResults.addAll(result); } System.out.println("7 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();
@Override public void run() { List<AlertStreamEvent> result = deduplicator.dedup(e8); if (result != null) { allResults.addAll(result); } System.out.println("8 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();
@Override public void run() { try { Thread.sleep(500); } catch (InterruptedException e) { } List<AlertStreamEvent> result = deduplicator.dedup(e5); if (result != null) { allResults.addAll(result); } System.out.println("5 >>>> " + ToStringBuilder.reflectionToString(result)); } }).start();