private DedupValue createDedupValue(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue) { DedupValue dedupValue; dedupValue = new DedupValue(); dedupValue.setFirstOccurrence(eventEniq.timestamp); int idx = event.getSchema().getColumnIndex(DOC_ID); if (idx >= 0) { dedupValue.setDocId(event.getData()[idx].toString()); } else { dedupValue.setDocId(""); } dedupValue.setCount(1); dedupValue.setCloseTime(0); dedupValue.setStateFieldValue(stateFieldValue); return dedupValue; }
private DedupValue updateDedupValue(DedupValue lastDedupValue, EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { if (lastDedupValue.getFirstOccurrence() >= eventEniq.timestamp) { // if dedup value happens later then event, dedup state changes. return null; } if (lastDedupValue.getStateFieldValue().equals(stateCloseValue) && eventEniq.timestamp < lastDedupValue.getCloseTime()) { DedupValue dv = createDedupValue(eventEniq, event, stateFieldValue); lastDedupValue.resetTo(dv); } else { // update lastDedupValue, set closeTime when close lastDedupValue.setStateFieldValue(stateFieldValue); if (stateFieldValue.equals(stateCloseValue)) { lastDedupValue.setCloseTime(eventEniq.timestamp); // when close an event, set closeTime for further check } } return lastDedupValue; }
private DedupValue updateCount(EventUniq eventEniq) { ConcurrentLinkedDeque<DedupValue> dedupValues = events.get(eventEniq); if (dedupValues == null || dedupValues.size() <= 0) { LOG.warn("{} No dedup values found for {}, cannot update count", this.publishName, eventEniq); return null; } else { DedupValue dedupValue = dedupValues.getLast(); dedupValue.setCount(dedupValue.getCount() + 1); String updateMsg = String.format( "%s Update count for dedup key %s, value %s and count %s", this.publishName, eventEniq, dedupValue.getStateFieldValue(), dedupValue.getCount()); if (LOG.isDebugEnabled()) { LOG.debug(updateMsg); } return dedupValue; } }
String colName = streamDefinition.getColumns().get(i).getName(); if (Objects.equal(colName, dedupStateField)) { event.getData()[i] = dedupValue.getStateFieldValue(); event.getData()[i] = dedupValue.getCount(); event.getData()[i] = dedupValue.getFirstOccurrence(); event.getData()[i] = dedupValue.getDocId();
@Override public boolean equals(Object dedupValue) { return Objects.equal(this.getStateFieldValue(), ((DedupValue) dedupValue).getStateFieldValue()); }
public synchronized DedupValue[] addOrUpdate(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { Map<EventUniq, ConcurrentLinkedDeque<DedupValue>> events = this.getEvents(); if (!events.containsKey(eventEniq) || (events.containsKey(eventEniq) && events.get(eventEniq).size() > 0 && !StringUtils.equalsIgnoreCase(stateFieldValue, events.get(eventEniq).getLast().getStateFieldValue()))) { DedupValue[] dedupValues = this.add(eventEniq, event, stateFieldValue, stateCloseValue); return dedupValues; } else { // update count this.updateCount(eventEniq); return null; } }
private DedupValue[] add(EventUniq eventEniq, AlertStreamEvent event, String stateFieldValue, String stateCloseValue) { DedupValue dedupValue = null; if (!events.containsKey(eventEniq)) { dedupValue = createDedupValue(eventEniq, event, stateFieldValue); ConcurrentLinkedDeque<DedupValue> dedupValues = new ConcurrentLinkedDeque<>(); dedupValues.add(dedupValue); // skip the event which put failed due to concurrency events.put(eventEniq, dedupValues); LOG.info("{} Add new dedup key {}, and value {}", this.publishName, eventEniq, dedupValues); } else if (!StringUtils.equalsIgnoreCase(stateFieldValue, events.get(eventEniq).getLast().getStateFieldValue())) { // existing a de-dup value, try update or reset DedupValue lastDedupValue = events.get(eventEniq).getLast(); dedupValue = updateDedupValue(lastDedupValue, eventEniq, event, stateFieldValue, stateCloseValue); LOG.info("{} Update dedup key {}, and value {}", this.publishName, eventEniq, dedupValue); } if (dedupValue == null) { return null; } return new DedupValue[] {dedupValue}; }