/** * singleton with init would be good for unit test as well, and it ensures that * initialization happens only once before you use it. * @param config * @param dao */ public void init(Map<String, Map<String, T>> initialAlertDefs, PolicyDefinitionDAO<T> dao, Config config){ if(!initialized){ synchronized(this){ if(!initialized){ internalInit(initialAlertDefs, dao, config); initialized = true; } } } }
@Override public void report() { PolicyDistroStatsLogReporter appender = new PolicyDistroStatsLogReporter(); appender.reportPolicyMembership(executorId + "_" + partitionSeq, policyEvaluators.keySet()); }
@SuppressWarnings("unchecked") public static <K extends AbstractPolicyDefinitionEntity> DynamicPolicyLoader<K> getInstanceOf(Class<K> clz) { if (maps.containsKey(clz)) { return maps.get(clz); } else { DynamicPolicyLoader<K> loader = new DynamicPolicyLoader<K>(); maps.putIfAbsent(clz, loader); return maps.get(clz); } }
@Override public void init() { StreamMetadataManager.getInstance().init(config, getAlertStreamSchemaDAO(config)); String application = config.getString(EagleConfigConstants.EAGLE_PROPS + "." + EagleConfigConstants.APPLICATION); try { initialAlertDefs = policyDefinitionDao.findActivePoliciesGroupbyExecutorId(site, application); int part = partitioner.partition(numPartitions, alertDef.getTags().get(Constants.POLICY_TYPE), alertDef.getTags().get(Constants.POLICY_ID)); if (part == partitionSeq) { tmpPolicyEvaluators.put(alertDef.getTags().get(Constants.POLICY_ID), createPolicyEvaluator(alertDef)); DynamicPolicyLoader<T> policyLoader = DynamicPolicyLoader.getInstanceOf(policyDefinitionClz); policyLoader.init(initialAlertDefs, policyDefinitionDao, config); String fullQualifiedAlertExecutorId = executorId + "_" + partitionSeq; policyLoader.addPolicyChangeListener(fullQualifiedAlertExecutorId, this); policyLoader.addPolicyDistributionReporter(fullQualifiedAlertExecutorId, this); LOG.info("Alert Executor created, partitionSeq: " + partitionSeq + " , numPartitions: " + numPartitions); LOG.info("All policy evaluators: " + policyEvaluators); initMetricReportor();
protected PolicyEvaluator<T> createPolicyEvaluator(T alertDef){ String policyType = alertDef.getTags().get(Constants.POLICY_TYPE); Class<? extends PolicyEvaluator> evalCls = PolicyManager.getInstance().getPolicyEvaluator(policyType); if(evalCls == null){ String msg = "No policy evaluator defined for policy type : " + policyType; try { policyDef = JsonSerDeserUtils.deserialize(alertDef.getPolicyDef(), AbstractPolicyDefinition.class, PolicyManager.getInstance().getPolicyModules(policyType)); PolicyEvaluationContext<T, K> context = new PolicyEvaluationContext<>(); context.policyId = alertDef.getTags().get("policyId"); context.alertExecutor = this; context.resultRender = this.getResultRender(); if (pe.isMarkdownEnabled()) // updating markdown details only if the policy is found invalid updateMarkdownDetails(alertDef, pe.isMarkdownEnabled(), pe.getMarkdownReason()); } catch(Exception ex) { LOG.error("Fail creating new policyEvaluator", ex);
protected PolicyEvaluator<AggregateDefinitionAPIEntity> createPolicyEvaluator(AggregateDefinitionAPIEntity alertDef) { String policyType = alertDef.getTags().get(Constants.POLICY_TYPE); Class<? extends PolicyEvaluator> evalCls = PolicyManager.getInstance().getPolicyEvaluator(policyType); if (evalCls == null) { String msg = "No policy evaluator defined for policy type : " + policyType; try { policyDef = JsonSerDeserUtils.deserialize(alertDef.getPolicyDef(), AbstractPolicyDefinition.class, PolicyManager.getInstance().getPolicyModules(policyType)); } catch (Exception ex) { LOG.error("Fail initial alert policy def: " + alertDef.getPolicyDef(), ex); PolicyEvaluationContext<AggregateDefinitionAPIEntity, AggregateEntity> context = new PolicyEvaluationContext<>(); context.policyId = alertDef.getTags().get("policyId"); context.alertExecutor = this;
@Override public void init() { String site = config.getString("eagleProps.site"); String application = config.getString("eagleProps.application"); Map<String, Map<String, AlertDefinitionAPIEntity>> initialAlertDefs; try { initialAlertDefs = dao.findActivePoliciesGroupbyExecutorId( site, application ); } catch (Exception ex) { LOG.error("fail to initialize initialAlertDefs: ", ex); throw new IllegalStateException("fail to initialize initialAlertDefs: ", ex); } if(initialAlertDefs == null || initialAlertDefs.isEmpty()){ LOG.warn("No alert definitions found for site: "+site+", application: "+ application); } try{ notificationManager = new NotificationPluginManagerImpl(config); }catch (Exception ex ){ LOG.error("Fail to initialize NotificationManager: ", ex); throw new IllegalStateException("Fail to initialize NotificationManager: ", ex); } DynamicPolicyLoader<AlertDefinitionAPIEntity> policyLoader = DynamicPolicyLoader.getInstanceOf(AlertDefinitionAPIEntity.class); policyLoader.init(initialAlertDefs, dao, config); for (String alertExecutorId : alertExecutorIdList) { policyLoader.addPolicyChangeListener(alertExecutorId, this); } }
@Override public void onPolicyUpdate(T newAlertDef) { AbstractPolicyDefinition policyDef = null; try { policyDef = JsonSerDeserUtils.deserialize(newAlertDef.getPolicyDef(), AbstractPolicyDefinition.class, PolicyManager.getInstance().getPolicyModules(newAlertDef.getTags().get(Constants.POLICY_TYPE))); } catch (Exception ex) { LOG.error("Initial policy def error, ", ex); } SiddhiRuntime previous = siddhiRuntime; siddhiRuntime = createSiddhiRuntime((SiddhiPolicyDefinition) policyDef); synchronized (previous) { if (!previous.markdownEnabled) // condition to check if previous SiddhiRuntime was started after policy validation previous.siddhiManager.getExecutionPlanRuntime(previous.executionPlanName).shutdown(); } }
@Override public void onPolicyChanged(Map<String, T> changed) { if(LOG.isDebugEnabled()) LOG.debug(executorId + ", partition " + partitionSeq + " policy changed : " + changed); for(T alertDef : changed.values()){ if(!accept(alertDef)) continue; LOG.info(executorId + ", partition " + partitionSeq + " policy really changed " + alertDef); synchronized(this.policyEvaluators) { PolicyEvaluator<T> pe = policyEvaluators.get(alertDef.getTags().get(Constants.POLICY_ID)); boolean previousMarkdown = pe.isMarkdownEnabled(); String previousMarkdownReason = pe.getMarkdownReason(); pe.onPolicyUpdate(alertDef); if (isMarkdownUpdateRequired(previousMarkdown, pe.isMarkdownEnabled(), previousMarkdownReason, pe.getMarkdownReason())) updateMarkdownDetails(alertDef, pe.isMarkdownEnabled(), pe.getMarkdownReason()); } } }
@Override public void onPolicyUpdate(AlertDefinitionAPIEntity newAlertDef) { LOG.info("onPolicyUpdate called"); AbstractPolicyDefinition policyDef = null; try { policyDef = JsonSerDeserUtils.deserialize(newAlertDef.getPolicyDef(), AbstractPolicyDefinition.class, PolicyManager.getInstance().getPolicyModules(newAlertDef.getTags().get("policyType"))); } catch (Exception ex) { LOG.error("initial policy def error, ", ex); } MLRuntime previous = mlRuntime; mlRuntime = newMLRuntime((MLPolicyDefinition) policyDef); synchronized (previous) { previous.mlAnomalyCallbacks = null; previous.mlAlgorithmEvaluators = null; previous.mlPolicyDef = null; } previous = null; }
String alertExecutorId = entry.getKey(); for (PolicyLifecycleMethods<T> policyLifecycleMethod : entry.getValue()) { Map<String, T> addedPolicies = (Map<String, T>)added.get(trimPartitionNum(alertExecutorId)); if(addedPolicies != null && addedPolicies.size() > 0){ policyLifecycleMethod.onPolicyCreated(addedPolicies); Map<String, T> changedPolicies = (Map<String, T>)changed.get(trimPartitionNum(alertExecutorId)); if(changedPolicies != null && changedPolicies.size() > 0){ policyLifecycleMethod.onPolicyChanged(changedPolicies); Map<String, T> deletedPolicies = (Map<String, T>)deleted.get(trimPartitionNum(alertExecutorId)); if(deletedPolicies != null && deletedPolicies.size() > 0){ policyLifecycleMethod.onPolicyDeleted(deletedPolicies); policyDistributionUpdateMethod.report();
if(LOG.isDebugEnabled()) LOG.debug("Current policyEvaluators: " + policyEvaluators.keySet().toString()); updateCounter(EAGLE_EVENT_COUNT, baseDimensions); try{ synchronized(this.policyEvaluators) { String policyId = entry.getKey(); PolicyEvaluator<T> evaluator = entry.getValue(); if (!evaluator.isMarkdownEnabled()) { // not evaluated for a marked down policy updateCounter(EAGLE_POLICY_EVAL_COUNT, getDimensions(policyId)); try { evaluator.evaluate(new ValuesArray(outputCollector, input.get(1), input.get(2))); } catch (Exception ex) { LOG.error("Got an exception, but continue to run " + input.get(2).toString(), ex); updateCounter(EAGLE_POLICY_EVAL_COUNT, getDimensions(policyId)); updateCounter(EAGLE_ALERT_FAIL_COUNT, baseDimensions);
@Override public void receive(Event[] events) { long timeStamp = System.currentTimeMillis(); List<K> alerts = new LinkedList<>(); PolicyEvaluationContext<T, K> siddhiContext = null; for (Event event : events) { Object[] data = event.getData(); List<Object> returns = SiddhiQueryCallbackImpl.getOutputObject(event.getData()); K alert = siddhiContext.resultRender.render(config, returns, siddhiContext, timeStamp); alerts.add(alert); if (siddhiContext == null) { siddhiContext = (PolicyEvaluationContext<T, K>) data[0]; } } if (siddhiContext != null) { siddhiContext.alertExecutor.onEvalEvents(siddhiContext, alerts); } } }
private PolicyManager(){ loader = ServiceLoader.load(PolicyEvaluatorServiceProvider.class); Iterator<PolicyEvaluatorServiceProvider> iter = loader.iterator(); while(iter.hasNext()){ PolicyEvaluatorServiceProvider factory = iter.next(); LOG.info("Supported policy type : " + factory.getPolicyType()); policyEvaluators.put(factory.getPolicyType(), factory.getPolicyEvaluator()); policyModules.put(factory.getPolicyType(), factory.getBindingModules()); } }
@SuppressWarnings("unchecked") @Override public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) { List<Object> rets = getOutputObject(inEvents[0].getData()); K alert = siddhiEvaluateContext.resultRender.render(config, rets, siddhiEvaluateContext, timeStamp); SiddhiEvaluationHandler<T, K> handler = siddhiEvaluateContext.alertExecutor; handler.onEvalEvents(siddhiEvaluateContext, Arrays.asList(alert)); } }
@Override public void flatMap(List<Object> input, Collector<Tuple2<String, AggregateEntity>> collector) { if (input.size() != 3) throw new IllegalStateException("AggregateExecutor always consumes exactly 3 fields: key, stream name and value(SortedMap)"); if (LOG.isDebugEnabled()) LOG.debug("Msg is coming " + input.get(2)); if (LOG.isDebugEnabled()) LOG.debug("Current policyEvaluators: " + evaluator); try { evaluator.evaluate(new ValuesArray(collector, input.get(1), input.get(2))); } catch (Exception ex) { LOG.error("Got an exception, but continue to run " + input.get(2).toString(), ex); } }
@Override public void onPolicyDeleted(Map<String, T> deleted) { if(LOG.isDebugEnabled()) LOG.debug(executorId + ", partition " + partitionSeq + " policy deleted : " + deleted); for(T alertDef : deleted.values()){ if(!accept(alertDef)) continue; LOG.info(executorId + ", partition " + partitionSeq + " policy really deleted " + alertDef); String policyId = alertDef.getTags().get(Constants.POLICY_ID); synchronized(this.policyEvaluators) { if (policyEvaluators.containsKey(policyId)) { PolicyEvaluator<T> pe = policyEvaluators.remove(alertDef.getTags().get(Constants.POLICY_ID)); pe.onPolicyDelete(); } } } }
public static boolean accept(AlertDefinitionAPIEntity alertDef, PolicyPartitioner partitioner, int numPartitions, int partitionSeq){ int targetPartitionSeq = partitioner.partition(numPartitions, alertDef.getTags().get(Constants.POLICY_TYPE), alertDef.getTags().get(Constants.POLICY_ID)); if(targetPartitionSeq == partitionSeq) return true; return false; } }
PolledConfigurationSource source = new DynamicPolicySource<T>(initialAlertDefs, dao, config);
/** * verify both alertExecutor logic name and partition id * @param alertDef alert definition * * @return whether accept the alert definition */ private boolean accept(T alertDef){ String executorID = alertDef.getTags().containsKey("executorId") ? alertDef.getTags().get("executorId") : alertDef.getTags().get("alertExecutorId"); if(!executorID.equals(executorId)) { if(LOG.isDebugEnabled()){ LOG.debug("alertDef does not belong to this alertExecutorId : " + executorId + ", alertDef : " + alertDef); } return false; } int targetPartitionSeq = partitioner.partition(numPartitions, alertDef.getTags().get(Constants.POLICY_TYPE), alertDef.getTags().get(Constants.POLICY_ID)); if(targetPartitionSeq == partitionSeq) return true; return false; }