private Map<String, String> createMetricsTypeMapping(ControllerServiceInitializationContext context) { return Arrays.stream(context.getPropertyValue(METRICS_TYPE_MAPPING).asString() .split(",")) .filter(StringUtils::isNotBlank) .map(s -> s.split(":")) .collect(Collectors.toMap(a -> a[0], a -> a[1])); }
/** * Gets the directory containing the logisland python dependencies (shipped with logisland) * @param context Logisland context * @throws Exception */ private void getLogislandDependenciesPathParam(ProcessContext context) throws Exception { // TODO: can this be found with correct default value pointing to resources in jar file? logislandDependenciesPath = context.getPropertyValue(LOGISLAND_DEPENDENCIES_PATH).asString(); logger.debug("Using python logisland dependencies path: " + logislandDependenciesPath); }
/** * Gets parameters needed for inline mode: imports, init and process methods code * @param context Logisland context * @throws Exception */ private void getInlineModeParams(ProcessContext context) throws Exception { scriptCodeImports = context.getPropertyValue(SCRIPT_CODE_IMPORTS).asString(); scriptCodeInit = context.getPropertyValue(SCRIPT_CODE_INIT).asString(); scriptCodeProcess = context.getPropertyValue(SCRIPT_CODE_PROCESS).asString(); }
@Override public Collection<Record> process(ProcessContext context, Collection<Record> records) { String fieldName = context.getPropertyValue(FIELD_NAME).asString(); String fieldValue = context.getPropertyValue(FIELD_VALUE).asString(); return records.stream() .filter(record -> record.hasField(fieldName) && record.getField(fieldName).asString().equals(fieldValue)) .collect(Collectors.toList()); }
@Override public Collection<Record> process(ProcessContext context, Collection<Record> records) { String fieldToFilterOn = context.getPropertyValue(FILTERING_FIELD).asString(); Collection<Record> outputRecords = records.stream().filter(record -> !record.hasField(fieldToFilterOn)).collect(Collectors.toList()); try { outputRecords.addAll( records.stream() .filter(distinctByKey(record -> record.getField(fieldToFilterOn))) .collect(Collectors.toList())); } catch (Exception ex) { logger.warn("issue while trying to remove field list {} : {}", context.getPropertyValue(FILTERING_FIELD).asString(), ex.toString()); } return outputRecords; }
@Override public Collection<Record> process(ProcessContext context, Collection<Record> records) { try { List<String> fieldsToRemove = Lists.newArrayList( context.getPropertyValue(FIELDS_TO_REMOVE).asString().split(",")); for (Record record : records) { new ArrayList<>(record.getAllFields()).forEach(field -> { String fieldName = field.getName(); if (fieldsToRemove.contains(fieldName)) { record.removeField(fieldName); } }); } } catch (Exception ex) { logger.warn("issue while trying to remove field list {} : {}", context.getPropertyValue(FIELDS_TO_REMOVE).asString(), ex.toString()); } return records; }
private boolean is_search_engine(String domain, ProcessContext context, Record record) { String es_search_engine_field = context.getPropertyValue(ES_SEARCH_ENGINE_FIELD).asString(); return has_domain_flag(domain, es_search_engine_field, context, record); }
@Override public Collection<Record> process(ProcessContext context, Collection<Record> records) { String property = context.getPropertyValue(PROPERTIES_FIELD).asString(); for (Record record : records) { extractAndParsePropertiesField(record, property); } return records; }
public void init(final ControllerServiceInitializationContext context) { this.context = context; final String redisMode = context.getPropertyValue(RedisUtils.REDIS_MODE).asString(); this.redisType = RedisType.fromDisplayName(redisMode); }
@Override public Collection<Record> process(ProcessContext context, Collection<Record> records) { SamplingAlgorithm algorithm = SamplingAlgorithm.valueOf( context.getPropertyValue(SAMPLING_ALGORITHM).asString().toUpperCase()); String valueFieldName = context.getPropertyValue(RECORD_VALUE_FIELD).asString(); String timeFieldName = context.getPropertyValue(RECORD_TIME_FIELD).asString(); int parameter = context.getPropertyValue(SAMPLING_PARAMETER).asInteger(); Sampler sampler = SamplerFactory.getSampler(algorithm, valueFieldName, timeFieldName, parameter); return sampler.sample(new ArrayList<>(records)).stream() .map(r -> { return r.setField("is_sampled", FieldType.BOOLEAN, true); }).collect(Collectors.toList()); }
private boolean is_social_network(String domain, ProcessContext context, Record record) { String es_social_network_field = context.getPropertyValue(ES_SOCIAL_NETWORK_FIELD).asString(); return has_domain_flag(domain, es_social_network_field, context, record); }
@Override @OnEnabled public void init(ControllerServiceInitializationContext context) throws InitializationException { try { pathName = context.getPropertyValue(ML_MODEL_FILE_PATH).asString(); } catch (Exception e){ throw new InitializationException(e); } }
@Override public void init(ProcessContext context) { this.clientService = context.getPropertyValue(HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class); if (context.getPropertyValue(RECORD_SCHEMA).isSet()) { serializer = SerializerProvider.getSerializer( context.getPropertyValue(RECORD_SERIALIZER).asString(), context.getPropertyValue(RECORD_SCHEMA).asString()); } else { serializer = SerializerProvider.getSerializer(context.getPropertyValue(RECORD_SERIALIZER).asString(), null); } }
private void updateRecord(ProcessContext context, Record record, Map<String, String> fieldsNameMapping) { String conflictPolicy = context.getPropertyValue(CONFLICT_RESOLUTION_POLICY).asString(); if ((fieldsNameMapping == null) || (fieldsNameMapping.keySet() == null)) { return; } fieldsNameMapping.keySet().forEach(addedFieldName -> { final String defaultValueToAdd = context.getPropertyValue(addedFieldName).evaluate(record).asString(); // field is already here if (record.hasField(addedFieldName)) { if (conflictPolicy.equals(OVERWRITE_EXISTING.getValue())) { overwriteObsoleteFieldValue(record, addedFieldName, defaultValueToAdd); } } else { record.setStringField(addedFieldName, defaultValueToAdd); } }); }
@Override public void init(final ProcessContext context) { clientService = context.getPropertyValue(HBASE_CLIENT_SERVICE).asControllerService(HBaseClientService.class); if(clientService == null) logger.error("HBase client service is not initialized!"); if (context.getPropertyValue(RECORD_SCHEMA).isSet()) { serializer = SerializerProvider.getSerializer( context.getPropertyValue(RECORD_SERIALIZER).asString(), context.getPropertyValue(RECORD_SCHEMA).asString()); } else { serializer = SerializerProvider.getSerializer(context.getPropertyValue(RECORD_SERIALIZER).asString(), null); } }
private void extractValueFields(String[] valueFields, Record outputRecord, Matcher valueMatcher, ProcessContext context) { String conflictPolicy = context.getPropertyValue(CONFLICT_RESOLUTION_POLICY).asString(); for (int i = 0; i < Math.min(valueMatcher.groupCount() + 1, valueFields.length); i++) { String content = valueMatcher.group(i + 1); String fieldName = valueFields[i]; if (content != null) { if (outputRecord.hasField(fieldName) && (outputRecord.getField(fieldName).asString() != null) && (! outputRecord.getField(fieldName).asString().isEmpty())) { if (conflictPolicy.equals(OVERWRITE_EXISTING.getValue())) { outputRecord.setStringField(fieldName, content.replaceAll("\"", "")); } } else { outputRecord.setStringField(fieldName, content.replaceAll("\"", "")); } } } }
@Override @OnEnabled public void init(ControllerServiceInitializationContext context) throws InitializationException { try { this.redisConnectionPool = new RedisConnectionPool(); this.redisConnectionPool.init(context); this.recordSerializer = getSerializer( context.getPropertyValue(RECORD_SERIALIZER).asString(), context.getPropertyValue(AVRO_SCHEMA).asString()); } catch (Exception e) { throw new InitializationException(e); } }
@Override protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { final boolean isAvroSerializer = validationContext.getPropertyValue(RECORD_SERIALIZER).asString().toLowerCase().contains("avro"); final boolean isAvroSchemaSet = validationContext.getPropertyValue(RECORD_SCHEMA).isSet(); final List<ValidationResult> problems = new ArrayList<>(); if (isAvroSerializer && !isAvroSchemaSet) { problems.add(new ValidationResult.Builder() .subject(RECORD_SERIALIZER.getDisplayName()) .valid(false) .explanation("an avro schema must be provided with an avro serializer") .build()); } return problems; }
@Override public void init(final ProcessContext context) { this.fieldsNameMapping = getFieldsNameMapping(context); this.nbSplitLimit = context.getPropertyValue(NB_SPLIT_LIMIT).asInteger(); this.isEnabledSplitCounter = context.getPropertyValue(ENABLE_SPLIT_COUNTER).asBoolean(); this.splitCounterSuffix = context.getPropertyValue(SPLIT_COUNTER_SUFFIX).asString(); }
@Override public void handleMatch(final Record record, final ProcessContext context, final MatchingRule matchingRule, MatchQuery.RecordTypeUpdatePolicy recordTypeUpdatePolicy) { this.outRecords.add( new StandardRecord(record) .setType( (recordTypeUpdatePolicy == MatchQuery.RecordTypeUpdatePolicy.overwrite) ? context.getPropertyValue(MatchQuery.OUTPUT_RECORD_TYPE).asString() : record.getType() ) .setStringField(ALERT_MATCH_NAME, matchingRule.getName()) .setStringField(ALERT_MATCH_QUERY, matchingRule.getLegacyQuery())); }