@Override public int hashCode() { return getMetadata() != null ? getMetadata().hashCode() : 0; }
@Override public LookupValue toValue(Map<String, Object> metadata) { return new EnrichmentValue(metadata); } })
@Override public void fromColumns(Iterable<Map.Entry<byte[], byte[]>> values) { for(Map.Entry<byte[], byte[]> cell : values) { if(Bytes.equals(cell.getKey(), VALUE_COLUMN_NAME_B)) { metadata = stringToValue(Bytes.toString(cell.getValue())); } } } public Map<String, Object> stringToValue(String s){
@Override public Iterable<Map.Entry<byte[], byte[]>> toColumns() { return AbstractConverter.toEntries( VALUE_COLUMN_NAME_B, Bytes.toBytes(valueToString(metadata)) ); }
@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; EnrichmentValue that = (EnrichmentValue) o; return getMetadata() != null ? getMetadata().equals(that.getMetadata()) : that.getMetadata() == null; }
@Override public LookupKV<EnrichmentKey, EnrichmentValue> fromResult(Result result, String columnFamily) throws IOException { return fromResult(result, columnFamily, new EnrichmentKey(), new EnrichmentValue()); } }
try { LookupKV<EnrichmentKey, EnrichmentValue> kv = lookup.get(new EnrichmentKey(enrichmentType, indicator), hbaseContext, true); if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) { return kv.getValue().getMetadata();
@Override public LookupKV<EnrichmentKey, EnrichmentValue> fromPut(Put put, String columnFamily) throws IOException { return fromPut(put, columnFamily, new EnrichmentKey(), new EnrichmentValue()); }
if (kv != null && kv.getValue() != null && kv.getValue().getMetadata() != null) { for (Map.Entry<String, Object> values : kv.getValue().getMetadata().entrySet()) { enriched.put(kv.getKey().type + "." + values.getKey(), values.getValue());
private EnrichmentValue getValue( JSONObject message , Set<String> keyColumns , Set<String> valueColumns ) { Map<String, Object> metadata = new HashMap<>(); if(valueColumns == null || valueColumns.isEmpty()) { for (Object kv : message.entrySet()) { Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv; if (!keyColumns.contains(entry.getKey())) { addMetadataEntry(metadata, entry); } } return new EnrichmentValue(metadata); } else { for (Object kv : message.entrySet()) { Map.Entry<Object, Object> entry = (Map.Entry<Object, Object>) kv; if (valueColumns.contains(entry.getKey())) { addMetadataEntry(metadata, entry); } } return new EnrichmentValue(metadata); } }
kv.getValue().getMetadata().put("source_type", "taxii"); kv.getValue().getMetadata().put("taxii_url", endpoint.toString()); kv.getValue().getMetadata().put("taxii_collection", collection); Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue()); HTableInterface table = getTable(hbaseTable);
final String indicatorType = typeStr + ":" + category; LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token) , new EnrichmentValue( new HashMap<String, Object>() {{ put("source-type", "STIX");
@Test public void testFilteredKeys() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), ImmutableList.of("user", "ip")); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertEquals("localhost", values.get(0).getValue().getMetadata().get("ip")); Assert.assertNull(values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(2, values.get(0).getValue().getMetadata().size()); }
@Override public Iterable<LookupKV> extract(final Hostname type, Map<String, Object> config) throws IOException { StringObjectPropertyType value = type.getHostnameValue(); String typeStr = getType(); if(config != null) { Object o = config.get(TYPE_CONFIG); if(o != null) { typeStr = o.toString(); } } List<LookupKV> ret = new ArrayList<>(); for(String token : StixExtractor.split(value)) { final String indicatorType = typeStr; LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token) , new EnrichmentValue(new HashMap<String, Object>() {{ put("source-type", "STIX"); put("indicator-type", indicatorType); put("source", type.toXMLString()); }} ) ); ret.add(results); } return ret; } @Override
@Test public void testBatchOneNormalPath() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertEquals("bar", values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(2, values.get(0).getValue().getMetadata().size()); }
@Override public Iterable<LookupKV> extract(URIObjectType type, Map<String, Object> config) throws IOException { List<LookupKV> ret = new ArrayList<>(); if(type != null) { AnyURIObjectPropertyType val = type.getValue(); if(val != null) { Object v = val.getValue(); if(v != null) { final String indicatorType = getType(); LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, v.toString()) , new EnrichmentValue( new HashMap<String, Object>() {{ put("source-type", "STIX"); put("uri", v.toString()); put("indicator-type", indicatorType); put("source", type.toXMLString()); }} ) ); ret.add(results); } } } return ret; }
@Test public void testFilteredKey() throws Exception { final String sensorType = "dummy"; SimpleHbaseEnrichmentWriter writer = new SimpleHbaseEnrichmentWriter(); WriterConfiguration configuration = createConfig(1, new HashMap<String, Object>(BASE_WRITER_CONFIG) {{ put(SimpleHbaseEnrichmentWriter.Configurations.KEY_COLUMNS.getKey(), "ip"); put(SimpleHbaseEnrichmentWriter.Configurations.VALUE_COLUMNS.getKey(), "user"); }} ); writer.configure(sensorType,configuration); writer.write( SENSOR_TYPE , configuration , null , new ArrayList<JSONObject>() {{ add(new JSONObject(ImmutableMap.of("ip", "localhost", "user", "cstella", "foo", "bar"))); }} ); List<LookupKV<EnrichmentKey, EnrichmentValue>> values = getValues(); Assert.assertEquals(1, values.size()); Assert.assertEquals("localhost", values.get(0).getKey().indicator); Assert.assertEquals("cstella", values.get(0).getValue().getMetadata().get("user")); Assert.assertNull(values.get(0).getValue().getMetadata().get("foo")); Assert.assertEquals(1, values.get(0).getValue().getMetadata().size()); }
@Override public Iterable<LookupKV> extract(final DomainName type, Map<String, Object> config) throws IOException { List<LookupKV> ret = new ArrayList<>(); String typeStr = getType(); if(config != null) { Object o = config.get(TYPE_CONFIG); if(o != null) { typeStr = o.toString(); } } final DomainNameTypeEnum domainType = type.getType(); if(domainType == null || SUPPORTED_TYPES.contains(domainType)) { StringObjectPropertyType value = type.getValue(); for (String token : StixExtractor.split(value)) { final String indicatorType = typeStr + ":" + DomainNameTypeEnum.FQDN; LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, token) , new EnrichmentValue( new HashMap<String, Object>() {{ put("source-type", "STIX"); put("indicator-type", indicatorType); put("source", type.toXMLString()); }} ) ); ret.add(results); } } return ret; } @Override
for (LookupKV<EnrichmentKey, EnrichmentValue> kv : result.getResult()) { Assert.assertTrue(validIndicators.contains(kv.getKey().indicator)); Assert.assertEquals(kv.getValue().getMetadata().get("source.type"), "dummy"); Assert.assertNotNull(kv.getValue().getMetadata().get("timestamp")); Assert.assertNotNull(kv.getValue().getMetadata().get("original_string")); Map<String, String> metadata = validMetadata.get(kv.getKey().indicator); for (Map.Entry<String, String> x : metadata.entrySet()) { Assert.assertEquals(kv.getValue().getMetadata().get(x.getKey()), x.getValue()); Assert.assertEquals(metadata.size() + 4, kv.getValue().getMetadata().size());
@Before public void setup() throws Exception { final MockHTable trackerTable = (MockHTable) MockHBaseTableProvider.addToCache(atTableName, cf); final MockHTable threatIntelTable = (MockHTable) MockHBaseTableProvider.addToCache(threatIntelTableName, cf); EnrichmentHelper.INSTANCE.load(threatIntelTable, cf, new ArrayList<LookupKV<EnrichmentKey, EnrichmentValue>>() {{ add(new LookupKV<>(new EnrichmentKey("10.0.2.3", "10.0.2.3"), new EnrichmentValue(new HashMap<>()))); }}); BloomAccessTracker bat = new BloomAccessTracker(threatIntelTableName, 100, 0.03); PersistentAccessTracker pat = new PersistentAccessTracker(threatIntelTableName, "0", trackerTable, cf, bat, 0L); lookup = new EnrichmentLookup(threatIntelTable, cf, pat); JSONParser jsonParser = new JSONParser(); expectedMessage = (JSONObject) jsonParser.parse(expectedMessageString); }