if (value instanceof Record) { final Record record = (Record) value; final Object ksqlValue = avroData.toConnectData(record.getSchema(), record).value(); genericRowValues.add( SchemaUtil.getOptionalValue(ksqlSchema.field(field.name()).schema(), ksqlValue));
@Override protected void customizeSchemaField(Field avroField, Record fieldType) { if (fieldType.getSchema().getName().equals(ARRAY_FIELD_TYPE)) { GenericData.EnumSymbol overrideStrategy = (GenericData.EnumSymbol) fieldType.get(OVERRIDE_STRATEGY); if (overrideStrategy != null && !overrideStrategy.toString().equalsIgnoreCase(OverrideStrategy.REPLACE.name())) { avroField.addProp(OVERRIDE_STRATEGY, overrideStrategy.toString().toLowerCase()); } } }
/** * Creates a GenericRecordBuilder by copying an existing record instance. * @param other the record instance to copy. */ public GenericRecordBuilder(Record other) { super(other.getSchema(), GenericData.get()); record = new GenericData.Record(other, /* deepCopy = */ true); // Set all fields in the RecordBuilder that are set in the record for (Field f : schema().getFields()) { Object value = other.get(f.pos()); // Only set the value if it is not null, if the schema type is null, // or if the schema type is a union that accepts nulls. if (isValidValue(f, value)) { set(f, data().deepCopy(f.schema(), value)); } } }
public static GenericData.Record convertEventAvro(GenericData.Record row) { String val1 = row.get("one").toString(); String val2 = row.get("two").toString(); GenericData.Record upd = new GenericData.Record(row.getSchema()); upd.put("one", val1); upd.put("two", "|" + val2 + "|"); return upd; }
private boolean isExistsProperty(GenericData.Record record) { Schema.Field field = record.getSchema().getField(fieldTop); if (field == null) { return false; } Object inner = record.get(fieldTop); if (!(inner instanceof GenericData.Record)) { return false; } return getter.isExistsPropertyAvro((GenericData.Record) inner); }
@Test() public void testInternalReferrer() throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); List<SchemaField> fields = builder.build().dependentFields.get("_referrer"); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(fields.stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null), new Schema.Field("_host", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); properties.put("_referrer", "https://test.com/"); properties.put("_host", "test.com"); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, InetAddress.getLocalHost(), null); assertNull(resp); assertNull(event.getAttribute("_referrer_source")); assertNull(event.getAttribute("_referrer_term")); assertEquals("internal", event.getAttribute("_referrer_medium")); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "chrome-user-agent") public void testUserAgentMapper(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { UserAgentEventMapper mapper = new UserAgentEventMapper(new WebsiteMapperConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_user_agent").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_user_agent", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Chrome", event.getAttribute("_user_agent_family")); assertEquals("Mac OS X", event.getAttribute("_os")); assertEquals("10", event.getAttribute("_os_version")); assertEquals("Other", event.getAttribute("_device_family")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_ip").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertTrue(event.properties().getSchema().getField("_country_code") != null); assertTrue(event.properties().getSchema().getField("_city") != null); assertTrue(event.properties().getSchema().getField("_timezone") != null); assertTrue(event.getAttribute("_latitude") instanceof Double); assertTrue(event.properties().getSchema().getField("_region") != null); assertTrue(event.getAttribute("_longitude") instanceof Double); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-referrer") public void testReferrer(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_referrer").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Google", event.getAttribute("_referrer_source")); assertEquals("test", event.getAttribute("_referrer_term")); assertEquals("search", event.getAttribute("_referrer_medium")); assertEquals("google.com", event.getAttribute("_referrer_domain")); assertEquals("/?q=test", event.getAttribute("_referrer_path")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Test(dataProvider = "google-ips") public void testConnectionTypeEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setConnectionTypeDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-Connection-Type-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_connection_type", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); // TODO: find a reliable ip that can be mapped. assertNull(event.getAttribute("connection_type")); GenericData.get().validate(properties.getSchema(), properties); }
public Event createEvent(String collection, Map<String, Object> properties) { List<SchemaField> cache = fieldCache.get(collection); List<SchemaField> fields; List<SchemaField> generatedSchema = generateSchema(properties); if (cache == null || !generatedSchema.stream().allMatch(f -> cache.contains(f))) { fields = metastore.getOrCreateCollectionFields(project, collection, ImmutableSet.copyOf(generatedSchema)); fieldCache.put(collection, fields); } else { fields = cache; } try { GenericData.Record record = new GenericData.Record(AvroUtil.convertAvroSchema(fields)); properties.forEach((key, value) -> record.put(key, cast(value, record.getSchema().getField(key).schema().getTypes().get(1).getType()))); return new Event(project, collection, Event.EventContext.empty(), fields, record); } catch (Exception e) { throw Throwables.propagate(e); } }
@Test(dataProvider = "google-ips") public void testIspEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig() .setAttributes("") .setIspDatabaseUrl(new URL("https://github.com/maxmind/MaxMind-DB/raw/master/test-data/GeoIP2-ISP-Test.mmdb"))); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); Record properties = new Record(Schema.createRecord(ImmutableList.of( new Schema.Field("_ip", Schema.create(NULL), null, null), new Schema.Field("__ip", Schema.create(STRING), null, null), new Schema.Field("_isp", Schema.create(STRING), null, null)))); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertEquals(event.getAttribute("_isp"), "Level 3 Communications"); GenericData.get().validate(properties.getSchema(), properties); }
if (value instanceof GenericData.Record) { final GenericData.Record avroRecord = (GenericData.Record) value; return normalizeValue(value, avroRecord.getSchema(), fieldName); final Schema recordSchema = record.getSchema(); final List<Field> recordFields = recordSchema.getFields(); final Map<String, Object> values = new HashMap<>(recordFields.size());
@Test public void testSchemaEvolution() throws Exception { List<ByteBuffer> buffers = new ArrayList<>(); List<Record> records = new ArrayList<>(); records.addAll(V1_RECORDS); records.addAll(V2_RECORDS); MessageEncoder<Record> v1Encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V1); MessageEncoder<Record> v2Encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V2); for (Record record : records) { if (record.getSchema() == SCHEMA_V1) { buffers.add(v1Encoder.encode(record)); } else { buffers.add(v2Encoder.encode(record)); } } Set<Record> allAsV2 = new HashSet<>(V2_RECORDS); allAsV2.add(V2_BUILDER.set("id", 1L).set("message", "m-1").clear("data").build()); allAsV2.add(V2_BUILDER.set("id", 2L).set("message", "m-2").clear("data").build()); allAsV2.add(V2_BUILDER.set("id", 4L).set("message", "m-4").clear("data").build()); allAsV2.add(V2_BUILDER.set("id", 6L).set("message", "m-6").clear("data").build()); BinaryMessageDecoder<Record> v2Decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); v2Decoder.addSchema(SCHEMA_V1); Set<Record> decodedUsingV2 = new HashSet<>(); for (ByteBuffer buffer : buffers) { decodedUsingV2.add(v2Decoder.decode(buffer)); } Assert.assertEquals(allAsV2, decodedUsingV2); }
/** * Creates a GenericRecordBuilder by copying an existing record instance. * @param other the record instance to copy. */ public GenericRecordBuilder(Record other) { super(other.getSchema(), GenericData.get()); record = new GenericData.Record(other, /* deepCopy = */ true); // Set all fields in the RecordBuilder that are set in the record for (Field f : schema().getFields()) { Object value = other.get(f.pos()); // Only set the value if it is not null, if the schema type is null, // or if the schema type is a union that accepts nulls. if (isValidValue(f, value)) { set(f, data().deepCopy(f.schema(), value)); } } }
@Override protected void customizeFormField(Record fieldType, Field field) { if (fieldType.getSchema().getName().equals(ARRAY_FIELD_TYPE)) { JsonNode overrideStrategyNode = field.getJsonProp(OVERRIDE_STRATEGY); Schema overrideStrategySchema = fieldType.getSchema().getField(OVERRIDE_STRATEGY).schema(); if (overrideStrategyNode != null && overrideStrategyNode.isTextual()) { fieldType.put(OVERRIDE_STRATEGY, new GenericData.EnumSymbol( overrideStrategySchema, overrideStrategyNode.asText().toUpperCase())); } else { fieldType.put(OVERRIDE_STRATEGY, new GenericData.EnumSymbol( overrideStrategySchema, OverrideStrategy.REPLACE.name())); } } }
@Override protected void customizeType(Record record, Schema fieldTypeSchema) { if (record != null && eventClassTypes.contains(record.getSchema().getName())) { JsonNode classTypeNode = fieldTypeSchema.getJsonProp(CLASS_TYPE); Schema enumSchema = record.getSchema().getField(CLASS_TYPE).schema(); if (classTypeNode != null && classTypeNode.isTextual()) { record.put(CLASS_TYPE, new GenericData.EnumSymbol(enumSchema, classTypeNode.asText().toUpperCase())); } else { record.put(CLASS_TYPE, new GenericData.EnumSymbol(enumSchema, OBJECT)); } } }
@Override protected void customizeType(Record record, Schema fieldTypeSchema) { if (record != null && record.getSchema().getName().equals(RECORD_FIELD_TYPE)) { JsonNode addressableNode = fieldTypeSchema.getJsonProp(ADDRESSABLE); if (addressableNode != null && addressableNode.isBoolean()) { record.put(ADDRESSABLE, addressableNode.asBoolean()); } else { record.put(ADDRESSABLE, true); } } }