private void setGeoFields(GenericRecord record, InetAddress address) { GeoLocation city = lookup.lookup(address); put(record,"_country_code", city.country); put(record,"_region", city.stateProv); put(record,"_city", city.city); put(record,"_latitude", city.coordination.latitude); put(record,"_longitude", city.coordination.longitude); } }
private GenericData.Array getArray(FieldType arrayElementType, String valueAsString) throws IOException { JsonParser parser = jsonFactory.createParser(valueAsString); List<Object> objects = new ArrayList<>(); JsonToken t = parser.getCurrentToken(); if (t != JsonToken.START_ARRAY) { return null; } else { t = parser.nextToken(); } for (; t != JsonToken.END_ARRAY; t = parser.nextToken()) { if (!t.isScalarValue()) { throw new JsonMappingException(String.format("Nested properties are not supported. ('%s' field)", arrayElementType.name())); } objects.add(getValue(arrayElementType, parser)); } return new GenericData.Array(generateAvroSchema(arrayElementType), objects); }
public static void put(GenericRecord properties, String key, Object value) { Schema union = properties.getSchema().getField(key).schema(); Schema.Type type; if (union.getType().equals(UNION)) { type = union.getTypes().get(1).getType(); } else { type = union.getType(); } properties.put(key, AvroUtil.cast(type, value)); }
public static byte[] exportAsAvro(QueryResult result) { Schema avroSchema = AvroUtil.convertAvroSchema(result.getMetadata()); ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter writer = new FilteredRecordWriter(avroSchema, GenericData.get()); BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); GenericData.Record record = new GenericData.Record(avroSchema); for (List<Object> row : result.getResult()) { List<SchemaField> metadata = result.getMetadata(); for (int i = 0; i < row.size(); i++) { record.put(i, getAvroValue(row.get(i), metadata.get(i).getType())); } try { writer.write(record, encoder); } catch (Exception e) { throw new RuntimeException("Couldn't serialize event", e); } } return out.toByteArray(); }
objectBuilder.add(AvroUtil.generateAvroField(field)); AvroUtil.put(event.properties(), attr, attrValue.value); event.properties(record, fields);
public static Schema generateAvroSchema(FieldType field) { return Schema.createUnion(Lists.newArrayList(Schema.create(NULL), getAvroSchema(field))); }
private Schema createNewSchema(Schema currentSchema, SchemaField newField) { List<Schema.Field> avroFields = currentSchema.getFields().stream() .filter(field -> field.schema().getType() != Schema.Type.NULL) .map(field -> new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue())) .collect(toList()); try { avroFields.add(AvroUtil.generateAvroField(newField)); } catch (SchemaParseException e) { throw new RakamException("Couldn't create new column: " + e.getMessage(), BAD_REQUEST); } conditionalMagicFields.keySet().stream() .filter(s -> !avroFields.stream().anyMatch(af -> af.name().equals(s))) .map(n -> new Schema.Field(n, Schema.create(NULL), "", null)) .forEach(x -> avroFields.add(x)); Schema avroSchema = Schema.createRecord("collection", null, null, false); avroSchema.setFields(avroFields); return avroSchema; }
public EventList deserialize(String project, String collection, SliceInput slice) throws IOException { String json = slice.readSlice(slice.readInt()).toStringUtf8(); Schema schema = new Schema.Parser().parse(json); int records = slice.readInt(); BinaryDecoder binaryDecoder = DecoderFactory.get().directBinaryDecoder(slice, null); List<SchemaField> fields = metastore.getCollection(project, collection); Schema avroSchema = AvroUtil.convertAvroSchema(fields); GenericDatumReader<GenericRecord> reader = new GenericDatumReader(schema, avroSchema); List<Event> list = new ArrayList<>(records); for (int i = 0; i < records; i++) { GenericRecord record = reader.read(null, binaryDecoder); list.add(new Event(project, collection, null, fields, record)); } return new EventList(Event.EventContext.empty(), project, list); } }
default: if (type.isMap()) { Schema union = Schema.createUnion(ImmutableList.of(Schema.create(Schema.Type.NULL), getAvroSchema(type.getMapValueType()))); return Schema.createMap(union); Schema union = Schema.createUnion(ImmutableList.of(Schema.create(Schema.Type.NULL), getAvroSchema(type.getArrayElementType()))); return Schema.createArray(union);
put(properties,"_user_agent_family", parsed.userAgent.family); put(properties,"_user_agent_version", parsed.userAgent.major); } catch (NumberFormatException e) { put(properties,"_device_family", parsed.device.family); put(properties, "_os", parsed.os.family); put(properties,"_os_version", parsed.os.major); } catch (Exception e) {
.mapToObj(i -> header.getKey().get(i).getType()).collect(Collectors.toList()); Schema schema = convertAvroSchema(fields); GenericData.Record record = new GenericData.Record(schema); int idx = 0;
public static Schema.Field generateAvroField(SchemaField field) { return new Schema.Field(field.getName(), generateAvroSchema(field.getType()), null, NullNode.getInstance()); }
put(record, "_referrer_medium", parse.medium != null ? parse.medium.toString().toLowerCase() : null); put(record, "_referrer_source", parse.source); put(record, "_referrer_term", parse.term); put(record, "_referrer_domain", referrerUri.getHost()); put(record, "_referrer_path", referrerUri.getPath() + (referrerUri.getQuery() == null ? "" : ("?" + referrerUri.getQuery())));
public Event createEvent(String collection, Map<String, Object> properties) { List<SchemaField> cache = fieldCache.get(collection); List<SchemaField> fields; List<SchemaField> generatedSchema = generateSchema(properties); if (cache == null || !generatedSchema.stream().allMatch(f -> cache.contains(f))) { fields = metastore.getOrCreateCollectionFields(project, collection, ImmutableSet.copyOf(generatedSchema)); fieldCache.put(collection, fields); } else { fields = cache; } try { GenericData.Record record = new GenericData.Record(AvroUtil.convertAvroSchema(fields)); properties.forEach((key, value) -> record.put(key, cast(value, record.getSchema().getField(key).schema().getTypes().get(1).getType()))); return new Event(project, collection, Event.EventContext.empty(), fields, record); } catch (Exception e) { throw Throwables.propagate(e); } }
generateAvroSchema(fieldType.convertToArrayType())); objects.add(next.value); while (iterator.hasNext()) {
private void setIsp(InetAddress address, GenericRecord properties) { IspResponse isp; try { isp = ispLookup.isp(address); } catch (AddressNotFoundException e) { return; } catch (Exception e) { LOGGER.error(e, "Error while searching for location information."); return; } put(properties,"_isp", isp.getIsp()); }
List<SchemaField> collection = metastore.getCollection(project, collectionName); Schema avroSchema = convertAvroSchema(collection); DatumWriter writer = new FilteredRecordWriter(avroSchema, data);
switch (attribute) { case "country_code": put(properties,"_country_code", city.getCountry().getIsoCode()); break; case "region": put(properties,"_region", city.getContinent().getName()); break; case "city": put(properties,"_city", city.getCity().getName()); break; case "latitude": put(properties,"_latitude", city.getLocation().getLatitude()); break; case "longitude": put(properties,"_longitude", city.getLocation().getLongitude()); break; case "timezone": put(properties,"_timezone", city.getLocation().getTimeZone()); break;
isNew = rakamSchema == null || rakamSchema.isEmpty(); rakamSchema = isNew ? ImmutableList.copyOf(constantFields) : rakamSchema; schema = new SimpleImmutableEntry<>(rakamSchema, convertAvroSchema(rakamSchema, conditionalMagicFields)); schemaCache.put(key, schema); Schema newAvroSchema = convertAvroSchema(rakamSchema, conditionalMagicFields);
private void setConnectionType(InetAddress address, GenericRecord properties) { ConnectionTypeResponse connectionType; try { connectionType = connectionTypeLookup.connectionType(address); } catch (AddressNotFoundException e) { return; } catch (Exception e) { LOGGER.error(e, "Error while searching for location information."); return; } ConnectionTypeResponse.ConnectionType connType = connectionType.getConnectionType(); if (connType != null) { put(properties,"_connection_type", connType.name()); } }