@Test /** Construct and use a different protocol whose "hello" method has an extra argument to check that schema is sent to parse request. */ public void testHandshake() throws IOException { Protocol protocol = new Protocol("Simple", "org.apache.avro.test"); List<Field> fields = new ArrayList<>(); fields.add(new Schema.Field("extra", Schema.create(Schema.Type.BOOLEAN), null, null)); fields.add(new Schema.Field("greeting", Schema.create(Schema.Type.STRING), null, null)); Protocol.Message message = protocol.createMessage("hello", null /* doc */, Schema.createRecord(fields), Schema.create(Schema.Type.STRING), Schema.createUnion(new ArrayList<>())); protocol.getMessages().put("hello", message); Transceiver t = new SocketTransceiver(new InetSocketAddress(server.getPort())); try { GenericRequestor r = new GenericRequestor(protocol, t); GenericRecord params = new GenericData.Record(message.getRequest()); params.put("extra", Boolean.TRUE); params.put("greeting", new Utf8("bob")); Utf8 response = (Utf8)r.request("hello", params); assertEquals(new Utf8("goodbye"), response); } finally { t.close(); } }
private Object serializeStruct(StructTypeInfo typeInfo, StructObjectInspector ssoi, Object o, Schema schema) throws AvroSerdeException { int size = schema.getFields().size(); List<? extends StructField> allStructFieldRefs = ssoi.getAllStructFieldRefs(); List<Object> structFieldsDataAsList = ssoi.getStructFieldsDataAsList(o); GenericData.Record record = new GenericData.Record(schema); ArrayList<TypeInfo> allStructFieldTypeInfos = typeInfo.getAllStructFieldTypeInfos(); for(int i = 0; i < size; i++) { Field field = schema.getFields().get(i); TypeInfo colTypeInfo = allStructFieldTypeInfos.get(i); StructField structFieldRef = allStructFieldRefs.get(i); Object structFieldData = structFieldsDataAsList.get(i); ObjectInspector fieldOI = structFieldRef.getFieldObjectInspector(); Object val = serialize(colTypeInfo, fieldOI, structFieldData, field.schema()); record.put(field.name(), val); } return record; }
switch (avroSchema.getType()) { case INT: { final LogicalType logicalType = avroSchema.getLogicalType(); if (logicalType == null) { return value; final LogicalType logicalType = avroSchema.getLogicalType(); if (logicalType == null) { return value; if (value instanceof GenericData.Record) { final GenericData.Record avroRecord = (GenericData.Record) value; return normalizeValue(value, avroRecord.getSchema(), fieldName); final Schema recordSchema = record.getSchema(); final List<Field> recordFields = recordSchema.getFields(); final Map<String, Object> values = new HashMap<>(recordFields.size()); for (final Field field : recordFields) { final Object avroFieldValue = record.get(field.name()); final Object fieldValue = normalizeValue(avroFieldValue, field.schema(), fieldName + "/" + field.name()); values.put(field.name(), fieldValue);
for (Field f : PROTOCOL.getType("TestRecord").getFields()) fields.add(new Field(f.name(), f.schema(), null, null)); fields.add(new Field("extra", Schema.create(Schema.Type.BOOLEAN), null, true)); Schema record = Schema.createRecord("TestRecord", null, "org.apache.avro.test", false); record.setFields(fields); Protocol protocol = new Protocol("Simple", "org.apache.avro.test"); List<Field> params = new ArrayList<>(); params.add(new Field("record", record, null, null)); protocol.createMessage("echo", null, Schema.createRecord(params), record, Schema.createUnion(new ArrayList<>())); try { GenericRequestor r = new GenericRequestor(protocol, t); GenericRecord args = new GenericData.Record(message.getRequest()); GenericRecord rec = new GenericData.Record(record); rec.put("name", new Utf8("foo")); rec.put("kind", new GenericData.EnumSymbol (PROTOCOL.getType("Kind"), "BAR")); rec.put("hash", new GenericData.Fixed (PROTOCOL.getType("MD5"), new byte[]{0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5}));
public static void fillComplexFullResyncDelta(GenericRecord delta) { GenericRecord testField2 = new GenericData.Record(getSchemaByFullName( delta.getSchema().getField("testField2").schema().getTypes(), "org.kaa.config.testRecordT")); testField2.put("testField3", 456); byte[] rawUuid = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; GenericFixed uuid = new GenericData.Fixed(delta.getSchema() .getField("__uuid").schema(), rawUuid); delta.put("testField1", "abc"); delta.put("testField2", testField2); delta.put("__uuid", uuid); }
for (final Schema.Field field : avroSchema.getFields()) { final boolean isSession = field.schema().getProp("session") != null; final boolean isSessionSiblingIntHash = field.schema().getProp("session-sibling-int-hash") != null; final String timeFormatFromLong = field.schema().getProp("format_as_time"); final String currentValue = (String) randomAvroMessage.get(field.name()); final String newCurrentValue = handleSessionisationOfValue(sessionManager, currentValue); sessionisationValue = newCurrentValue; final Object value = randomAvroMessage.get(field.name()); if (value instanceof Record) { final Record record = (Record) value; final Object ksqlValue = avroData.toConnectData(record.getSchema(), record).value(); genericRowValues.add( SchemaUtil.getOptionalValue(ksqlSchema.field(field.name()).schema(), ksqlValue)); randomAvroMessage.getSchema().getField(key).schema(), randomAvroMessage.get(key)).value().toString();
@Test public void testByteBufferDeepCopy() { // Test that a deep copy of a byte buffer respects the byte buffer // limits and capacity. byte[] buffer_value = {0, 1, 2, 3, 0, 0, 0}; ByteBuffer buffer = ByteBuffer.wrap(buffer_value, 1, 4); Schema schema = Schema.createRecord("my_record", "doc", "mytest", false); Field byte_field = new Field("bytes", Schema.create(Type.BYTES), null, null); schema.setFields(Arrays.asList(byte_field)); GenericRecord record = new GenericData.Record(schema); record.put(byte_field.name(), buffer); GenericRecord copy = GenericData.get().deepCopy(schema, record); ByteBuffer buffer_copy = (ByteBuffer) copy.get(byte_field.name()); assertEquals(buffer, buffer_copy); }
@Override public Iterable<GenericRecord> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { GenericRecord genericRecord = new GenericData.Record(outputSchema); BiMap<String, String> inversedViewOfFieldsRenameMap = this.fieldsRenameMap.inverse(); for (Schema.Field field : outputSchema.getFields()) { String curFieldName = field.name(); String originalFieldName = inversedViewOfFieldsRenameMap.containsKey(curFieldName) ? inversedViewOfFieldsRenameMap.get(curFieldName) : curFieldName; if (this.nonMapFields.contains(originalFieldName)) { genericRecord.put(curFieldName, inputRecord.get(originalFieldName)); } else { genericRecord.put(curFieldName, AvroUtils.getFieldValue(inputRecord, Joiner.on('.').join(this.mapFieldName, originalFieldName)).or("")); } } return new SingleRecordIterable<>(genericRecord); } }
@Override public Iterable<GenericRecord> convertRecord(Schema outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException { GenericRecord outputRecord = new GenericData.Record(outputSchema); for (Field field : inputRecord.getSchema().getFields()) { outputRecord.put(field.name(), convertFieldValue(outputSchema, field, inputRecord, workUnit)); } return new SingleRecordIterable<>(outputRecord); }
@Override public GenericRecord partitionForRecord(GenericRecord record) { GenericRecord partition = new GenericData.Record(SCHEMA); partition.put(SCHEMA_STRING, record.getSchema().toString()); return partition; } }
@Test public void testToStringIsJson() throws JsonParseException, IOException { Field stringField = new Field("string", Schema.create(Type.STRING), null, null); Field enumField = new Field("enum", Schema.createEnum("my_enum", "doc", null, Arrays.asList("a", "b", "c")), null, null); Schema schema = Schema.createRecord("my_record", "doc", "mytest", false); schema.setFields(Arrays.asList(stringField, enumField)); GenericRecord r = new GenericData.Record(schema); // \u2013 is EN DASH r.put(stringField.name(), "hello\nthere\"\tyou\u2013}"); r.put(enumField.name(), new GenericData.EnumSymbol(enumField.schema(),"a")); String json = r.toString(); JsonFactory factory = new JsonFactory(); JsonParser parser = factory.createJsonParser(json); ObjectMapper mapper = new ObjectMapper(); // will throw exception if string is not parsable json mapper.readTree(parser); }
private GenericRecord convertRowToAvroRecord(Schema schema, Row row) { final List<Schema.Field> fields = schema.getFields(); final int length = fields.size(); final GenericRecord record = new GenericData.Record(schema); for (int i = 0; i < length; i++) { final Schema.Field field = fields.get(i); record.put(i, convertFlinkType(field.schema(), row.getField(i))); } return record; }
@Test(dataProvider = "google-ips") public void testEventMapper(Map<String, Object> props, InetAddress address) throws Exception { MaxmindGeoIPEventMapper mapper = new MaxmindGeoIPEventMapper(new MaxmindGeoIPModuleConfig()); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_ip").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_ip", Schema.create(NULL), null, null)) .build(); Record properties = new Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, EventMapper.RequestParams.EMPTY_PARAMS, address, null); assertTrue(resp == null); assertTrue(event.properties().getSchema().getField("_country_code") != null); assertTrue(event.properties().getSchema().getField("_city") != null); assertTrue(event.properties().getSchema().getField("_timezone") != null); assertTrue(event.getAttribute("_latitude") instanceof Double); assertTrue(event.properties().getSchema().getField("_region") != null); assertTrue(event.getAttribute("_longitude") instanceof Double); GenericData.get().validate(properties.getSchema(), properties); }
@Test public void testEquals() { Schema s = recordSchema(); GenericRecord r0 = new GenericData.Record(s); GenericRecord r1 = new GenericData.Record(s); GenericRecord r2 = new GenericData.Record(s); Collection<CharSequence> l0 = new ArrayDeque<>(); List<CharSequence> l1 = new ArrayList<>(); GenericArray<CharSequence> l2 = new GenericData.Array<>(1, s.getFields().get(0).schema()); String foo = "foo"; l0.add(new StringBuffer(foo)); l1.add(foo); l2.add(new Utf8(foo)); r0.put(0, l0); r1.put(0, l1); r2.put(0, l2); assertEquals(r0, r1); assertEquals(r0, r2); assertEquals(r1, r2); }
public Event createEvent(String collection, Map<String, Object> properties) { List<SchemaField> cache = fieldCache.get(collection); List<SchemaField> fields; List<SchemaField> generatedSchema = generateSchema(properties); if (cache == null || !generatedSchema.stream().allMatch(f -> cache.contains(f))) { fields = metastore.getOrCreateCollectionFields(project, collection, ImmutableSet.copyOf(generatedSchema)); fieldCache.put(collection, fields); } else { fields = cache; } try { GenericData.Record record = new GenericData.Record(AvroUtil.convertAvroSchema(fields)); properties.forEach((key, value) -> record.put(key, cast(value, record.getSchema().getField(key).schema().getTypes().get(1).getType()))); return new Event(project, collection, Event.EventContext.empty(), fields, record); } catch (Exception e) { throw Throwables.propagate(e); } }
@Test(dataProvider = "google-referrer") public void testReferrer(Map<String, Object> props, EventMapper.RequestParams headers) throws Exception { ReferrerEventMapper mapper = new ReferrerEventMapper(); FieldDependencyBuilder builder = new FieldDependencyBuilder(); mapper.addFieldDependency(builder); builder.build(); ImmutableList<Schema.Field> build = ImmutableList.<Schema.Field>builder() .addAll(builder.build().dependentFields.get("_referrer").stream() .map(AvroUtil::generateAvroField).collect(Collectors.toList())) .add(new Schema.Field("_referrer", Schema.create(NULL), null, null)) .build(); GenericData.Record properties = new GenericData.Record(Schema.createRecord(build)); props.forEach(properties::put); Event event = new Event("testproject", "testcollection", null, null, properties); List<Cookie> resp = mapper.map(event, headers, InetAddress.getLocalHost(), null); assertEquals("Google", event.getAttribute("_referrer_source")); assertEquals("test", event.getAttribute("_referrer_term")); assertEquals("search", event.getAttribute("_referrer_medium")); assertEquals("google.com", event.getAttribute("_referrer_domain")); assertEquals("/?q=test", event.getAttribute("_referrer_path")); assertNull(resp); GenericData.get().validate(properties.getSchema(), properties); }
@Override public String getProperty(String key) throws Exception { if(key == null) { throw new NullPointerException("key"); } Schema schema = _protocol.getMessages().get(ProtocolConstants.MSG_META).getRequest(); GenericRecord req = new GenericData.Record(schema); req.put("src", _sourceUtf8); req.put("opt", ProtocolConstants.OPT_GET_PROPERTY_UTF8); req.put("key", new Utf8(key)); Utf8 res = (Utf8)send(ProtocolConstants.MSG_META, req); return res == null ? null : res.toString(); }
public Object respond(Message message, Object request) throws AvroRemoteException { GenericRecord params = (GenericRecord)request; if ("hello".equals(message.getName())) { LOG.info("hello: "+params.get("greeting")); return new Utf8("goodbye"); } if ("echo".equals(message.getName())) { Object record = params.get("record"); LOG.info("echo: "+record); return record; } if ("echoBytes".equals(message.getName())) { Object data = params.get("data"); LOG.info("echoBytes: "+data); return data; } if ("error".equals(message.getName())) { if (throwUndeclaredError) throw new RuntimeException("foo"); GenericRecord error = new GenericData.Record(PROTOCOL.getType("TestError")); error.put("message", new Utf8("an error")); throw new AvroRemoteException(error); } throw new AvroRuntimeException("unexpected message: "+message.getName()); }
@Test public void testHello() throws IOException { GenericRecord params = new GenericData.Record(PROTOCOL.getMessages().get("hello").getRequest()); params.put("greeting", new Utf8("bob")); Utf8 response = (Utf8)requestor.request("hello", params); assertEquals(new Utf8("goodbye"), response); }
private GenericRecord generateRecordWithArrays() { ArrayList<Schema.Field> fields = new ArrayList<Schema.Field>(); String fieldName = "array1"; Schema fieldSchema = Schema.createArray(Schema.create(Schema.Type.STRING)); String docString = "doc"; fields.add(new Schema.Field(fieldName, fieldSchema, docString, null)); Schema schema = Schema.createRecord("name", docString, "test", false); schema.setFields(fields); GenericData.Record record = new GenericData.Record(schema); GenericData.Array<String> arr = new GenericData.Array<>(2, fieldSchema); arr.add("foobar"); arr.add("foobaz"); record.put("array1", arr); return record; } }