public void writeNext() throws IOException { GenericData.Record nextRecord = new GenericData.Record(_avroSchema); for (String column : _generatorMap.keySet()) { nextRecord.put(column, _generatorMap.get(column).next()); } _recordWriter.append(nextRecord); }
private void addString(Record doc, String avroField, String val) { if (val == null) { return; } doc.put(avroField, val); totalTextIndexed += val.length(); }
/** * TODO: Currently generates a static schema avro record. **/ public static GenericRecord generateRandomAvroRecord() { ArrayList<Schema.Field> fields = new ArrayList<Schema.Field>(); String fieldName = "field1"; Schema fieldSchema = Schema.create(Schema.Type.STRING); String docString = "doc"; fields.add(new Schema.Field(fieldName, fieldSchema, docString, null)); Schema schema = Schema.createRecord("name", docString, "test",false); schema.setFields(fields); GenericData.Record record = new GenericData.Record(schema); record.put("field1", "foobar"); return record; }
/** * Sets the value of a field. * @param field the field to set. * @param pos the position of the field. * @param value the value to set. * @return a reference to the RecordBuilder. */ private GenericRecordBuilder set(Field field, int pos, Object value) { validate(field, value); record.put(pos, value); fieldSetFlags()[pos] = true; return this; }
/** * Sets the value of a field. * @param field the field to set. * @param pos the position of the field. * @param value the value to set. * @return a reference to the RecordBuilder. */ private GenericRecordBuilder set(Field field, int pos, Object value) { validate(field, value); record.put(pos, value); fieldSetFlags()[pos] = true; return this; }
@Before public void setUp() { writtenRecord = new GenericData.Record(writer); writtenRecord.put("a", "record"); writtenRecord.put("b", 21); innerRecord = new GenericData.Record(inner); innerRecord.put("c11", 1); innerRecord.put("c12", "hello"); evolvedRecord = new GenericData.Record(evolved); evolvedRecord.put("a", "record"); evolvedRecord.put("b", 21); evolvedRecord.put("c", innerRecord); }
private <T> Record defaultRecordWithSchema(Schema schema, String key, T value) { Record data = new GenericData.Record(schema); data.put(key, value); return data; }
@Test(expected=AvroRuntimeException.class) public void testRecordPutInvalidField() throws Exception { Schema s = Schema.createRecord("schemaName", "schemaDoc", "namespace", false); List<Schema.Field> fields = new ArrayList<>(); fields.add(new Schema.Field("someFieldName", s, "docs", null)); s.setFields(fields); Record r = new GenericData.Record(s); r.put("invalidFieldName", "someValue"); }
@Override protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { long count = 0; for (LongWritable value: values) { count += value.get(); } result.datum().put("key", key.toString()); result.datum().put("value", count); context.write(result, NullWritable.get()); } }
@Override public GenericRecord next() { GenericData.Record testRecord = new GenericData.Record(schema); String testContent = "hello world" + currentIndex; GenericData.Record dataRecord = new GenericData.Record(dataRecordSchema); dataRecord.put("data", ByteBuffer.wrap(testContent.getBytes(Charset.forName("UTF-8")))); dataRecord.put("flags", 0); testRecord.put("key", "hello" + currentIndex); testRecord.put("data", dataRecord); currentIndex++; return testRecord; } };
/** * Clears the value of the given field. * @param pos the position of the field to clear. * @return a reference to the RecordBuilder. */ protected GenericRecordBuilder clear(int pos) { record.put(pos, null); fieldSetFlags()[pos] = false; return this; }
@Override protected void customizeType(Record record, Schema fieldTypeSchema) { if (record != null && record.getSchema().getName().equals(RECORD_FIELD_TYPE)) { JsonNode addressableNode = fieldTypeSchema.getJsonProp(ADDRESSABLE); if (addressableNode != null && addressableNode.isBoolean()) { record.put(ADDRESSABLE, addressableNode.asBoolean()); } else { record.put(ADDRESSABLE, true); } } }
/** * Clears the value of the given field. * @param pos the position of the field to clear. * @return a reference to the RecordBuilder. */ protected GenericRecordBuilder clear(int pos) { record.put(pos, null); fieldSetFlags()[pos] = false; return this; }
@Test public void testMapInRecord() throws Exception { String json = "{\"type\":\"record\", \"name\":\"Test\", \"fields\":" +"[{\"name\":\"f\", \"type\": {\"type\":\"map\", \"values\":\"long\"}}]}"; Schema schema = Schema.parse(json); HashMap<Utf8,Long> map = new HashMap<>(); map.put(new Utf8("a"), 1L); GenericData.Record record = new GenericData.Record(schema); record.put("f", map); check(new File(DIR.getRoot(), name.getMethodName()),json, "{\"f\":{\"a\":1}}", record, false); }
@Override public GenericRecord next() { GenericData.Record testRecord = new GenericData.Record(schema); String testContent = "hello world" + currentIndex; GenericData.Record dataRecord = new GenericData.Record(dataRecordSchema); dataRecord.put("data", ByteBuffer.wrap(testContent.getBytes(Charset.forName("UTF-8")))); dataRecord.put("flags", 0); testRecord.put("key", "hello" + currentIndex); testRecord.put("data", dataRecord); currentIndex++; return testRecord; } };
@Test public void testWithAvroRecord() throws IOException { Schema recordSchema = new Schema.Parser().parse(getClass().getClassLoader().getResourceAsStream("converter/fieldPickInput.avsc")); GenericData.Record record = new GenericData.Record(recordSchema); record.put("name", "foo"); RecordAccessor accessor = RecordAccessorProviderFactory.getRecordAccessorForObject(record); Assert.assertNotNull(accessor); Assert.assertEquals(accessor.getAsString("name"), "foo"); }
private static byte[] writeVersion0with1Present(Map<Integer, String> versions, Schema s0) { GenericData.Record record = new GenericData.Record(s0); record.put("original", new Utf8("Abhinay")); AvroVersionedGenericSerializer serializer = new AvroVersionedGenericSerializer(versions); return serializer.toBytes(record); }
private static byte[] writeVersion0(Schema s0) { GenericData.Record record = new GenericData.Record(s0); record.put("original", new Utf8("Abhinay")); AvroVersionedGenericSerializer serializer = new AvroVersionedGenericSerializer(s0.toString()); return serializer.toBytes(record); }
public void testRoundtripAvroWithGenericRecord() throws Exception { String jsonSchema = "{\"name\": \"Compact Disk\", \"type\": \"record\", " + "\"fields\": [" + "{\"name\": \"name\", \"type\": \"string\", \"order\": \"ascending\"}" + "]}"; AvroGenericSerializer serializer = new AvroGenericSerializer(jsonSchema); Record record = new Record(Schema.parse(jsonSchema)); // we need to use a Utf8 instance to map to a String. record.put("name", new Utf8("Hello")); byte[] bytes = serializer.toBytes(record); assertTrue(serializer.toObject(bytes).equals(record)); }