/** * Creates a ParquetWriterFactory for the given type. The Parquet writers will use Avro * to reflectively create a schema for the type and use that schema to write the columnar data. * * @param type The class of the type to write. */ public static <T> ParquetWriterFactory<T> forReflectRecord(Class<T> type) { final String schemaString = ReflectData.get().getSchema(type).toString(); final ParquetBuilder<T> builder = (out) -> createAvroParquetWriter(schemaString, ReflectData.get(), out); return new ParquetWriterFactory<>(builder); }
private void flush() throws IOException { OutputStream out = fs.create(new Path(dir, "part-"+(part++)+EXT)); try { writer.writeTo(out); } finally { out.close(); } writer = new AvroColumnWriter<>(schema, meta, ReflectData.get()); }
@Test public void testNoPackage() throws Exception { Class<?> noPackage = Class.forName("NoPackage"); Schema s = ReflectData.get().getSchema(noPackage); assertEquals(noPackage.getName(), ReflectData.getClassName(s)); }
/** Test that the error message contains the name of the class. */ @Test public void testReflectFieldError() throws Exception { Object datum = ""; try { ReflectData.get().getField(datum, "notAFieldOfString", 0); } catch (AvroRuntimeException e) { assertTrue(e.getMessage().contains(datum.getClass().getName())); } }
@Test public void testP4() throws Exception { Protocol p = ReflectData.get().getProtocol(P4.class); Protocol.Message message = p.getMessages().get("foo"); assertEquals(Schema.Type.INT, message.getResponse().getType()); Field field = message.getRequest().getField("x"); assertEquals(Schema.Type.INT, field.schema().getType()); }
@Override public boolean canHandle(final Class clazz) { if ("java.lang.Class".equals(clazz.getName())) { return false; } try { ReflectData.get().getSchema(clazz); } catch (final RuntimeException e) { return false; } return true; }
@Test public void testUnionWithMap() { Schema s = new Schema.Parser().parse ("[\"null\", {\"type\":\"map\",\"values\":\"float\"}]"); GenericData data = ReflectData.get(); assertEquals(1, data.resolveUnion(s, new HashMap<String,Float>())); }
@InterfaceAudience.Private @Override public Schema getSchema(Object t) { return ReflectData.get().getSchema(t.getClass()); }
@Test public void testUnionWithCollection() { Schema s = new Schema.Parser().parse ("[\"null\", {\"type\":\"array\",\"items\":\"float\"}]"); GenericData data = ReflectData.get(); assertEquals(1, data.resolveUnion(s, new ArrayList<Float>())); }
@Override public Schema retrieveWriterSchema(Object source) { Class<?> clazz; try { clazz = Class.forName("org.apache.hadoop.hive.hbase.avro.Employee"); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } return ReflectData.get().getSchema(clazz); }
@Test public void testUnionWithBytes() { Schema s = new Schema.Parser().parse ("[\"null\", \"bytes\"]"); GenericData data = ReflectData.get(); assertEquals(1, data.resolveUnion(s, ByteBuffer.wrap(new byte[]{1}))); }
@Override public Schema retrieveReaderSchema(Object source) { Class<?> clazz; try { clazz = Class.forName("org.apache.hadoop.hive.hbase.avro.Employee"); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } return ReflectData.get().getSchema(clazz); }
@Test public void testUnionWithMapWithUtf8Keys() { Schema s = new Schema.Parser().parse ("[\"null\", {\"type\":\"map\",\"values\":\"float\"}]"); GenericData data = ReflectData.get(); HashMap<Utf8,Float> map = new HashMap<Utf8,Float>(); map.put(new Utf8("foo"), 1.0f); assertEquals(1, data.resolveUnion(s, map)); }
private Schema nullableSchema(Class<?> type) { return Schema.createUnion(Arrays.asList( Schema.create(Schema.Type.NULL), ReflectData.get().getSchema(type))); } }
@Test public void testUnionWithEnum() { Schema s = new Schema.Parser().parse ("[\"null\", {\"type\":\"enum\",\"name\":\"E\",\"namespace\":" + "\"org.apache.avro.reflect.TestReflect\",\"symbols\":[\"A\",\"B\"]}]"); GenericData data = ReflectData.get(); assertEquals(1, data.resolveUnion(s, E.A)); }
@Test public void testReflectMatchStructure() throws SchemaValidationException { testValidatorPasses(builder.canBeReadStrategy().validateAll(), circleSchemaDifferentNames, ReflectData.get().getSchema(Circle.class)); }
@Test public void testR12() throws Exception { Schema s = ReflectData.get().getSchema(R12.class); assertEquals(Schema.Type.INT, s.getField("x").schema().getType()); assertEquals(Schema.parse ("{\"type\":\"array\",\"items\":[\"null\",\"string\"]}"), s.getField("strings").schema()); }
private void writeLinesFile(File dir) throws IOException { DatumWriter<Text> writer = new ReflectDatumWriter<>(); DataFileWriter<Text> out = new DataFileWriter<>(writer); File linesFile = new File(dir+"/lines.avro"); dir.mkdirs(); out.create(ReflectData.get().getSchema(Text.class), linesFile); for (String line : WordCountUtil.LINES) out.append(new Text(line)); out.close(); }
@Test public void testR10() throws Exception { Schema r10Schema = ReflectData.get().getSchema(R10.class); assertEquals(Schema.Type.STRING, r10Schema.getType()); assertEquals(R10.class.getName(), r10Schema.getProp("java-class")); checkReadWrite(new R10("foo"), r10Schema); }
@Test public void test() throws Exception{ Schema schema = ReflectData.get().getSchema(X.class); ByteArrayOutputStream bout = new ByteArrayOutputStream(); writeOneXAsAvro(schema, bout); X record = readOneXFromAvro(schema, bout); String expected = getmd5(content); String actual = getmd5(record.content); assertEquals("md5 for result differed from input",expected,actual); }