/** * Creates a GenericRecordBuilder for building Record instances. * @param schema the schema associated with the record class. */ public GenericRecordBuilder(Schema schema) { super(schema, GenericData.get()); record = new GenericData.Record(schema); }
/** * Creates a ParquetWriterFactory that accepts and writes Avro generic types. * The Parquet writers will use the given schema to build and write the columnar data. * * @param schema The schema of the generic type. */ public static ParquetWriterFactory<GenericRecord> forGenericRecord(Schema schema) { final String schemaString = schema.toString(); final ParquetBuilder<GenericRecord> builder = (out) -> createAvroParquetWriter(schemaString, GenericData.get(), out); return new ParquetWriterFactory<>(builder); }
@Override public int compareTo(GenericEnumSymbol that) { return GenericData.get().compare(this, that, schema); } }
private Object deserializeUnion(Object datum, Schema fileSchema, Schema recordSchema, UnionTypeInfo columnType) throws AvroSerdeException { // Calculate tags individually since the schema can evolve and can have different tags. In worst case, both schemas are same // and we would end up doing calculations twice to get the same tag int fsTag = GenericData.get().resolveUnion(fileSchema, datum); // Determine index of value from fileSchema int rsTag = GenericData.get().resolveUnion(recordSchema, datum); // Determine index of value from recordSchema Object desered = worker(datum, fileSchema == null ? null : fileSchema.getTypes().get(fsTag), recordSchema.getTypes().get(rsTag), columnType.getAllUnionObjectTypeInfos().get(rsTag)); return new StandardUnionObjectInspector.StandardUnion((byte)rsTag, desered); }
@Override public int compareTo(GenericArray<T> that) { return GenericData.get().compare(this, that, this.getSchema()); } @Override
@Override public int compareTo(GenericArray<T> that) { return GenericData.get().compare(this, that, this.getSchema()); } @Override
@Test(expected = BadHeaderException.class) public void testByteBufferMissingFullHeader() throws Exception { MessageEncoder<Record> encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V2); MessageDecoder<Record> decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); ByteBuffer buffer = encoder.encode(V2_RECORDS.get(0)); buffer.limit(8); decoder.decode(buffer); }
@Test(expected = BadHeaderException.class) public void testByteBufferBadVersionByte() throws Exception { MessageEncoder<Record> encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V2); MessageDecoder<Record> decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); ByteBuffer buffer = encoder.encode(V2_RECORDS.get(0)); buffer.array()[1] = 0x00; decoder.decode(buffer); }
@Test(expected = BadHeaderException.class) public void testByteBufferBadMarkerByte() throws Exception { MessageEncoder<Record> encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V2); MessageDecoder<Record> decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); ByteBuffer buffer = encoder.encode(V2_RECORDS.get(0)); buffer.array()[0] = 0x00; decoder.decode(buffer); }
@Test(expected = AvroRuntimeException.class) public void testByteBufferMissingPayload() throws Exception { MessageEncoder<Record> encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V2); MessageDecoder<Record> decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); ByteBuffer buffer = encoder.encode(V2_RECORDS.get(0)); buffer.limit(12); decoder.decode(buffer); }
@Test(expected = MissingSchemaException.class) public void testCompatibleReadFailsWithoutSchema() throws Exception { MessageEncoder<Record> v1Encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V1); BinaryMessageDecoder<Record> v2Decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); ByteBuffer v1Buffer = v1Encoder.encode(V1_RECORDS.get(3)); v2Decoder.decode(v1Buffer); }
@Test public void testBufferCopy() throws Exception { MessageEncoder<Record> encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V1); ByteBuffer b0 = encoder.encode(V1_RECORDS.get(0)); ByteBuffer b1 = encoder.encode(V1_RECORDS.get(1)); Assert.assertNotEquals(b0.array(), b1.array()); MessageDecoder<Record> decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V1); // bytes are not changed by reusing the encoder Assert.assertEquals("Buffer was copied, decode(b0) should be record 0", V1_RECORDS.get(0), decoder.decode(b0)); }
@Test public void testCopyDecimalRaw() { testCopy(LogicalTypes.decimal(9, 2).addToSchema(Schema.create(Schema.Type.BYTES)), ByteBuffer.wrap(new BigDecimal("-34.34").unscaledValue().toByteArray()), GenericData.get()); // no conversions }
@Test public void testByteBufferRoundTrip() throws Exception { MessageEncoder<Record> encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V2); MessageDecoder<Record> decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); Record copy = decoder.decode(encoder.encode(V2_RECORDS.get(0))); Assert.assertNotSame("Copy should not be the same object", copy, V2_RECORDS.get(0)); Assert.assertEquals("Record should be identical after round-trip", V2_RECORDS.get(0), copy); }
@Test public void testCopyUuidRaw() { testCopy(LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)), UUID.randomUUID().toString(), // use raw type GenericData.get()); // with no conversions }
@Test public void testToStringEscapesControlCharsInBytes() throws Exception { GenericData data = GenericData.get(); ByteBuffer bytes = ByteBuffer.wrap(new byte[] {'a', '\n', 'b'}); assertEquals("{\"bytes\": \"a\\nb\"}", data.toString(bytes)); assertEquals("{\"bytes\": \"a\\nb\"}", data.toString(bytes)); }
@Test public void testCompatibleReadWithSchema() throws Exception { MessageEncoder<Record> v1Encoder = new BinaryMessageEncoder<>(GenericData.get(), SCHEMA_V1); BinaryMessageDecoder<Record> v2Decoder = new BinaryMessageDecoder<>(GenericData.get(), SCHEMA_V2); v2Decoder.addSchema(SCHEMA_V1); ByteBuffer v1Buffer = v1Encoder.encode(V1_RECORDS.get(3)); Record record = v2Decoder.decode(v1Buffer); Assert.assertEquals( V2_BUILDER.set("id", 6L).set("message", "m-6").clear("data").build(), record); }
@Test public void testToStringFixed() throws Exception { GenericData data = GenericData.get(); assertEquals("[97, 10, 98]", data.toString(new GenericData.Fixed( Schema.createFixed("test", null, null, 3), new byte[] {'a', '\n', 'b'}))); }
@Test public void validateRequiresGenericSymbolForEnumSchema() { final Schema schema = Schema.createEnum("my_enum", "doc", "namespace", Arrays.asList("ONE","TWO","THREE")); final GenericData gd = GenericData.get(); /* positive cases */ assertTrue(gd.validate(schema, new GenericData.EnumSymbol(schema, "ONE"))); assertTrue(gd.validate(schema, new GenericData.EnumSymbol(schema, anEnum.ONE))); /* negative cases */ assertFalse("We don't expect GenericData to allow a String datum for an enum schema", gd.validate(schema, "ONE")); assertFalse("We don't expect GenericData to allow a Java Enum for an enum schema", gd.validate(schema, anEnum.ONE)); }
@Test public void testToStringNanInfinity() throws Exception { GenericData data = GenericData.get(); assertEquals("\"Infinity\"",data.toString(Float.POSITIVE_INFINITY)); assertEquals("\"-Infinity\"",data.toString(Float.NEGATIVE_INFINITY)); assertEquals("\"NaN\"", data.toString(Float.NaN)); assertEquals("\"Infinity\"",data.toString(Double.POSITIVE_INFINITY)); assertEquals("\"-Infinity\"",data.toString(Double.NEGATIVE_INFINITY)); assertEquals("\"NaN\"", data.toString(Double.NaN)); }