@Override public DatumWriter createDatumWriter(Schema schema) { return new SpecificDatumWriter(schema, this); }
@Override public DatumWriter createDatumWriter(Schema schema) { return new SpecificDatumWriter(schema, this); }
@InterfaceAudience.Private @Override public DatumWriter getWriter(Class<SpecificRecord> clazz) { return new SpecificDatumWriter(); }
public AvroSerializer(Schema schema, SchemaVersionWriter schemaVersionWriter) throws IOException { this.closer = Closer.create(); this.byteArrayOutputStream = new ByteArrayOutputStream(); this.out = this.closer.register(new DataOutputStream(this.byteArrayOutputStream)); this.encoder = getEncoder(schema, this.out); this.schemaVersionWriter = schemaVersionWriter; this.writer = new SpecificDatumWriter<>(schema); }
/** * Instantiates a new Avro json converter based on class. * * @param schema the schema * @param typeParameterClass the type parameter class */ public AvroJsonConverter(Schema schema, Class<T> typeParameterClass) { super(); this.schema = schema; avroReader = new SpecificDatumReader<T>(typeParameterClass); avroWriter = new SpecificDatumWriter<T>(typeParameterClass); }
/** * Instantiates a new Avro byte array converter based on class. * * @param typeParameterClass the type parameter class */ public AvroByteArrayConverter(Class<T> typeParameterClass) { super(); avroReader = new SpecificDatumReader<T>(typeParameterClass); avroWriter = new SpecificDatumWriter<T>(typeParameterClass); }
@SuppressWarnings("unchecked") private void readObject(ObjectInputStream inputStream) throws ClassNotFoundException, IOException { recordClazz = (Class<? extends SpecificRecord>) inputStream.readObject(); schemaString = (String) inputStream.readObject(); if (recordClazz != null) { schema = SpecificData.get().getSchema(recordClazz); } else { schema = new Schema.Parser().parse(schemaString); } datumWriter = new SpecificDatumWriter<>(schema); arrayOutputStream = new ByteArrayOutputStream(); encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); } }
protected SpecificDatumWriter<T> newWriter() { return new SpecificDatumWriter<>(schema); } @Override
@Override protected DatumWriter<Object> getDatumWriter(Schema schema) { return new SpecificDatumWriter<>(schema, getSpecificData()); }
@SuppressWarnings("OptionalUsedAsFieldOrParameterType") private static <T> AvroFactory<T> fromSpecific(Class<T> type, ClassLoader cl, Optional<Schema> previousSchema) { SpecificData specificData = new SpecificData(cl); Schema newSchema = specificData.getSchema(type); return new AvroFactory<>( specificData, newSchema, new SpecificDatumReader<>(previousSchema.orElse(newSchema), newSchema, specificData), new SpecificDatumWriter<>(newSchema, specificData) ); }
/** * Creates an Avro serialization schema for the given specific record class. * * @param recordClazz Avro record class used to serialize Flink's row to Avro's record */ public AvroRowSerializationSchema(Class<? extends SpecificRecord> recordClazz) { Preconditions.checkNotNull(recordClazz, "Avro record class must not be null."); this.recordClazz = recordClazz; this.schema = SpecificData.get().getSchema(recordClazz); this.schemaString = schema.toString(); this.datumWriter = new SpecificDatumWriter<>(schema); this.arrayOutputStream = new ByteArrayOutputStream(); this.encoder = EncoderFactory.get().binaryEncoder(arrayOutputStream, null); }
@Override public void writeExternal(ObjectOutput out) throws IOException { new SpecificDatumWriter(getSchema()) .write(this, SpecificData.getEncoder(out)); }
public byte[] toBytes(T object) { ByteArrayOutputStream output = new ByteArrayOutputStream(); Encoder encoder = new BinaryEncoder(output); SpecificDatumWriter<T> datumWriter = null; try { datumWriter = new SpecificDatumWriter<T>(clazz); datumWriter.write(object, encoder); encoder.flush(); } catch(IOException e) { throw new SerializationException(e); } finally { SerializationUtils.close(output); } return output.toByteArray(); }
@Override public void writeExternal(ObjectOutput out) throws IOException { new SpecificDatumWriter(getSchema()) .write(this, SpecificData.getEncoder(out)); }
public AvroFileWriter(LogFilePath logFilePath, CompressionCodec codec) throws IOException { file = new File(logFilePath.getLogFilePath()); file.getParentFile().mkdirs(); LOG.debug("Creating Brand new Writer for path {}", logFilePath.getLogFilePath()); topic = logFilePath.getTopic(); Schema schema = schemaRegistryClient.getSchema(topic); SpecificDatumWriter specificDatumWriter= new SpecificDatumWriter(schema); writer = new DataFileWriter(specificDatumWriter); writer.setCodec(getCodecFactory(codec)); writer.create(schema, file); }
public AvroParquetFileReader(LogFilePath logFilePath, CompressionCodec codec) throws IOException { Path path = new Path(logFilePath.getLogFilePath()); String topic = logFilePath.getTopic(); Schema schema = schemaRegistryClient.getSchema(topic); reader = AvroParquetReader.<GenericRecord>builder(path).build(); writer = new SpecificDatumWriter(schema); offset = logFilePath.getOffset(); }
public static byte[] serializeRecord(StringablesRecord stringablesRecord) throws IOException { SpecificDatumWriter<StringablesRecord> datumWriter = new SpecificDatumWriter<>(StringablesRecord.SCHEMA$); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(stringablesRecord, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
public static byte[] serializeRecord(FooBarSpecificRecord fooBarSpecificRecord) throws IOException { SpecificDatumWriter<FooBarSpecificRecord> datumWriter = new SpecificDatumWriter<>(FooBarSpecificRecord.SCHEMA$); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null); datumWriter.write(fooBarSpecificRecord, encoder); encoder.flush(); return byteArrayOutputStream.toByteArray(); }
@SuppressWarnings("unchecked") <T extends SpecificRecordBase> void verifySerDeAndStandardMethods(T original) { final SpecificDatumWriter<T> datumWriterFromSchema = new SpecificDatumWriter<>(original.getSchema()); final SpecificDatumReader<T> datumReaderFromSchema = new SpecificDatumReader<>(original.getSchema(), original.getSchema()); verifySerDeAndStandardMethods(original, datumWriterFromSchema, datumReaderFromSchema); final SpecificDatumWriter<T> datumWriterFromClass = new SpecificDatumWriter(original.getClass()); final SpecificDatumReader<T> datumReaderFromClass = new SpecificDatumReader(original.getClass()); verifySerDeAndStandardMethods(original, datumWriterFromClass, datumReaderFromClass); }
private byte[] getSerializedMessage(IndexedRecord message, Schema schema) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); SpecificDatumWriter<IndexedRecord> writer = new SpecificDatumWriter<>(); DataFileWriter<IndexedRecord> dfw = null; try { dfw = new DataFileWriter<>(writer).create(schema, baos); dfw.append(message); } finally { if (dfw != null) { dfw.close(); } } return baos.toByteArray(); }