Refine search
/** * Decode binary data. * * @param data the data * @param reuse the reuse * @return the decoded object * @throws IOException Signals that an I/O exception has occurred. */ public T decodeBinary(byte[] data, T reuse) throws IOException { binaryDecoder = DecoderFactory.get().binaryDecoder(data, binaryDecoder); return datumReader.read(reuse, binaryDecoder); }
/** Initialize the stream without reading from it. */ void initialize(InputStream in, Header header) throws IOException { this.header = header; this.codec = resolveCodec(); reader.setSchema(header.schema); }
@Override public void readExternal(java.io.ObjectInput in) throws java.io.IOException { READER$.read(this, SpecificData.getDecoder(in)); }
@Test(expected=AvroTypeException.class) public void testNoDefaultField() throws Exception { Schema expected = Schema.parse("{\"type\":\"record\", \"name\":\"Foo\", \"fields\":"+ "[{\"name\":\"f\", \"type\": \"string\"}]}"); DatumReader<Object> in = new GenericDatumReader<>(ACTUAL, expected); in.read(null, DecoderFactory.get().binaryDecoder( new ByteArrayInputStream(new byte[0]), null)); }
+ " not set and has no default value"); if (json.isNull() && (field.schema().getType() == Type.NULL || (field.schema().getType() == Type.UNION && field.schema().getTypes().get(0).getType() == Type.NULL))) { return null; encoder.flush(); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(baos.toByteArray(), null); defaultValue = createDatumReader(field.schema()).read(null, decoder);
/** * Decode json data. * * @param data the data * @param reuse the reuse * @return the decoded object * @throws IOException Signals that an I/O exception has occurred. */ public T decodeJson(String data, T reuse) throws IOException { jsonDecoder = DecoderFactory.get().jsonDecoder(this.schema, data, true); return datumReader.read(null, jsonDecoder); }
public static void checkDirectBinary(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writer.setSchema(schema); Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); writer.write(datum, encoder); // no flush for direct byte[] data = out.toByteArray(); reader.setSchema(schema); Object decoded = reader.read(null, DecoderFactory.get() .binaryDecoder(data, null)); assertEquals("Decoded data does not match.", datum, decoded); }
@Test public void testSerialize() throws IOException { Schema writerSchema = Schema.create(Schema.Type.STRING); AvroSerializer<CharSequence> serializer = new AvroSerializer<>(writerSchema); Schema readerSchema = Schema.create(Schema.Type.STRING); DatumReader<CharSequence> datumReader = new GenericDatumReader<>(readerSchema); Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); CharSequence record = null; record = datumReader.read(record, decoder); assertEquals("record1", record.toString()); record = datumReader.read(record, decoder); assertEquals("record2", record.toString());
private GenericRecord serializeWithWriterThenDeserializeWithReader(Schema writerSchema, GenericRecord datum, Schema readerSchema) throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); DatumWriter<Object> datumWriter = new GenericDatumWriter<>(writerSchema); datumWriter.write(datum, encoder); encoder.flush(); byte[] bytes = baos.toByteArray(); Decoder decoder = DecoderFactory.get().resolvingDecoder( writerSchema, readerSchema, DecoderFactory.get().binaryDecoder(bytes, null)); DatumReader<Object> datumReader = new GenericDatumReader<>(readerSchema); return (GenericRecord)datumReader.read(null, decoder); }
private void checkNumeric(String type, Object value) throws Exception { String def = "{\"type\":\"record\",\"name\":\"X\",\"fields\":" +"[{\"type\":\""+type+"\",\"name\":\"n\"}]}"; Schema schema = Schema.parse(def); DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema); String[] records = {"{\"n\":1}", "{\"n\":1.0}"}; for (String record : records) { Decoder decoder = DecoderFactory.get().jsonDecoder(schema, record); GenericRecord r = reader.read(null, decoder); Assert.assertEquals(value, r.get("n")); } }
private Object jsonToAvro(String jsonString, Schema schema) throws Exception { DatumReader<Object> reader = new GenericDatumReader<>(schema); Object object = reader.read(null, DecoderFactory.get().jsonDecoder(schema, jsonString)); if (schema.getType().equals(Schema.Type.STRING)) { object = object.toString(); } return object; }
private static void checkJson(Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out); writer.setSchema(schema); writer.write(datum, encoder); writer.write(datum, encoder); encoder.flush(); byte[] data = out.toByteArray(); reader.setSchema(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data)); Object decoded = reader.read(null, decoder); assertEquals("Decoded data does not match.", datum, decoded); decoded = reader.read(decoded, decoder); assertEquals("Decoded data does not match.", datum, decoded); }
public static void checkBinaryJson(String json) throws Exception { Object node = Json.parseJson(json); ByteArrayOutputStream out = new ByteArrayOutputStream(); DatumWriter<Object> writer = new Json.ObjectWriter(); Encoder encoder = EncoderFactory.get().binaryEncoder(out, null); encoder = EncoderFactory.get().validatingEncoder(Json.SCHEMA, encoder); writer.write(node, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); DatumReader<Object> reader = new Json.ObjectReader(); Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); decoder = DecoderFactory.get().validatingDecoder(Json.SCHEMA, decoder); Object decoded = reader.read(null, decoder); assertEquals("Decoded json does not match.", Json.toString(node), Json.toString(decoded)); }
helper.getSchema()) : new GenericDatumReader<GenericRecord>(helper.getSchema(), helper.getTargetSchema()); InputStream inStream = new ByteArrayInputStream(message.getPayload()); JsonDecoder jsonDecoder = DecoderFactory.get().jsonDecoder(helper.getSchema(), inStream); datum = (GenericRecord) reader.read(datum, jsonDecoder); ByteArrayOutputStream output = new ByteArrayOutputStream(); GenericDatumWriter<GenericRecord> writer = helper.getTargetSchema() == null ? new GenericDatumWriter<GenericRecord>( return new KafkaAvroMessageDecoder.CamusAvroWrapper((GenericData.Record) avroReader.read(null, this.decoderFactory.binaryDecoder(output.toByteArray(), 0, output.toByteArray().length, null))); } catch (RuntimeException e) { log.error("Caught exception while parsing JSON string '" + payloadString + "'.");
this.vin = DecoderFactory.get().binaryDecoder(in, vin); byte[] magic = new byte[DataFileConstants.MAGIC.length]; try { header.schema = Schema.parse(getMetaString(DataFileConstants.SCHEMA),false); this.codec = resolveCodec(); reader.setSchema(header.schema);
@Override public D decode(InputStream stream, D reuse) { BinaryDecoder decoder = DecoderFactory.get() .directBinaryDecoder(stream, DECODER.get()); DECODER.set(decoder); try { return reader.read(reuse, decoder); } catch (IOException e) { throw new AvroRuntimeException("Decoding datum failed", e); } } }
/** * Change the schema of an Avro record. * @param record The Avro record whose schema is to be changed. * @param newSchema The target schema. It must be compatible as reader schema with record.getSchema() as writer schema. * @return a new Avro record with the new schema. * @throws IOException if conversion failed. */ public static GenericRecord convertRecordSchema(GenericRecord record, Schema newSchema) throws IOException { if (record.getSchema().equals(newSchema)) { return record; } try { BinaryDecoder decoder = new DecoderFactory().binaryDecoder(recordToByteArray(record), null); DatumReader<GenericRecord> reader = new GenericDatumReader<>(record.getSchema(), newSchema); return reader.read(null, decoder); } catch (IOException e) { throw new IOException( String.format("Cannot convert avro record to new schema. Origianl schema = %s, new schema = %s", record.getSchema(), newSchema), e); } }
@Override public GenericRow decode(byte[] payload, int offset, int length, GenericRow destination) { try { GenericData.Record avroRecord = _reader.read(null, _decoderFactory.binaryDecoder(payload, offset, length, null)); return _rowGenerator.transform(avroRecord, destination); } catch (Exception e) { LOGGER.error("Caught exception", e); throw new RuntimeException(e); } } }
@Override public Object readResponse(Schema writer, Schema reader, Decoder in) throws IOException { return getDatumReader(writer, reader).read(null, in); }