/** * When we want to consume SinkRecord which generated by debezium-connector-mysql, it should not * throw error "org.apache.avro.SchemaParseException: Illegal character in: server-id" */ @Test public void shouldValidateSourceInfoSchema() { org.apache.kafka.connect.data.Schema kafkaSchema = SourceInfo.SCHEMA; Schema avroSchema = avroData.fromConnectSchema(kafkaSchema); assertTrue(avroSchema != null); }
@Test @FixFor("DBZ-650") public void shouldCreateSchemaWithNestedObject() { for (Entry<String, BsonValue> entry : val.entrySet()) { converter.addFieldSchema(entry, builder); } Schema finalSchema = builder.build(); final org.apache.avro.Schema avroSchema = avroData.fromConnectSchema(finalSchema); assertThat(avroSchema.toString()).isEqualTo( "{\"type\":\"record\",\"name\":\"complex\",\"fields\":[" + "{\"name\":\"_id\",\"type\":[\"null\",\"int\"],\"default\":null}," + "{\"name\":\"s1\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"s1\",\"namespace\":\"complex\",\"fields\":[" + "{\"name\":\"s1f1\",\"type\":[\"null\",\"string\"],\"default\":null}," + "{\"name\":\"s1f2\",\"type\":[\"null\",\"string\"],\"default\":null}]," + "\"connect.name\":\"complex.s1\"}],\"default\":null}," + "{\"name\":\"s2\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"s2\",\"namespace\":\"complex\",\"fields\":[" + "{\"name\":\"s2f1\",\"type\":[\"null\",\"string\"],\"default\":null}," + "{\"name\":\"s2f2\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"s2f2\",\"namespace\":\"complex.s2\",\"fields\":[" + "{\"name\":\"in1\",\"type\":[\"null\",\"int\"],\"default\":null}]," + "\"connect.name\":\"complex.s2.s2f2\"}],\"default\":null}]," + "\"connect.name\":\"complex.s2\"}],\"default\":null}]," + "\"connect.name\":\"complex\"}"); }
@Override public void alterSchema(String database, String tableName, Schema schema) throws HiveMetaStoreException { Table table = hiveMetaStore.getTable(database, tableName); table.getParameters().put(AVRO_SCHEMA_LITERAL, avroData.fromConnectSchema(schema).toString()); hiveMetaStore.alterTable(table); }
@Override public RecordWriter<SinkRecord> getRecordWriter( Configuration conf, final String fileName, SinkRecord record, final AvroData avroData) throws IOException { final Schema avroSchema = avroData.fromConnectSchema(record.valueSchema()); CompressionCodecName compressionCodecName = CompressionCodecName.SNAPPY; int blockSize = 256 * 1024 * 1024; int pageSize = 64 * 1024; Path path = new Path(fileName); final ParquetWriter<GenericRecord> writer = new AvroParquetWriter<>(path, avroSchema, compressionCodecName, blockSize, pageSize, true, conf); return new RecordWriter<SinkRecord>() { @Override public void write(SinkRecord record) throws IOException { Object value = avroData.fromConnectData(record.valueSchema(), record.value()); writer.write((GenericRecord) value); } @Override public void close() throws IOException { writer.close(); } }; } }
@Override public RecordWriter<SinkRecord> getRecordWriter(Configuration conf, final String fileName, SinkRecord record, final AvroData avroData) throws IOException { DatumWriter<Object> datumWriter = new GenericDatumWriter<>(); final DataFileWriter<Object> writer = new DataFileWriter<>(datumWriter); Path path = new Path(fileName); final Schema schema = record.valueSchema(); final FSDataOutputStream out = path.getFileSystem(conf).create(path); org.apache.avro.Schema avroSchema = avroData.fromConnectSchema(schema); writer.create(avroSchema, out); return new RecordWriter<SinkRecord>(){ @Override public void write(SinkRecord record) throws IOException { log.trace("Sink record: {}", record.toString()); Object value = avroData.fromConnectData(schema, record.value()); writer.append(value); } @Override public void close() throws IOException { writer.close(); } }; } }
/** * When we want to consume SinkRecord which generated by debezium-connector-mysql, it should not * throw error "org.apache.avro.SchemaParseException: Illegal character in: server-id" */ @Test public void shouldValidateSourceInfoSchema() { org.apache.kafka.connect.data.Schema kafkaSchema = SourceInfo.SCHEMA; Schema avroSchema = avroData.fromConnectSchema(kafkaSchema); assertTrue(avroSchema != null); }
private Table constructAvroTable(String database, String tableName, Schema schema, Partitioner partitioner) throws HiveMetaStoreException { Table table = new Table(database, tableName); table.setTableType(TableType.EXTERNAL_TABLE); table.getParameters().put("EXTERNAL", "TRUE"); String tablePath = FileUtils.hiveDirectoryName(url, topicsDir, tableName); table.setDataLocation(new Path(tablePath)); table.setSerializationLib(avroSerde); try { table.setInputFormatClass(avroInputFormat); table.setOutputFormatClass(avroOutputFormat); } catch (HiveException e) { throw new HiveMetaStoreException("Cannot find input/output format:", e); } List<FieldSchema> columns = HiveSchemaConverter.convertSchema(schema); table.setFields(columns); table.setPartCols(partitioner.partitionFields()); table.getParameters().put(AVRO_SCHEMA_LITERAL, avroData.fromConnectSchema(schema).toString()); return table; } }