keySchema, oldKey, envelope.schema(), envelope.delete(valueBefore, origin, ts)); consumer.accept(record); ++count; keySchema, key, envelope.schema(), envelope.create(valueAfter, origin, ts)); consumer.accept(record); ++count; keySchema, key, envelope.schema(), envelope.update(valueBefore, valueAfter, origin, ts)); consumer.accept(record); ++count;
@Override public int read(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.read(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
private SourceRecord createCreateRecord() { final Schema recordSchema = SchemaBuilder.struct().field("id", SchemaBuilder.int8()).build(); Envelope envelope = Envelope.defineSchema() .withName("dummy.Envelope") .withRecord(recordSchema) .withSource(SchemaBuilder.struct().build()) .build(); final Struct before = new Struct(recordSchema); before.put("id", (byte)1); final Struct payload = envelope.create(before, null, System.nanoTime()); return new SourceRecord(new HashMap<>(), new HashMap<>(), "dummy", envelope.schema(), payload); }
@Override public int insert(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.create(value, origin, ts)); consumer.accept(record); return 1; } return 0; }
private void emitUpdateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object[] newColumnValues = getNewColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); // regular update if (Objects.equals(oldKey, newKey)) { Struct envelope = tableSchema.getEnvelopeSchema().update(oldValue, newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.UPDATE, newKey, envelope, offsetContext); } // PK update -> emit as delete and re-insert with new key else { Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); } }
private SourceRecord createDeleteRecord() { final Schema recordSchema = SchemaBuilder.struct().field("id", SchemaBuilder.int8()).build(); Envelope envelope = Envelope.defineSchema() .withName("dummy.Envelope") .withRecord(recordSchema) .withSource(SchemaBuilder.struct().build()) .build(); final Struct before = new Struct(recordSchema); before.put("id", (byte)1); final Struct payload = envelope.delete(before, null, System.nanoTime()); return new SourceRecord(new HashMap<>(), new HashMap<>(), "dummy", envelope.schema(), payload); }
@Override public int delete(SourceInfo source, Object[] row, int rowNumber, int numberOfRows, BitSet includedColumns, long ts, BlockingConsumer<SourceRecord> consumer) throws InterruptedException { int count = 0; Object key = tableSchema.keyFromColumnData(row); Struct value = tableSchema.valueFromColumnData(row); if (value != null || key != null) { Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = source.partition(); Map<String, Object> offset = source.offsetForRow(rowNumber, numberOfRows); Struct origin = source.struct(id); // Send a delete message ... SourceRecord record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, envelope.schema(), envelope.delete(value, origin, ts)); consumer.accept(record); ++count; // And send a tombstone ... if (emitTombstoneOnDelete) { record = new SourceRecord(partition, getSourceRecordOffset(offset), topicName, partitionNum, keySchema, key, null, null); consumer.accept(record); ++count; } } return count; }
@Test public void shouldBuildWithSimpleOptionalTypesForBeforeAndAfter() { Envelope env = Envelope.defineSchema() .withName("someName") .withRecord(Schema.OPTIONAL_STRING_SCHEMA) .withSource(Schema.OPTIONAL_INT64_SCHEMA) .build(); assertThat(env.schema()).isNotNull(); assertThat(env.schema().name()).isEqualTo("someName"); assertThat(env.schema().doc()).isNull(); assertThat(env.schema().version()).isNull(); assertOptionalField(env, Envelope.FieldName.AFTER, Schema.OPTIONAL_STRING_SCHEMA); assertOptionalField(env, Envelope.FieldName.BEFORE, Schema.OPTIONAL_STRING_SCHEMA); assertOptionalField(env, Envelope.FieldName.SOURCE, Schema.OPTIONAL_INT64_SCHEMA); assertRequiredField(env, Envelope.FieldName.OPERATION, Schema.STRING_SCHEMA); }
private void emitDeleteRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); }
private void emitCreateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); }
private void emitReadRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().read(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.READ, newKey, envelope, offsetContext); }
protected void assertRequiredField(Envelope env, String fieldName, Schema expectedSchema) { assertField(env.schema().field(fieldName),fieldName,expectedSchema, false); }
Envelope envelope = Envelope.defineSchema() .withName(schemaNameAdjuster.adjust(envelopSchemaName)) .withRecord(valSchema)
throw new IllegalStateException("The envelope schema is missing field(s) " + String.join(", ", missingFields)); return new Envelope(builder.build());
private void emitUpdateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object[] newColumnValues = getNewColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); // regular update if (Objects.equals(oldKey, newKey)) { Struct envelope = tableSchema.getEnvelopeSchema().update(oldValue, newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.UPDATE, newKey, envelope, offsetContext); } // PK update -> emit as delete and re-insert with new key else { Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); } }
protected void generateCreateRecord(TableId tableId, Object[] rowData, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException { if (rowData == null || rowData.length == 0) { logger.warn("no new values found for table '{}' from update message at '{}';skipping record" , tableId, sourceInfo); return; } TableSchema tableSchema = schema().schemaFor(tableId); assert tableSchema != null; Object key = tableSchema.keyFromColumnData(rowData); Struct value = tableSchema.valueFromColumnData(rowData); if (value == null) { logger.warn("no values found for table '{}' from create message at '{}'; skipping record" , tableId, sourceInfo); return; } Schema keySchema = tableSchema.keySchema(); Map<String, ?> partition = sourceInfo.partition(); Map<String, ?> offset = sourceInfo.offset(); String topicName = topicSelector().topicNameFor(tableId); Envelope envelope = tableSchema.getEnvelopeSchema(); SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.create(value, sourceInfo.source(), clock().currentTimeInMillis())); if (logger.isDebugEnabled()) { logger.debug("sending create event '{}' to topic '{}'", record, topicName); } recordConsumer.accept(new ChangeEvent(record, lastCompletelyProcessedLsn)); }
private void emitDeleteRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] oldColumnValues = getOldColumnValues(); Object oldKey = tableSchema.keyFromColumnData(oldColumnValues); Struct oldValue = tableSchema.valueFromColumnData(oldColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().delete(oldValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.DELETE, oldKey, envelope, offsetContext); }
private void emitCreateRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().create(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.CREATE, newKey, envelope, offsetContext); }
private void emitReadRecord(Receiver receiver, TableSchema tableSchema) throws InterruptedException { Object[] newColumnValues = getNewColumnValues(); Object newKey = tableSchema.keyFromColumnData(newColumnValues); Struct newValue = tableSchema.valueFromColumnData(newColumnValues); Struct envelope = tableSchema.getEnvelopeSchema().read(newValue, offsetContext.getSourceInfo(), clock.currentTimeInMillis()); receiver.changeRecord(tableSchema, Operation.READ, newKey, envelope, offsetContext); }