/** * Verify that the given {@link SourceRecord} is a {@link Operation#CREATE INSERT/CREATE} record without primary key. * * @param record the source record; may not be null */ public static void isValidInsert(SourceRecord record) { isValidInsert(record, false); }
/** * Validate that a {@link SourceRecord}'s key and value can each be converted to a byte[] and then back to an equivalent * {@link SourceRecord}. * * @param record the record to validate; may not be null */ protected void validate(SourceRecord record) { VerifyRecord.isValid(record); }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#DELETE DELETE} record without PK. * matches the expected value. * * @param record the source record; may not be null * @param pkField the single field defining the primary key of the struct; may not be null * @param pk the expected integer value of the primary key in the struct */ public static void isValidDelete(SourceRecord record) { isValidDelete(record, false); }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#UPDATE UPDATE} record, and that the integer key * matches the expected value. * * @param record the source record; may not be null * @param pkField the single field defining the primary key of the struct; may not be null * @param pk the expected integer value of the primary key in the struct */ public static void isValidUpdate(SourceRecord record, String pkField, int pk) { hasValidKey(record, pkField, pk); isValidUpdate(record, true); }
/** * Verify that the given {@link SourceRecord} is a valid tombstone, meaning it has a valid non-null key with key schema * but null value and value schema. * * @param record the source record; may not be null * @param pkField the single field defining the primary key of the struct; may not be null * @param pk the expected integer value of the primary key in the struct */ public static void isValidTombstone(SourceRecord record, String pkField, int pk) { hasValidKey(record, pkField, pk); isValidTombstone(record); }
/** * Verify that the given {@link SourceRecord} is a {@link Operation#DELETE DELETE} record, and that the integer key * matches the expected value. * * @param record the source record; may not be null * @param pkField the single field defining the primary key of the struct; may not be null * @param pk the expected integer value of the primary key in the struct */ public static void isValidDelete(SourceRecord record, String pkField, int pk) { hasValidKey(record, pkField, pk); isValidDelete(record, true); }
@Test public void intTypes() throws Exception { int expectedRecordCount = 0; if (insertRecordsDuringTest()) { insertIntTypes(); } Testing.debug("Inserted"); expectedRecordCount++; final SourceRecords records = consumeRecordsByTopic(expectedRecordCount); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.TYPE_INT"); assertThat(testTableRecords).hasSize(expectedRecordCount); SourceRecord record = testTableRecords.get(0); VerifyRecord.isValid(record); // insert if (insertRecordsDuringTest()) { VerifyRecord.isValidInsert(record, "ID", 1); } else { VerifyRecord.isValidRead(record, "ID", 1); } Struct after = (Struct) ((Struct)record.value()).get("after"); assertRecord(after, EXPECTED_INT); }
@Test public void shouldReceiveChangesForUpdatesWithPKChanges() throws Exception { consumer = testConsumer(3); recordsProducer.start(consumer, blackHole); executeAndWait("UPDATE test_table SET text = 'update', pk = 2"); String topicName = topicName("public.test_table"); // first should be a delete of the old pk SourceRecord deleteRecord = consumer.remove(); assertEquals(topicName, deleteRecord.topic()); VerifyRecord.isValidDelete(deleteRecord, PK_FIELD, 1); // followed by a tombstone of the old pk SourceRecord tombstoneRecord = consumer.remove(); assertEquals(topicName, tombstoneRecord.topic()); VerifyRecord.isValidTombstone(tombstoneRecord, PK_FIELD, 1); // and finally insert of the new value SourceRecord insertRecord = consumer.remove(); assertEquals(topicName, insertRecord.topic()); VerifyRecord.isValidInsert(insertRecord, PK_FIELD, 2); }
String topicName = topicName("public.test_table"); assertEquals(topicName, updatedRecord.topic()); VerifyRecord.isValidUpdate(updatedRecord, PK_FIELD, 1); VerifyRecord.isValidUpdate(updatedRecord, PK_FIELD, 1); executeAndWait(statements); updatedRecord = consumer.remove(); VerifyRecord.isValidUpdate(updatedRecord, PK_FIELD, 1); updatedRecord = consumer.remove(); VerifyRecord.isValidInsert(updatedRecord, PK_FIELD, 2); assertRecordSchemaAndValues( Collections.singletonList(new SchemaAndValueField("modtype", SchemaBuilder.OPTIONAL_INT32_SCHEMA, 1)), updatedRecord, Envelope.FieldName.AFTER); executeAndWait(statements); updatedRecord = consumer.remove(); VerifyRecord.isValidUpdate(updatedRecord, PK_FIELD, 2); assertRecordSchemaAndValues( Collections.singletonList(new SchemaAndValueField("modtype", SchemaBuilder.OPTIONAL_INT16_SCHEMA, (short)1)), updatedRecord, Envelope.FieldName.BEFORE);
VerifyRecord.isValidInsert(testTableRecords.get(0), "ID", 1); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertThat(after.get("ID")).isEqualTo(1); VerifyRecord.isValidUpdate(testTableRecords.get(1), "ID", 1); Struct before = (Struct) ((Struct)testTableRecords.get(1).value()).get("before"); assertThat(before.get("ID")).isEqualTo(1); VerifyRecord.isValidDelete(testTableRecords.get(2), "ID", 1); before = (Struct) ((Struct)testTableRecords.get(2).value()).get("before"); assertThat(before.get("ID")).isEqualTo(1); assertThat(before.get("REGISTERED")).isEqualTo(toMicroSecondsSinceEpoch(LocalDateTime.of(2018, 3, 23, 0, 0, 0))); VerifyRecord.isValidTombstone(testTableRecords.get(3)); VerifyRecord.isValidInsert(testTableRecords.get(4), "ID", 2); after = (Struct) ((Struct)testTableRecords.get(4).value()).get("after"); assertThat(after.get("ID")).isEqualTo(2); VerifyRecord.isValidDelete(testTableRecords.get(5), "ID", 2); before = (Struct) ((Struct)testTableRecords.get(5).value()).get("before"); assertThat(before.get("ID")).isEqualTo(2); assertThat(before.get("REGISTERED")).isEqualTo(toMicroSecondsSinceEpoch(LocalDateTime.of(2018, 3, 23, 0, 0, 0))); VerifyRecord.isValidTombstone(testTableRecords.get(6));
@Test public void shouldReceiveChangesForDeletes() throws Exception { // add a new entry and remove both String statements = "INSERT INTO test_table (text) VALUES ('insert2');" + "DELETE FROM test_table WHERE pk > 0;"; consumer = testConsumer(5); recordsProducer.start(consumer, blackHole); executeAndWait(statements); String topicPrefix = "public.test_table"; String topicName = topicName(topicPrefix); assertRecordInserted(topicPrefix, PK_FIELD, 2); // first entry removed SourceRecord record = consumer.remove(); assertEquals(topicName, record.topic()); VerifyRecord.isValidDelete(record, PK_FIELD, 1); // followed by a tombstone record = consumer.remove(); assertEquals(topicName, record.topic()); VerifyRecord.isValidTombstone(record, PK_FIELD, 1); // second entry removed record = consumer.remove(); assertEquals(topicName, record.topic()); VerifyRecord.isValidDelete(record, PK_FIELD, 2); // followed by a tombstone record = consumer.remove(); assertEquals(topicName, record.topic()); VerifyRecord.isValidTombstone(record, PK_FIELD, 2); }
@Test public void shouldSnapshotTablesInLexicographicalOrder() throws Exception{ config = simpleConfig() .build(); context = new MySqlTaskContext(config, new Filters.Builder(config).build()); context.start(); reader = new SnapshotReader("snapshot", context); reader.uponCompletion(completed::countDown); reader.generateInsertEvents(); // Start the snapshot ... reader.start(); // Poll for records ... // Testing.Print.enable(); List<SourceRecord> records; LinkedHashSet<String> tablesInOrder = new LinkedHashSet<>(); LinkedHashSet<String> tablesInOrderExpected = getTableNamesInSpecifiedOrder("Products", "customers", "dbz_342_timetest", "orders", "products_on_hand"); while ((records = reader.poll()) != null) { records.forEach(record -> { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); if (record.value() != null) tablesInOrder.add(getTableNameFromSourceRecord.apply(record)); }); } assertArrayEquals(tablesInOrder.toArray(), tablesInOrderExpected.toArray()); }
private void assertReadRecord(SourceRecord record, Map<String, List<SchemaAndValueField>> expectedValuesByTopicName) { VerifyRecord.isValidRead(record, PK_FIELD, 1); String topicName = record.topic().replace(TestHelper.TEST_SERVER + ".", ""); List<SchemaAndValueField> expectedValuesAndSchemasForTopic = expectedValuesByTopicName.get(topicName); assertNotNull("No expected values for " + topicName + " found", expectedValuesAndSchemasForTopic); assertRecordSchemaAndValues(expectedValuesAndSchemasForTopic, record, Envelope.FieldName.AFTER); }
@Test @FixFor("DBZ-582") public void shouldReceiveChangesForUpdatesWithPKChangesWithoutTombstone() throws Exception { PostgresConnectorConfig config = new PostgresConnectorConfig(TestHelper.defaultConfig() .with(PostgresConnectorConfig.INCLUDE_UNKNOWN_DATATYPES, true) .with(CommonConnectorConfig.TOMBSTONES_ON_DELETE, false) .build() ); setupRecordsProducer(config); consumer = testConsumer(2); recordsProducer.start(consumer, blackHole); executeAndWait("UPDATE test_table SET text = 'update', pk = 2"); String topicName = topicName("public.test_table"); // first should be a delete of the old pk SourceRecord deleteRecord = consumer.remove(); assertEquals(topicName, deleteRecord.topic()); VerifyRecord.isValidDelete(deleteRecord, PK_FIELD, 1); // followed by insert of the new value SourceRecord insertRecord = consumer.remove(); assertEquals(topicName, insertRecord.topic()); VerifyRecord.isValidInsert(insertRecord, PK_FIELD, 2); }
private void assertSchemaContent(String[] fields, Schema[] types, Schema keySchema) { IntStream.range(0, fields.length).forEach(i -> { String fieldName = fields[i].trim(); Field field = keySchema.field(Strings.unquoteIdentifierPart(fieldName)); assertNotNull(fieldName + " not found in schema", field); VerifyRecord.assertConnectSchemasAreEqual(fieldName, types[i], field.schema()); }); }
executeAndWait("UPDATE test_table set text='b' WHERE id=1"); SourceRecord updatedRecord = consumer.remove(); VerifyRecord.isValidUpdate(updatedRecord); executeAndWait("DELETE FROM test_table WHERE id=1"); SourceRecord deletedRecord = consumer.remove(); VerifyRecord.isValidDelete(deletedRecord);
/** * Verify that the given {@link SourceRecord} is a {@link Operation#CREATE INSERT/CREATE} record, and that the integer key * matches the expected value. * * @param record the source record; may not be null * @param pkField the single field defining the primary key of the struct; may not be null * @param pk the expected integer value of the primary key in the struct */ public static void isValidInsert(SourceRecord record, String pkField, int pk) { hasValidKey(record, pkField, pk); isValidInsert(record, true); }
VerifyRecord.isValidInsert(first, PK_FIELD, 2); assertEquals(topicName("s1.a"), first.topic()); assertRecordOffsetAndSnapshotSource(first, false, false); VerifyRecord.isValidInsert(second, PK_FIELD, 2); assertEquals(topicName("s2.a"), second.topic()); assertRecordOffsetAndSnapshotSource(second, false, false); int counterVal = counter.getAndIncrement(); int expectedPk = (counterVal % 3) + 1; //each table has 3 entries keyed 1-3 VerifyRecord.isValidRead(record, PK_FIELD, expectedPk); assertRecordOffsetAndSnapshotSource(record, true, counterVal == (expectedRecordsCount - 1)); assertSourceInfo(record); VerifyRecord.isValidInsert(first, PK_FIELD, 4); assertRecordOffsetAndSnapshotSource(first, false, false); assertSourceInfo(first, "test_database", "s1", "a"); VerifyRecord.isValidInsert(second, PK_FIELD, 4); assertRecordOffsetAndSnapshotSource(second, false, false); assertSourceInfo(second, "test_database", "s2", "a");
/** * Verify that the given {@link SourceRecord} is a {@link Operation#UPDATE UPDATE} record without PK. * * @param record the source record; may not be null * @param pkField the single field defining the primary key of the struct; may not be null * @param pk the expected integer value of the primary key in the struct */ public static void isValidUpdate(SourceRecord record) { isValidUpdate(record, false); }
private void assertSourceRecordMatch(SourceRecord actual, SourceRecord expected, Predicate<String> ignoreFields, Map<String, RecordValueComparator> comparatorsByName, Map<String, RecordValueComparator> comparatorsBySchemaName) { try { VerifyRecord.isValid(actual); } catch (AssertionError e) { throw new AssertionError("Actual source record is not valid: " + e.getMessage()); } try { VerifyRecord.isValid(expected); } catch (AssertionError e) { throw new AssertionError("Expected source record is not valid: " + e.getMessage()); } VerifyRecord.assertEquals(actual, expected, ignoreFields, comparatorsByName, comparatorsBySchemaName); }