/** * Verify that the given {@link SourceRecord} is a {@link Operation#CREATE READ} record, and that the integer key * matches the expected value. * * @param record the source record; may not be null * @param pkField the single field defining the primary key of the struct; may not be null * @param pk the expected integer value of the primary key in the struct */ public static void isValidRead(SourceRecord record, String pkField, int pk) { hasValidKey(record, pkField, pk); isValidRead(record); }
private void assertReadRecord(SourceRecord record, Map<String, List<SchemaAndValueField>> expectedValuesByTopicName) { VerifyRecord.isValidRead(record, PK_FIELD, 1); String topicName = record.topic().replace(TestHelper.TEST_SERVER + ".", ""); List<SchemaAndValueField> expectedValuesAndSchemasForTopic = expectedValuesByTopicName.get(topicName); assertNotNull("No expected values for " + topicName + " found", expectedValuesAndSchemasForTopic); assertRecordSchemaAndValues(expectedValuesAndSchemasForTopic, record, Envelope.FieldName.AFTER); }
private void assertRecordsFromSnapshot(int expectedCount, int...pks) throws InterruptedException { SourceRecords actualRecords = consumeRecordsByTopic(expectedCount); assertThat(actualRecords.allRecordsInOrder().size()).isEqualTo(expectedCount); // we have 2 schemas/topics that we expect int expectedCountPerSchema = expectedCount / 2; List<SourceRecord> recordsForTopicS1 = actualRecords.recordsForTopic(topicName("s1.a")); assertThat(recordsForTopicS1.size()).isEqualTo(expectedCountPerSchema); IntStream.range(0, expectedCountPerSchema) .forEach(i -> VerifyRecord.isValidRead(recordsForTopicS1.remove(0), PK_FIELD, pks[i])); List<SourceRecord> recordsForTopicS2 = actualRecords.recordsForTopic(topicName("s2.a")); assertThat(recordsForTopicS2.size()).isEqualTo(expectedCountPerSchema); IntStream.range(0, expectedCountPerSchema) .forEach(i -> VerifyRecord.isValidRead(recordsForTopicS2.remove(0), PK_FIELD, pks[i + expectedCountPerSchema])); }
@Test public void fpTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_fp"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_FP); }
@Test public void dateTimeTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_time"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_DATE_TIME); }
@Test public void otherTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_xml"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_XML); }
@Test public void intTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_int"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_INT); }
@Test public void stringTypes() throws Exception { Testing.debug("Inserted"); final SourceRecords records = consumeRecordsByTopic(EXPECTED_RECORD_COUNT); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.dbo.type_string"); assertThat(testTableRecords).hasSize(1); // insert VerifyRecord.isValidRead(testTableRecords.get(0)); Struct after = (Struct) ((Struct)testTableRecords.get(0).value()).get("after"); assertRecord(after, EXPECTED_STRING); }
@Test @FixFor("DBZ-878") public void shouldReplaceInvalidTopicNameCharacters() throws Exception { String setupStmt = SETUP_TABLES_STMT + "CREATE TABLE s1.\"dbz_878_some|test@data\" (pk SERIAL, aa integer, PRIMARY KEY(pk));" + "INSERT INTO s1.\"dbz_878_some|test@data\" (aa) VALUES (123);"; TestHelper.execute(setupStmt); Configuration.Builder configBuilder = TestHelper.defaultConfig() .with(PostgresConnectorConfig.SNAPSHOT_MODE, INITIAL.getValue()) .with(PostgresConnectorConfig.DROP_SLOT_ON_STOP, Boolean.TRUE) .with(PostgresConnectorConfig.SCHEMA_WHITELIST, "s1") .with(PostgresConnectorConfig.TABLE_WHITELIST, "s1\\.dbz_878_some\\|test@data"); start(PostgresConnector.class, configBuilder.build()); assertConnectorIsRunning(); SourceRecords actualRecords = consumeRecordsByTopic(1); List<SourceRecord> records = actualRecords.recordsForTopic(topicName("s1.dbz_878_some_test_data")); assertThat(records.size()).isEqualTo(1); SourceRecord record = records.get(0); VerifyRecord.isValidRead(record, PK_FIELD, 1); String sourceTable = ((Struct)record.value()).getStruct("source").getString("table"); assertThat(sourceTable).isEqualTo("dbz_878_some|test@data"); }
AtomicInteger pkValue = new AtomicInteger(1); recordsForS1a.forEach(record -> { VerifyRecord.isValidRead(record, PK_FIELD, pkValue.getAndIncrement()); assertFieldAbsent(record, "bb"); });
int counterVal = counter.getAndIncrement(); int expectedPk = (counterVal % 3) + 1; //each table has 3 entries keyed 1-3 VerifyRecord.isValidRead(record, PK_FIELD, expectedPk); assertRecordOffsetAndSnapshotSource(record, true, counterVal == (expectedRecordsCount - 1)); assertSourceInfo(record);
private void assertReadRecord(SourceRecord record, Map<String, List<SchemaAndValueField>> expectedValuesByTopicName) { VerifyRecord.isValidRead(record, PK_FIELD, 1); String topicName = record.topic().replace(TestHelper.TEST_SERVER + ".", ""); List<SchemaAndValueField> expectedValuesAndSchemasForTopic = expectedValuesByTopicName.get(topicName); assertNotNull("No expected values for " + topicName + " found", expectedValuesAndSchemasForTopic); assertRecordSchemaAndValues(expectedValuesAndSchemasForTopic, record, Envelope.FieldName.AFTER); }
private void assertRecordsFromSnapshot(int expectedCount, int...pks) throws InterruptedException { SourceRecords actualRecords = consumeRecordsByTopic(expectedCount); assertThat(actualRecords.allRecordsInOrder().size()).isEqualTo(expectedCount); // we have 2 schemas/topics that we expect int expectedCountPerSchema = expectedCount / 2; List<SourceRecord> recordsForTopicS1 = actualRecords.recordsForTopic(topicName("s1.a")); assertThat(recordsForTopicS1.size()).isEqualTo(expectedCountPerSchema); IntStream.range(0, expectedCountPerSchema) .forEach(i -> VerifyRecord.isValidRead(recordsForTopicS1.remove(0), PK_FIELD, pks[i])); List<SourceRecord> recordsForTopicS2 = actualRecords.recordsForTopic(topicName("s2.a")); assertThat(recordsForTopicS2.size()).isEqualTo(expectedCountPerSchema); IntStream.range(0, expectedCountPerSchema) .forEach(i -> VerifyRecord.isValidRead(recordsForTopicS2.remove(0), PK_FIELD, pks[i + expectedCountPerSchema])); }
@Test public void intTypes() throws Exception { int expectedRecordCount = 0; if (insertRecordsDuringTest()) { insertIntTypes(); } Testing.debug("Inserted"); expectedRecordCount++; final SourceRecords records = consumeRecordsByTopic(expectedRecordCount); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.TYPE_INT"); assertThat(testTableRecords).hasSize(expectedRecordCount); SourceRecord record = testTableRecords.get(0); VerifyRecord.isValid(record); // insert if (insertRecordsDuringTest()) { VerifyRecord.isValidInsert(record, "ID", 1); } else { VerifyRecord.isValidRead(record, "ID", 1); } Struct after = (Struct) ((Struct)record.value()).get("after"); assertRecord(after, EXPECTED_INT); }
@Test public void stringTypes() throws Exception { int expectedRecordCount = 0; if (insertRecordsDuringTest()) { insertStringTypes(); } Testing.debug("Inserted"); expectedRecordCount++; final SourceRecords records = consumeRecordsByTopic(expectedRecordCount); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.TYPE_STRING"); assertThat(testTableRecords).hasSize(expectedRecordCount); SourceRecord record = testTableRecords.get(0); VerifyRecord.isValid(record); // insert if (insertRecordsDuringTest()) { VerifyRecord.isValidInsert(record, "ID", 1); } else { VerifyRecord.isValidRead(record, "ID", 1); } Struct after = (Struct) ((Struct)record.value()).get("after"); assertRecord(after, EXPECTED_STRING); }
@Test public void timeTypes() throws Exception { int expectedRecordCount = 0; if (insertRecordsDuringTest()) { insertTimeTypes(); } Testing.debug("Inserted"); expectedRecordCount++; final SourceRecords records = consumeRecordsByTopic(expectedRecordCount); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.TYPE_TIME"); assertThat(testTableRecords).hasSize(expectedRecordCount); SourceRecord record = testTableRecords.get(0); VerifyRecord.isValid(record); // insert if (insertRecordsDuringTest()) { VerifyRecord.isValidInsert(record, "ID", 1); } else { VerifyRecord.isValidRead(record, "ID", 1); } Struct after = (Struct) ((Struct)record.value()).get("after"); assertRecord(after, EXPECTED_TIME); }
@Test public void fpTypes() throws Exception { int expectedRecordCount = 0; if (insertRecordsDuringTest()) { insertFpTypes(); } Testing.debug("Inserted"); expectedRecordCount++; final SourceRecords records = consumeRecordsByTopic(expectedRecordCount); List<SourceRecord> testTableRecords = records.recordsForTopic("server1.DEBEZIUM.TYPE_FP"); assertThat(testTableRecords).hasSize(expectedRecordCount); SourceRecord record = testTableRecords.get(0); VerifyRecord.isValid(record); // insert if (insertRecordsDuringTest()) { VerifyRecord.isValidInsert(record, "ID", 1); } else { VerifyRecord.isValidRead(record, "ID", 1); } Struct after = (Struct) ((Struct)record.value()).get("after"); assertRecord(after, EXPECTED_FP); }
AtomicInteger pkValue = new AtomicInteger(1); recordsForS1a.forEach(record -> { VerifyRecord.isValidRead(record, PK_FIELD, pkValue.getAndIncrement()); assertFieldAbsent(record, "bb"); });
@Test @FixFor("DBZ-878") public void shouldReplaceInvalidTopicNameCharacters() throws Exception { String setupStmt = SETUP_TABLES_STMT + "CREATE TABLE s1.\"dbz_878_some|test@data\" (pk SERIAL, aa integer, PRIMARY KEY(pk));" + "INSERT INTO s1.\"dbz_878_some|test@data\" (aa) VALUES (123);"; TestHelper.execute(setupStmt); Configuration.Builder configBuilder = TestHelper.defaultConfig() .with(PostgresConnectorConfig.SNAPSHOT_MODE, INITIAL.getValue()) .with(PostgresConnectorConfig.DROP_SLOT_ON_STOP, Boolean.TRUE) .with(PostgresConnectorConfig.SCHEMA_WHITELIST, "s1") .with(PostgresConnectorConfig.TABLE_WHITELIST, "s1\\.dbz_878_some\\|test@data"); start(PostgresConnector.class, configBuilder.build()); assertConnectorIsRunning(); SourceRecords actualRecords = consumeRecordsByTopic(1); List<SourceRecord> records = actualRecords.recordsForTopic(topicName("s1.dbz_878_some_test_data")); assertThat(records.size()).isEqualTo(1); SourceRecord record = records.get(0); VerifyRecord.isValidRead(record, PK_FIELD, 1); String sourceTable = ((Struct)record.value()).getStruct("source").getString("table"); assertThat(sourceTable).isEqualTo("dbz_878_some|test@data"); }
VerifyRecord.isValidRead(record1, "ID", 1); Struct after = (Struct) ((Struct)record1.value()).get("after"); assertThat(after.get("ID")).isEqualTo(1); VerifyRecord.isValidRead(record2, "ID", 2); after = (Struct) ((Struct)record2.value()).get("after"); assertThat(after.get("ID")).isEqualTo(2);