public Collection collection(String catalog, String tableName) { return collection(new TableId(catalog, null, tableName)); }
/** * Create a KeyValueStore that removes from the topic names the supplied prefix to obtain the {@link TableId}. * * @param prefix the prefix after which all of the topic name forms the table ID; may not be null * @return the key value store */ public static KeyValueStore createForTopicsBeginningWith(String prefix) { return new KeyValueStore(prefixedWith(prefix)); }
protected int consumeAtLeast(int minNumber, long timeout, TimeUnit unit) throws InterruptedException { AtomicInteger counter = new AtomicInteger(); List<SourceRecord> records = null; long startTime = System.currentTimeMillis(); while (counter.get() < minNumber && (System.currentTimeMillis() - startTime) < unit.toMillis(timeout)) { records = reader.poll(); if (records != null) { records.forEach(record -> { counter.incrementAndGet(); VerifyRecord.isValid(record); store.add(record); schemaChanges.add(record); }); Testing.print("" + counter.get() + " records"); } } return counter.get(); }
/** * Create a KeyValueStore that uses the supplied regular expression and group number to extract the {@link TableId} from * the topic name. * * @param regex the regular expression that identifies the table ID within the topic name; may not be null * @param groupNumber the group number in the regex for the table ID string * @return the key value store */ public static KeyValueStore createForTopicsMatching(String regex, int groupNumber) { return new KeyValueStore(fromRegex(regex, groupNumber)); }
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); while ((records = reader.poll()) != null) { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); store.add(record); schemaChanges.add(record); System.out.println(record); assertThat(store.databases()).containsOnly(DATABASE.getDatabaseName(), OTHER_DATABASE.getDatabaseName()); // 2 databases assertThat(store.collectionCount()).isEqualTo(9); // 2 databases Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName()); assertThat(products.numberOfCreates()).isEqualTo(0); assertThat(products.numberOfUpdates()).isEqualTo(0); Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand"); assertThat(products_on_hand.numberOfCreates()).isEqualTo(0); assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0); Collection customers = store.collection(DATABASE.getDatabaseName(), "customers"); assertThat(customers.numberOfCreates()).isEqualTo(0); assertThat(customers.numberOfUpdates()).isEqualTo(0); Collection orders = store.collection(DATABASE.getDatabaseName(), "orders"); assertThat(orders.numberOfCreates()).isEqualTo(0); assertThat(orders.numberOfUpdates()).isEqualTo(0);
assertThat(consumed).isGreaterThanOrEqualTo(expected); store.sourceRecords().forEach(System.out::println); assertThat(store.collectionCount()).isEqualTo(5); Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName()); assertThat(products.numberOfCreates()).isEqualTo(9); assertThat(products.numberOfUpdates()).isEqualTo(0); assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1); Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand"); assertThat(products_on_hand.numberOfCreates()).isEqualTo(9); assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0); assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1); Collection customers = store.collection(DATABASE.getDatabaseName(), "customers"); assertThat(customers.numberOfCreates()).isEqualTo(4); assertThat(customers.numberOfUpdates()).isEqualTo(0); assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1); Collection orders = store.collection(DATABASE.getDatabaseName(), "orders"); assertThat(orders.numberOfCreates()).isEqualTo(5); assertThat(orders.numberOfUpdates()).isEqualTo(0);
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); while ((records = reader.poll()) != null) { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); store.add(record); schemaChanges.add(record); }); assertThat(store.collectionCount()).isEqualTo(0);
assertThat(store.collectionCount()).isEqualTo(5); Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName()); assertThat(products.numberOfCreates()).isEqualTo(9); assertThat(products.numberOfUpdates()).isEqualTo(0); assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1); Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand"); assertThat(products_on_hand.numberOfCreates()).isEqualTo(9); assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0); assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1); Collection customers = store.collection(DATABASE.getDatabaseName(), "customers"); assertThat(customers.numberOfCreates()).isEqualTo(4); assertThat(customers.numberOfUpdates()).isEqualTo(0); assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1); Collection orders = store.collection(DATABASE.getDatabaseName(), "orders"); assertThat(orders.numberOfCreates()).isEqualTo(5); assertThat(orders.numberOfUpdates()).isEqualTo(0);
@Test(expected = ConnectException.class) public void shouldCreateSnapshotSchemaOnlyRecovery_exception() throws Exception { config = simpleConfig().with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.SCHEMA_ONLY_RECOVERY).build(); context = new MySqlTaskContext(config, new Filters.Builder(config).build()); context.start(); reader = new SnapshotReader("snapshot", context); reader.uponCompletion(completed::countDown); reader.generateInsertEvents(); // Start the snapshot ... reader.start(); // Poll for records ... // Testing.Print.enable(); List<SourceRecord> records = null; KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); while ((records = reader.poll()) != null) { records.forEach(record -> { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); store.add(record); schemaChanges.add(record); }); } // should fail because we have no existing binlog information }
@Before public void beforeEach() { Testing.Files.delete(DB_HISTORY_PATH); DATABASE.createAndInitialize(); this.store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); this.schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); }
public void add(SourceRecord record) { TableId tableId = tableIdFromTopic.apply(record.topic()); if (tableId != null) { this.sourceRecords.add(record); getOrCreate(tableId).add(record); } }
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); while ((records = reader.poll()) != null) { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); store.add(record); schemaChanges.add(record); }); assertThat(store.collectionCount()).isEqualTo(5); Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName()); assertThat(products.numberOfCreates()).isEqualTo(9); assertThat(products.numberOfUpdates()).isEqualTo(0); Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand"); assertThat(products_on_hand.numberOfCreates()).isEqualTo(9); assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0); Collection customers = store.collection(DATABASE.getDatabaseName(), "customers"); assertThat(customers.numberOfCreates()).isEqualTo(4); assertThat(customers.numberOfUpdates()).isEqualTo(0); Collection orders = store.collection(DATABASE.getDatabaseName(), "orders"); assertThat(orders.numberOfCreates()).isEqualTo(5); assertThat(orders.numberOfUpdates()).isEqualTo(0); Collection timetest = store.collection(DATABASE.getDatabaseName(), "dbz_342_timetest"); assertThat(timetest.numberOfCreates()).isEqualTo(1);
assertThat(consumed).isGreaterThanOrEqualTo(expected); store.sourceRecords().forEach(System.out::println); assertThat(store.collectionCount()).isEqualTo(5); Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName()); assertThat(products.numberOfCreates()).isEqualTo(9); assertThat(products.numberOfUpdates()).isEqualTo(0); assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1); Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand"); assertThat(products_on_hand.numberOfCreates()).isEqualTo(9); assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0); assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1); Collection customers = store.collection(DATABASE.getDatabaseName(), "customers"); assertThat(customers.numberOfCreates()).isEqualTo(4); assertThat(customers.numberOfUpdates()).isEqualTo(0); assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1); Collection orders = store.collection(DATABASE.getDatabaseName(), "orders"); assertThat(orders.numberOfCreates()).isEqualTo(5); assertThat(orders.numberOfUpdates()).isEqualTo(0);
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); while ((records = reader.poll()) != null) { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); store.add(record); schemaChanges.add(record); }); assertThat(store.collectionCount()).isEqualTo(0);
assertThat(store.collectionCount()).isEqualTo(5); Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName()); assertThat(products.numberOfCreates()).isEqualTo(9); assertThat(products.numberOfUpdates()).isEqualTo(0); assertThat(products.numberOfValueSchemaChanges()).isEqualTo(1); Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand"); assertThat(products_on_hand.numberOfCreates()).isEqualTo(9); assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0); assertThat(products_on_hand.numberOfValueSchemaChanges()).isEqualTo(1); Collection customers = store.collection(DATABASE.getDatabaseName(), "customers"); assertThat(customers.numberOfCreates()).isEqualTo(4); assertThat(customers.numberOfUpdates()).isEqualTo(0); assertThat(customers.numberOfValueSchemaChanges()).isEqualTo(1); Collection orders = store.collection(DATABASE.getDatabaseName(), "orders"); assertThat(orders.numberOfCreates()).isEqualTo(5); assertThat(orders.numberOfUpdates()).isEqualTo(0);
@Test(expected = ConnectException.class) public void shouldCreateSnapshotSchemaOnlyRecovery_exception() throws Exception { config = simpleConfig().with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.SCHEMA_ONLY_RECOVERY).build(); context = new MySqlTaskContext(config, new Filters.Builder(config).build()); context.start(); reader = new SnapshotReader("snapshot", context); reader.uponCompletion(completed::countDown); reader.generateInsertEvents(); // Start the snapshot ... reader.start(); // Poll for records ... // Testing.Print.enable(); List<SourceRecord> records = null; KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); while ((records = reader.poll()) != null) { records.forEach(record -> { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); store.add(record); schemaChanges.add(record); }); } // should fail because we have no existing binlog information }
@Before public void beforeEach() { Testing.Files.delete(DB_HISTORY_PATH); DATABASE.createAndInitialize(); this.store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); this.schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); }
KeyValueStore store = KeyValueStore.createForTopicsBeginningWith(DATABASE.getServerName() + "."); SchemaChangeHistory schemaChanges = new SchemaChangeHistory(DATABASE.getServerName()); while ((records = reader.poll()) != null) { VerifyRecord.isValid(record); VerifyRecord.hasNoSourceQuery(record); store.add(record); schemaChanges.add(record); }); assertThat(store.collectionCount()).isEqualTo(5); Collection products = store.collection(DATABASE.getDatabaseName(), productsTableName()); assertThat(products.numberOfCreates()).isEqualTo(9); assertThat(products.numberOfUpdates()).isEqualTo(0); Collection products_on_hand = store.collection(DATABASE.getDatabaseName(), "products_on_hand"); assertThat(products_on_hand.numberOfCreates()).isEqualTo(9); assertThat(products_on_hand.numberOfUpdates()).isEqualTo(0); Collection customers = store.collection(DATABASE.getDatabaseName(), "customers"); assertThat(customers.numberOfCreates()).isEqualTo(4); assertThat(customers.numberOfUpdates()).isEqualTo(0); Collection orders = store.collection(DATABASE.getDatabaseName(), "orders"); assertThat(orders.numberOfCreates()).isEqualTo(5); assertThat(orders.numberOfUpdates()).isEqualTo(0); Collection timetest = store.collection(DATABASE.getDatabaseName(), "dbz_342_timetest"); assertThat(timetest.numberOfCreates()).isEqualTo(1);
public Collection collection(String fullyQualifiedName) { return collection(TableId.parse(fullyQualifiedName)); }