@Override public void put(TableReference tableRef, Map<Cell, byte[]> values, long timestamp) throws KeyAlreadyExistsException { Validate.isTrue(timestamp != Long.MAX_VALUE); Validate.isTrue(timestamp >= 0); Validate.isTrue(!tableRef.equals(TransactionConstants.TRANSACTION_TABLE), TRANSACTION_ERROR); if (values.isEmpty()) { return; } delegate.put(tableRef, values, timestamp); }
/** * Tables that are eligible for migration. Non-transactional tables and potential old checkpoint tables cannot be * migrated since the migrator reads and writes data transactionally, and migrating a checkpoint table would * interfere with the migration. We still want to drop and recreate these tables to enable manual migration. */ public static Set<TableReference> getMigratableTableNames(KeyValueService kvs, Set<TableReference> skipTables, TableReference checkpointTable) { Set<TableReference> tableNames = getCreatableTables(kvs, skipTables); tableNames.removeAll(TargetedSweepSchema.INSTANCE.getLatestSchema().getTableDefinitions().keySet()); tableNames.removeAll(AtlasDbConstants.HIDDEN_TABLES); tableNames.removeIf(tableRef -> tableRef.equals(checkpointTable)); return tableNames; }
@Override public TimestampService createTimestampService( KeyValueService rawKvs, Optional<TableReference> timestampTable, boolean initializeAsync) { Preconditions.checkArgument(!timestampTable.isPresent() || timestampTable.get().equals(AtlasDbConstants.TIMESTAMP_TABLE), "***ERROR:This can cause severe data corruption.***\nUnexpected timestamp table found: %s" + "\nThis can happen if you configure the timelock server to use Cassandra KVS for timestamp" + " persistence, which is unsupported.\nWe recommend using the default paxos timestamp" + " persistence. However, if you are need to persist the timestamp service state in the" + " database, please specify a valid DbKvs config in the timestampBoundPersister block." + "\nNote that if the service has already been running, you will have to migrate the timestamp" + " table to Postgres/Oracle and rename it to %s.", timestampTable.map(TableReference::getQualifiedName).orElse("unknown table"), AtlasDbConstants.TIMELOCK_TIMESTAMP_TABLE); AtlasDbVersion.ensureVersionReported(); Preconditions.checkArgument(rawKvs instanceof CassandraKeyValueService, "TimestampService must be created from an instance of" + " CassandraKeyValueService, found %s", rawKvs.getClass()); return PersistentTimestampServiceImpl.create( CassandraTimestampBoundStore.create((CassandraKeyValueService) rawKvs, initializeAsync), initializeAsync); }
@Override public TimestampService createTimestampService( KeyValueService rawKvs, Optional<TableReference> timestampTable, boolean initializeAsync) { if (initializeAsync) { log.warn("Asynchronous initialization not implemented, will initialize synchronousy."); } Preconditions.checkArgument(!timestampTable.isPresent() || timestampTable.get().equals(AtlasDbConstants.TIMESTAMP_TABLE), "***ERROR:This can cause severe data corruption.***\nUnexpected timestamp table found: " + timestampTable.map(TableReference::getQualifiedName).orElse("unknown table") + "\nThis can happen if you configure the timelock server to use JDBC KVS for timestamp" + " persistence, which is unsupported.\nWe recommend using the default paxos timestamp" + " persistence. However, if you are need to persist the timestamp service state in the" + " database, please specify a valid DbKvs config in the timestampBoundPersister block." + "\nNote that if the service has already been running, you will have to migrate the timestamp" + " table to Postgres/Oracle and rename it to %s.", AtlasDbConstants.TIMELOCK_TIMESTAMP_TABLE); AtlasDbVersion.ensureVersionReported(); return PersistentTimestampServiceImpl.create(JdbcTimestampBoundStore.create((JdbcKeyValueService) rawKvs)); } }
return; Validate.isTrue(!tableRef.equals(TransactionConstants.TRANSACTION_TABLE), TRANSACTION_ERROR);
private void thenTableIsChosenAtLeastOnce(TableReference table) { Assert.assertTrue("expected table " + table + " to be chosen at least once, but wasn't!", tablesToSweep.stream() .filter(Optional::isPresent) .map(Optional::get) .map(TableToSweep::getTableRef) .anyMatch(chosenTable -> chosenTable.equals(table))); }
@Override public void dropTables(Set<TableReference> tableRefs) { Set<TableReference> tableNames = getShortTableReferencesForExistingTables(tableRefs); delegate().dropTables(tableNames); // We're purposely updating the table mappings after all drops are complete for (TableReference tableRef : tableRefs) { // Handles the edge case of deleting _namespace when clearing the kvs if (tableRef.equals(AtlasDbConstants.NAMESPACE_TABLE)) { break; } tableMapper.removeTable(tableRef); } }
@Override public void put(TableReference tableRef, Map<Cell, byte[]> values, long timestamp) { if (tableRef.equals(TransactionConstants.TRANSACTION_TABLE)) { SortedMap<LockDescriptor, LockMode> mapToAssertLockHeld = Maps.newTreeMap(); SortedMap<LockDescriptor, LockMode> mapToAssertLockNotHeld = Maps.newTreeMap();
boolean storeNewMapping(TableReference table, int id) { SweepIdToNameColumn column = SweepIdToNameColumn.of(id); SweepIdToNameColumnValue value = SweepIdToNameColumnValue.of(column, table.getQualifiedName()); Cell cell = Cell.create(rowAsBytes, value.persistColumnName()); CheckAndSetRequest request = CheckAndSetRequest.newCell(ID_TO_NAME, cell, value.persistValue()); try { kvs.checkAndSet(request); return true; } catch (CheckAndSetException e) { return get(id).get().equals(table); } }
@Test public void smokeTest() throws Exception { createTable(TABLE_1, SweepStrategy.CONSERVATIVE); createTable(TABLE_2, SweepStrategy.THOROUGH); createTable(TABLE_3, SweepStrategy.NOTHING); putManyCells(TABLE_1, 100, 110); putManyCells(TABLE_1, 103, 113); putManyCells(TABLE_1, 105, 115); putManyCells(TABLE_2, 101, 111); putManyCells(TABLE_2, 104, 114); putManyCells(TABLE_3, 120, 130); try (SingleLockService sweepLocks = backgroundSweeper.createSweepLocks()) { for (int i = 0; i < 50; ++i) { backgroundSweeper.checkConfigAndRunSweep(sweepLocks); } } verifyTableSwept(TABLE_1, 75, true); verifyTableSwept(TABLE_2, 58, false); List<SweepPriority> priorities = txManager.runTaskReadOnly( tx -> SweepPriorityStoreImpl.create(kvs, SweepTableFactory.of(), false).loadNewPriorities(tx)); Assert.assertTrue(priorities.stream().anyMatch(p -> p.tableRef().equals(TABLE_1))); Assert.assertTrue(priorities.stream().anyMatch(p -> p.tableRef().equals(TABLE_2))); }
TransactionConstants.TRANSACTION_TABLE); for (TableReference tableRef : allTables) { if (tableRef.equals(TransactionConstants.TRANSACTION_TABLE)) { result.add( AtlasRowLockDescriptor.of(
@Override public void put(TableReference tableRef, Map<Cell, byte[]> values, long timestamp) throws KeyAlreadyExistsException { Validate.isTrue(timestamp != Long.MAX_VALUE); Validate.isTrue(timestamp >= 0); Validate.isTrue(!tableRef.equals(TransactionConstants.TRANSACTION_TABLE), TRANSACTION_ERROR); if (values.isEmpty()) { return; } delegate.put(tableRef, values, timestamp); }
/** * Tables that are eligible for migration. Non-transactional tables and potential old checkpoint tables cannot be * migrated since the migrator reads and writes data transactionally, and migrating a checkpoint table would * interfere with the migration. We still want to drop and recreate these tables to enable manual migration. */ public static Set<TableReference> getMigratableTableNames(KeyValueService kvs, Set<TableReference> skipTables, TableReference checkpointTable) { Set<TableReference> tableNames = getCreatableTables(kvs, skipTables); tableNames.removeAll(TargetedSweepSchema.INSTANCE.getLatestSchema().getTableDefinitions().keySet()); tableNames.removeAll(AtlasDbConstants.HIDDEN_TABLES); tableNames.removeIf(tableRef -> tableRef.equals(checkpointTable)); return tableNames; }
@Override public TimestampService createTimestampService( KeyValueService rawKvs, Optional<TableReference> timestampTable, boolean initializeAsync) { Preconditions.checkArgument(!timestampTable.isPresent() || timestampTable.get().equals(AtlasDbConstants.TIMESTAMP_TABLE), "***ERROR:This can cause severe data corruption.***\nUnexpected timestamp table found: %s" + "\nThis can happen if you configure the timelock server to use Cassandra KVS for timestamp" + " persistence, which is unsupported.\nWe recommend using the default paxos timestamp" + " persistence. However, if you are need to persist the timestamp service state in the" + " database, please specify a valid DbKvs config in the timestampBoundPersister block." + "\nNote that if the service has already been running, you will have to migrate the timestamp" + " table to Postgres/Oracle and rename it to %s.", timestampTable.map(TableReference::getQualifiedName).orElse("unknown table"), AtlasDbConstants.TIMELOCK_TIMESTAMP_TABLE); AtlasDbVersion.ensureVersionReported(); Preconditions.checkArgument(rawKvs instanceof CassandraKeyValueService, "TimestampService must be created from an instance of" + " CassandraKeyValueService, found %s", rawKvs.getClass()); return PersistentTimestampServiceImpl.create( CassandraTimestampBoundStore.create((CassandraKeyValueService) rawKvs, initializeAsync), initializeAsync); }
return; Validate.isTrue(!tableRef.equals(TransactionConstants.TRANSACTION_TABLE), TRANSACTION_ERROR);
@Override public void dropTables(Set<TableReference> tableRefs) { Set<TableReference> tableNames = getShortTableReferencesForExistingTables(tableRefs); delegate().dropTables(tableNames); // We're purposely updating the table mappings after all drops are complete for (TableReference tableRef : tableRefs) { // Handles the edge case of deleting _namespace when clearing the kvs if (tableRef.equals(AtlasDbConstants.NAMESPACE_TABLE)) { break; } tableMapper.removeTable(tableRef); } }
@Override public void put(TableReference tableRef, Map<Cell, byte[]> values, long timestamp) { if (tableRef.equals(TransactionConstants.TRANSACTION_TABLE)) { SortedMap<LockDescriptor, LockMode> mapToAssertLockHeld = Maps.newTreeMap(); SortedMap<LockDescriptor, LockMode> mapToAssertLockNotHeld = Maps.newTreeMap();
boolean storeNewMapping(TableReference table, int id) { SweepIdToNameColumn column = SweepIdToNameColumn.of(id); SweepIdToNameColumnValue value = SweepIdToNameColumnValue.of(column, table.getQualifiedName()); Cell cell = Cell.create(rowAsBytes, value.persistColumnName()); CheckAndSetRequest request = CheckAndSetRequest.newCell(ID_TO_NAME, cell, value.persistValue()); try { kvs.checkAndSet(request); return true; } catch (CheckAndSetException e) { return get(id).get().equals(table); } }
@Test public void smokeTest() throws Exception { createTable(TABLE_1, SweepStrategy.CONSERVATIVE); createTable(TABLE_2, SweepStrategy.THOROUGH); createTable(TABLE_3, SweepStrategy.NOTHING); putManyCells(TABLE_1, 100, 110); putManyCells(TABLE_1, 103, 113); putManyCells(TABLE_1, 105, 115); putManyCells(TABLE_2, 101, 111); putManyCells(TABLE_2, 104, 114); putManyCells(TABLE_3, 120, 130); try (SingleLockService sweepLocks = backgroundSweeper.createSweepLocks()) { for (int i = 0; i < 50; ++i) { backgroundSweeper.checkConfigAndRunSweep(sweepLocks); } } verifyTableSwept(TABLE_1, 75, true); verifyTableSwept(TABLE_2, 58, false); List<SweepPriority> priorities = txManager.runTaskReadOnly( tx -> SweepPriorityStoreImpl.create(kvs, SweepTableFactory.of(), false).loadNewPriorities(tx)); Assert.assertTrue(priorities.stream().anyMatch(p -> p.tableRef().equals(TABLE_1))); Assert.assertTrue(priorities.stream().anyMatch(p -> p.tableRef().equals(TABLE_2))); }
TransactionConstants.TRANSACTION_TABLE); for (TableReference tableRef : allTables) { if (tableRef.equals(TransactionConstants.TRANSACTION_TABLE)) { result.add( AtlasRowLockDescriptor.of(