@Test public void configHasSweepEnabledByDefault() { AtlasDbRuntimeConfig config = AtlasDbRuntimeConfig.defaultRuntimeConfig(); assertTrue(config.sweep().enabled()); }
private static void assertNoSpuriousTimeLockBlockInRuntimeConfig( AtlasDbConfig config, AtlasDbRuntimeConfig initialRuntimeConfig) { // Note: The other direction (timelock install config without a runtime block) should be maintained for // backwards compatibility. if (remoteTimestampAndLockOrLeaderBlocksPresent(config) && initialRuntimeConfig.timelockRuntime().isPresent()) { throw new IllegalStateException("Found a service configured not to use timelock, with a timelock" + " block in the runtime config! This is unexpected. If you wish to use non-timelock services," + " please remove the timelock block from the runtime config; if you wish to use timelock," + " please remove the leader, remote timestamp or remote lock configuration blocks."); } }
@Test public void configWithoutSweepHasSweepDisabled() throws Exception { AtlasDbRuntimeConfig config = AtlasDbRuntimeConfig.withSweepDisabled(); assertFalse(config.sweep().enabled()); }
@Test public void configWithoutSpecifyingStreamPersistenceConfigHasDefaultConfig() { AtlasDbRuntimeConfig config = AtlasDbRuntimeConfig.defaultRuntimeConfig(); assertThat(config.streamStorePersistence()).isEqualTo(StreamStorePersistenceConfiguration.DEFAULT_CONFIG); } }
@Test public void doesNotSweepDuringMigration() { assertThat(fromServices.getAtlasDbRuntimeConfig().sweep().enabled()).isFalse(); assertThat(fromServices.getAtlasDbRuntimeConfig().targetedSweep().enabled()).isFalse(); assertThat(toServices.getAtlasDbRuntimeConfig().sweep().enabled()).isFalse(); assertThat(toServices.getAtlasDbRuntimeConfig().targetedSweep().enabled()).isFalse(); }
MetricsManager metricsManager = MetricsManagers.of(globalMetricsRegistry(), globalTaggedMetricRegistry()); AtlasDbRuntimeConfig defaultRuntime = AtlasDbRuntimeConfig.defaultRuntimeConfig(); Supplier<AtlasDbRuntimeConfig> runtimeConfigSupplier = () -> runtimeConfigSupplier().get().orElse(defaultRuntime); config().initializeAsync(), new TimestampCache(metricsManager.getRegistry(), () -> runtimeConfigSupplier.get().getTimestampCacheSize()), targetedSweep, callbacks, validateLocksOnReads(), () -> runtimeConfigSupplier.get().transaction()), closeables);
cleaner, new TimestampCache(metricsManager.getRegistry(), () -> config.atlasDbRuntimeConfig().getTimestampCacheSize()), config.allowAccessToHiddenTables(), config.atlasDbConfig().keyValueService().concurrentGetRangesThreadPoolSize(), new NamedThreadFactory(TransactionManagerModule.class + "-delete-executor", true)), true, () -> config.atlasDbRuntimeConfig().transaction());
@Test public void canDeserializeRuntimeConfig() throws IOException { AtlasDbRuntimeConfig runtimeConfig = AtlasDbConfigs.OBJECT_MAPPER.readValue(TEST_RUNTIME_CONFIG_FILE, AtlasDbRuntimeConfig.class); assertThat(runtimeConfig.timestampClient().enableTimestampBatching()).isTrue(); assertThat(runtimeConfig.timelockRuntime()).isPresent(); assertThat(runtimeConfig.timelockRuntime().get().serversList().servers()) .containsExactlyInAnyOrder( "https://foo1:12345", "https://foo2:8421", "https://foo3:9421"); assertThat(runtimeConfig.timelockRuntime().get().serversList().sslConfiguration()).satisfies( sslConfiguration -> sslConfiguration.ifPresent(this::assertSslConfigDeserializedCorrectly)); assertThat(runtimeConfig.streamStorePersistence()).satisfies( persistenceConfig -> { assertThat(persistenceConfig.numBlocksToWriteBeforePause()).isEqualTo(7); assertThat(persistenceConfig.writePauseDurationMillis()).isEqualTo(77); }); assertThat(runtimeConfig.sweep().sweepPriorityOverrides()).satisfies( overrides -> { assertThat(overrides.priorityTables()).containsExactlyInAnyOrder("atlas.mission_critical_table"); assertThat(overrides.blacklistTables()).containsExactlyInAnyOrder( "atlas.bad_table", "atlas2.immutable_log"); }); }
AdjustableSweepBatchConfigSource sweepBatchConfigSource = AdjustableSweepBatchConfigSource.create( metricsManager, () -> getSweepBatchConfig(runtimeConfigSupplier.get().sweep())); metricsManager, sweepBatchConfigSource, () -> runtimeConfigSupplier.get().sweep().enabled(), () -> runtimeConfigSupplier.get().sweep().sweepThreads(), () -> runtimeConfigSupplier.get().sweep().pauseMillis(), () -> runtimeConfigSupplier.get().sweep().sweepPriorityOverrides(), persistentLockManager, specificTableSweeper);
protected AtlasDbRuntimeConfig getAtlasDbRuntimeConfig() { if (runtimeConfig == null) { if (runtimeConfigFile != null) { runtimeConfig = parseAtlasDbConfig(runtimeConfigFile, AtlasDbRuntimeConfig.class, runtimeConfigRoot); } else { runtimeConfig = AtlasDbRuntimeConfig.defaultRuntimeConfig(); } } return runtimeConfig; }
@Test public void canDeserializeRuntimeConfigWithZeroServers() throws IOException { AtlasDbRuntimeConfig runtimeConfig = AtlasDbConfigs.OBJECT_MAPPER.readValue(TEST_RUNTIME_CONFIG_NO_SERVERS_FILE, AtlasDbRuntimeConfig.class); assertThat(runtimeConfig.timestampClient().enableTimestampBatching()).isTrue(); assertThat(runtimeConfig.timelockRuntime()).isPresent(); assertThat(runtimeConfig.timelockRuntime().get().serversList().servers()).isEmpty(); assertThat(runtimeConfig.timelockRuntime().get().serversList().sslConfiguration()).isEmpty(); } }
public AtlasDbServices connectFromServices() throws IOException { AtlasDbConfig fromConfig = AtlasDbConfigs.load(fromConfigFile, configRoot, AtlasDbConfig.class); ServicesConfigModule scm = ServicesConfigModule.create( makeOfflineIfNecessary(fromConfig), AtlasDbRuntimeConfig.withSweepDisabled()); return DaggerAtlasDbServices.builder().servicesConfigModule(scm).build(); }
@Value.Derived public ServiceDiscoveringAtlasSupplier atlasDbSupplier(MetricsManager metrics) { return new ServiceDiscoveringAtlasSupplier( metrics, atlasDbConfig().keyValueService(), () -> atlasDbRuntimeConfig().keyValueService(), atlasDbConfig().leader(), atlasDbConfig().namespace(), Optional.empty(), atlasDbConfig().initializeAsync(), adapter()); }
MetricsManager metricsManager = MetricsManagers.of(globalMetricsRegistry(), globalTaggedMetricRegistry()); AtlasDbRuntimeConfig defaultRuntime = AtlasDbRuntimeConfig.defaultRuntimeConfig(); Supplier<AtlasDbRuntimeConfig> runtimeConfigSupplier = () -> runtimeConfigSupplier().get().orElse(defaultRuntime); config().initializeAsync(), new TimestampCache(metricsManager.getRegistry(), () -> runtimeConfigSupplier.get().getTimestampCacheSize()), targetedSweep, callbacks, validateLocksOnReads(), () -> runtimeConfigSupplier.get().transaction()), closeables);
cleaner, new TimestampCache( metricsManager.getRegistry(), () -> config.atlasDbRuntimeConfig().getTimestampCacheSize()), config.allowAccessToHiddenTables(), config.atlasDbConfig().keyValueService().concurrentGetRangesThreadPoolSize(), PTExecutors.newSingleThreadExecutor(true), true, () -> config.atlasDbRuntimeConfig().transaction());
@Provides @Singleton @Named("kvs") public KeyValueService provideWrappedKeyValueService(@Named("rawKvs") KeyValueService rawKvs, TimestampService tss, ServicesConfig config, MetricsManager metricsManager) { config.adapter().setTimestampService(tss); KvsProfilingLogger.setSlowLogThresholdMillis(config.atlasDbConfig().getKvsSlowLogThresholdMillis()); KeyValueService kvs = ProfilingKeyValueService.create(rawKvs); kvs = TracingKeyValueService.create(kvs); kvs = AtlasDbMetrics.instrument(metricsManager.getRegistry(), KeyValueService.class, kvs); kvs = ValidatingQueryRewritingKeyValueService.create(kvs); SweepConfig sweepConfig = config.atlasDbRuntimeConfig().sweep(); kvs = SweepStatsKeyValueService.create( kvs, tss, sweepConfig::writeThreshold, sweepConfig::writeSizeThreshold); TransactionTables.createTables(kvs); ImmutableSet<Schema> schemas = ImmutableSet.<Schema>builder() .add(SweepSchema.INSTANCE.getLatestSchema()) .add(CompactSchema.INSTANCE.getLatestSchema()) .addAll(config.schemas()).build(); for (Schema schema : schemas) { Schemas.createTablesAndIndexes(schema, kvs); } return kvs; }
@Test public void daggerCanInstantiateAtlas() throws URISyntaxException, IOException { File config = new File(getResourcePath("simple_atlas_config.yml")); ServicesConfigModule servicesConfigModule = ServicesConfigModule.create( AtlasDbConfigs.load(config, AtlasDbConfig.class), AtlasDbRuntimeConfig.defaultRuntimeConfig()); DaggerAtlasDbServices.builder() .servicesConfigModule(servicesConfigModule) .build(); }
when(runtimeConfig.timestampClient()).thenReturn(ImmutableTimestampClientConfig.of(false)); when(runtimeConfig.timelockRuntime()).thenReturn(Optional.empty());
public AtlasDbServices connectToServices() throws IOException { AtlasDbConfig toConfig = toConfigFile != null ? AtlasDbConfigs.load(toConfigFile, configRoot, AtlasDbConfig.class) : AtlasDbConfigs.loadFromString(inlineConfig, null, AtlasDbConfig.class); ServicesConfigModule scm = ServicesConfigModule.create( makeOfflineIfNecessary(toConfig), AtlasDbRuntimeConfig.withSweepDisabled()); return DaggerAtlasDbServices.builder().servicesConfigModule(scm).build(); }
@Value.Derived public ServiceDiscoveringAtlasSupplier atlasDbSupplier(MetricsManager metrics) { return new ServiceDiscoveringAtlasSupplier( metrics, atlasDbConfig().keyValueService(), () -> atlasDbRuntimeConfig().keyValueService(), atlasDbConfig().leader(), atlasDbConfig().namespace(), Optional.empty(), atlasDbConfig().initializeAsync(), adapter()); }