How to use

BookKeeperConfig

in

io.pravega.segmentstore.storage.impl.bookkeeper

@Test
public void testDefaultValues() {
  BookKeeperConfig cfg = BookKeeperConfig.builder()
      .build();
  Assert.assertEquals("localhost:2181", cfg.getZkAddress());
  Assert.assertEquals(Duration.ofMillis(10000), cfg.getZkSessionTimeout());
  Assert.assertEquals(Duration.ofMillis(10000), cfg.getZkConnectionTimeout());
  Assert.assertEquals("/segmentstore/containers", cfg.getZkMetadataPath());
  Assert.assertEquals(2, cfg.getZkHierarchyDepth());
  Assert.assertEquals(5, cfg.getMaxWriteAttempts());
  Assert.assertEquals(3, cfg.getBkEnsembleSize());
  Assert.assertEquals(2, cfg.getBkAckQuorumSize());
  Assert.assertEquals(3, cfg.getBkWriteQuorumSize());
  Assert.assertEquals(5000, cfg.getBkWriteTimeoutMillis());
  Assert.assertEquals(5000, cfg.getBkReadTimeoutMillis());
  Assert.assertEquals(1024 * 1024 * 1024, cfg.getBkLedgerMaxSize());
  Assert.assertEquals(0, cfg.getBKPassword().length);
  Assert.assertEquals("", cfg.getBkLedgerPath());
  Assert.assertEquals(false, cfg.isTLSEnabled());
  Assert.assertEquals("config/client.truststore.jks", cfg.getTlsTrustStore());
  Assert.assertEquals("", cfg.getTlsTrustStorePasswordPath());
}

/**
 * Creates a new Ledger in BookKeeper.
 *
 * @return A LedgerHandle for the new ledger.
 * @throws DataLogNotAvailableException If BookKeeper is unavailable or the ledger could not be created because an
 *                                      insufficient number of Bookies are available. The causing exception is wrapped
 *                                      inside it.
 * @throws DurableDataLogException      If another exception occurred. The causing exception is wrapped inside it.
 */
static LedgerHandle create(BookKeeper bookKeeper, BookKeeperConfig config) throws DurableDataLogException {
  try {
    return Exceptions.handleInterruptedCall(() ->
        bookKeeper.createLedger(
            config.getBkEnsembleSize(),
            config.getBkWriteQuorumSize(),
            config.getBkAckQuorumSize(),
            LEDGER_DIGEST_TYPE,
            config.getBKPassword()));
  } catch (BKException.BKNotEnoughBookiesException bkEx) {
    throw new DataLogNotAvailableException("Unable to create new BookKeeper Ledger.", bkEx);
  } catch (BKException bkEx) {
    throw new DurableDataLogException("Unable to create new BookKeeper Ledger.", bkEx);
  }
}

private BookKeeper startBookKeeperClient() throws Exception {
  // These two are in Seconds, not Millis.
  int writeTimeout = (int) Math.ceil(this.config.getBkWriteTimeoutMillis() / 1000.0);
  int readTimeout = (int) Math.ceil(this.config.getBkReadTimeoutMillis() / 1000.0);
  ClientConfiguration config = new ClientConfiguration()
      .setClientTcpNoDelay(true)
      .setAddEntryTimeout(writeTimeout)
      .setReadEntryTimeout(readTimeout)
      .setGetBookieInfoTimeout(readTimeout)
      .setClientConnectTimeoutMillis((int) this.config.getZkConnectionTimeout().toMillis())
      .setZkTimeout((int) this.config.getZkConnectionTimeout().toMillis());
  if (this.config.isTLSEnabled()) {
    config = (ClientConfiguration) config.setTLSProvider("OpenSSL");
    config = config.setTLSTrustStore(this.config.getTlsTrustStore());
    config.setTLSTrustStorePasswordPath(this.config.getTlsTrustStorePasswordPath());
  }
  String metadataServiceUri = "zk://" + this.config.getZkAddress();
  if (this.config.getBkLedgerPath().isEmpty()) {
    metadataServiceUri += "/" + this.namespace + "/bookkeeper/ledgers";
  } else {
    metadataServiceUri += this.config.getBkLedgerPath();
  }
  config.setMetadataServiceUri(metadataServiceUri);
  return new BookKeeper(config);
}

/**
 * Creates a new instance of the BookKeeper log class.
 *
 * @param containerId     The Id of the Container whose BookKeeperLog to open.
 * @param zkClient        A reference to the CuratorFramework client to use.
 * @param bookKeeper      A reference to the BookKeeper client to use.
 * @param config          Configuration to use.
 * @param executorService An Executor to use for async operations.
 */
BookKeeperLog(int containerId, CuratorFramework zkClient, BookKeeper bookKeeper, BookKeeperConfig config, ScheduledExecutorService executorService) {
  Preconditions.checkArgument(containerId >= 0, "containerId must be a non-negative integer.");
  this.zkClient = Preconditions.checkNotNull(zkClient, "zkClient");
  this.bookKeeper = Preconditions.checkNotNull(bookKeeper, "bookKeeper");
  this.config = Preconditions.checkNotNull(config, "config");
  this.executorService = Preconditions.checkNotNull(executorService, "executorService");
  this.closed = new AtomicBoolean();
  this.logNodePath = HierarchyUtils.getPath(containerId, this.config.getZkHierarchyDepth());
  this.traceObjectId = String.format("Log[%d]", containerId);
  this.writes = new WriteQueue();
  val retry = createRetryPolicy(this.config.getMaxWriteAttempts(), this.config.getBkWriteTimeoutMillis());
  this.writeProcessor = new SequentialAsyncProcessor(this::processWritesSync, retry, this::handleWriteProcessorFailures, this.executorService);
  this.rolloverProcessor = new SequentialAsyncProcessor(this::rollover, retry, this::handleRolloverFailure, this.executorService);
  this.metrics = new BookKeeperMetrics.BookKeeperLog(containerId);
  this.metricReporter = this.executorService.scheduleWithFixedDelay(this::reportMetrics, REPORT_INTERVAL, REPORT_INTERVAL, TimeUnit.MILLISECONDS);
}

  @Test
  public void testZkHierarchyDepth() {
    AssertExtensions.assertThrows("BookKeeperConfig did not throw InvalidPropertyValueException",
        () -> BookKeeperConfig.builder()
            .with(BookKeeperConfig.ZK_HIERARCHY_DEPTH, -1)
            .build(),
        ex -> ex instanceof InvalidPropertyValueException);
  }
}

@Test
public void testZkServers() {
  // When multiple servers are specified and separated by comma, it should replace it by semicolon
  BookKeeperConfig cfg1 = BookKeeperConfig.builder()
      .with(BookKeeperConfig.ZK_ADDRESS, "foo:12345,bar:54321")
      .build();
  Assert.assertEquals("foo:12345;bar:54321", cfg1.getZkAddress());
  // No changes should be made to the following configs
  BookKeeperConfig cfg2 = BookKeeperConfig.builder()
      .with(BookKeeperConfig.ZK_ADDRESS, "foo:12345")
      .build();
  Assert.assertEquals("foo:12345", cfg2.getZkAddress());
  BookKeeperConfig cfg3 = BookKeeperConfig.builder()
      .with(BookKeeperConfig.ZK_ADDRESS, "10.20.30.40:12345;bar:2181")
      .build();
  Assert.assertEquals("10.20.30.40:12345;bar:2181", cfg3.getZkAddress());
}

/**
 * Opens a ledger for reading. This operation does not fence out the ledger.
 *
 * @param ledgerId   The Id of the Ledger to open.
 * @param bookKeeper A references to the BookKeeper client to use.
 * @param config     Configuration to use.
 * @return A LedgerHandle for the newly opened ledger.
 * @throws DurableDataLogException If an exception occurred. The causing exception is wrapped inside it.
 */
static LedgerHandle openRead(long ledgerId, BookKeeper bookKeeper, BookKeeperConfig config) throws DurableDataLogException {
  try {
    return Exceptions.handleInterruptedCall(
        () -> bookKeeper.openLedgerNoRecovery(ledgerId, LEDGER_DIGEST_TYPE, config.getBKPassword()));
  } catch (BKException bkEx) {
    throw new DurableDataLogException(String.format("Unable to open-read ledger %d.", ledgerId), bkEx);
  }
}

/**
 * Collects an ordered list of Writes to execute to BookKeeper.
 *
 * @return The list of Writes to execute.
 */
private List<Write> getWritesToExecute() {
  // Calculate how much estimated space there is in the current ledger.
  final long maxTotalSize = this.config.getBkLedgerMaxSize() - getWriteLedger().ledger.getLength();
  // Get the writes to execute from the queue.
  List<Write> toExecute = this.writes.getWritesToExecute(maxTotalSize);
  // Check to see if any writes executed on closed ledgers, in which case they either need to be failed (if deemed
  // appropriate, or retried).
  if (handleClosedLedgers(toExecute)) {
    // If any changes were made to the Writes in the list, re-do the search to get a more accurate list of Writes
    // to execute (since some may have changed Ledgers, more writes may not be eligible for execution).
    toExecute = this.writes.getWritesToExecute(maxTotalSize);
  }
  return toExecute;
}

if (attemptCount > this.config.getMaxWriteAttempts()) {

.builder()
.with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + BK_PORT.get())
.with(BookKeeperConfig.MAX_WRITE_ATTEMPTS, MAX_WRITE_ATTEMPTS)

/**
 * Opens a ledger. This operation also fences out the ledger in case anyone else was writing to it.
 *
 * @param ledgerId   The Id of the Ledger to open.
 * @param bookKeeper A references to the BookKeeper client to use.
 * @param config     Configuration to use.
 * @return A LedgerHandle for the newly opened ledger.
 * @throws DurableDataLogException If an exception occurred. The causing exception is wrapped inside it.
 */
static LedgerHandle openFence(long ledgerId, BookKeeper bookKeeper, BookKeeperConfig config) throws DurableDataLogException {
  try {
    return Exceptions.handleInterruptedCall(
        () -> bookKeeper.openLedger(ledgerId, LEDGER_DIGEST_TYPE, config.getBKPassword()));
  } catch (BKException bkEx) {
    throw new DurableDataLogException(String.format("Unable to open-fence ledger %d.", ledgerId), bkEx);
  }
}

if (!l.isClosed() && l.getLength() < this.config.getBkLedgerMaxSize()) {

@Test
public void testQuorumSize() {
  AssertExtensions.assertThrows("BookKeeperConfig did not throw InvalidPropertyValueException",
      () -> BookKeeperConfig.builder()
          .with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, 3)
          .with(BookKeeperConfig.BK_WRITE_QUORUM_SIZE, 2)
          .build(),
      ex -> ex instanceof InvalidPropertyValueException);
}

/**
 * Tests the BookKeeperLogFactory and its initialization.
 */
@Test
public void testFactoryInitialize() {
  BookKeeperConfig bkConfig = BookKeeperConfig
      .builder()
      .with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + BK_PORT.get())
      .with(BookKeeperConfig.BK_LEDGER_MAX_SIZE, WRITE_MAX_LENGTH * 10) // Very frequent rollovers.
      .with(BookKeeperConfig.ZK_METADATA_PATH, this.zkClient.get().getNamespace())
      .build();
  @Cleanup
  val factory = new BookKeeperLogFactory(bkConfig, this.zkClient.get(), executorService());
  AssertExtensions.assertThrows("",
      factory::initialize,
      ex -> ex instanceof DataLogNotAvailableException &&
          ex.getCause() instanceof BKException.ZKException
  );
}

b.include(ServiceConfig.builder()
            .with(ServiceConfig.CONTAINER_COUNT, testConfig.getContainerCount()));
b.include(BookKeeperConfig.builder()
             .with(BookKeeperConfig.ZK_ADDRESS, TestConfig.LOCALHOST + ":" + testConfig.getZkPort())
             .with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, bkWriteQuorum)

.builder()
.with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + zkPort)
.with(BookKeeperConfig.ZK_METADATA_PATH, logMetaNamespace)

/**
 * Creates a new Context to be used by the BookKeeper command.
 *
 * @return A new Context.
 * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized.
 */
protected Context createContext() throws DurableDataLogException {
  val serviceConfig = getServiceConfig();
  val bkConfig = getCommandArgs().getState().getConfigBuilder()
                  .include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL()))
                  .build().getConfig(BookKeeperConfig::builder);
  val zkClient = createZKClient();
  val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor());
  try {
    factory.initialize();
  } catch (DurableDataLogException ex) {
    zkClient.close();
    throw ex;
  }
  val bkAdmin = new BookKeeperAdmin(factory.getBookKeeperClient());
  return new Context(serviceConfig, bkConfig, zkClient, factory, bkAdmin);
}

/**
 * Creates a new Context to be used by the BookKeeper command.
 *
 * @return A new Context.
 * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized.
 */
@Override
protected Context createContext() throws DurableDataLogException {
  val serviceConfig = getServiceConfig();
  val containerConfig = getCommandArgs().getState().getConfigBuilder().build().getConfig(ContainerConfig::builder);
  val bkConfig = getCommandArgs().getState().getConfigBuilder()
                  .include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL()))
                  .build().getConfig(BookKeeperConfig::builder);
  val zkClient = createZKClient();
  val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor());
  try {
    factory.initialize();
  } catch (DurableDataLogException ex) {
    zkClient.close();
    throw ex;
  }
  val bkAdmin = new BookKeeperAdmin(factory.getBookKeeperClient());
  return new Context(serviceConfig, containerConfig, bkConfig, zkClient, factory, bkAdmin);
}

/**
 * Generates a new ServiceBuilderConfig.Builder with hardcoded defaults, in case these are not supplied via other means,
 * such as a config file or System Properties.
 */
private static ServiceBuilderConfig.Builder getDefaultServiceBuilderConfig() {
  return ServiceBuilderConfig
      .builder()
      .include(ServiceConfig.builder()
                 .with(ServiceConfig.THREAD_POOL_SIZE, 80)
                 .with(ServiceConfig.CACHE_POLICY_MAX_TIME, 600)
                 .with(ServiceConfig.CACHE_POLICY_MAX_SIZE, 4 * 1024 * 1024 * 1024L)
                 .with(ServiceConfig.CERT_FILE, "../config/cert.pem")
                 .with(ServiceConfig.KEY_FILE, "../config/key.pem"))
      .include(DurableLogConfig.builder()
                   .with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 100)
                   .with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 100)
                   .with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 100 * 1024 * 1024L))
      .include(ReadIndexConfig.builder()
                  .with(ReadIndexConfig.MEMORY_READ_MIN_LENGTH, 128 * 1024))
      .include(ContainerConfig.builder()
                  .with(ContainerConfig.SEGMENT_METADATA_EXPIRATION_SECONDS,
                      ContainerConfig.MINIMUM_SEGMENT_METADATA_EXPIRATION_SECONDS)
                  .with(ContainerConfig.MAX_ACTIVE_SEGMENT_COUNT, 500))
      // This is for those tests that use BookKeeper for Tier1.
      .include(BookKeeperConfig.builder()
                   .with(BookKeeperConfig.BK_LEDGER_MAX_SIZE, Integer.MAX_VALUE)
                   .with(BookKeeperConfig.ZK_METADATA_PATH, "/pravega/selftest/segmentstore/containers")
                   .with(BookKeeperConfig.BK_LEDGER_PATH, TestConfig.BK_LEDGER_PATH));
}