@Override protected void startUp() throws Exception { // Start BookKeeper. this.bookKeeperService = BookKeeperAdapter.startBookKeeperOutOfProcess(this.testConfig, this.logId); // Create a ZK client. this.zkClient = CuratorFrameworkFactory .builder() .connectString("localhost:" + this.testConfig.getZkPort()) .namespace("pravega") .retryPolicy(new ExponentialBackoffRetry(1000, 5)) .sessionTimeoutMs(5000) .connectionTimeoutMs(5000) .build(); this.zkClient.start(); // Create a BK client. this.logFactory = new BookKeeperLogFactory(this.bkConfig, this.zkClient, this.executor); this.logFactory.initialize(); }
@Override public void initialize() throws DurableDataLogException { Preconditions.checkState(this.bookKeeper.get() == null, "BookKeeperLogFactory is already initialized."); try { this.bookKeeper.set(startBookKeeperClient()); } catch (IllegalArgumentException | NullPointerException ex) { // Most likely a configuration issue; re-throw as is. close(); throw ex; } catch (Throwable ex) { if (!Exceptions.mustRethrow(ex)) { // Make sure we close anything we may have opened. close(); } // ZooKeeper not reachable, some other environment issue. throw new DataLogNotAvailableException("Unable to establish connection to ZooKeeper or BookKeeper.", ex); } }
@Override protected DurableDataLog createDurableDataLog() { return this.factory.get().createDurableDataLog(CONTAINER_ID); }
/** * Creates a new Context to be used by the BookKeeper command. * * @return A new Context. * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized. */ protected Context createContext() throws DurableDataLogException { val serviceConfig = getServiceConfig(); val bkConfig = getCommandArgs().getState().getConfigBuilder() .include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL())) .build().getConfig(BookKeeperConfig::builder); val zkClient = createZKClient(); val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor()); try { factory.initialize(); } catch (DurableDataLogException ex) { zkClient.close(); throw ex; } val bkAdmin = new BookKeeperAdmin(factory.getBookKeeperClient()); return new Context(serviceConfig, bkConfig, zkClient, factory, bkAdmin); }
private ServiceBuilder attachDataLogFactory(ServiceBuilder builder) { if (this.config.getBookieCount() > 0) { // We were instructed to start at least one Bookie. this.zkClient = CuratorFrameworkFactory .builder() .connectString("localhost:" + this.config.getZkPort()) .namespace("pravega") .retryPolicy(new ExponentialBackoffRetry(1000, 5)) .sessionTimeoutMs(5000) .connectionTimeoutMs(5000) .build(); this.zkClient.start(); return builder.withDataLogFactory(setup -> { BookKeeperConfig bkConfig = setup.getConfig(BookKeeperConfig::builder); return new BookKeeperLogFactory(bkConfig, this.zkClient, setup.getCoreExecutor()); }); } else { // No Bookies -> InMemory Tier1. return builder.withDataLogFactory(setup -> new InMemoryDurableDataLogFactory(setup.getCoreExecutor())); } }
@Override @SneakyThrows(BKException.class) public void close() { this.logFactory.close(); this.zkClient.close(); // There is no need to close the BK Admin object since it doesn't own anything; however it does have a close() // method and it's a good idea to invoke it. Exceptions.handleInterrupted(this.bkAdmin::close); } }
private void deleteCandidates(List<Long> deletionCandidates, Collection<Long> referencedLedgerIds, Context context) { for (long ledgerId : deletionCandidates) { if (referencedLedgerIds.contains(ledgerId)) { output("Not deleting Ledger %d because is is now referenced.", ledgerId); continue; } try { Exceptions.handleInterrupted(() -> context.logFactory.getBookKeeperClient().deleteLedger(ledgerId)); output("Deleted Ledger %d.", ledgerId); } catch (Exception ex) { output("FAILED to delete Ledger %d: %s.", ledgerId, ex.getMessage()); } } }
private void checkForExtraLogs(Context context) throws Exception { val maxLogId = context.serviceConfig.getContainerCount() * 10; for (int logId = context.serviceConfig.getContainerCount(); logId < maxLogId; logId++) { @Cleanup DebugLogWrapper log = context.logFactory.createDebugLogWrapper(logId); val m = log.fetchMetadata(); if (m != null) { throw new Exception(String.format("Discovered BookKeeperLog %d which is beyond the maximum log id (%d) as specified in the configuration.", logId, context.serviceConfig.getContainerCount() - 1)); } } }
/** * Creates a new Context to be used by the BookKeeper command. * * @return A new Context. * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized. */ @Override protected Context createContext() throws DurableDataLogException { val serviceConfig = getServiceConfig(); val containerConfig = getCommandArgs().getState().getConfigBuilder().build().getConfig(ContainerConfig::builder); val bkConfig = getCommandArgs().getState().getConfigBuilder() .include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL())) .build().getConfig(BookKeeperConfig::builder); val zkClient = createZKClient(); val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor()); try { factory.initialize(); } catch (DurableDataLogException ex) { zkClient.close(); throw ex; } val bkAdmin = new BookKeeperAdmin(factory.getBookKeeperClient()); return new Context(serviceConfig, containerConfig, bkConfig, zkClient, factory, bkAdmin); }
private void attachDataLogFactory(ServiceBuilder builder) { builder.withDataLogFactory(setup -> { switch (this.serviceConfig.getDataLogTypeImplementation()) { case BOOKKEEPER: return new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), this.zkClient, setup.getCoreExecutor()); case INMEMORY: return new InMemoryDurableDataLogFactory(setup.getCoreExecutor()); default: throw new IllegalStateException("Unsupported storage implementation: " + this.serviceConfig.getDataLogTypeImplementation()); } }); }
@Override protected void shutDown() { this.logs.values().forEach(DurableDataLog::close); this.logs.clear(); BookKeeperLogFactory lf = this.logFactory; if (lf != null) { lf.close(); this.logFactory = null; } stopBookKeeper(); CuratorFramework zkClient = this.zkClient; if (zkClient != null) { zkClient.close(); this.zkClient = null; } Runtime.getRuntime().removeShutdownHook(this.stopBookKeeperProcess); }
val bkAdmin = new BookKeeperAdmin(context.logFactory.getBookKeeperClient()); val allLedgerIds = new ArrayList<Long>(); bkAdmin.listLedgers().forEach(allLedgerIds::add);
private void collectAllReferencedLedgerIds(Collection<Long> referencedLedgerIds, Context context) throws Exception { referencedLedgerIds.clear(); for (int logId = 0; logId < context.serviceConfig.getContainerCount(); logId++) { @Cleanup DebugLogWrapper log = context.logFactory.createDebugLogWrapper(logId); val m = log.fetchMetadata(); if (m == null) { continue; } for (val lm : m.getLedgers()) { referencedLedgerIds.add(lm.getLedgerId()); } } }
val factory = new BookKeeperLogFactory(this.config.get(), this.zkClient.get(), executorService()); factory.initialize(); this.factory.set(factory);
/** * Tests the BookKeeperLogFactory and its initialization. */ @Test public void testFactoryInitialize() { BookKeeperConfig bkConfig = BookKeeperConfig .builder() .with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + BK_PORT.get()) .with(BookKeeperConfig.BK_LEDGER_MAX_SIZE, WRITE_MAX_LENGTH * 10) // Very frequent rollovers. .with(BookKeeperConfig.ZK_METADATA_PATH, this.zkClient.get().getNamespace()) .build(); @Cleanup val factory = new BookKeeperLogFactory(bkConfig, this.zkClient.get(), executorService()); AssertExtensions.assertThrows("", factory::initialize, ex -> ex instanceof DataLogNotAvailableException && ex.getCause() instanceof BKException.ZKException ); }
@After public void tearDown() { val factory = this.factory.getAndSet(null); if (factory != null) { factory.close(); } val zkClient = this.zkClient.getAndSet(null); if (zkClient != null) { zkClient.close(); } }
if (shouldExist.test(i)) { Ledgers.openFence(e.getKey(), this.factory.get().getBookKeeperClient(), this.config.get()); } else { AssertExtensions.assertThrows( "Ledger not deleted from BookKeeper.", () -> Ledgers.openFence(e.getKey(), this.factory.get().getBookKeeperClient(), this.config.get()), ex -> true);
boolean success = false; try { log = this.logFactory.createDurableDataLog(id); this.logs.put(logName, log); log.initialize(timeout);
@Override public void execute() throws Exception { ensureArgCount(0); // Loop through all known log ids and fetch their metadata. @Cleanup val context = createContext(); for (int logId = 0; logId < context.serviceConfig.getContainerCount(); logId++) { @Cleanup DebugLogWrapper log = context.logFactory.createDebugLogWrapper(logId); val m = log.fetchMetadata(); outputLogSummary(logId, m); } }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder configBuilder, int instanceId) { ServiceBuilderConfig builderConfig = getBuilderConfig(configBuilder, instanceId); return ServiceBuilder .newInMemoryBuilder(builderConfig) .withCacheFactory(setup -> new RocksDBCacheFactory(builderConfig.getConfig(RocksDBConfig::builder))) .withStorageFactory(setup -> new HDFSStorageFactory(setup.getConfig(HDFSStorageConfig::builder), setup.getStorageExecutor())) .withDataLogFactory(setup -> new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), getBookkeeper().getZkClient(), setup.getCoreExecutor())); }