public PersistentOfflineTopicStats getEstimatedUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, String managedLedgerName) throws Exception { return estimateUnloadedTopicBacklog(factory, TopicName.get("persistent://" + managedLedgerName)); }
@Test public void testOfflineTopicBacklog() throws Exception { ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); factoryConf.setMaxCacheSize(0); ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1) .setMetadataAckQuorumSize(1); ManagedLedger ledger = factory.open("property/cluster/namespace/my-ledger", config); ManagedCursor cursor = ledger.openCursor("c1"); int N = 1; for (int i = 0; i < N; i++) { String entry = "entry-" + i; ledger.addEntry(entry.getBytes()); } List<Entry> entries = cursor.readEntries(N); assertEquals(N, entries.size()); entries.forEach(e -> e.release()); ledger.close(); ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog( DigestType.CRC32, "".getBytes(Charsets.UTF_8), "", false); PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.getEstimatedUnloadedTopicBacklog( (ManagedLedgerFactoryImpl) factory, "property/cluster/namespace/my-ledger"); factory.shutdown(); assertNotNull(offlineTopicStats); }
public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, TopicName topicName) throws Exception { String managedLedgerName = topicName.getPersistenceNamingEncoding(); long numberOfEntries = 0; long totalSize = 0; final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers = new ConcurrentSkipListMap<>(); final PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(managedLedgerName, brokerName); // calculate total managed ledger size and number of entries without loading the topic readLedgerMeta(factory, topicName, ledgers); for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { numberOfEntries += ls.getEntries(); totalSize += ls.getSize(); if (accurate) { offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); } } offlineTopicStats.totalMessages = numberOfEntries; offlineTopicStats.storageSize = totalSize; if (log.isDebugEnabled()) { log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); } // calculate per cursor message backlog calculateCursorBacklogs(factory, topicName, ledgers, offlineTopicStats); offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); return offlineTopicStats; }
log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); if (lastAckedMessagePosition == null) { log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", cursorName, range); long cursorBacklog = getNumberOfEntries(range, ledgers); offlineTopicStats.messageBacklog += cursorBacklog; offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(config.getDigestType(), config.getPassword(), pulsar().getAdvertisedAddress(), false); offlineTopicStats = offlineTopicBacklog .estimateUnloadedTopicBacklog((ManagedLedgerFactoryImpl) pulsar().getManagedLedgerFactory(), dn); pulsar().getBrokerService().cacheOfflineTopicStats(dn, offlineTopicStats); } catch (Exception exception) {
ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(config.getDigestType(), config.getPassword(), pulsar().getAdvertisedAddress(), false); offlineTopicStats = offlineTopicBacklog.estimateUnloadedTopicBacklog( (ManagedLedgerFactoryImpl) pulsar().getManagedLedgerFactory(), topicName); pulsar().getBrokerService().cacheOfflineTopicStats(topicName, offlineTopicStats);
log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); if (lastAckedMessagePosition == null) { log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", cursorName, range); long cursorBacklog = getNumberOfEntries(range, ledgers); offlineTopicStats.messageBacklog += cursorBacklog; offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
public PersistentOfflineTopicStats getEstimatedUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, String managedLedgerName) throws Exception { return estimateUnloadedTopicBacklog(factory, TopicName.get("persistent://" + managedLedgerName)); }
public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, DestinationName dn) throws Exception { String managedLedgerName = dn.getPersistenceNamingEncoding(); long numberOfEntries = 0; long totalSize = 0; final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers = new ConcurrentSkipListMap<>(); final PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(managedLedgerName, brokerName); // calculate total managed ledger size and number of entries without loading the topic readLedgerMeta(factory, dn, ledgers); for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { numberOfEntries += ls.getEntries(); totalSize += ls.getSize(); if (accurate) { offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); } } offlineTopicStats.totalMessages = numberOfEntries; offlineTopicStats.storageSize = totalSize; if (log.isDebugEnabled()) { log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); } // calculate per cursor message backlog calculateCursorBacklogs(factory, dn, ledgers, offlineTopicStats); offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); return offlineTopicStats; }
log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId); PositionImpl lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName); if (lastAckedMessagePosition == null) { log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position", cursorName, range); long cursorBacklog = getNumberOfEntries(range, ledgers); offlineTopicStats.messageBacklog += cursorBacklog; offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
public PersistentOfflineTopicStats getEstimatedUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, String managedLedgerName) throws Exception { return estimateUnloadedTopicBacklog(factory, DestinationName.get("persistent://" + managedLedgerName)); }
public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, TopicName topicName) throws Exception { String managedLedgerName = topicName.getPersistenceNamingEncoding(); long numberOfEntries = 0; long totalSize = 0; final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers = new ConcurrentSkipListMap<>(); final PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(managedLedgerName, brokerName); // calculate total managed ledger size and number of entries without loading the topic readLedgerMeta(factory, topicName, ledgers); for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { numberOfEntries += ls.getEntries(); totalSize += ls.getSize(); if (accurate) { offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); } } offlineTopicStats.totalMessages = numberOfEntries; offlineTopicStats.storageSize = totalSize; if (log.isDebugEnabled()) { log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); } // calculate per cursor message backlog calculateCursorBacklogs(factory, topicName, ledgers, offlineTopicStats); offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); return offlineTopicStats; }