log.debug("[{}] Deleting ledger {}", name, ls); bookKeeper.asyncDeleteLedger(ls.getLedgerId(), (rc, ctx1) -> { switch (rc) { case BKException.Code.NoSuchLedgerExistsException: log.warn("[{}] Ledger {} not found when deleting it", name, ls.getLedgerId()); log.warn("[{}] Failed to delete ledger {} -- {}", name, ls.getLedgerId(), BKException.getMessage(rc)); int toDelete = ledgersToDelete.get();
long estimateBacklogFromPosition(PositionImpl pos) { synchronized (this) { LedgerInfo ledgerInfo = ledgers.get(pos.getLedgerId()); if (ledgerInfo == null) { return getTotalSize(); // position no longer in managed ledger, so return total size } long sizeBeforePosLedger = ledgers.values().stream().filter(li -> li.getLedgerId() < pos.getLedgerId()) .mapToLong(li -> li.getSize()).sum(); long size = getTotalSize() - sizeBeforePosLedger; if (pos.getLedgerId() == currentLedger.getId()) { return size - consumedLedgerSize(currentLedgerSize, currentLedgerEntries, pos.getEntryId()); } else { return size - consumedLedgerSize(ledgerInfo.getSize(), ledgerInfo.getEntries(), pos.getEntryId()); } } }
public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, TopicName topicName) throws Exception { String managedLedgerName = topicName.getPersistenceNamingEncoding(); long numberOfEntries = 0; long totalSize = 0; final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers = new ConcurrentSkipListMap<>(); final PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(managedLedgerName, brokerName); // calculate total managed ledger size and number of entries without loading the topic readLedgerMeta(factory, topicName, ledgers); for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) { numberOfEntries += ls.getEntries(); totalSize += ls.getSize(); if (accurate) { offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId()); } } offlineTopicStats.totalMessages = numberOfEntries; offlineTopicStats.storageSize = totalSize; if (log.isDebugEnabled()) { log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize); } // calculate per cursor message backlog calculateCursorBacklogs(factory, topicName, ledgers, offlineTopicStats); offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis()); return offlineTopicStats; }
size -= ledgers.values().stream().filter(li -> li.getLedgerId() < slowestConsumerLedgerId) .mapToLong(li -> li.getSize()).sum();
PositionImpl.get(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0)); ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(), ledger.getLedgersInfoAsList().get(1).getLedgerId()));
result = result && (hasLedgerId() == other.hasLedgerId()); if (hasLedgerId()) { result = result && (getLedgerId() == other.getLedgerId());
long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId(); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); Assert.assertTrue(bkc.getLedgers().contains(firstLedgerId));
long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId(); long secondLedgerId = ledger.getLedgersInfoAsList().get(1).getLedgerId(); long thirdLedgerId = ledger.getLedgersInfoAsList().get(2).getLedgerId(); long fourthLedgerId = ledger.getLedgersInfoAsList().get(3).getLedgerId(); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(firstLedgerId, thirdLedgerId));
ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(), ledger.getLedgersInfoAsList().get(1).getLedgerId(), ledger.getLedgersInfoAsList().get(2).getLedgerId()));
hash = (37 * hash) + LEDGERID_FIELD_NUMBER; hash = (53 * hash) + com.google.protobuf.Internal.hashLong( getLedgerId());
promise.complete(ImmutableSet.of(ledger.getLedgersInfoAsList().get(failIndex).getLedgerId())); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
@Test public void testOffloadDelete() throws Exception { Set<Pair<Long, UUID>> deleted = ConcurrentHashMap.newKeySet(); CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>(); Set<Pair<Long, UUID>> failedOffloads = ConcurrentHashMap.newKeySet(); MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(0, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); ManagedCursor cursor = ledger.openCursor("foobar"); for (int i = 0; i < 15; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 1); Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete()); long firstLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId(); long secondLedger = ledger.getLedgersInfoAsList().get(1).getLedgerId(); cursor.markDelete(ledger.getLastConfirmedEntry()); assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 1); Assert.assertEquals(ledger.getLedgersInfoAsList().get(0).getLedgerId(), secondLedger); assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedger)); }
/** * Get the entry position that come before the specified position in the message stream, using information from the * ledger list and each ledger entries count. * * @param position * the current position * @return the previous position */ PositionImpl getPreviousPosition(PositionImpl position) { if (position.getEntryId() > 0) { return PositionImpl.get(position.getLedgerId(), position.getEntryId() - 1); } // The previous position will be the last position of an earlier ledgers NavigableMap<Long, LedgerInfo> headMap = ledgers.headMap(position.getLedgerId(), false); if (headMap.isEmpty()) { // There is no previous ledger, return an invalid position in the current ledger return PositionImpl.get(position.getLedgerId(), -1); } // We need to find the most recent non-empty ledger for (long ledgerId : headMap.descendingKeySet()) { LedgerInfo li = headMap.get(ledgerId); if (li.getEntries() > 0) { return PositionImpl.get(li.getLedgerId(), li.getEntries() - 1); } } // in case there are only empty ledgers, we return a position in the first one return PositionImpl.get(headMap.firstEntry().getKey(), -1); }
@Test public void testOffloadSamePositionTwice() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
@Test public void testOffload() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
@Test public void offloadAsSoonAsClosed() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setOffloadAutoTriggerSizeThresholdBytes(0); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); for (int i = 0; i < 11; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId())); for (int i = 0; i < 10; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(), ledger.getLedgersInfoAsList().get(1).getLedgerId())); }
public Builder mergeFrom(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo other) { if (other == org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo.getDefaultInstance()) return this; if (other.hasLedgerId()) { setLedgerId(other.getLedgerId()); } if (other.hasEntries()) { setEntries(other.getEntries()); } if (other.hasSize()) { setSize(other.getSize()); } if (other.hasTimestamp()) { setTimestamp(other.getTimestamp()); } if (other.hasOffloadContext()) { mergeOffloadContext(other.getOffloadContext()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }
@Test public void testAutoTriggerOffload() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setOffloadAutoTriggerSizeThresholdBytes(100); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); // Ledger will roll twice, offload will run on first ledger after second closed for (int i = 0; i < 25; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); // offload should eventually be triggered assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId())); }
@Override public void operationComplete(ManagedLedgerInfo result, Stat version) { // Update the list ManagedLedgerInfo.Builder info = ManagedLedgerInfo.newBuilder(result); info.clearLedgerInfo(); info.addLedgerInfo(LedgerInfo.newBuilder().setLedgerId(l1info.getLedgerId()).build()); info.addLedgerInfo(l2info); store.asyncUpdateLedgerIds("my_test_ledger", info.build(), version, new MetaStoreCallback<Void>() { @Override public void operationComplete(Void result, Stat version) { counter.countDown(); } @Override public void operationFailed(MetaStoreException e) { counter.countDown(); } }); }
private ReadOnlyCursor createReadOnlyCursor(PositionImpl startPosition) { if (ledgers.isEmpty()) { lastConfirmedEntry = PositionImpl.earliest; } else if (ledgers.lastEntry().getValue().getEntries() > 0) { // Last ledger has some of the entries lastConfirmedEntry = new PositionImpl(ledgers.lastKey(), ledgers.lastEntry().getValue().getEntries() - 1); } else { // Last ledger is empty. If there is a previous ledger, position on the last entry of that ledger if (ledgers.size() > 1) { long lastLedgerId = ledgers.lastKey(); LedgerInfo li = ledgers.headMap(lastLedgerId, false).lastEntry().getValue(); lastConfirmedEntry = new PositionImpl(li.getLedgerId(), li.getEntries() - 1); } else { lastConfirmedEntry = PositionImpl.earliest; } } ReadOnlyCursorImpl cursor = new ReadOnlyCursorImpl(bookKeeper, config, this, startPosition, "read-only-cursor"); return cursor; }