/** * Returns a LogMetadata class with the exact contents of this instance, but the enabled flag set to false. No changes * are performed on this instance. * * @return This instance, if isEnabled() == false, of a new instance of the LogMetadata class which will have * isEnabled() == false, otherwise. */ LogMetadata asDisabled() { return this.enabled ? new LogMetadata(this.epoch, false, this.ledgers, this.truncationAddress, this.updateVersion.get()) : this; }
@Override public void enable() throws DurableDataLogException { Exceptions.checkNotClosed(this.closed.get(), this); synchronized (this.lock) { Preconditions.checkState(this.writeLedger == null, "BookKeeperLog is already initialized; cannot re-enable."); assert this.logMetadata == null : "writeLedger == null but logMetadata != null"; // Load existing metadata. Inexistent metadata means the BookKeeperLog has never been accessed, and therefore // enabled by default. LogMetadata metadata = loadMetadata(); Preconditions.checkState(metadata != null && !metadata.isEnabled(), "BookKeeperLog is already enabled."); metadata = metadata.asEnabled(); persistMetadata(metadata, false); log.info("{}: Enabled (Epoch = {}, UpdateVersion = {}).", this.traceObjectId, metadata.getEpoch(), metadata.getUpdateVersion()); } }
/** * Determines which Ledger Ids are safe to delete from BookKeeper. * * @param oldMetadata A pointer to the previous version of the metadata, that contains all Ledgers eligible for * deletion. Only those Ledgers that do not exist in currentMetadata will be selected. * @param currentMetadata A pointer to the current version of the metadata. No Ledger that is referenced here will * be selected. * @return A List that contains Ledger Ids to remove. May be empty. */ @GuardedBy("lock") private List<Long> getLedgerIdsToDelete(LogMetadata oldMetadata, LogMetadata currentMetadata) { if (oldMetadata == null) { return Collections.emptyList(); } val existingIds = currentMetadata.getLedgers().stream() .map(LedgerMetadata::getLedgerId) .collect(Collectors.toSet()); return oldMetadata.getLedgers().stream() .map(LedgerMetadata::getLedgerId) .filter(id -> !existingIds.contains(id)) .collect(Collectors.toList()); }
@Override public void disable() throws DurableDataLogException { // Get the current metadata, disable it, and then persist it back. synchronized (this.lock) { ensurePreconditions(); LogMetadata metadata = getLogMetadata(); Preconditions.checkState(metadata.isEnabled(), "BookKeeperLog is already disabled."); metadata = this.logMetadata.asDisabled(); persistMetadata(metadata, false); this.logMetadata = metadata; log.info("{}: Disabled (Epoch = {}, UpdateVersion = {}).", this.traceObjectId, metadata.getEpoch(), metadata.getUpdateVersion()); } // Close this instance of the BookKeeperLog. This ensures the proper cancellation of any ongoing writes. close(); }
expectedLedgerIds.add(ledgerId); if (metadata == null) { metadata = new LogMetadata(ledgerId).withUpdateVersion(i); } else { metadata.withUpdateVersion(i); metadata = metadata.addLedger(ledgerId); Assert.assertEquals("Unexpected epoch.", expectedEpoch, metadata.getEpoch()); Assert.assertEquals("Unexpected update version.", i, metadata.getUpdateVersion()); checkLedgerIds(expectedLedgerIds, metadata); for (int i = 1; i < metadata.getLedgers().size(); i++) { Assert.assertEquals("Sequence is not incremented.", metadata.getLedgers().get(i - 1).getSequence() + 1, metadata.getLedgers().get(i).getSequence()); val lm = metadata.getLedger(ledgerId); if (expectedLedgerIds.contains(ledgerId)) { Assert.assertNotNull("Existing LedgerMetadata was not found.", lm); val truncatedMetadata = metadata.truncate(new LedgerAddress(0, ledgerId, 123)); .filter(i -> i % 3 > 0) .collect(Collectors.toMap(i -> i, i -> i % 3 == 1 ? Ledgers.NO_ENTRY_ID : 1000)); val m = metadata.updateLedgerStatus(lacs); for (long ledgerId : expectedLedgerIds) { Assert.assertEquals("markEmptyLedgers modified base metadata", LedgerMetadata.Status.Unknown, metadata.getLedger(ledgerId).getStatus()); long lac = lacs.getOrDefault(ledgerId, Long.MIN_VALUE);
/** * Tests serialization/deserialization. */ @Test(timeout = 5000) public void testSerialization() throws Exception { Supplier<Long> nextLedgerId = new AtomicLong()::incrementAndGet; LogMetadata m1 = null; val lacs = new HashMap<Long, Long>(); for (int i = 0; i < LEDGER_COUNT; i++) { long ledgerId = nextLedgerId.get() * 2; if (m1 == null) { m1 = new LogMetadata(ledgerId).withUpdateVersion(i); } else { m1 = m1.addLedger(ledgerId).withUpdateVersion(i); } if (i % 2 == 0) { // Every other Ledger, update the LastAddConfirmed. lacs.put((long) i, (long) i + 1); } } m1 = m1.updateLedgerStatus(lacs); val serialization = LogMetadata.SERIALIZER.serialize(m1); val m2 = LogMetadata.SERIALIZER.deserialize(serialization); Assert.assertEquals("Unexpected epoch.", m1.getEpoch(), m2.getEpoch()); Assert.assertEquals("Unexpected TruncationAddress.", m1.getTruncationAddress().getSequence(), m2.getTruncationAddress().getSequence()); Assert.assertEquals("Unexpected TruncationAddress.", m1.getTruncationAddress().getLedgerId(), m2.getTruncationAddress().getLedgerId()); AssertExtensions.assertListEquals("Unexpected ledgers.", m1.getLedgers(), m2.getLedgers(), (l1, l2) -> l1.getSequence() == l2.getSequence() && l1.getLedgerId() == l2.getLedgerId() && l1.getStatus() == l2.getStatus()); }
if (!oldMetadata.isEnabled()) { throw new DataLogDisabledException("BookKeeperLog is disabled. Cannot initialize."); val emptyLedgerIds = Ledgers.fenceOut(oldMetadata.getLedgers(), this.bookKeeper, this.config, this.traceObjectId); oldMetadata = oldMetadata.updateLedgerStatus(emptyLedgerIds); LedgerMetadata ledgerMetadata = newMetadata.getLedger(newLedger.getId()); assert ledgerMetadata != null : "cannot find newly added ledger metadata"; this.writeLedger = new WriteLedger(newLedger, ledgerMetadata); log.info("{}: Initialized (Epoch = {}, UpdateVersion = {}).", this.traceObjectId, newMetadata.getEpoch(), newMetadata.getUpdateVersion());
val empty = new LogMetadata(1).truncate(new LedgerAddress(2, 2, 2)); Assert.assertNull("Unexpected result from empty metadata", empty.getNextAddress(new LedgerAddress(1, 1, 0), Long.MAX_VALUE)); long ledgerId = nextLedgerId.get() * 2; if (m == null) { m = new LogMetadata(ledgerId).withUpdateVersion(i); } else { m.withUpdateVersion(i); m = m.addLedger(ledgerId); val firstLedgerId = m.getLedgers().get(0).getLedgerId(); val firstLedgerSeq = m.getLedgers().get(0).getSequence(); LedgerAddress a = m.getNextAddress(new LedgerAddress(firstLedgerSeq - 1, firstLedgerId - 1, 1234), Long.MAX_VALUE); Assert.assertEquals("Unexpected ledger id when input address less than first ledger.", firstLedgerId, a.getLedgerId()); Assert.assertEquals("Unexpected entry id when input address less than first ledger.", 0, a.getEntryId()); val secondLedgerId = m.getLedgers().get(1).getLedgerId(); val secondLedgerSeq = m.getLedgers().get(1).getSequence(); LedgerAddress truncationAddress = new LedgerAddress(secondLedgerSeq, secondLedgerId, 1); val m2 = m.truncate(truncationAddress); a = m2.getNextAddress(new LedgerAddress(firstLedgerSeq, firstLedgerId, 0), Long.MAX_VALUE); Assert.assertEquals("Unexpected result when input address less than truncation address.", 0, truncationAddress.compareTo(a)); a = m.getNextAddress(new LedgerAddress(firstLedgerSeq, firstLedgerId, 0), 2); Assert.assertEquals("Unexpected ledger id when result should be in the same ledger.", firstLedgerId, a.getLedgerId()); Assert.assertEquals("Unexpected entry id when result should be in the same ledger.", 1, a.getEntryId()); for (int i = 0; i < m.getLedgers().size(); i++) {
if (create) { currentMetadata = new LogMetadata(newLedger.getId()); } else { currentMetadata = currentMetadata.addLedger(newLedger.getId()); if (clearEmptyLedgers) { currentMetadata = currentMetadata.removeEmptyLedgers(Ledgers.MIN_FENCE_LEDGER_COUNT);
val newMetadata = oldMetadata.truncate(upToAddress); val ledgerIdsToKeep = newMetadata.getLedgers().stream().map(LedgerMetadata::getLedgerId).collect(Collectors.toSet()); val ledgersToDelete = oldMetadata.getLedgers().stream().filter(lm -> !ledgerIdsToKeep.contains(lm.getLedgerId())).iterator(); while (ledgersToDelete.hasNext()) { val lm = ledgersToDelete.next();
.forPath(this.logNodePath, serializedMetadata); metadata.withUpdateVersion(0); } else { this.zkClient.setData() .withVersion(metadata.getUpdateVersion()) .forPath(this.logNodePath, serializedMetadata); metadata.withUpdateVersion(metadata.getUpdateVersion() + 1);
@Override public DurableDataLog.ReadItem getNext() throws DurableDataLogException { Exceptions.checkNotClosed(this.closed.get(), this); if (this.currentLedger == null) { // First time we call this. Locate the first ledger based on the metadata truncation address. We don't know // how many entries are in that first ledger, so open it anyway so we can figure out. openNextLedger(this.metadata.getNextAddress(this.metadata.getTruncationAddress(), Long.MAX_VALUE)); } while (this.currentLedger != null && (!this.currentLedger.canRead())) { // We have reached the end of the current ledger. Find next one, and skip over empty ledgers). val lastAddress = new LedgerAddress(this.currentLedger.metadata, this.currentLedger.handle.getLastAddConfirmed()); Ledgers.close(this.currentLedger.handle); openNextLedger(this.metadata.getNextAddress(lastAddress, this.currentLedger.handle.getLastAddConfirmed())); } // Try to read from the current reader. if (this.currentLedger == null || this.currentLedger.reader == null) { return null; } return new LogReader.ReadItem(this.currentLedger.reader.nextElement(), this.currentLedger.metadata); }
LedgerMetadata metadata = this.metadata.getLedger(address.getLedgerId()); assert metadata != null : "no LedgerMetadata could be found with valid LedgerAddress " + address; val allMetadatas = this.metadata.getLedgers();
private void write00(LogMetadata m, RevisionDataOutput output) throws IOException { output.writeBoolean(m.isEnabled()); output.writeCompactLong(m.getEpoch()); output.writeCompactLong(m.truncationAddress.getSequence()); output.writeCompactLong(m.truncationAddress.getLedgerId()); output.writeCollection(m.ledgers, this::writeLedger00); }
@Override public long getEpoch() { return this.logMetadata.getEpoch(); }
LedgerMetadata ledgerMetadata = metadata.getLedger(newLedger.getId()); assert ledgerMetadata != null : "cannot find newly added ledger metadata"; log.debug("{}: Rollover: updated metadata '{}.", this.traceObjectId, metadata);
@Override public long getEpoch() { ensurePreconditions(); return getLogMetadata().getEpoch(); }
/** * Returns a LogMetadata class with the exact contents of this instance, but the enabled flag set to true. No changes * are performed on this instance. * * @return This instance, if isEnabled() == true, of a new instance of the LogMetadata class which will have * isEnabled() == true, otherwise. */ LogMetadata asEnabled() { return this.enabled ? this : new LogMetadata(this.epoch, true, this.ledgers, this.truncationAddress, this.updateVersion.get()); }
private void checkLedgerIds(List<Long> expectedLedgerIds, LogMetadata metadata) { val actualLedgerIds = metadata.getLedgers().stream().map(LedgerMetadata::getLedgerId).collect(Collectors.toList()); AssertExtensions.assertListEquals("Unexpected ledger ids.", expectedLedgerIds, actualLedgerIds, Long::equals); } }
/** * Creates a new instance of the LogMetadata class which contains all the ledgers after (and including) the given address. * * @param upToAddress The address to truncate to. * @return A new instance of the LogMetadata class. */ LogMetadata truncate(LedgerAddress upToAddress) { Preconditions.checkState(this.enabled, "Log is not enabled. Cannot perform any modifications on it."); // Exclude all those Ledgers that have a LedgerId less than the one we are given. An optimization to this would // involve trimming out the ledger which has a matching ledger id and the entry is is the last one, but that would // involve opening the Ledger in BookKeeper and inspecting it, which would take too long. val newLedgers = this.ledgers.stream().filter(lm -> lm.getLedgerId() >= upToAddress.getLedgerId()).collect(Collectors.toList()); return new LogMetadata(this.epoch, this.enabled, Collections.unmodifiableList(newLedgers), upToAddress, this.updateVersion.get()); }