public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { if (((bitField0_ & 0x00000001) == 0x00000001)) { output.writeInt64(1, ledgerId_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { output.writeInt64(2, entries_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { output.writeInt64(3, size_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { output.writeInt64(4, timestamp_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { output.writeMessage(5, getOffloadContext()); } unknownFields.writeTo(output); }
public int getSerializedSize() { int size = memoizedSize; if (size != -1) return size; size = 0; if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(1, ledgerId_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(2, entries_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(3, size_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(4, timestamp_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream .computeMessageSize(5, getOffloadContext()); } size += unknownFields.getSerializedSize(); memoizedSize = size; return size; }
public static Map<String, String> getOffloadDriverMetadata(LedgerInfo ledgerInfo) { Map<String, String> metadata = Maps.newHashMap(); if (ledgerInfo.hasOffloadContext()) { OffloadContext ctx = ledgerInfo.getOffloadContext(); if (ctx.hasDriverMetadata()) { OffloadDriverMetadata driverMetadata = ctx.getDriverMetadata(); if (driverMetadata.getPropertiesCount() > 0) { driverMetadata.getPropertiesList().forEach(kv -> metadata.put(kv.getKey(), kv.getValue())); } } } return metadata; }
.filter(e -> e.getOffloadContext().getComplete()).count(), 0); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().hasUidMsb()).count(), 1); Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().hasUidMsb());
.filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers());
hash = (53 * hash) + getOffloadContext().hashCode();
.filter(e -> e.getOffloadContext().getComplete()).count(), 0); try { ledger.offloadPrefix(p); .filter(e -> e.getOffloadContext().getComplete()).count(), 0); .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
return transformLedgerInfo(ledgerId, (oldInfo) -> { UUID existingUuid = new UUID(oldInfo.getOffloadContext().getUidMsb(), oldInfo.getOffloadContext().getUidLsb()); if (existingUuid.equals(uuid)) { LedgerInfo.Builder builder = oldInfo.toBuilder();
return transformLedgerInfo(ledgerId, (oldInfo) -> { if (oldInfo.getOffloadContext().hasUidMsb()) { UUID oldUuid = new UUID(oldInfo.getOffloadContext().getUidMsb(), oldInfo.getOffloadContext().getUidLsb()); log.info("[{}] Found previous offload attempt for ledger {}, uuid {}" + ", cleaning up", name, ledgerId, uuid);
.filter(e -> e.getOffloadContext().getComplete()).count(), 0); Assert.assertEquals(offloader.offloadedLedgers().size(), 0);
@Test public void testOffloadDelete() throws Exception { Set<Pair<Long, UUID>> deleted = ConcurrentHashMap.newKeySet(); CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>(); Set<Pair<Long, UUID>> failedOffloads = ConcurrentHashMap.newKeySet(); MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(0, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); ManagedCursor cursor = ledger.openCursor("foobar"); for (int i = 0; i < 15; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 1); Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete()); long firstLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId(); long secondLedger = ledger.getLedgersInfoAsList().get(1).getLedgerId(); cursor.markDelete(ledger.getLastConfirmedEntry()); assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 1); Assert.assertEquals(ledger.getLedgersInfoAsList().get(0).getLedgerId(), secondLedger); assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedger)); }
.filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 2); Assert.assertFalse(ledger.getLedgersInfoAsList().get(failIndex).getOffloadContext().getComplete());
@Test public void testOffloadSamePositionTwice() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
@Test public void testOffload() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
@Test public void testOffloadClosedManagedLedger() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); for (int i = 0; i < 21; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Position p = ledger.getLastConfirmedEntry(); ledger.close(); try { ledger.offloadPrefix(p); Assert.fail("Should fail because ML is closed"); } catch (ManagedLedgerException.ManagedLedgerAlreadyClosedException e) { // expected } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 0); Assert.assertEquals(offloader.offloadedLedgers().size(), 0); }
public Builder mergeFrom(org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo other) { if (other == org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo.getDefaultInstance()) return this; if (other.hasLedgerId()) { setLedgerId(other.getLedgerId()); } if (other.hasEntries()) { setEntries(other.getEntries()); } if (other.hasSize()) { setSize(other.getSize()); } if (other.hasTimestamp()) { setTimestamp(other.getTimestamp()); } if (other.hasOffloadContext()) { mergeOffloadContext(other.getOffloadContext()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }
public final boolean isInitialized() { byte isInitialized = memoizedIsInitialized; if (isInitialized == 1) return true; if (isInitialized == 0) return false; if (!hasLedgerId()) { memoizedIsInitialized = 0; return false; } if (hasOffloadContext()) { if (!getOffloadContext().isInitialized()) { memoizedIsInitialized = 0; return false; } } memoizedIsInitialized = 1; return true; }
public static Map<String, String> getOffloadDriverMetadata(LedgerInfo ledgerInfo, Map<String, String> defaultOffloadDriverMetadata) { if (ledgerInfo.hasOffloadContext()) { OffloadContext ctx = ledgerInfo.getOffloadContext(); if (ctx.hasDriverMetadata()) { OffloadDriverMetadata driverMetadata = ctx.getDriverMetadata(); if (driverMetadata.getPropertiesCount() > 0) { Map<String, String> metadata = Maps.newHashMap(); driverMetadata.getPropertiesList().forEach(kv -> metadata.put(kv.getKey(), kv.getValue())); return metadata; } } } return defaultOffloadDriverMetadata; }
private void asyncDeleteLedger(long ledgerId, LedgerInfo info) { if (!info.getOffloadContext().getBookkeeperDeleted()) { // only delete if it hasn't been previously deleted for offload asyncDeleteLedger(ledgerId, DEFAULT_LEDGER_DELETE_RETRIES); } if (info.getOffloadContext().hasUidMsb()) { UUID uuid = new UUID(info.getOffloadContext().getUidMsb(), info.getOffloadContext().getUidLsb()); cleanupOffloaded(ledgerId, uuid, OffloadUtils.getOffloadDriverName(info, config.getLedgerOffloader().getOffloadDriverName()), OffloadUtils.getOffloadDriverMetadata(info, config.getLedgerOffloader().getOffloadDriverMetadata()), "Trimming"); } }
public static String getOffloadDriverName(LedgerInfo ledgerInfo, String defaultDriverName) { if (ledgerInfo.hasOffloadContext()) { OffloadContext ctx = ledgerInfo.getOffloadContext(); if (ctx.hasDriverMetadata()) { OffloadDriverMetadata driverMetadata = ctx.getDriverMetadata(); if (driverMetadata.hasName()) { return driverMetadata.getName(); } } } return defaultDriverName; }