@Test public void testCursorRecoveryForEmptyLedgers() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testCursorRecoveryForEmptyLedgers"); ManagedCursor c1 = ledger.openCursor("c1"); assertEquals(ledger.getLedgersInfoAsList().size(), 1); assertEquals(c1.getMarkDeletedPosition(), ledger.lastConfirmedEntry); c1.close(); ledger.close(); ledger = (ManagedLedgerImpl) factory.open("testCursorRecoveryForEmptyLedgers"); c1 = ledger.openCursor("c1"); assertEquals(ledger.getLedgersInfoAsList().size(), 1); assertEquals(c1.getMarkDeletedPosition(), ledger.lastConfirmedEntry); }
@Test public void discardEmptyLedgersOnError() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 1); bkc.failNow(BKException.Code.NoBookieAvailableException); zkc.failNow(Code.CONNECTIONLOSS); try { ledger.addEntry("entry".getBytes()); fail("Should have received exception"); } catch (ManagedLedgerException e) { // Ok } assertEquals(ledger.getLedgersInfoAsList().size(), 0); // Next write should fail as well try { ledger.addEntry("entry".getBytes()); fail("Should have received exception"); } catch (ManagedLedgerException e) { // Ok } assertEquals(ledger.getLedgersInfoAsList().size(), 0); assertEquals(ledger.getNumberOfEntries(), 0); }
@Test public void testMinimumRolloverTime() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); conf.setMinimumRolloverTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf); ledger.openCursor("c1"); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); Thread.sleep(1000); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 2); }
@Test public void discardEmptyLedgersOnClose() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("entry".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); c1.close(); ledger.close(); // re-open ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry and the current writing ledger c1.close(); ledger.close(); // re-open, now the previous empty ledger should have been discarded ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry, and the current // writing ledger }
@Test public void testMaximumRolloverTime() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(5); conf.setMinimumRolloverTime(1, TimeUnit.SECONDS); conf.setMaximumRolloverTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_maxtime_ledger", conf); ledger.openCursor("c1"); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); Thread.sleep(2000); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 2); }
@Test public void testOffload() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
@Test public void testOffloadSamePositionTwice() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
ledger.addEntry(content.getBytes()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4); promise.complete(ImmutableSet.of(ledger.getLedgersInfoAsList().get(failIndex).getLedgerId())); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 2); Assert.assertFalse(ledger.getLedgersInfoAsList().get(failIndex).getOffloadContext().getComplete());
@Test public void testOffloadDelete() throws Exception { Set<Pair<Long, UUID>> deleted = ConcurrentHashMap.newKeySet(); CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>(); Set<Pair<Long, UUID>> failedOffloads = ConcurrentHashMap.newKeySet(); MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(0, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); ManagedCursor cursor = ledger.openCursor("foobar"); for (int i = 0; i < 15; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 1); Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete()); long firstLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId(); long secondLedger = ledger.getLedgersInfoAsList().get(1).getLedgerId(); cursor.markDelete(ledger.getLastConfirmedEntry()); assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 1); Assert.assertEquals(ledger.getLedgersInfoAsList().get(0).getLedgerId(), secondLedger); assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedger)); }
ledger.addEntry(content.getBytes()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4); long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId(); long secondLedgerId = ledger.getLedgersInfoAsList().get(1).getLedgerId(); long thirdLedgerId = ledger.getLedgersInfoAsList().get(2).getLedgerId(); long fourthLedgerId = ledger.getLedgersInfoAsList().get(3).getLedgerId(); Assert.assertEquals(firstUnoffloaded.getEntryId(), 0); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
@Test public void testAutoTriggerOffload() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setOffloadAutoTriggerSizeThresholdBytes(100); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); // Ledger will roll twice, offload will run on first ledger after second closed for (int i = 0; i < 25; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); // offload should eventually be triggered assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId())); }
@Test public void testOffloadClosedManagedLedger() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); for (int i = 0; i < 21; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Position p = ledger.getLastConfirmedEntry(); ledger.close(); try { ledger.offloadPrefix(p); Assert.fail("Should fail because ML is closed"); } catch (ManagedLedgerException.ManagedLedgerAlreadyClosedException e) { // expected } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 0); Assert.assertEquals(offloader.offloadedLedgers().size(), 0); }
ledger.addEntry(content.getBytes()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 0); Assert.assertEquals(offloader.offloadedLedgers().size(), 0);
@Test public void cursorReadsWithDiscardedEmptyLedgers() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); ManagedCursor c1 = ledger.openCursor("c1"); Position p1 = c1.getReadPosition(); c1.close(); ledger.close(); // re-open ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); c1 = ledger.openCursor("c1"); assertEquals(c1.getNumberOfEntries(), 0); assertEquals(c1.hasMoreEntries(), false); ledger.addEntry("entry".getBytes()); assertEquals(c1.getNumberOfEntries(), 1); assertEquals(c1.hasMoreEntries(), true); assertEquals(ledger.getLedgersInfoAsList().size(), 1); List<Entry> entries = c1.readEntries(1); assertEquals(entries.size(), 1); entries.forEach(e -> e.release()); assertEquals(c1.hasMoreEntries(), false); assertEquals(c1.readEntries(1).size(), 0); c1.seek(p1); assertEquals(c1.hasMoreEntries(), true); assertEquals(c1.getNumberOfEntries(), 1); entries = c1.readEntries(1); assertEquals(entries.size(), 1); entries.forEach(e -> e.release()); assertEquals(c1.readEntries(1).size(), 0); }
@Test public void offloadAsSoonAsClosed() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setOffloadAutoTriggerSizeThresholdBytes(0); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); for (int i = 0; i < 11; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId())); for (int i = 0; i < 10; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(), ledger.getLedgersInfoAsList().get(1).getLedgerId())); }
@Test public void testOffloadNewML() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); try { ledger.offloadPrefix(ledger.getLastConfirmedEntry()); } catch (ManagedLedgerException.InvalidCursorPositionException e) { // expected } // add one entry and try again ledger.addEntry("foobar".getBytes()); Position p = ledger.getLastConfirmedEntry(); Assert.assertEquals(p, ledger.offloadPrefix(ledger.getLastConfirmedEntry())); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 1); Assert.assertEquals(offloader.offloadedLedgers().size(), 0); }
@Test(enabled = true) public void testNoRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(0); config.setMaxEntriesPerLedger(1); // Default is no-retention ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1noretention"); ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config); c1 = ml.openCursor("c1noretention"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); // sleep for trim Thread.sleep(1000); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() <= 1); assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length); }
@Test public void testRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(10); config.setMaxEntriesPerLedger(1); config.setRetentionTime(1, TimeUnit.HOURS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test public void testInfiniteRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(-1); config.setRetentionTime(-1, TimeUnit.HOURS); config.setMaxEntriesPerLedger(1); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test public void testDeletionAfterRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(0); config.setMaxEntriesPerLedger(1); config.setRetentionTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1noretention"); ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config); c1 = ml.openCursor("c1noretention"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); // let retention expire Thread.sleep(1000); ml.internalTrimConsumedLedgers(CompletableFuture.completedFuture(null)); assertTrue(ml.getLedgersInfoAsList().size() <= 1); assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length); ml.close(); }