@Override public Position addEntry(byte[] data) throws InterruptedException, ManagedLedgerException { return addEntry(data, 0, data.length); }
@Test public void testMinimumRolloverTime() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); conf.setMinimumRolloverTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf); ledger.openCursor("c1"); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); Thread.sleep(1000); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 2); }
@Test public void testMaximumRolloverTime() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(5); conf.setMinimumRolloverTime(1, TimeUnit.SECONDS); conf.setMaximumRolloverTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_maxtime_ledger", conf); ledger.openCursor("c1"); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); Thread.sleep(2000); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 2); }
@Test public void addEntryWithOffset() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("012345678".getBytes(), 2, 3); List<Entry> entries = c1.readEntries(1); assertEquals(entries.get(0).getLength(), 3); Entry entry = entries.get(0); assertEquals(new String(entry.getData()), "234"); entry.release(); }
@Test public void testEstimatedBacklogSize() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testEstimatedBacklogSize"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry(new byte[1024]); Position position2 = ledger.addEntry(new byte[1024]); ledger.addEntry(new byte[1024]); ledger.addEntry(new byte[1024]); Position lastPosition = ledger.addEntry(new byte[1024]); long backlog = ledger.getEstimatedBacklogSize(); assertEquals(backlog, 1024 * 5); List<Entry> entries = c1.readEntries(2); entries.forEach(Entry::release); c1.markDelete(position2); backlog = ledger.getEstimatedBacklogSize(); assertEquals(backlog, 1024 * 3); entries = c1.readEntries(3); entries.forEach(Entry::release); c1.markDelete(lastPosition); backlog = ledger.getEstimatedBacklogSize(); assertEquals(backlog, 0); }
@Test public void testGetNextValidPosition() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetNextValidPosition", conf); ManagedCursor c1 = ledger.openCursor("c1"); PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes()); PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes()); PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes()); assertEquals(ledger.getNextValidPosition((PositionImpl) c1.getMarkDeletedPosition()), p1); assertEquals(ledger.getNextValidPosition(p1), p2); assertEquals(ledger.getNextValidPosition(p3), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)); }
@Test public void discardEmptyLedgersOnError() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 1); bkc.failNow(BKException.Code.NoBookieAvailableException); zkc.failNow(Code.CONNECTIONLOSS); try { ledger.addEntry("entry".getBytes()); fail("Should have received exception"); } catch (ManagedLedgerException e) { // Ok } assertEquals(ledger.getLedgersInfoAsList().size(), 0); // Next write should fail as well try { ledger.addEntry("entry".getBytes()); fail("Should have received exception"); } catch (ManagedLedgerException e) { // Ok } assertEquals(ledger.getLedgersInfoAsList().size(), 0); assertEquals(ledger.getNumberOfEntries(), 0); }
@Test public void totalSizeTest() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry(new byte[10], 1, 8); assertEquals(ledger.getTotalSize(), 8); PositionImpl p2 = (PositionImpl) ledger.addEntry(new byte[12], 2, 5); assertEquals(ledger.getTotalSize(), 13); c1.markDelete(new PositionImpl(p2.getLedgerId(), -1)); // Wait for background trimming Thread.sleep(400); assertEquals(ledger.getTotalSize(), 5); }
@Test public void discardEmptyLedgersOnClose() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("entry".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); c1.close(); ledger.close(); // re-open ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry and the current writing ledger c1.close(); ledger.close(); // re-open, now the previous empty ledger should have been discarded ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry, and the current // writing ledger }
@Test public void testConsumerSubscriptionInitializePosition() throws Exception{ final int MAX_ENTRY_PER_LEDGER = 2; ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(MAX_ENTRY_PER_LEDGER); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("lastest_earliest_ledger", config); final int totalInsertedEntries = 20; for (int i = 0; i < totalInsertedEntries; i++) { String content = "entry" + i; // 5 bytes ledger.addEntry(content.getBytes()); } // Open Cursor also adds cursor into activeCursor-container ManagedCursor latestCursor = ledger.openCursor("c1", InitialPosition.Latest); ManagedCursor earliestCursor = ledger.openCursor("c2", InitialPosition.Earliest); // Since getReadPosition returns the next position, we decrease the entryId by 1 PositionImpl p1 = (PositionImpl) latestCursor.getReadPosition(); PositionImpl p2 = (PositionImpl) earliestCursor.getReadPosition(); Pair<PositionImpl, Long> latestPositionAndCounter = ledger.getLastPositionAndCounter(); Pair<PositionImpl, Long> earliestPositionAndCounter = ledger.getFirstPositionAndCounter(); assertEquals(latestPositionAndCounter.getLeft().getNext(), p1); assertEquals(earliestPositionAndCounter.getLeft().getNext(), p2); assertEquals(latestPositionAndCounter.getRight().longValue(), totalInsertedEntries); assertEquals(earliestPositionAndCounter.getRight().longValue(), totalInsertedEntries - earliestCursor.getNumberOfEntriesInBacklog()); ledger.close(); }
@Test public void testOffload() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
@Test public void testOffloadSamePositionTwice() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); int i = 0; for (; i < 25; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()) .map(e -> e.getLedgerId()).collect(Collectors.toSet()), offloader.offloadedLedgers()); }
@Test public void testInfiniteRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(-1); config.setRetentionTime(-1, TimeUnit.HOURS); config.setMaxEntriesPerLedger(1); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test public void testRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(10); config.setMaxEntriesPerLedger(1); config.setRetentionTime(1, TimeUnit.HOURS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test void verifyNoCacheIfNoConsumer() throws Exception { ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig(); config.setMaxCacheSize(7 * 10); config.setCacheEvictionWatermark(0.8); factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config); EntryCacheManager cacheManager = factory.getEntryCacheManager(); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("ledger"); EntryCache cache1 = ledger.entryCache; for (int i = 0; i < 10; i++) { ledger.addEntry(("entry-" + i).getBytes()); } assertEquals(cache1.getSize(), 0); assertEquals(cacheManager.getSize(), 0); cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS); assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 7 * 10); assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 0); assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0); assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0); assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0); assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0); }
@Test(enabled = true) public void testNoRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(0); config.setMaxEntriesPerLedger(1); // Default is no-retention ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1noretention"); ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config); c1 = ml.openCursor("c1noretention"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); // sleep for trim Thread.sleep(1000); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() <= 1); assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length); }
@Test public void testDeletionAfterRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(0); config.setMaxEntriesPerLedger(1); config.setRetentionTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1noretention"); ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config); c1 = ml.openCursor("c1noretention"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); // let retention expire Thread.sleep(1000); ml.internalTrimConsumedLedgers(CompletableFuture.completedFuture(null)); assertTrue(ml.getLedgersInfoAsList().size() <= 1); assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length); ml.close(); }
@Test public void testAutoTriggerOffload() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setOffloadAutoTriggerSizeThresholdBytes(100); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); // Ledger will roll twice, offload will run on first ledger after second closed for (int i = 0; i < 25; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3); // offload should eventually be triggered assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId())); }
@Test public void testOffloadNewML() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); try { ledger.offloadPrefix(ledger.getLastConfirmedEntry()); } catch (ManagedLedgerException.InvalidCursorPositionException e) { // expected } // add one entry and try again ledger.addEntry("foobar".getBytes()); Position p = ledger.getLastConfirmedEntry(); Assert.assertEquals(p, ledger.offloadPrefix(ledger.getLastConfirmedEntry())); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 1); Assert.assertEquals(offloader.offloadedLedgers().size(), 0); }
@Test public void offloadAsSoonAsClosed() throws Exception { MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setOffloadAutoTriggerSizeThresholdBytes(0); config.setRetentionTime(10, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); for (int i = 0; i < 11; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId())); for (int i = 0; i < 10; i++) { ledger.addEntry(buildEntry(10, "entry-" + i)); } assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2); Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(), ledger.getLedgersInfoAsList().get(1).getLedgerId())); }