@Override public ManagedCursor openCursor(String cursorName) throws InterruptedException, ManagedLedgerException { return openCursor(cursorName, InitialPosition.Latest); }
@Test public void addEntryWithOffset() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("012345678".getBytes(), 2, 3); List<Entry> entries = c1.readEntries(1); assertEquals(entries.get(0).getLength(), 3); Entry entry = entries.get(0); assertEquals(new String(entry.getData()), "234"); entry.release(); }
@Test public void testCursorRecoveryForEmptyLedgers() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testCursorRecoveryForEmptyLedgers"); ManagedCursor c1 = ledger.openCursor("c1"); assertEquals(ledger.getLedgersInfoAsList().size(), 1); assertEquals(c1.getMarkDeletedPosition(), ledger.lastConfirmedEntry); c1.close(); ledger.close(); ledger = (ManagedLedgerImpl) factory.open("testCursorRecoveryForEmptyLedgers"); c1 = ledger.openCursor("c1"); assertEquals(ledger.getLedgersInfoAsList().size(), 1); assertEquals(c1.getMarkDeletedPosition(), ledger.lastConfirmedEntry); }
@Test public void testMinimumRolloverTime() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); conf.setMinimumRolloverTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf); ledger.openCursor("c1"); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); Thread.sleep(1000); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 2); }
@Test public void testMaximumRolloverTime() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(5); conf.setMinimumRolloverTime(1, TimeUnit.SECONDS); conf.setMaximumRolloverTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_maxtime_ledger", conf); ledger.openCursor("c1"); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); Thread.sleep(2000); ledger.addEntry("data".getBytes()); ledger.addEntry("data".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 2); }
@Test public void testConsumerSubscriptionInitializePosition() throws Exception{ final int MAX_ENTRY_PER_LEDGER = 2; ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(MAX_ENTRY_PER_LEDGER); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("lastest_earliest_ledger", config); final int totalInsertedEntries = 20; for (int i = 0; i < totalInsertedEntries; i++) { String content = "entry" + i; // 5 bytes ledger.addEntry(content.getBytes()); } // Open Cursor also adds cursor into activeCursor-container ManagedCursor latestCursor = ledger.openCursor("c1", InitialPosition.Latest); ManagedCursor earliestCursor = ledger.openCursor("c2", InitialPosition.Earliest); // Since getReadPosition returns the next position, we decrease the entryId by 1 PositionImpl p1 = (PositionImpl) latestCursor.getReadPosition(); PositionImpl p2 = (PositionImpl) earliestCursor.getReadPosition(); Pair<PositionImpl, Long> latestPositionAndCounter = ledger.getLastPositionAndCounter(); Pair<PositionImpl, Long> earliestPositionAndCounter = ledger.getFirstPositionAndCounter(); assertEquals(latestPositionAndCounter.getLeft().getNext(), p1); assertEquals(earliestPositionAndCounter.getLeft().getNext(), p2); assertEquals(latestPositionAndCounter.getRight().longValue(), totalInsertedEntries); assertEquals(earliestPositionAndCounter.getRight().longValue(), totalInsertedEntries - earliestCursor.getNumberOfEntriesInBacklog()); ledger.close(); }
@Test public void discardEmptyLedgersOnClose() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("entry".getBytes()); assertEquals(ledger.getLedgersInfoAsList().size(), 1); c1.close(); ledger.close(); // re-open ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry and the current writing ledger c1.close(); ledger.close(); // re-open, now the previous empty ledger should have been discarded ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry, and the current // writing ledger }
@Test public void testGetNextValidPosition() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetNextValidPosition", conf); ManagedCursor c1 = ledger.openCursor("c1"); PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes()); PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes()); PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes()); assertEquals(ledger.getNextValidPosition((PositionImpl) c1.getMarkDeletedPosition()), p1); assertEquals(ledger.getNextValidPosition(p1), p2); assertEquals(ledger.getNextValidPosition(p3), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)); }
@Test // (timeOut = 20000) public void asyncOpenClosedLedger() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-closed-ledger"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); c1.close(); assertEquals(ledger.getNumberOfEntries(), 1); ledger.setFenced(); final CountDownLatch counter = new CountDownLatch(1); class Result { ManagedLedger instance1 = null; } final Result result = new Result(); factory.asyncOpen("my-closed-ledger", new OpenLedgerCallback() { @Override public void openLedgerComplete(ManagedLedger ledger, Object ctx) { result.instance1 = ledger; counter.countDown(); } @Override public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { } }, null); counter.await(); assertNotNull(result.instance1); ManagedCursor c2 = result.instance1.openCursor("c1"); List<Entry> entries = c2.readEntries(1); assertEquals(entries.size(), 1); entries.forEach(e -> e.release()); }
@Test public void cursorReadsWithDiscardedEmptyLedgers() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); ManagedCursor c1 = ledger.openCursor("c1"); Position p1 = c1.getReadPosition(); c1.close(); ledger.close(); // re-open ledger = (ManagedLedgerImpl) factory.open("my_test_ledger"); c1 = ledger.openCursor("c1"); assertEquals(c1.getNumberOfEntries(), 0); assertEquals(c1.hasMoreEntries(), false); ledger.addEntry("entry".getBytes()); assertEquals(c1.getNumberOfEntries(), 1); assertEquals(c1.hasMoreEntries(), true); assertEquals(ledger.getLedgersInfoAsList().size(), 1); List<Entry> entries = c1.readEntries(1); assertEquals(entries.size(), 1); entries.forEach(e -> e.release()); assertEquals(c1.hasMoreEntries(), false); assertEquals(c1.readEntries(1).size(), 0); c1.seek(p1); assertEquals(c1.hasMoreEntries(), true); assertEquals(c1.getNumberOfEntries(), 1); entries = c1.readEntries(1); assertEquals(entries.size(), 1); entries.forEach(e -> e.release()); assertEquals(c1.readEntries(1).size(), 0); }
@Test public void totalSizeTest() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry(new byte[10], 1, 8); assertEquals(ledger.getTotalSize(), 8); PositionImpl p2 = (PositionImpl) ledger.addEntry(new byte[12], 2, 5); assertEquals(ledger.getTotalSize(), 13); c1.markDelete(new PositionImpl(p2.getLedgerId(), -1)); // Wait for background trimming Thread.sleep(400); assertEquals(ledger.getTotalSize(), 5); }
@Test public void testRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(10); config.setMaxEntriesPerLedger(1); config.setRetentionTime(1, TimeUnit.HOURS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test(enabled = true) public void testNoRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(0); config.setMaxEntriesPerLedger(1); // Default is no-retention ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1noretention"); ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config); c1 = ml.openCursor("c1noretention"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); // sleep for trim Thread.sleep(1000); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() <= 1); assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length); }
@Test public void testInfiniteRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(-1); config.setRetentionTime(-1, TimeUnit.HOURS); config.setMaxEntriesPerLedger(1); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test public void testDeletionAfterRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(0); config.setMaxEntriesPerLedger(1); config.setRetentionTime(1, TimeUnit.SECONDS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1noretention"); ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config); c1 = ml.openCursor("c1noretention"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); // let retention expire Thread.sleep(1000); ml.internalTrimConsumedLedgers(CompletableFuture.completedFuture(null)); assertTrue(ml.getLedgersInfoAsList().size() <= 1); assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length); ml.close(); }
@Test public void testEstimatedBacklogSize() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testEstimatedBacklogSize"); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry(new byte[1024]); Position position2 = ledger.addEntry(new byte[1024]); ledger.addEntry(new byte[1024]); ledger.addEntry(new byte[1024]); Position lastPosition = ledger.addEntry(new byte[1024]); long backlog = ledger.getEstimatedBacklogSize(); assertEquals(backlog, 1024 * 5); List<Entry> entries = c1.readEntries(2); entries.forEach(Entry::release); c1.markDelete(position2); backlog = ledger.getEstimatedBacklogSize(); assertEquals(backlog, 1024 * 3); entries = c1.readEntries(3); entries.forEach(Entry::release); c1.markDelete(lastPosition); backlog = ledger.getEstimatedBacklogSize(); assertEquals(backlog, 0); }
@Test public void testOffloadDelete() throws Exception { Set<Pair<Long, UUID>> deleted = ConcurrentHashMap.newKeySet(); CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>(); Set<Pair<Long, UUID>> failedOffloads = ConcurrentHashMap.newKeySet(); MockLedgerOffloader offloader = new MockLedgerOffloader(); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setMaxEntriesPerLedger(10); config.setMinimumRolloverTime(0, TimeUnit.SECONDS); config.setRetentionTime(0, TimeUnit.MINUTES); config.setLedgerOffloader(offloader); ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config); ManagedCursor cursor = ledger.openCursor("foobar"); for (int i = 0; i < 15; i++) { String content = "entry-" + i; ledger.addEntry(content.getBytes()); } Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); ledger.offloadPrefix(ledger.getLastConfirmedEntry()); Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2); Assert.assertEquals(ledger.getLedgersInfoAsList().stream() .filter(e -> e.getOffloadContext().getComplete()).count(), 1); Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete()); long firstLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId(); long secondLedger = ledger.getLedgersInfoAsList().get(1).getLedgerId(); cursor.markDelete(ledger.getLastConfirmedEntry()); assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 1); Assert.assertEquals(ledger.getLedgersInfoAsList().get(0).getLedgerId(), secondLedger); assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedger)); }
/** * When auto-replication is triggered, if there were no writes on the ML during the grace period, auto-replication * will close the ledger an re-replicate it. After that, the next write will get a FencedException. We should * recover from this condition by creating a new ledger and retrying the write. */ @Test public void ledgerFencedByAutoReplication() throws Exception { ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", config); ManagedCursor c1 = ledger.openCursor("c1"); PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes()); // Trigger the closure of the data ledger bkc.openLedger(p1.getLedgerId(), BookKeeper.DigestType.CRC32C, new byte[] {}); ledger.addEntry("entry-2".getBytes()); assertEquals(2, c1.getNumberOfEntries()); assertEquals(2, c1.getNumberOfEntriesInBacklog()); PositionImpl p3 = (PositionImpl) ledger.addEntry("entry-3".getBytes()); // Now entry-2 should have been written before entry-3 assertEquals(3, c1.getNumberOfEntries()); assertEquals(3, c1.getNumberOfEntriesInBacklog()); assertTrue(p1.getLedgerId() != p3.getLedgerId()); factory.shutdown(); }
EntryCache entryCache = ledger.entryCache; ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); ManagedCursorImpl c2 = (ManagedCursorImpl) ledger.openCursor("c2");
@Test public void previousPosition() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("my_cursor"); Position p0 = cursor.getMarkDeletedPosition(); // This is expected because p0 is already an "invalid" position (since no entry has been mark-deleted yet) assertEquals(ledger.getPreviousPosition((PositionImpl) p0), p0); // Force to close an empty ledger ledger.close(); ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); // again ledger.close(); ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); PositionImpl pBeforeWriting = ledger.getLastPosition(); PositionImpl p1 = (PositionImpl) ledger.addEntry("entry".getBytes()); ledger.close(); ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); Position p2 = ledger.addEntry("entry".getBytes()); Position p3 = ledger.addEntry("entry".getBytes()); Position p4 = ledger.addEntry("entry".getBytes()); assertEquals(ledger.getPreviousPosition(p1), pBeforeWriting); assertEquals(ledger.getPreviousPosition((PositionImpl) p2), p1); assertEquals(ledger.getPreviousPosition((PositionImpl) p3), p2); assertEquals(ledger.getPreviousPosition((PositionImpl) p4), p3); }