@BeforeMethod public void setUp(Method method) throws Exception { LOG.info(">>>>>> starting {}", method); try { // start bookkeeper service startBookKeeper(); } catch (Exception e) { LOG.error("Error setting up", e); throw e; } ManagedLedgerFactoryConfig conf = new ManagedLedgerFactoryConfig(); factory = new ManagedLedgerFactoryImpl(bkc, zkc, conf); zkc.create("/managed-ledgers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); }
@Test public void errorInRecovering4() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.addEntry("entry".getBytes()); ledger.close(); factory = new ManagedLedgerFactoryImpl(bkc, zkc); zkc.failAfter(1, Code.CONNECTIONLOSS); try { ledger = factory.open("my_test_ledger"); fail("should fail"); } catch (ManagedLedgerException e) { // ok } // It should be fine now ledger = factory.open("my_test_ledger"); }
@Test public void errorInRecovering3() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.addEntry("entry".getBytes()); ledger.close(); factory = new ManagedLedgerFactoryImpl(bkc, zkc); bkc.failAfter(1, BKException.Code.LedgerFencedException); try { ledger = factory.open("my_test_ledger"); fail("should fail"); } catch (ManagedLedgerException e) { // ok } // It should be fine now ledger = factory.open("my_test_ledger"); }
@Test(timeOut = 20000) void errorRecoveringCursor2() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.openCursor("c1"); ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); bkc.failAfter(4, BKException.Code.MetadataVersionException); try { ledger = factory2.open("my_test_ledger"); fail("should have failed"); } catch (ManagedLedgerException e) { // ok } factory2.shutdown(); }
@Test public void errorInRecovering() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.addEntry("entry".getBytes()); ledger.close(); factory = new ManagedLedgerFactoryImpl(bkc, zkc); bkc.failNow(BKException.Code.LedgerFencedException); try { ledger = factory.open("my_test_ledger"); fail("should fail"); } catch (ManagedLedgerException e) { // ok } // It should be fine now ledger = factory.open("my_test_ledger"); }
@Test public void errorInRecovering2() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.addEntry("entry".getBytes()); ledger.close(); factory = new ManagedLedgerFactoryImpl(bkc, zkc); bkc.failAfter(1, BKException.Code.LedgerFencedException); try { ledger = factory.open("my_test_ledger"); fail("should fail"); } catch (ManagedLedgerException e) { // ok } // It should be fine now ledger = factory.open("my_test_ledger"); }
@Test public void errorInRecovering5() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.addEntry("entry".getBytes()); ledger.close(); factory = new ManagedLedgerFactoryImpl(bkc, zkc); zkc.failAfter(2, Code.CONNECTIONLOSS); try { ledger = factory.open("my_test_ledger"); fail("should fail"); } catch (ManagedLedgerException e) { // ok } // It should be fine now ledger = factory.open("my_test_ledger"); }
@Test public void errorInRecovering6() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.openCursor("c1"); ledger.addEntry("entry".getBytes()); ledger.close(); factory = new ManagedLedgerFactoryImpl(bkc, zkc); zkc.failAfter(3, Code.CONNECTIONLOSS); try { ledger = factory.open("my_test_ledger"); fail("should fail"); } catch (ManagedLedgerException e) { // ok } // It should be fine now ledger = factory.open("my_test_ledger"); }
@Test(timeOut = 20000) public void closeAndReopen() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ManagedCursor cursor = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.close(); log.info("Closing ledger and reopening"); // / Reopen the same managed-ledger ManagedLedgerFactoryImpl factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ledger = factory2.open("my_test_ledger"); cursor = ledger.openCursor("c1"); assertEquals(ledger.getNumberOfEntries(), 2); assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length * 2); List<Entry> entries = cursor.readEntries(100); assertEquals(entries.size(), 1); entries.forEach(e -> e.release()); ledger.close(); factory2.shutdown(); }
@Test(timeOut = 20000) public void testReopenAndCleanup() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ledger.openCursor("c1"); ledger.addEntry("data".getBytes(Encoding)); ledger.close(); Thread.sleep(100); assertEquals(bkc.getLedgers().size(), 1); factory.shutdown(); factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ledger = factory.open("my_test_ledger"); ledger.openCursor("c1"); Thread.sleep(100); assertEquals(bkc.getLedgers().size(), 2); ledger.close(); factory.open("my_test_ledger", new ManagedLedgerConfig()).delete(); Thread.sleep(100); assertEquals(bkc.getLedgers().size(), 0); factory.shutdown(); }
@Test(timeOut = 20000) void errorRecoveringCursor3() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); Position p1 = ledger.addEntry("entry".getBytes()); ledger.addEntry("entry".getBytes()); ManagedCursor c1 = ledger.openCursor("c1"); Position p3 = ledger.addEntry("entry".getBytes()); assertEquals(c1.getReadPosition(), p3); ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); bkc.failAfter(4, BKException.Code.ReadException); ledger = factory2.open("my_test_ledger"); c1 = ledger.openCursor("c1"); // Verify the ManagedCursor was rewind back to the snapshotted position assertEquals(c1.getReadPosition(), p3); factory2.shutdown(); }
@Test(timeOut = 20000) void errorRecoveringCursor() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); Position p1 = ledger.addEntry("entry".getBytes()); ledger.addEntry("entry".getBytes()); ManagedCursor c1 = ledger.openCursor("c1"); Position p3 = ledger.addEntry("entry".getBytes()); assertEquals(c1.getReadPosition(), p3); ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); bkc.failAfter(3, BKException.Code.LedgerRecoveryException); ledger = factory2.open("my_test_ledger"); c1 = ledger.openCursor("c1"); // Verify the ManagedCursor was rewind back to the snapshotted position assertEquals(c1.getReadPosition(), p3); factory2.shutdown(); }
@Test void doubleInsert() throws Exception { ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig(); config.setMaxCacheSize(10); config.setCacheEvictionWatermark(0.8); factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config); EntryCacheManager cacheManager = factory.getEntryCacheManager(); EntryCache cache1 = cacheManager.getEntryCache(ml1); assertEquals(cache1.insert(EntryImpl.create(1, 1, new byte[4])), true); assertEquals(cache1.insert(EntryImpl.create(1, 0, new byte[3])), true); assertEquals(cache1.getSize(), 7); assertEquals(cacheManager.getSize(), 7); assertEquals(cache1.insert(EntryImpl.create(1, 0, new byte[5])), false); assertEquals(cache1.getSize(), 7); assertEquals(cacheManager.getSize(), 7); }
@Test public void handleCursorRecoveryFailure() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); ManagedCursor cursor = ledger.openCursor("my-cursor"); Position p0 = cursor.getMarkDeletedPosition(); Position p1 = ledger.addEntry("entry-1".getBytes()); cursor.markDelete(p1); // Re-open from a different factory ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, zkc); bkc.failAfter(3, BKException.Code.LedgerRecoveryException); ledger = factory2.open("my_test_ledger"); cursor = ledger.openCursor("my-cursor"); // Since the cursor was rewind, it will be back to p0 assertEquals(cursor.getMarkDeletedPosition(), p0); factory2.shutdown(); }
@Test public void testSimple() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc); ManagedLedgerConfig mlConfig = new ManagedLedgerConfig(); mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1); // set the data ledger size mlConfig.setMaxEntriesPerLedger(100); // set the metadata ledger size to 1 to kick off many ledger switching cases mlConfig.setMetadataMaxEntriesPerLedger(2); ManagedLedger ledger = factory.open("ml-simple-ledger", mlConfig); ledger.addEntry("test".getBytes()); factory.shutdown(); }
@Test public void testRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(10); config.setMaxEntriesPerLedger(1); config.setRetentionTime(1, TimeUnit.HOURS); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test public void testInfiniteRetention() throws Exception { ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(-1); config.setRetentionTime(-1, TimeUnit.HOURS); config.setMaxEntriesPerLedger(1); ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); ManagedCursor c1 = ml.openCursor("c1"); ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); // reopen ml ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config); c1 = ml.openCursor("c1"); ml.addEntry("shortmessage".getBytes()); c1.skipEntries(1, IndividualDeletedEntries.Exclude); ml.close(); assertTrue(ml.getLedgersInfoAsList().size() > 1); assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length); }
@Test public void testSimpleRead() throws Exception { ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig(); factoryConf.setMaxCacheSize(0); ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf); ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1) .setMetadataAckQuorumSize(1); ManagedLedger ledger = factory.open("my-ledger", config); ManagedCursor cursor = ledger.openCursor("c1"); int N = 1; for (int i = 0; i < N; i++) { String entry = "entry-" + i; ledger.addEntry(entry.getBytes()); } List<Entry> entries = cursor.readEntries(N); assertEquals(N, entries.size()); entries.forEach(e -> e.release()); factory.shutdown(); }
@Test(timeOut = 20000) void testNumberOfEntriesWithReopen() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1)); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ManagedCursor c2 = ledger.openCursor("c2"); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ManagedCursor c3 = ledger.openCursor("c3"); ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1)); c1 = ledger.openCursor("c1"); c2 = ledger.openCursor("c2"); c3 = ledger.openCursor("c3"); assertEquals(c1.getNumberOfEntries(), 2); assertEquals(c1.hasMoreEntries(), true); assertEquals(c2.getNumberOfEntries(), 1); assertEquals(c2.hasMoreEntries(), true); assertEquals(c3.getNumberOfEntries(), 0); assertEquals(c3.hasMoreEntries(), false); factory2.shutdown(); }
@Test(timeOut = 20000) void testRateLimitMarkDelete() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setThrottleMarkDelete(1); // Throttle to 1/s ManagedLedger ledger = factory.open("my_test_ledger", config); ManagedCursor c1 = ledger.openCursor("c1"); Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding)); assertEquals(c1.getNumberOfEntriesInBacklog(), 3); c1.markDelete(p1); c1.markDelete(p2); c1.markDelete(p3); assertEquals(c1.getNumberOfEntriesInBacklog(), 0); // Re-open to recover from storage ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle()); ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig()); c1 = ledger.openCursor("c1"); // Only the 1st mark-delete was persisted assertEquals(c1.getNumberOfEntriesInBacklog(), 2); factory2.shutdown(); }