PositionImpl startReadOperationOnLedger(PositionImpl position) { long ledgerId = ledgers.ceilingKey(position.getLedgerId()); if (ledgerId != position.getLedgerId()) { // The ledger pointed by this position does not exist anymore. It was deleted because it was empty. We need // to skip on the next available ledger position = new PositionImpl(ledgerId, 0); } return position; }
@Override void asyncReadEntry(PositionImpl position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) { this.getLedgerHandle(position.getLedgerId()).thenAccept((ledger) -> { asyncReadEntry(ledger, position, callback, ctx); }).exceptionally((ex) -> { log.error("[{}] Error opening ledger for reading at position {} - {}", new Object[]{this.name, position, ex.getMessage()}); callback.readEntryFailed(ManagedLedgerException.getManagedLedgerException(ex.getCause()), ctx); return null; }); }
void asyncReadEntry(PositionImpl position, ReadEntryCallback callback, Object ctx) { LedgerHandle currentLedger = this.currentLedger; if (log.isDebugEnabled()) { log.debug("[{}] Reading entry ledger {}: {}", name, position.getLedgerId(), position.getEntryId()); } if (position.getLedgerId() == currentLedger.getId()) { LedgerHandle ledger = currentLedger; asyncReadEntry(ledger, position, callback, ctx); } else { getLedgerHandle(position.getLedgerId()).thenAccept(ledger -> { asyncReadEntry(ledger, position, callback, ctx); }).exceptionally(ex -> { log.error("[{}] Error opening ledger for reading at position {} - {}", name, position, ex.getMessage()); callback.readEntryFailed(ManagedLedgerException.getManagedLedgerException(ex.getCause()), ctx); return null; }); } }
void updateCursor(ManagedCursorImpl cursor, PositionImpl newPosition) { Pair<PositionImpl, PositionImpl> pair = cursors.cursorUpdated(cursor, newPosition); if (pair == null) { // Cursor has been removed in the meantime trimConsumedLedgersInBackground(); return; } PositionImpl previousSlowestReader = pair.getLeft(); PositionImpl currentSlowestReader = pair.getRight(); if (previousSlowestReader.compareTo(currentSlowestReader) == 0) { // The slowest consumer has not changed position. Nothing to do right now return; } // Only trigger a trimming when switching to the next ledger if (previousSlowestReader.getLedgerId() != newPosition.getLedgerId()) { trimConsumedLedgersInBackground(); } }
public static EntryImpl create(PositionImpl position, ByteBuf data) { EntryImpl entry = RECYCLER.get(); entry.ledgerId = position.getLedgerId(); entry.entryId = position.getEntryId(); entry.data = data; entry.data.retain(); entry.setRefCnt(1); return entry; }
long estimateBacklogFromPosition(PositionImpl pos) { synchronized (this) { LedgerInfo ledgerInfo = ledgers.get(pos.getLedgerId()); if (ledgerInfo == null) { return getTotalSize(); // position no longer in managed ledger, so return total size } long sizeBeforePosLedger = ledgers.values().stream().filter(li -> li.getLedgerId() < pos.getLedgerId()) .mapToLong(li -> li.getSize()).sum(); long size = getTotalSize() - sizeBeforePosLedger; if (pos.getLedgerId() == currentLedger.getId()) { return size - consumedLedgerSize(currentLedgerSize, currentLedgerEntries, pos.getEntryId()); } else { return size - consumedLedgerSize(ledgerInfo.getSize(), ledgerInfo.getEntries(), pos.getEntryId()); } } }
PositionImpl getNextValidPosition(final PositionImpl position) { PositionImpl nextPosition = position.getNext(); while (!isValidPosition(nextPosition)) { Long nextLedgerId = ledgers.ceilingKey(nextPosition.getLedgerId() + 1); if (nextLedgerId == null) { return null; } nextPosition = PositionImpl.get(nextLedgerId.longValue(), 0); } return nextPosition; }
NonDurableCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, String cursorName, PositionImpl startCursorPosition) { super(bookkeeper, config, ledger, cursorName); // Compare with "latest" position marker by using only the ledger id. Since the C++ client is using 48bits to // store the entryId, it's not able to pass a Long.max() as entryId. In this case there's no point to require // both ledgerId and entryId to be Long.max() if (startCursorPosition == null || startCursorPosition.getLedgerId() == PositionImpl.latest.getLedgerId()) { // Start from last entry initializeCursorPosition(ledger.getLastPositionAndCounter()); } else if (startCursorPosition.equals(PositionImpl.earliest)) { // Start from invalid ledger to read from first available entry recoverCursor(ledger.getPreviousPosition(ledger.getFirstPosition())); } else { // Since the cursor is positioning on the mark-delete position, we need to take 1 step back from the desired // read-position recoverCursor(startCursorPosition); } log.info("[{}] Created non-durable cursor read-position={} mark-delete-position={}", ledger.getName(), readPosition, markDeletePosition); }
private ManagedLedgerInfo getManagedLedgerInfo() { ManagedLedgerInfo.Builder mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()); if (state == State.Terminated) { mlInfo.setTerminatedPosition(NestedPositionInfo.newBuilder().setLedgerId(lastConfirmedEntry.getLedgerId()) .setEntryId(lastConfirmedEntry.getEntryId())); } return mlInfo.build(); }
@Test(timeOut = 20000) public void deleteWithErrors1() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger"); PositionImpl position = (PositionImpl) ledger.addEntry("dummy-entry-1".getBytes(Encoding)); assertEquals(ledger.getNumberOfEntries(), 1); // Force delete a ledger and test that deleting the ML still happens // without errors bkc.deleteLedger(position.getLedgerId()); ledger.delete(); }
@Test public void simpleTest() { PositionImpl pos = new PositionImpl(1, 2); assertEquals(pos.getLedgerId(), 1); assertEquals(pos.getEntryId(), 2); assertEquals(pos, new PositionImpl(1, 2)); assertFalse(pos.equals(new PositionImpl(1, 3))); assertFalse(pos.equals(new PositionImpl(3, 2))); assertFalse(pos.equals("1:2")); }
void checkReadCompletion() { if (entries.size() < count && cursor.hasMoreEntries()) { // We still have more entries to read from the next ledger, schedule a new async operation if (nextReadPosition.getLedgerId() != readPosition.getLedgerId()) { cursor.ledger.startReadOperationOnLedger(nextReadPosition); } // Schedule next read in a different thread cursor.ledger.getExecutor().execute(safeRun(() -> { readPosition = cursor.ledger.startReadOperationOnLedger(nextReadPosition); cursor.ledger.asyncReadEntries(OpReadEntry.this); })); } else { // The reading was already completed, release resources and trigger callback try { cursor.readOperationCompleted(); } finally { cursor.ledger.getExecutor().executeOrdered(cursor.ledger.getName(), safeRun(() -> { callback.readEntriesComplete(entries, ctx); recycle(); })); } } }
@Test public void hashes() throws Exception { PositionImpl p1 = new PositionImpl(5, 15); PositionImpl p2 = new PositionImpl(PositionInfo.parseFrom(p1.getPositionInfo().toByteArray())); assertEquals(p2.getLedgerId(), 5); assertEquals(p2.getEntryId(), 15); assertEquals(new PositionImpl(5, 15).hashCode(), p2.hashCode()); } }
@Test(timeOut = 20000) void seekPosition2() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); PositionImpl seekPosition = (PositionImpl) ledger.addEntry("dummy-entry-3".getBytes(Encoding)); ledger.addEntry("dummy-entry-4".getBytes(Encoding)); ledger.addEntry("dummy-entry-5".getBytes(Encoding)); ledger.addEntry("dummy-entry-6".getBytes(Encoding)); cursor.seek(new PositionImpl(seekPosition.getLedgerId(), seekPosition.getEntryId())); }
@Test(timeOut = 20000) void testResetCursor() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.latest); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); try { cursor.resetCursor(resetPosition); moveStatus.set(true); } catch (Exception e) { log.warn("error in reset cursor", e.getCause()); } assertTrue(moveStatus.get()); assertTrue(cursor.getReadPosition().equals(resetPosition)); cursor.close(); ledger.close(); }
@Test(timeOut = 20000) void seekPosition() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); cursor.seek(new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 1)); }
@Test(timeOut = 20000) void testResetCursor() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.openCursor("trc1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); try { cursor.resetCursor(resetPosition); moveStatus.set(true); } catch (Exception e) { log.warn("error in reset cursor", e.getCause()); } assertTrue(moveStatus.get()); assertTrue(cursor.getReadPosition().equals(resetPosition)); cursor.close(); ledger.close(); }
@Test public void testGetNextValidPosition() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetNextValidPosition", conf); ManagedCursor c1 = ledger.openCursor("c1"); PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes()); PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes()); PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes()); assertEquals(ledger.getNextValidPosition((PositionImpl) c1.getMarkDeletedPosition()), p1); assertEquals(ledger.getNextValidPosition(p1), p2); assertEquals(ledger.getNextValidPosition(p3), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)); }
@Test(timeOut = 20000) void markDeleteSkippingMessage() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.earliest); Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl p4 = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); assertEquals(cursor.getNumberOfEntries(), 4); cursor.markDelete(p1); assertEquals(cursor.hasMoreEntries(), true); assertEquals(cursor.getNumberOfEntries(), 3); assertEquals(cursor.getReadPosition(), p2); List<Entry> entries = cursor.readEntries(1); assertEquals(entries.size(), 1); assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-2"); entries.forEach(e -> e.release()); cursor.markDelete(p4); assertEquals(cursor.hasMoreEntries(), false); assertEquals(cursor.getNumberOfEntries(), 0); assertEquals(cursor.getReadPosition(), new PositionImpl(p4.getLedgerId(), p4.getEntryId() + 1)); }
@Test public void totalSizeTest() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf); ManagedCursor c1 = ledger.openCursor("c1"); ledger.addEntry(new byte[10], 1, 8); assertEquals(ledger.getTotalSize(), 8); PositionImpl p2 = (PositionImpl) ledger.addEntry(new byte[12], 2, 5); assertEquals(ledger.getTotalSize(), 13); c1.markDelete(new PositionImpl(p2.getLedgerId(), -1)); // Wait for background trimming Thread.sleep(400); assertEquals(ledger.getTotalSize(), 5); }