@Override public long getNumberOfActiveEntries() { long totalEntries = getNumberOfEntries(); PositionImpl pos = cursors.getSlowestReaderPosition(); if (pos == null) { // If there are no consumers, there are no active entries return 0; } else { // The slowest consumer will be in the first ledger in the list. We need to subtract the entries it has // already consumed in order to get the active entries count. return totalEntries - (pos.getEntryId() + 1); } }
public static EntryImpl create(PositionImpl position, ByteBuf data) { EntryImpl entry = RECYCLER.get(); entry.ledgerId = position.getLedgerId(); entry.entryId = position.getEntryId(); entry.data = data; entry.data.retain(); entry.setRefCnt(1); return entry; }
void asyncReadEntry(PositionImpl position, ReadEntryCallback callback, Object ctx) { LedgerHandle currentLedger = this.currentLedger; if (log.isDebugEnabled()) { log.debug("[{}] Reading entry ledger {}: {}", name, position.getLedgerId(), position.getEntryId()); } if (position.getLedgerId() == currentLedger.getId()) { LedgerHandle ledger = currentLedger; asyncReadEntry(ledger, position, callback, ctx); } else { getLedgerHandle(position.getLedgerId()).thenAccept(ledger -> { asyncReadEntry(ledger, position, callback, ctx); }).exceptionally(ex -> { log.error("[{}] Error opening ledger for reading at position {} - {}", name, position, ex.getMessage()); callback.readEntryFailed(ManagedLedgerException.getManagedLedgerException(ex.getCause()), ctx); return null; }); } }
long estimateBacklogFromPosition(PositionImpl pos) { synchronized (this) { LedgerInfo ledgerInfo = ledgers.get(pos.getLedgerId()); if (ledgerInfo == null) { return getTotalSize(); // position no longer in managed ledger, so return total size } long sizeBeforePosLedger = ledgers.values().stream().filter(li -> li.getLedgerId() < pos.getLedgerId()) .mapToLong(li -> li.getSize()).sum(); long size = getTotalSize() - sizeBeforePosLedger; if (pos.getLedgerId() == currentLedger.getId()) { return size - consumedLedgerSize(currentLedgerSize, currentLedgerEntries, pos.getEntryId()); } else { return size - consumedLedgerSize(ledgerInfo.getSize(), ledgerInfo.getEntries(), pos.getEntryId()); } } }
@Override public boolean hasMoreEntries() { // If writer and reader are on the same ledger, we just need to compare the entry id to know if we have more // entries. // If they are on different ledgers we have 2 cases : // * Writer pointing to valid entry --> should return true since we have available entries // * Writer pointing to "invalid" entry -1 (meaning no entries in that ledger) --> Need to check if the reader // is // at the last entry in the previous ledger PositionImpl writerPosition = ledger.getLastPosition(); if (writerPosition.getEntryId() != -1) { return readPosition.compareTo(writerPosition) <= 0; } else { // Fall back to checking the number of entries to ensure we are at the last entry in ledger and no ledgers // are in the middle return getNumberOfEntries() > 0; } }
private List<MLDataFormats.MessageRange> buildIndividualDeletedMessageRanges() { lock.readLock().lock(); try { if (individualDeletedMessages.isEmpty()) { return Collections.emptyList(); } MLDataFormats.NestedPositionInfo.Builder nestedPositionBuilder = MLDataFormats.NestedPositionInfo .newBuilder(); MLDataFormats.MessageRange.Builder messageRangeBuilder = MLDataFormats.MessageRange.newBuilder(); return individualDeletedMessages.asRanges().stream().limit(config.getMaxUnackedRangesToPersist()) .map(positionRange -> { PositionImpl p = positionRange.lowerEndpoint(); nestedPositionBuilder.setLedgerId(p.getLedgerId()); nestedPositionBuilder.setEntryId(p.getEntryId()); messageRangeBuilder.setLowerEndpoint(nestedPositionBuilder.build()); p = positionRange.upperEndpoint(); nestedPositionBuilder.setLedgerId(p.getLedgerId()); nestedPositionBuilder.setEntryId(p.getEntryId()); messageRangeBuilder.setUpperEndpoint(nestedPositionBuilder.build()); return messageRangeBuilder.build(); }).collect(Collectors.toList()); } finally { lock.readLock().unlock(); } }
private ManagedLedgerInfo getManagedLedgerInfo() { ManagedLedgerInfo.Builder mlInfo = ManagedLedgerInfo.newBuilder().addAllLedgerInfo(ledgers.values()); if (state == State.Terminated) { mlInfo.setTerminatedPosition(NestedPositionInfo.newBuilder().setLedgerId(lastConfirmedEntry.getLedgerId()) .setEntryId(lastConfirmedEntry.getEntryId())); } return mlInfo.build(); }
@Test public void simpleTest() { PositionImpl pos = new PositionImpl(1, 2); assertEquals(pos.getLedgerId(), 1); assertEquals(pos.getEntryId(), 2); assertEquals(pos, new PositionImpl(1, 2)); assertFalse(pos.equals(new PositionImpl(1, 3))); assertFalse(pos.equals(new PositionImpl(3, 2))); assertFalse(pos.equals("1:2")); }
@Override public AtomicBoolean call() throws Exception { barrier.await(); final AtomicBoolean moveStatus = new AtomicBoolean(false); CountDownLatch countDownLatch = new CountDownLatch(1); final PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - (5 * idx)); cursor.asyncResetCursor(resetPosition, new AsyncCallbacks.ResetCursorCallback() { @Override public void resetComplete(Object ctx) { moveStatus.set(true); PositionImpl pos = (PositionImpl) ctx; log.info("move to [{}] completed for consumer [{}]", pos.toString(), idx); countDownLatch.countDown(); } @Override public void resetFailed(ManagedLedgerException exception, Object ctx) { moveStatus.set(false); PositionImpl pos = (PositionImpl) ctx; log.warn("move to [{}] failed for consumer [{}]", pos.toString(), idx); countDownLatch.countDown(); } }); countDownLatch.await(); assertTrue(cursor.getReadPosition().equals(resetPosition)); cursor.close(); return moveStatus; } }));
@Test(timeOut = 20000) public void spanningMultipleLedgers() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(10); ManagedLedger ledger = factory.open("my_test_ledger", config); assertEquals(ledger.getNumberOfEntries(), 0); assertEquals(ledger.getTotalSize(), 0); ManagedCursor cursor = ledger.openCursor("c1"); for (int i = 0; i < 11; i++) ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding)); List<Entry> entries = cursor.readEntries(100); assertEquals(entries.size(), 11); assertEquals(cursor.hasMoreEntries(), false); PositionImpl first = (PositionImpl) entries.get(0).getPosition(); PositionImpl last = (PositionImpl) entries.get(entries.size() - 1).getPosition(); entries.forEach(e -> e.release()); log.info("First={} Last={}", first, last); assertTrue(first.getLedgerId() < last.getLedgerId()); assertEquals(first.getEntryId(), 0); assertEquals(last.getEntryId(), 0); // Read again, from next ledger id entries = cursor.readEntries(100); assertEquals(entries.size(), 0); assertEquals(cursor.hasMoreEntries(), false); ledger.close(); }
@Test(timeOut = 20000) void seekPosition2() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); PositionImpl seekPosition = (PositionImpl) ledger.addEntry("dummy-entry-3".getBytes(Encoding)); ledger.addEntry("dummy-entry-4".getBytes(Encoding)); ledger.addEntry("dummy-entry-5".getBytes(Encoding)); ledger.addEntry("dummy-entry-6".getBytes(Encoding)); cursor.seek(new PositionImpl(seekPosition.getLedgerId(), seekPosition.getEntryId())); }
@Test public void hashes() throws Exception { PositionImpl p1 = new PositionImpl(5, 15); PositionImpl p2 = new PositionImpl(PositionInfo.parseFrom(p1.getPositionInfo().toByteArray())); assertEquals(p2.getLedgerId(), 5); assertEquals(p2.getEntryId(), 15); assertEquals(new PositionImpl(5, 15).hashCode(), p2.hashCode()); } }
@Test(timeOut = 20000) void testResetCursor() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.latest); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); try { cursor.resetCursor(resetPosition); moveStatus.set(true); } catch (Exception e) { log.warn("error in reset cursor", e.getCause()); } assertTrue(moveStatus.get()); assertTrue(cursor.getReadPosition().equals(resetPosition)); cursor.close(); ledger.close(); }
@Test public void testGetNextValidPosition() throws Exception { ManagedLedgerConfig conf = new ManagedLedgerConfig(); conf.setMaxEntriesPerLedger(1); ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetNextValidPosition", conf); ManagedCursor c1 = ledger.openCursor("c1"); PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes()); PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes()); PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes()); assertEquals(ledger.getNextValidPosition((PositionImpl) c1.getMarkDeletedPosition()), p1); assertEquals(ledger.getNextValidPosition(p1), p2); assertEquals(ledger.getNextValidPosition(p3), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1)); }
@Test(timeOut = 20000) void seekPosition() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); cursor.seek(new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 1)); }
@Test(timeOut = 20000) void testResetCursor() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.openCursor("trc1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); final AtomicBoolean moveStatus = new AtomicBoolean(false); PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2); try { cursor.resetCursor(resetPosition); moveStatus.set(true); } catch (Exception e) { log.warn("error in reset cursor", e.getCause()); } assertTrue(moveStatus.get()); assertTrue(cursor.getReadPosition().equals(resetPosition)); cursor.close(); ledger.close(); }
@Test(timeOut = 20000) void seekPosition3() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("c1"); ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl seekPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); Position entry5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding)); Position entry6 = ledger.addEntry("dummy-entry-6".getBytes(Encoding)); cursor.seek(new PositionImpl(seekPosition.getLedgerId(), seekPosition.getEntryId())); assertEquals(cursor.getReadPosition(), seekPosition); List<Entry> entries = cursor.readEntries(1); assertEquals(entries.size(), 1); assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-4"); entries.forEach(e -> e.release()); cursor.seek(entry5.getNext()); assertEquals(cursor.getReadPosition(), entry6); entries = cursor.readEntries(1); assertEquals(entries.size(), 1); assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-6"); entries.forEach(e -> e.release()); }
@Test(timeOut = 20000) void markDeleteSkippingMessage() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.openCursor("c1"); Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl p4 = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); assertEquals(cursor.getNumberOfEntries(), 4); cursor.markDelete(p1); assertEquals(cursor.hasMoreEntries(), true); assertEquals(cursor.getNumberOfEntries(), 3); assertEquals(cursor.getReadPosition(), p2); List<Entry> entries = cursor.readEntries(1); assertEquals(entries.size(), 1); assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-2"); entries.forEach(e -> e.release()); cursor.markDelete(p4); assertEquals(cursor.hasMoreEntries(), false); assertEquals(cursor.getNumberOfEntries(), 0); assertEquals(cursor.getReadPosition(), new PositionImpl(p4.getLedgerId(), p4.getEntryId() + 1)); }
@Test(timeOut = 20000) void markDeleteSkippingMessage() throws Exception { ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10)); ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.earliest); Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding)); Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding)); ledger.addEntry("dummy-entry-3".getBytes(Encoding)); PositionImpl p4 = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding)); assertEquals(cursor.getNumberOfEntries(), 4); cursor.markDelete(p1); assertEquals(cursor.hasMoreEntries(), true); assertEquals(cursor.getNumberOfEntries(), 3); assertEquals(cursor.getReadPosition(), p2); List<Entry> entries = cursor.readEntries(1); assertEquals(entries.size(), 1); assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-2"); entries.forEach(e -> e.release()); cursor.markDelete(p4); assertEquals(cursor.hasMoreEntries(), false); assertEquals(cursor.getNumberOfEntries(), 0); assertEquals(cursor.getReadPosition(), new PositionImpl(p4.getLedgerId(), p4.getEntryId() + 1)); }
protected void asyncReadEntry(ReadHandle ledger, PositionImpl position, ReadEntryCallback callback, Object ctx) { long timeout = config.getReadEntryTimeoutSeconds(); boolean checkTimeout = timeout > 0; if (checkTimeout) { // set readOpCount to uniquely validate if ReadEntryCallbackWrapper is already recycled long readOpCount = READ_OP_COUNT_UPDATER.incrementAndGet(this); ReadEntryCallbackWrapper readCallback = ReadEntryCallbackWrapper.create(name, position.getLedgerId(), position.getEntryId(), callback, readOpCount, ctx); final ScheduledFuture<?> task = scheduledExecutor.schedule(() -> { // validate ReadEntryCallbackWrapper object is not recycled by bk-client callback (by validating // readOpCount) and fail the callback if read is not completed yet if (readCallback.readOpCount == readOpCount && ReadEntryCallbackWrapper.READ_COMPLETED_UPDATER.get(readCallback) == FALSE) { log.warn("[{}]-{} read entry timeout for {} after {} sec", this.name, ledger.getId(), position, timeout); readCallback.readEntryFailed(createManagedLedgerException(BKException.Code.TimeoutException), readOpCount); } }, timeout, TimeUnit.SECONDS); readCallback.task = task; entryCache.asyncReadEntry(ledger, position, readCallback, readOpCount); } else { entryCache.asyncReadEntry(ledger, position, callback, ctx); } }