@Override public void operationComplete() { assertEquals(cursor.getMarkDeletedPosition(), lastPosition); assertEquals(cursor.getReadPosition(), nextPosition); assertEquals(cursor.getNumberOfEntries(), 0L); }
@Override public void readEntriesComplete(List<Entry> returnedEntries, Object ctx) { // Filter the returned entries for individual deleted messages int entriesSize = returnedEntries.size(); final PositionImpl lastPosition = (PositionImpl) returnedEntries.get(entriesSize - 1).getPosition(); if (log.isDebugEnabled()) { log.debug("[{}][{}] Read entries succeeded batch_size={} cumulative_size={} requested_count={}", cursor.ledger.getName(), cursor.getName(), returnedEntries.size(), entries.size(), count); } List<Entry> filteredEntries = cursor.filterReadEntries(returnedEntries); entries.addAll(filteredEntries); // if entries have been filtered out then try to skip reading of already deletedMessages in that range final Position nexReadPosition = entriesSize != filteredEntries.size() ? cursor.getNextAvailablePosition(lastPosition) : lastPosition.getNext(); updateReadPosition(nexReadPosition); checkReadCompletion(); }
cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName); cursor.recover(new VoidCallback() { @Override public void operationComplete() {
void initialize(PositionImpl position, final VoidCallback callback) { recoveredCursor(position, Collections.emptyMap(), null); if (log.isDebugEnabled()) { log.debug("[{}] Consumer {} cursor initialized with counters: consumed {} mdPos {} rdPos {}", ledger.getName(), name, messagesConsumedCounter, markDeletePosition, readPosition); } createNewMetadataLedger(new VoidCallback() { @Override public void operationComplete() { STATE_UPDATER.set(ManagedCursorImpl.this, State.Open); callback.operationComplete(); } @Override public void operationFailed(ManagedLedgerException exception) { callback.operationFailed(exception); } }); }
ManagedCursorImpl cursor = (ManagedCursorImpl) c; CursorStats cs = new CursorStats(); cs.markDeletePosition = cursor.getMarkDeletedPosition().toString(); cs.readPosition = cursor.getReadPosition().toString(); cs.waitingReadOp = cursor.hasPendingReadRequest(); cs.pendingReadOps = cursor.getPendingReadOpsCount(); cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter(); cs.cursorLedger = cursor.getCursorLedger(); cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry(); cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages(); cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp()); cs.state = cursor.getState(); cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage(); cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange(); cs.properties = cursor.getProperties(); stats.cursors.put(cursor.getName(), cs); }); return stats;
ManagedCursorImpl cursor = (ManagedCursorImpl) c; CursorStats cs = new CursorStats(); cs.markDeletePosition = cursor.getMarkDeletedPosition().toString(); cs.readPosition = cursor.getReadPosition().toString(); cs.waitingReadOp = cursor.hasPendingReadRequest(); cs.pendingReadOps = cursor.getPendingReadOpsCount(); cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter(); cs.cursorLedger = cursor.getCursorLedger(); cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry(); cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages(); cs.lastLedgerSwitchTimestamp = DATE_FORMAT.format(Instant.ofEpochMilli(cursor.getLastLedgerSwitchTimestamp())); cs.state = cursor.getState(); stats.cursors.put(cursor.getName(), cs); }); return stats;
boolean hasMoreEntries(PositionImpl position) { PositionImpl lastPositionInLedger = ledger.getLastPosition(); if (position.compareTo(lastPositionInLedger) <= 0) { return getNumberOfEntries(Range.closed(position, lastPositionInLedger)) > 0; } return false; }
@Override public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { cursor.readOperationCompleted(); })); } else if (cursor.config.isAutoSkipNonRecoverableData() && exception instanceof NonRecoverableLedgerException) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", cursor.ledger.getName(), cursor.getName(), readPosition, exception.getMessage()); final Position nexReadPosition = cursor.getNextLedgerPosition(readPosition.getLedgerId()); if (!(exception instanceof TooManyRequestsException)) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", cursor.ledger.getName(), cursor.getName(), readPosition, exception.getMessage()); } else { if (log.isDebugEnabled()) { log.debug("[{}][{}] read throttled failed from ledger at position:{}", cursor.ledger.getName(), cursor.getName(), readPosition);
void createNewMetadataLedger() { createNewMetadataLedger(new VoidCallback() { @Override public void operationComplete() { // We now have a new ledger where we can write synchronized (pendingMarkDeleteOps) { flushPendingMarkDeletes(); // Resume normal mark-delete operations STATE_UPDATER.set(ManagedCursorImpl.this, State.Open); } } @Override public void operationFailed(ManagedLedgerException exception) { log.error("[{}][{}] Metadata ledger creation failed", ledger.getName(), name, exception); synchronized (pendingMarkDeleteOps) { while (!pendingMarkDeleteOps.isEmpty()) { MarkDeleteEntry entry = pendingMarkDeleteOps.poll(); entry.callback.markDeleteFailed(exception, entry.ctx); } // At this point we don't have a ledger ready STATE_UPDATER.set(ManagedCursorImpl.this, State.NoLedger); } } }); }
/** * * @return Whether the cursor responded to the notification */ void notifyEntriesAvailable() { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Received ml notification", ledger.getName(), name); } OpReadEntry opReadEntry = WAITING_READ_OP_UPDATER.getAndSet(this, null); if (opReadEntry != null) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Received notification of new messages persisted, reading at {} -- last: {}", ledger.getName(), name, opReadEntry.readPosition, ledger.lastConfirmedEntry); log.debug("[{}] Consumer {} cursor notification: other counters: consumed {} mdPos {} rdPos {}", ledger.getName(), name, messagesConsumedCounter, markDeletePosition, readPosition); } PENDING_READ_OPS_UPDATER.incrementAndGet(this); opReadEntry.readPosition = (PositionImpl) getReadPosition(); ledger.asyncReadEntries(opReadEntry); } else { // No one is waiting to be notified. Ignore if (log.isDebugEnabled()) { log.debug("[{}] [{}] Received notification but had no pending read operation", ledger.getName(), name); } } }
@Override public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, ReadEntryCallback callback, Object ctx) { checkArgument(n > 0); if (STATE_UPDATER.get(this) == State.Closed) { callback.readEntryFailed(new ManagedLedgerException("Cursor was already closed"), ctx); return; } PositionImpl startPosition = ledger.getNextValidPosition(markDeletePosition); PositionImpl endPosition = ledger.getLastPosition(); if (startPosition.compareTo(endPosition) <= 0) { long numOfEntries = getNumberOfEntries(Range.closed(startPosition, endPosition)); if (numOfEntries >= n) { long deletedMessages = 0; if (deletedEntries == IndividualDeletedEntries.Exclude) { deletedMessages = getNumIndividualDeletedEntriesToSkip(n); } PositionImpl positionAfterN = ledger.getPositionAfterN(markDeletePosition, n + deletedMessages, PositionBound.startExcluded); ledger.asyncReadEntry(positionAfterN, callback, ctx); } else { callback.readEntryComplete(null, ctx); } } else { callback.readEntryComplete(null, ctx); } }
log.debug("[{}] Creating new cursor: {}", name, cursorName); final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, config, this, cursorName); CompletableFuture<ManagedCursor> cursorFuture = new CompletableFuture<>(); uninitializedCursors.put(cursorName, cursorFuture); cursor.initialize(getLastPosition(), new VoidCallback() { @Override public void operationComplete() {
if (i % 2 == 0) { c1.delete(addedPositions.get(i)); assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2); assertEquals(cursorLedgerId.get(), c1.getCursorLedger()); bkc.asyncOpenLedger(c1.getCursorLedger(), DigestType.CRC32C, "".getBytes(), (rc, lh, ctx) -> { if (rc == BKException.Code.OK) { long lastEntry = lh.getLastAddConfirmed(); c1 = (ManagedCursorImpl) ledger.openCursor("c1"); assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2); List<Entry> entries = c1.readEntries(totalAddEntries); assertEquals(entries.size(), totalAddEntries / 2);
void updateReadPosition(Position newReadPosition) { nextReadPosition = (PositionImpl) newReadPosition; cursor.setReadPosition(nextReadPosition); }
cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName);
@Override public void readEntriesFailed(ManagedLedgerException status, Object ctx) { cursor.readOperationCompleted(); if (!entries.isEmpty()) { // There were already some entries that were read before, we can return them cursor.ledger.getExecutor().submit(safeRun(() -> { callback.readEntriesComplete(entries, ctx); recycle(); })); } else { if (!(status instanceof TooManyRequestsException)) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", cursor.ledger.getName(), cursor.getName(), readPosition, status.getMessage()); } else { if (log.isDebugEnabled()) { log.debug("[{}][{}] read throttled failed from ledger at position:{}", cursor.ledger.getName(), cursor.getName(), readPosition); } } callback.readEntriesFailed(status, ctx); cursor.ledger.mbean.recordReadEntriesError(); recycle(); } }
final PositionImpl position = (PositionImpl) getMarkDeletedPosition(); persistPosition(lh, position, new VoidCallback() { @Override public void operationComplete() {
assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0); List<Entry> entries = c1.readEntries(10); assertEquals(entries.size(), 10); entries.forEach(e -> e.release()); assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0); entries = c2.readEntries(10); assertEquals(entries.size(), 10); c2.setReadPosition(pos); ledger.discardEntriesFromCache(c2, pos); entries.forEach(e -> e.release());
@Override public void clearBacklog() throws InterruptedException, ManagedLedgerException { class Result { ManagedLedgerException exception = null; } final Result result = new Result(); final CountDownLatch counter = new CountDownLatch(1); asyncClearBacklog(new ClearBacklogCallback() { @Override public void clearBacklogComplete(Object ctx) { counter.countDown(); } @Override public void clearBacklogFailed(ManagedLedgerException exception, Object ctx) { result.exception = exception; counter.countDown(); } }, null); if (!counter.await(ManagedLedgerImpl.AsyncOperationTimeoutSeconds, TimeUnit.SECONDS)) { throw new ManagedLedgerException("Timeout during clear backlog operation"); } if (result.exception != null) { throw result.exception; } }
@Override public long getNumberOfEntriesInBacklog() { if (log.isDebugEnabled()) { log.debug("[{}] Consumer {} cursor ml-entries: {} -- deleted-counter: {} other counters: mdPos {} rdPos {}", ledger.getName(), name, ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.get(ledger), messagesConsumedCounter, markDeletePosition, readPosition); } long backlog = ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.get(ledger) - messagesConsumedCounter; if (backlog < 0) { // In some case the counters get incorrect values, fall back to the precise backlog count backlog = getNumberOfEntries(Range.closed(markDeletePosition, ledger.getLastPosition())) - 1; } return backlog; }