boolean hasMoreEntries(PositionImpl position) { PositionImpl lastPos = lastConfirmedEntry; boolean result = position.compareTo(lastPos) <= 0; if (log.isDebugEnabled()) { log.debug("[{}] hasMoreEntries: pos={} lastPos={} res={}", name, position, lastPos, result); } return result; }
/** * Get the last position written in the managed ledger, alongside with the associated counter. */ Pair<PositionImpl, Long> getLastPositionAndCounter() { PositionImpl pos; long count; do { pos = lastConfirmedEntry; count = ENTRIES_ADDED_COUNTER_UPDATER.get(this); // Ensure no entry was written while reading the two values } while (pos.compareTo(lastConfirmedEntry) != 0); return Pair.of(pos, count); }
@Override public void seek(Position newReadPositionInt) { checkArgument(newReadPositionInt instanceof PositionImpl); PositionImpl newReadPosition = (PositionImpl) newReadPositionInt; lock.writeLock().lock(); try { if (newReadPosition.compareTo(markDeletePosition) <= 0) { // Make sure the newReadPosition comes after the mark delete position newReadPosition = ledger.getNextValidPosition(markDeletePosition); } PositionImpl oldReadPosition = readPosition; readPosition = newReadPosition; } finally { lock.writeLock().unlock(); } }
boolean hasMoreEntries(PositionImpl position) { PositionImpl lastPositionInLedger = ledger.getLastPosition(); if (position.compareTo(lastPositionInLedger) <= 0) { return getNumberOfEntries(Range.closed(position, lastPositionInLedger)) > 0; } return false; }
private void recoverCursor(PositionImpl mdPosition) { Pair<PositionImpl, Long> lastEntryAndCounter = ledger.getLastPositionAndCounter(); this.readPosition = ledger.getNextValidPosition(mdPosition); markDeletePosition = mdPosition; // Initialize the counter such that the difference between the messages written on the ML and the // messagesConsumed is equal to the current backlog (negated). long initialBacklog = readPosition.compareTo(lastEntryAndCounter.getLeft()) < 0 ? ledger.getNumberOfEntries(Range.closed(readPosition, lastEntryAndCounter.getLeft())) : 0; messagesConsumedCounter = lastEntryAndCounter.getRight() - initialBacklog; }
@Override public long getNumberOfEntriesSinceFirstNotAckedMessage() { // sometimes for already caught up consumer: due to race condition markDeletePosition > readPosition. so, // validate it before preparing range PositionImpl markDeletePosition = this.markDeletePosition; PositionImpl readPosition = this.readPosition; return (markDeletePosition.compareTo(readPosition) < 0) ? ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, readPosition)) : 0; }
/** * Push the item down towards the bottom of the tree (highest reading position). */ private void siftDown(final Item item) { while (true) { Item j = null; Item right = getRight(item); if (right != null && right.position.compareTo(item.position) < 0) { Item left = getLeft(item); if (left != null && left.position.compareTo(right.position) < 0) { j = left; } else { j = right; } } else { Item left = getLeft(item); if (left != null && left.position.compareTo(item.position) < 0) { j = left; } } if (j != null) { swap(item, j); } else { break; } } }
/** * Get the first position written in the managed ledger, alongside with the associated counter */ Pair<PositionImpl, Long> getFirstPositionAndCounter() { PositionImpl pos; long count; Pair<PositionImpl, Long> lastPositionAndCounter; do { pos = getFirstPosition(); lastPositionAndCounter = getLastPositionAndCounter(); count = lastPositionAndCounter.getRight() - getNumberOfEntries(Range.openClosed(pos, lastPositionAndCounter.getLeft())); } while (pos.compareTo(getFirstPosition()) != 0 || lastPositionAndCounter.getLeft().compareTo(getLastPosition()) != 0); return Pair.of(pos, count); }
/** * Push the item up towards the the root of the tree (lowest reading position). */ private void siftUp(Item item) { Item parent = getParent(item); while (item.idx > 0 && parent.position.compareTo(item.position) > 0) { swap(item, parent); parent = getParent(item); } }
@Test public void comparisons() { PositionImpl pos1_1 = new PositionImpl(1, 1); PositionImpl pos2_5 = new PositionImpl(2, 5); PositionImpl pos10_0 = new PositionImpl(10, 0); PositionImpl pos10_1 = new PositionImpl(10, 1); assertEquals(0, pos1_1.compareTo(pos1_1)); assertEquals(-1, pos1_1.compareTo(pos2_5)); assertEquals(-1, pos1_1.compareTo(pos10_0)); assertEquals(-1, pos1_1.compareTo(pos10_1)); assertEquals(+1, pos2_5.compareTo(pos1_1)); assertEquals(0, pos2_5.compareTo(pos2_5)); assertEquals(-1, pos2_5.compareTo(pos10_0)); assertEquals(-1, pos2_5.compareTo(pos10_1)); assertEquals(+1, pos10_0.compareTo(pos1_1)); assertEquals(+1, pos10_0.compareTo(pos2_5)); assertEquals(0, pos10_0.compareTo(pos10_0)); assertEquals(-1, pos10_0.compareTo(pos10_1)); assertEquals(+1, pos10_1.compareTo(pos1_1)); assertEquals(+1, pos10_1.compareTo(pos2_5)); assertEquals(+1, pos10_1.compareTo(pos10_0)); assertEquals(0, pos10_1.compareTo(pos10_1)); }
@Override public boolean hasMoreEntries() { // If writer and reader are on the same ledger, we just need to compare the entry id to know if we have more // entries. // If they are on different ledgers we have 2 cases : // * Writer pointing to valid entry --> should return true since we have available entries // * Writer pointing to "invalid" entry -1 (meaning no entries in that ledger) --> Need to check if the reader // is // at the last entry in the previous ledger PositionImpl writerPosition = ledger.getLastPosition(); if (writerPosition.getEntryId() != -1) { return readPosition.compareTo(writerPosition) <= 0; } else { // Fall back to checking the number of entries to ensure we are at the last entry in ledger and no ledgers // are in the middle return getNumberOfEntries() > 0; } }
/** * Signal that a cursor position has been updated and that the container must re-order the cursor list. * * @param cursor * @return a pair of positions, representing the previous slowest consumer and the new slowest consumer (after the * update). */ public Pair<PositionImpl, PositionImpl> cursorUpdated(ManagedCursor cursor, Position newPosition) { checkNotNull(cursor); long stamp = rwLock.writeLock(); try { Item item = cursors.get(cursor.getName()); if (item == null) { return null; } PositionImpl previousSlowestConsumer = heap.get(0).position; // When the cursor moves forward, we need to push it toward the // bottom of the tree and push it up if a reset was done item.position = (PositionImpl) newPosition; if (item.idx == 0 || getParent(item).position.compareTo(item.position) <= 0) { siftDown(item); } else { siftUp(item); } PositionImpl newSlowestConsumer = heap.get(0).position; return Pair.of(previousSlowestConsumer, newSlowestConsumer); } finally { rwLock.unlockWrite(stamp); } }
void updateCursor(ManagedCursorImpl cursor, PositionImpl newPosition) { Pair<PositionImpl, PositionImpl> pair = cursors.cursorUpdated(cursor, newPosition); if (pair == null) { // Cursor has been removed in the meantime trimConsumedLedgersInBackground(); return; } PositionImpl previousSlowestReader = pair.getLeft(); PositionImpl currentSlowestReader = pair.getRight(); if (previousSlowestReader.compareTo(currentSlowestReader) == 0) { // The slowest consumer has not changed position. Nothing to do right now return; } // Only trigger a trimming when switching to the next ledger if (previousSlowestReader.getLedgerId() != newPosition.getLedgerId()) { trimConsumedLedgersInBackground(); } }
long getNumIndividualDeletedEntriesToSkip(long numEntries) { long totalEntriesToSkip = 0; long deletedMessages = 0; lock.readLock().lock(); try { PositionImpl startPosition = markDeletePosition; PositionImpl endPosition = null; for (Range<PositionImpl> r : individualDeletedMessages.asRanges()) { endPosition = r.lowerEndpoint(); if (startPosition.compareTo(endPosition) <= 0) { Range<PositionImpl> range = Range.openClosed(startPosition, endPosition); long entries = ledger.getNumberOfEntries(range); if (totalEntriesToSkip + entries >= numEntries) { break; } totalEntriesToSkip += entries; deletedMessages += ledger.getNumberOfEntries(r); startPosition = r.upperEndpoint(); } else { if (log.isDebugEnabled()) { log.debug("[{}] deletePosition {} moved ahead without clearing deleteMsgs {} for cursor {}", ledger.getName(), markDeletePosition, r.lowerEndpoint(), name); } } } } finally { lock.readLock().unlock(); } return deletedMessages; }
@Override public long getNumberOfEntries() { if (readPosition.compareTo(ledger.getLastPosition().getNext()) > 0) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] Read position {} is ahead of last position {}. There are no entries to read", ledger.getName(), name, readPosition, ledger.getLastPosition()); } return 0; } else { return getNumberOfEntries(Range.closedOpen(readPosition, ledger.getLastPosition().getNext())); } }
public ReadOnlyCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, PositionImpl startPosition, String cursorName) { super(bookkeeper, config, ledger, cursorName); if (startPosition.equals(PositionImpl.earliest)) { readPosition = ledger.getFirstPosition().getNext(); } else { readPosition = startPosition; } if (ledger.getLastPosition().compareTo(readPosition) <= 0) { messagesConsumedCounter = 0; } else { messagesConsumedCounter = -getNumberOfEntries(Range.closed(readPosition, ledger.getLastPosition())); } this.state = State.NoLedger; }
/** * If we fail to recover the cursor ledger, we want to still open the ML and rollback. * * @param info */ private PositionImpl getRollbackPosition(ManagedCursorInfo info) { PositionImpl firstPosition = ledger.getFirstPosition(); PositionImpl snapshottedPosition = new PositionImpl(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId()); if (firstPosition == null) { // There are no ledgers in the ML, any position is good return snapshottedPosition; } else if (snapshottedPosition.compareTo(firstPosition) < 0) { // The snapshotted position might be pointing to a ledger that was already deleted return firstPosition; } else { return snapshottedPosition; } }
/** * Checks given position is part of deleted-range and returns next position of upper-end as all the messages are * deleted up to that point. * * @param position * @return next available position */ public PositionImpl getNextAvailablePosition(PositionImpl position) { Range<PositionImpl> range = individualDeletedMessages.rangeContaining(position); if (range != null) { PositionImpl nextPosition = range.upperEndpoint().getNext(); return (nextPosition != null && nextPosition.compareTo(position) > 0) ? nextPosition : position.getNext(); } return position.getNext(); }
@Override public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, ReadEntryCallback callback, Object ctx) { checkArgument(n > 0); if (STATE_UPDATER.get(this) == State.Closed) { callback.readEntryFailed(new ManagedLedgerException("Cursor was already closed"), ctx); return; } PositionImpl startPosition = ledger.getNextValidPosition(markDeletePosition); PositionImpl endPosition = ledger.getLastPosition(); if (startPosition.compareTo(endPosition) <= 0) { long numOfEntries = getNumberOfEntries(Range.closed(startPosition, endPosition)); if (numOfEntries >= n) { long deletedMessages = 0; if (deletedEntries == IndividualDeletedEntries.Exclude) { deletedMessages = getNumIndividualDeletedEntriesToSkip(n); } PositionImpl positionAfterN = ledger.getPositionAfterN(markDeletePosition, n + deletedMessages, PositionBound.startExcluded); ledger.asyncReadEntry(positionAfterN, callback, ctx); } else { callback.readEntryComplete(null, ctx); } } else { callback.readEntryComplete(null, ctx); } }
private void recoveredCursor(PositionImpl position, Map<String, Long> properties, LedgerHandle recoveredFromCursorLedger) { // if the position was at a ledger that didn't exist (since it will be deleted if it was previously empty), // we need to move to the next existing ledger if (!ledger.ledgerExists(position.getLedgerId())) { Long nextExistingLedger = ledger.getNextValidLedger(position.getLedgerId()); if (nextExistingLedger == null) { log.info("[{}] [{}] Couldn't find next next valid ledger for recovery {}", ledger.getName(), name, position); } position = nextExistingLedger != null ? PositionImpl.get(nextExistingLedger, -1) : position; } if (position.compareTo(ledger.getLastPosition()) > 0) { log.warn("[{}] [{}] Current position {} is ahead of last position {}", ledger.getName(), name, position, ledger.getLastPosition()); position = PositionImpl.get(ledger.getLastPosition()); } log.info("[{}] Cursor {} recovered to position {}", ledger.getName(), name, position); messagesConsumedCounter = -getNumberOfEntries(Range.openClosed(position, ledger.getLastPosition())); markDeletePosition = position; readPosition = ledger.getNextValidPosition(position); lastMarkDeleteEntry = new MarkDeleteEntry(markDeletePosition, properties, null, null); // assign cursor-ledger so, it can be deleted when new ledger will be switched this.cursorLedger = recoveredFromCursorLedger; STATE_UPDATER.set(this, State.NoLedger); }