minOffset = Math.max(minOffset, consumeQueue.getMinOffsetInQueue()); maxOffset = Math.min(maxOffset, consumeQueue.getMaxOffsetInQueue()); SelectMappedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(nextOffset); if (bufferConsumeQueue != null) { try {
this.minLogicOffset = result.getMappedFile().getFileFromOffset() + i; log.info("Compute logical min offset: {}, topic: {}, queueId: {}", this.getMinOffsetInQueue(), this.topic, this.queueId); if (isExtAddr(tagsCode)) { minExtAddr = tagsCode; if (isExtReadEnable()) { this.consumeQueueExt.truncateByMinAddress(minExtAddr);
continue; ConsumeQueue logic = new ConsumeQueue( topic, queueId, this); this.putConsumeQueue(topic, queueId, logic); if (!logic.load()) { return false;
public long getMessageTotalInQueue() { return this.getMaxOffsetInQueue() - this.getMinOffsetInQueue(); }
private void recoverTopicQueueTable() { HashMap<String/* topic-queueid */, Long/* offset */> table = new HashMap<String, Long>(1024); long minPhyOffset = this.commitLog.getMinOffset(); for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) { for (ConsumeQueue logic : maps.values()) { String key = logic.getTopic() + "-" + logic.getQueueId(); table.put(key, logic.getMaxOffsetInQueue()); logic.correctMinOffset(minPhyOffset); } } this.commitLog.setTopicQueueTable(table); }
@Override public long getEarliestMessageTime(String topic, int queueId) { ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); if (logicQueue != null) { long minLogicOffset = logicQueue.getMinLogicOffset(); SelectMappedBufferResult result = logicQueue.getIndexBuffer(minLogicOffset / ConsumeQueue.CQ_STORE_UNIT_SIZE); return getStoreTime(result); } return -1; }
response.setBody(body.encode()); body.setMaxQueueIndex(consumeQueue.getMaxOffsetInQueue()); body.setMinQueueIndex(consumeQueue.getMinOffsetInQueue()); SelectMappedBufferResult result = consumeQueue.getIndexBuffer(requestHeader.getIndex()); if (result == null) { response.setRemark(String.format("Index %d of %d@%s is not exist!", requestHeader.getIndex(), requestHeader.getQueueId(), requestHeader.getTopic())); one.setTagsCode(result.getByteBuffer().getLong()); if (!consumeQueue.isExtAddr(one.getTagsCode())) { queues.add(one); continue; ConsumeQueueExt.CqExtUnit cqExtUnit = consumeQueue.getExt(one.getTagsCode()); if (cqExtUnit != null) { one.setExtendDataJson(JSON.toJSONString(cqExtUnit));
SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(this.offset); if (bufferCQ != null) { try { long tagsCode = bufferCQ.getByteBuffer().getLong(); if (cq.isExtAddr(tagsCode)) { if (cq.getExt(tagsCode, cqExtUnit)) { tagsCode = cqExtUnit.getTagsCode(); } else { long cqMinOffset = cq.getMinOffsetInQueue(); if (offset < cqMinOffset) { failScheduleOffset = cqMinOffset; log.error("schedule CQ offset invalid. offset=" + offset + ", cqMinOffset=" + cqMinOffset + ", queueId=" + cq.getQueueId());
while (itQT.hasNext()) { Entry<Integer, ConsumeQueue> nextQT = itQT.next(); long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset(); nextQT.getValue().getTopic(), nextQT.getValue().getQueueId(), nextQT.getValue().getMaxPhysicOffset(), nextQT.getValue().getMinLogicOffset()); } else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) { log.info( maxCLOffsetInConsumeQueue); DefaultMessageStore.this.commitLog.removeQueueFromTopicQueueTable(nextQT.getValue().getTopic(), nextQT.getValue().getQueueId()); nextQT.getValue().destroy(); itQT.remove();
long queueOffsetOrigin = getMaxOffsetInQueue(); this.maxPhysicOffset = offset; if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; topic, queueId, getMaxOffsetInQueue(), queueOffsetOrigin); mappedFile.truncateDirty(mappedFile.getWrotePosition()); return; mappedFile.setFlushedPosition(pos); this.maxPhysicOffset = offset; if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; topic, queueId, getMaxOffsetInQueue(), queueOffsetOrigin); return; topic, queueId, getMaxOffsetInQueue(), queueOffsetOrigin); return; topic, queueId, getMaxOffsetInQueue(), queueOffsetOrigin); if (isExtReadEnable()) { this.consumeQueueExt.truncateByMaxAddress(maxExtAddr);
for (int i = 0; i < maxRetries && canWrite; i++) { long tagsCode = request.getTagsCode(); if (isExtWriteEnable()) { ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit(); cqExtUnit.setFilterBitMap(request.getBitMap()); if (isExtAddr(extAddr)) { tagsCode = extAddr; } else { boolean result = this.putMessagePositionInfo(request.getCommitLogOffset(), request.getMsgSize(), tagsCode, request.getConsumeQueueOffset()); if (result) {
this.maxPhysicOffset = offset; if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; mappedFile.setFlushedPosition(pos); this.maxPhysicOffset = offset; if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; if (isExtReadEnable()) { this.consumeQueueExt.truncateByMaxAddress(maxExtAddr);
@Override public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) { ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); if (logicQueue != null) { SelectMappedBufferResult result = logicQueue.getIndexBuffer(consumeQueueOffset); return getStoreTime(result); } return -1; }
@Override public int cleanUnusedTopic(Set<String> topics) { Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator(); while (it.hasNext()) { Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next(); String topic = next.getKey(); if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) { ConcurrentMap<Integer, ConsumeQueue> queueTable = next.getValue(); for (ConsumeQueue cq : queueTable.values()) { cq.destroy(); log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned", cq.getTopic(), cq.getQueueId() ); this.commitLog.removeQueueFromTopicQueueTable(cq.getTopic(), cq.getQueueId()); } it.remove(); log.info("cleanUnusedTopic: {},topic destroyed", topic); } } return 0; }
public long getMaxOffsetInQueue(String topic, int queueId) { ConsumeQueue logic = this.findConsumeQueue(topic, queueId); if (logic != null) { long offset = logic.getMaxOffsetInQueue(); return offset; } return 0; }
public long getMinOffsetInQueue(String topic, int queueId) { ConsumeQueue logic = this.findConsumeQueue(topic, queueId); if (logic != null) { return logic.getMinOffsetInQueue(); } return -1; }
private void checkSelf() { this.commitLog.checkSelf(); Iterator<Entry<String, ConcurrentMap<Integer, ConsumeQueue>>> it = this.consumeQueueTable.entrySet().iterator(); while (it.hasNext()) { Entry<String, ConcurrentMap<Integer, ConsumeQueue>> next = it.next(); Iterator<Entry<Integer, ConsumeQueue>> itNext = next.getValue().entrySet().iterator(); while (itNext.hasNext()) { Entry<Integer, ConsumeQueue> cq = itNext.next(); cq.getValue().checkSelf(); } } }
public int deleteExpiredFile(long offset) { int cnt = this.mappedFileQueue.deleteExpiredFileByOffset(offset, CQ_STORE_UNIT_SIZE); this.correctMinOffset(offset); return cnt; }
public void destroyLogics() { for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) { for (ConsumeQueue logic : maps.values()) { logic.destroy(); } } }
private void deleteExpiredFiles() { int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); if (minOffset > this.lastPhysicalMinOffset) { this.lastPhysicalMinOffset = minOffset; ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable; for (ConcurrentMap<Integer, ConsumeQueue> maps : tables.values()) { for (ConsumeQueue logic : maps.values()) { int deleteCount = logic.deleteExpiredFile(minOffset); if (deleteCount > 0 && deleteLogicsFilesInterval > 0) { try { Thread.sleep(deleteLogicsFilesInterval); } catch (InterruptedException ignored) { } } } } DefaultMessageStore.this.indexService.deleteExpiredFile(minOffset); } }