public static boolean isRealSubject(final String subject) { return !Strings.isNullOrEmpty(subject) && !isRetrySubject(subject) && !isDeadRetrySubject(subject); }
AckSendQueue(String brokerGroupName, String subject, String group, AckService ackService, BrokerService brokerService, SendMessageBack sendMessageBack, boolean isBroadcast) { this.brokerGroupName = brokerGroupName; this.subject = subject; this.group = group; this.ackService = ackService; this.brokerService = brokerService; this.sendMessageBack = sendMessageBack; this.isBroadcast = isBroadcast; String realSubject = RetrySubjectUtils.isRetrySubject(subject) ? RetrySubjectUtils.getRealSubject(subject) : subject; this.retrySubject = RetrySubjectUtils.buildRetrySubject(realSubject, group); this.deadRetrySubject = RetrySubjectUtils.buildDeadRetrySubject(realSubject, group); this.pullBatchSize = PullSubjectsConfig.get().getPullBatchSize(subject); }
public BaseMessage(String messageId, String subject) { Preconditions.checkArgument(!Strings.isNullOrEmpty(messageId), "message id should not empty"); Preconditions.checkArgument(!Strings.isNullOrEmpty(subject), "message subject should not empty"); Preconditions.checkArgument(messageId.length() <= MAX_MESSAGE_ID_LEN, "messageId长度不能超过" + MAX_MESSAGE_ID_LEN + "个字符"); if (RetrySubjectUtils.isRealSubject(subject)) { Preconditions.checkArgument(subject.length() <= MAX_MESSAGE_ID_LEN, "subject长度不能超过" + MAX_MESSAGE_ID_LEN + "个字符"); } this.messageId = messageId; this.subject = subject; long time = System.currentTimeMillis(); setProperty(keys.qmq_createTime, time); }
public ConsumeParam(String subject, String group, boolean isBroadcast, boolean isConsumeMostOnce, TagType tagType, Set<String> tags, String clientId) { this.subject = subject; this.group = group; this.originSubject = RetrySubjectUtils.isRetrySubject(subject) ? RetrySubjectUtils.getRealSubject(subject) : subject; this.retrySubject = RetrySubjectUtils.buildRetrySubject(originSubject, group); this.consumerId = clientId; this.isBroadcast = isBroadcast; this.isConsumeMostOnce = isConsumeMostOnce; this.tagType = tagType; this.tags = tags; }
private void monitorMessageReceived(long receiveTime, String subject) { if (RetrySubjectUtils.isRetrySubject(subject)) { String[] subjectAndGroup = RetrySubjectUtils.parseSubjectAndGroup(subject); if (subjectAndGroup == null || subjectAndGroup.length != 2) return; QMon.consumerErrorCount(subjectAndGroup[0], subjectAndGroup[1]); return; } if (RetrySubjectUtils.isDeadRetrySubject(subject)) { String[] subjectAndGroup = RetrySubjectUtils.parseSubjectAndGroup(subject); if (subjectAndGroup == null || subjectAndGroup.length != 2) return; QMon.consumerErrorCount(subjectAndGroup[0], subjectAndGroup[1]); QMon.deadLetterQueueCount(subjectAndGroup[0], subjectAndGroup[1]); return; } QMon.receivedMessagesCountInc(subject); QMon.produceTime(subject, System.currentTimeMillis() - receiveTime); }
private void putNeedRetryMessages(String subject, String group, String consumerId, List<SegmentBuffer> needRetryMessages) { try { for (SegmentBuffer buffer : needRetryMessages) { final ByteBuf message = Unpooled.wrappedBuffer(buffer.getBuffer()); final RawMessage rawMessage = QMQSerializer.deserializeRawMessage(message); if (!RetrySubjectUtils.isRetrySubject(subject)) { final String retrySubject = RetrySubjectUtils.buildRetrySubject(subject, group); rawMessage.setSubject(retrySubject); } final PutMessageResult putMessageResult = storage.appendMessage(rawMessage); if (putMessageResult.getStatus() != PutMessageStatus.SUCCESS) { LOG.error("put message error, consumer:{} {} {}, status:{}", subject, group, consumerId, putMessageResult.getStatus()); throw new RuntimeException("put retry message error"); } } } finally { needRetryMessages.forEach(SegmentBuffer::release); } QMon.putNeedRetryMessagesCountInc(subject, group, needRetryMessages.size()); }
public void heartbeat(String consumerId, String subject, String group) { final String realSubject = RetrySubjectUtils.getRealSubject(subject); final String retrySubject = RetrySubjectUtils.buildRetrySubject(realSubject, group); refreshSubscriber(realSubject, group, consumerId); refreshSubscriber(retrySubject, group, consumerId); }
private void logTimes(List<PulledMessage> pulledMessages) { for (PulledMessage pulledMessage : pulledMessages) { int times = pulledMessage.times(); if (times > MAX_MESSAGE_RETRY_THRESHOLD) { LOGGER.warn("这是第 {} 次收到同一条消息,请注意检查逻辑是否有问题. subject={}, msgId={}", times, RetrySubjectUtils.getRealSubject(pulledMessage.getSubject()), pulledMessage.getMessageId()); } } }
public synchronized GetMessageResult pollMessages(final int maxMessages) { enableLagMonitor(); long currentSequence = nextSequence.get(); if (RetrySubjectUtils.isRetrySubject(subject)) { return storage.pollMessages(subject, currentSequence, maxMessages, this::isDelayReached); } else { final GetMessageResult result = storage.pollMessages(subject, currentSequence, maxMessages); long actualSequence = result.getNextBeginOffset() - result.getSegmentBuffers().size(); long delta = actualSequence - currentSequence; if (delta > 0) { QMon.expiredMessagesCountInc(subject, group, delta); LOG.error("next sequence skipped. subject: {}, group: {}, nextSequence: {}, result: {}", subject, group, currentSequence, result); } return result; } }
@Override public synchronized void regist(String subject, String group, RegistParam param) { registPullEntry(subject, group, param, new AlwaysPullStrategy()); registPullEntry(RetrySubjectUtils.buildRetrySubject(subject, group), group, param, new WeightPullStrategy()); }
private synchronized void changeOnOffline(String subject, String group, boolean isOnline, StatusSource src) { final String realSubject = RetrySubjectUtils.getRealSubject(subject); final String retrySubject = RetrySubjectUtils.buildRetrySubject(realSubject, group); final String key = MapKeyBuilder.buildSubscribeKey(realSubject, group); final PullEntry pullEntry = pullEntryMap.get(key); changeOnOffline(pullEntry, isOnline, src); final PullEntry retryPullEntry = pullEntryMap.get(MapKeyBuilder.buildSubscribeKey(retrySubject, group)); changeOnOffline(retryPullEntry, isOnline, src); final DefaultPullConsumer pullConsumer = pullConsumerMap.get(key); if (pullConsumer == null) return; if (isOnline) { pullConsumer.online(src); } else { pullConsumer.offline(src); } }
public PullConsumer getOrCreateDefault(String subject, String group, boolean isBroadcast) { final String realSubject = RetrySubjectUtils.getRealSubject(subject); final String key = MapKeyBuilder.buildSubscribeKey(realSubject, group); PullConsumer consumer = pullConsumerMap.get(key); if (consumer != null) { return consumer; } pullConsumerMapLock.lock(); try { consumer = pullConsumerMap.get(key); if (consumer != null) { return consumer; } PullConsumer consumerImpl = createDefaultPullConsumer(realSubject, group, isBroadcast); pullConsumerMap.put(key, consumerImpl); return consumerImpl; } catch (Exception e) { if (e instanceof DuplicateListenerException) { throw new CreatePullConsumerException("已经使用了onMessage方式处理的主题不能再纯拉模式", realSubject, group); } throw e; } finally { pullConsumerMapLock.unlock(); } }
public static String[] parseSubjectAndGroup(String subject) { if (!isRetrySubject(subject) && !isDeadRetrySubject(subject)) { return null; } final List<String> parts = RETRY_SUBJECT_SPLITTER.splitToList(subject); if (parts.size() != 3) { return null; } else { return new String[]{parts.get(1), parts.get(2)}; } } }
private void handleResponse(final Datagram response) { final short responseCode = response.getHeader().getCode(); if (responseCode == CommandCode.NO_MESSAGE) { callback.onCompleted(responseCode, Collections.<BaseMessage>emptyList()); } else if (responseCode != CommandCode.SUCCESS) { monitorPullError(request.getSubject(), request.getGroup()); callback.onCompleted(responseCode, Collections.<BaseMessage>emptyList()); } else { List<BaseMessage> messages = deserializeBaseMessage(response.getBody()); if (messages == null) { messages = Collections.emptyList(); } monitorPullCount(request.getSubject(), request.getGroup(), messages.size()); for (BaseMessage message : messages) { if (message.getMaxRetryNum() < 0) { String realSubject = RetrySubjectUtils.getRealSubject(message.getSubject()); message.setMaxRetryNum(PullSubjectsConfig.get().getMaxRetryNum(realSubject).get()); } } callback.onCompleted(responseCode, messages); } }
private void updateConsumerState(MetaInfoResponse response) { updateLock.lock(); try { if (isStale(response.getTimestamp(), lastUpdateTimestamp)) { LOGGER.debug("skip response {}", response); return; } lastUpdateTimestamp = response.getTimestamp(); final String subject = response.getSubject(); final String consumerGroup = response.getConsumerGroup(); if (RetrySubjectUtils.isRealSubject(subject) && !Strings.isNullOrEmpty(consumerGroup)) { boolean online = response.getOnOfflineState() == OnOfflineState.ONLINE; LOGGER.debug("消费者状态发生变更 {}/{}:{}", subject, consumerGroup, online); triggerConsumerStateChanged(subject, consumerGroup, online); } } catch (Exception e) { LOGGER.error("update meta info exception. response={}", response, e); } finally { updateLock.unlock(); } }
public static Optional<String> getSubject(final String retrySubject) { if (!isRetrySubject(retrySubject) && !isDeadRetrySubject(retrySubject)) { return Optional.absent(); } final List<String> parts = RETRY_SUBJECT_SPLITTER.splitToList(retrySubject); if (parts.size() != 3) { return Optional.absent(); } else { return Optional.of(parts.get(1)); } }
private MetaInfoResponse handleClientRegister(final MetaInfoRequest request) { final String realSubject = RetrySubjectUtils.getRealSubject(request.getSubject()); final int clientRequestType = request.getRequestType(); try { if (ClientRequestType.ONLINE.getCode() == clientRequestType) { store.insertClientMetaInfo(request); } final List<BrokerGroup> brokerGroups = subjectRouter.route(realSubject, request); List<BrokerGroup> removedReadonlyGroups = readonlyBrokerGroupManager.disableReadonlyBrokerGroup(realSubject, request.getClientTypeCode(), brokerGroups); final List<BrokerGroup> filteredBrokerGroups = filterBrokerGroups(removedReadonlyGroups); final OnOfflineState clientState = offlineStateManager.queryClientState(request.getClientId(), request.getSubject(), request.getConsumerGroup()); ClientLogUtils.log(realSubject, "client register response, request:{}, realSubject:{}, brokerGroups:{}, clientState:{}", request, realSubject, filteredBrokerGroups, clientState); return buildResponse(request, offlineStateManager.getLastUpdateTimestamp(), clientState, new BrokerCluster(filteredBrokerGroups)); } catch (Exception e) { LOG.error("process exception. {}", request, e); return buildResponse(request, -2, OnOfflineState.OFFLINE, new BrokerCluster(new ArrayList<>())); } }
private void monitorMessageReceived(long receiveTime, String subject) { if (RetrySubjectUtils.isRetrySubject(subject) || RetrySubjectUtils.isDeadRetrySubject(subject)) { QMon.receivedRetryMessagesCountInc(subject); } QMon.receivedMessagesCountInc(subject); QMon.produceTime(subject, System.currentTimeMillis() - receiveTime); }