public RedisDynoQueue(Clock clock, String redisKeyPrefix, String queueName, Set<String> allShards, String shardName, int unackScheduleInMS, ShardingStrategy shardingStrategy) { this.clock = clock; this.redisKeyPrefix = redisKeyPrefix; this.queueName = queueName; this.allShards = ImmutableList.copyOf(allShards.stream().collect(Collectors.toList())); this.shardName = shardName; this.messageStoreKey = redisKeyPrefix + ".MESSAGE." + queueName; this.myQueueShard = getQueueShardKey(queueName, shardName); this.shardingStrategy = shardingStrategy; ObjectMapper om = new ObjectMapper(); om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); om.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); om.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); om.setSerializationInclusion(Include.NON_NULL); om.setSerializationInclusion(Include.NON_EMPTY); om.disable(SerializationFeature.INDENT_OUTPUT); this.om = om; this.monitor = new QueueMonitor(queueName, shardName); this.prefetchedIds = new ConcurrentLinkedQueue<>(); schedulerForUnacksProcessing = Executors.newScheduledThreadPool(1); schedulerForUnacksProcessing.scheduleAtFixedRate(() -> processUnacks(), unackScheduleInMS, unackScheduleInMS, TimeUnit.MILLISECONDS); logger.info(RedisDynoQueue.class.getName() + " is ready to serve " + queueName); }
@Override public Map<String, Map<String, Long>> shardSizes() { Stopwatch sw = monitor.size.start(); Map<String, Map<String, Long>> shardSizes = new HashMap<>(); try { return execute("shardSizes", "(a shard in) " + queueName, () -> { for (String shard : allShards) { long size = nonQuorumConn.zcard(getQueueShardKey(queueName, shard)); long uacked = nonQuorumConn.zcard(getUnackKey(queueName, shard)); Map<String, Long> shardDetails = new HashMap<>(); shardDetails.put("size", size); shardDetails.put("uacked", uacked); shardSizes.put(shard, shardDetails); } return shardSizes; }); } finally { sw.stop(); } }
@Override public List<String> push(final List<Message> messages) { Stopwatch sw = monitor.start(monitor.push, messages.size()); try { execute("push", "(a shard in) " + queueName, () -> { for (Message message : messages) { String json = om.writeValueAsString(message); quorumConn.hset(messageStoreKey, message.getId(), json); double priority = message.getPriority() / 100.0; double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; String shard = shardingStrategy.getNextShard(allShards, message); String queueShard = getQueueShardKey(queueName, shard); quorumConn.zadd(queueShard, score, message.getId()); } return messages; }); return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList()); } finally { sw.stop(); } }
@Override public long size() { Stopwatch sw = monitor.size.start(); try { return execute("size", "(a shard in) " + queueName, () -> { long size = 0; for (String shard : allShards) { size += nonQuorumConn.zcard(getQueueShardKey(queueName, shard)); } return size; }); } finally { sw.stop(); } }
@Override public void clear() { execute("clear", "(a shard in) " + queueName, () -> { for (String shard : allShards) { String queueShard = getQueueShardKey(queueName, shard); String unackShard = getUnackKey(queueName, shard); quorumConn.del(queueShard); quorumConn.del(unackShard); } quorumConn.del(messageStoreKey); return null; }); }
@Override public boolean setTimeout(String messageId, long timeout) { return execute("setTimeout", "(a shard in) " + queueName, () -> { String json = nonQuorumConn.hget(messageStoreKey, messageId); if(json == null) { return false; } Message message = om.readValue(json, Message.class); message.setTimeout(timeout); for (String shard : allShards) { String queueShard = getQueueShardKey(queueName, shard); Double score = quorumConn.zscore(queueShard, messageId); if(score != null) { double priorityd = message.getPriority() / 100; double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; ZAddParams params = ZAddParams.zAddParams().xx(); quorumConn.zadd(queueShard, newScore, messageId, params); json = om.writeValueAsString(message); quorumConn.hset(messageStoreKey, message.getId(), json); return true; } } return false; }); }
@Override public boolean remove(String messageId) { Stopwatch sw = monitor.remove.start(); try { return execute("remove", "(a shard in) " + queueName, () -> { for (String shard : allShards) { String unackShardKey = getUnackKey(queueName, shard); quorumConn.zrem(unackShardKey, messageId); String queueShardKey = getQueueShardKey(queueName, shard); Long removed = quorumConn.zrem(queueShardKey, messageId); Long msgRemoved = quorumConn.hdel(messageStoreKey, messageId); if (removed > 0 && msgRemoved > 0) { return true; } } return false; }); } finally { sw.stop(); } }