@Override public void push(String queueName, String id, long offsetTimeInSecond) { Message msg = new Message(id, null); msg.setTimeout(offsetTimeInSecond, TimeUnit.SECONDS); queues.get(queueName).push(Collections.singletonList(msg)); }
@Override public List<com.netflix.conductor.core.events.queue.Message> pollMessages(String queueName, int count, int timeout) { List<Message> msgs = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msgs.stream() .map(msg -> new com.netflix.conductor.core.events.queue.Message(msg.getId(), msg.getPayload(), null)) .collect(Collectors.toList()); }
List<Message> messages = new LinkedList<>(); for (int i = 0; i < count; i++) { Message msg = new Message("" + i, "Hello World-" + i); msg.setPriority(count - i); messages.add(msg); Message found = rdq.get(msg.getId()); assertNotNull(found); assertEquals(msg.getId(), found.getId()); assertEquals(msg.getTimeout(), found.getTimeout()); assertEquals(messages.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()), messages3.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList())); assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); messages3.stream().forEach(System.out::println); assertTrue(rdq.ack(msg.getId())); assertFalse(rdq.ack(msg.getId()));
@Test public void test(){ Message msg = new Message(); msg.setPayload("payload"); msg.setTimeout(10, TimeUnit.SECONDS); assertEquals(msg.toString(), 10*1000, msg.getTimeout()); msg.setTimeout(10); assertEquals(msg.toString(), 10, msg.getTimeout()); }
@Test(expected=IllegalArgumentException.class) public void testPrioirty(){ Message msg = new Message(); msg.setPriority(-1); }
@Override public boolean setTimeout(String messageId, long timeout) { RedisConnection jedis = connPool.getResource(); try { String json = jedis.hget(messageStoreKey(messageId), messageId); if (json == null) { return false; } Message message = om.readValue(json, Message.class); message.setTimeout(timeout); Double score = jedis.zscore(myQueueShard, messageId); if (score != null) { double priorityd = message.getPriority() / 100.0; double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; jedis.zadd(myQueueShard, newScore, messageId); json = om.writeValueAsString(message); jedis.hset(messageStoreKey(message.getId()), message.getId(), json); return true; } return false; } catch (Exception e) { throw new RuntimeException(e); } finally { jedis.close(); } }
@Override public List<String> push(final List<Message> messages) { Stopwatch sw = monitor.start(monitor.push, messages.size()); try { execute("push", "(a shard in) " + queueName, () -> { for (Message message : messages) { String json = om.writeValueAsString(message); quorumConn.hset(messageStoreKey, message.getId(), json); double priority = message.getPriority() / 100.0; double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; String shard = shardingStrategy.getNextShard(allShards, message); String queueShard = getQueueShardKey(queueName, shard); quorumConn.zadd(queueShard, score, message.getId()); } return messages; }); return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList()); } finally { sw.stop(); } }
Message msg = new Message(); msg.setId("" + i); msg.setPayload("" + i); messages.add(msg);
@Override public void push(String queueName, List<com.netflix.conductor.core.events.queue.Message> messages) { List<Message> msgs = messages.stream() .map(msg -> new Message(msg.getId(), msg.getPayload())) .collect(Collectors.toList()); queues.get(queueName).push(msgs); }
@Override public void ack(List<Message> messages) { for(Message message : messages) { ack(message.getId()); } }
List<Message> messages = new LinkedList<>(); for (int i = 0; i < count; i++) { Message msg = new Message("" + i, "Hello World-" + i); msg.setPriority(count - i); messages.add(msg); Message found = rdq.get(msg.getId()); assertNotNull(found); assertEquals(msg.getId(), found.getId()); assertEquals(msg.getTimeout(), found.getTimeout()); assertEquals(messages.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList()), messages3.stream().map(msg -> msg.getId()).sorted().collect(Collectors.toList())); assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); messages3.stream().forEach(System.out::println); int bucketCounts = 0; assertTrue(rdq.ack(msg.getId())); assertFalse(rdq.ack(msg.getId()));
@Test(expected=IllegalArgumentException.class) public void testPrioirty2(){ Message msg = new Message(); msg.setPriority(100); } }
@Override public boolean setTimeout(String messageId, long timeout) { return execute("setTimeout", "(a shard in) " + queueName, () -> { String json = nonQuorumConn.hget(messageStoreKey, messageId); if(json == null) { return false; } Message message = om.readValue(json, Message.class); message.setTimeout(timeout); for (String shard : allShards) { String queueShard = getQueueShardKey(queueName, shard); Double score = quorumConn.zscore(queueShard, messageId); if(score != null) { double priorityd = message.getPriority() / 100; double newScore = Long.valueOf(clock.millis() + timeout).doubleValue() + priorityd; ZAddParams params = ZAddParams.zAddParams().xx(); quorumConn.zadd(queueShard, newScore, messageId, params); json = om.writeValueAsString(message); quorumConn.hset(messageStoreKey, message.getId(), json); return true; } } return false; }); }
@Override public List<String> push(final List<Message> messages) { Stopwatch sw = monitor.start(monitor.push, messages.size()); RedisConnection conn = connPool.getResource(); try { Pipe pipe = conn.pipelined(); for (Message message : messages) { String json = om.writeValueAsString(message); pipe.hset(messageStoreKey(message.getId()), message.getId(), json); double priority = message.getPriority() / 100.0; double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() + priority; pipe.zadd(myQueueShard, score, message.getId()); } pipe.sync(); pipe.close(); return messages.stream().map(msg -> msg.getId()).collect(Collectors.toList()); } catch (Exception e) { throw new RuntimeException(e); } finally { conn.close(); sw.stop(); } }
@Override public void push(String queueName, List<com.netflix.conductor.core.events.queue.Message> messages) { List<Message> msgs = messages.stream() .map(msg -> new Message(msg.getId(), msg.getPayload())) .collect(Collectors.toList()); queues.get(queueName).push(msgs); }
@Override public String getNextShard(List<String> allShards, Message message) { int hashCodeAbs = Math.abs(message.getId().hashCode()); int calculatedShard = (hashCodeAbs % allShards.size()); return allShards.get(calculatedShard); } }
List<Message> messages = new LinkedList<>(); for (int i = 0; i < count; i++) { Message msg = new Message("" + i, "Hello World-" + i); msg.setPriority(count - i); messages.add(msg); Message found = rdq.get(msg.getId()); assertNotNull(found); assertEquals(msg.getId(), found.getId()); assertEquals(msg.getTimeout(), found.getTimeout()); assertEquals(10, messages3.size()); assertEquals(messages, messages3); assertEquals(10, messages3.stream().map(msg -> msg.getId()).collect(Collectors.toSet()).size()); messages3.stream().forEach(System.out::println); assertTrue(dynoClient.hlen(messageKey) == 10); assertTrue(rdq.ack(msg.getId())); assertFalse(rdq.ack(msg.getId())); assertEquals(max, msg.getPriority()); rdq.remove(msg.getId()); max--;
@Override public void run() { List<Message> messages = new LinkedList<>(); for (int i = 0; i < 10; i++) { Message msg = new Message(UUID.randomUUID().toString(), "Hello World-" + i); msg.setPriority(new Random().nextInt(98)); messages.add(msg); } if(published.get() >= count) { publishLatch.countDown(); return; } published.addAndGet(messages.size()); rdq.push(messages); } };
@Override public boolean pushIfNotExists(String queueName, String id, long offsetTimeInSecond) { DynoQueue queue = queues.get(queueName); if (queue.get(id) != null) { return false; } Message msg = new Message(id, null); msg.setTimeout(offsetTimeInSecond, TimeUnit.SECONDS); queue.push(Collections.singletonList(msg)); return true; }
@Override public List<com.netflix.conductor.core.events.queue.Message> pollMessages(String queueName, int count, int timeout) { List<Message> msgs = queues.get(queueName).pop(count, timeout, TimeUnit.MILLISECONDS); return msgs.stream() .map(msg -> new com.netflix.conductor.core.events.queue.Message(msg.getId(), msg.getPayload(), null)) .collect(Collectors.toList()); }