/** * Creates a {@link CloseableIterator} of {@link RawMessage} based on the given fetch request. */ private CloseableIterator<RawMessage> fetchMessages(GenericRecord fetchRequest, TopicId topicId) throws IOException, TopicNotFoundException { MessageFetcher fetcher = messagingService.prepareFetch(topicId); Object startFrom = fetchRequest.get("startFrom"); if (startFrom != null) { if (startFrom instanceof ByteBuffer) { // start message id is specified fetcher.setStartMessage(Bytes.toBytes((ByteBuffer) startFrom), (Boolean) fetchRequest.get("inclusive")); } else if (startFrom instanceof Long) { // start by timestamp is specified fetcher.setStartTime((Long) startFrom); } else { // This shouldn't happen as it's guaranteed by the schema LOG.warn("Ignore unrecognized type for startFrom. Type={}, Value={}", startFrom.getClass(), startFrom); } } Integer limit = (Integer) fetchRequest.get("limit"); if (limit != null) { fetcher.setLimit(limit); } ByteBuffer encodedTx = (ByteBuffer) fetchRequest.get("transaction"); if (encodedTx != null) { fetcher.setTransaction(TRANSACTION_CODEC.decode(ByteBuffers.getByteArray(encodedTx))); } return fetcher.fetch(); }
/** * Creates a {@link CloseableIterator} of {@link RawMessage} based on the given fetch request. */ private CloseableIterator<RawMessage> fetchMessages(GenericRecord fetchRequest, TopicId topicId) throws IOException, TopicNotFoundException { MessageFetcher fetcher = messagingService.prepareFetch(topicId); Object startFrom = fetchRequest.get("startFrom"); if (startFrom != null) { if (startFrom instanceof ByteBuffer) { // start message id is specified fetcher.setStartMessage(Bytes.toBytes((ByteBuffer) startFrom), (Boolean) fetchRequest.get("inclusive")); } else if (startFrom instanceof Long) { // start by timestamp is specified fetcher.setStartTime((Long) startFrom); } else { // This shouldn't happen as it's guaranteed by the schema LOG.warn("Ignore unrecognized type for startFrom. Type={}, Value={}", startFrom.getClass(), startFrom); } } Integer limit = (Integer) fetchRequest.get("limit"); if (limit != null) { fetcher.setLimit(limit); } ByteBuffer encodedTx = (ByteBuffer) fetchRequest.get("transaction"); if (encodedTx != null) { fetcher.setTransaction(TRANSACTION_CODEC.decode(ByteBuffers.getByteArray(encodedTx))); } return fetcher.fetch(); }
private List<AuditMessage> fetchAuditMessages() throws TopicNotFoundException, IOException { List<AuditMessage> result = new ArrayList<>(); try (CloseableIterator<RawMessage> iterator = messagingService.prepareFetch(auditTopic).fetch()) { while (iterator.hasNext()) { RawMessage message = iterator.next(); result.add(GSON.fromJson(new String(message.getPayload(), StandardCharsets.UTF_8), AuditMessage.class)); } } return result; } }
@Override public CloseableIterator<Message> fetch(String namespace, String topic, int limit, @Nullable String afterMessageId) throws IOException, TopicNotFoundException { co.cask.cdap.messaging.MessageFetcher fetcher = messagingService .prepareFetch(new NamespaceId(namespace).topic(topic)) .setLimit(limit); if (afterMessageId != null) { fetcher.setStartMessage(Bytes.fromHexString(afterMessageId), false); } if (transaction != null) { fetcher.setTransaction(transaction); } return new MessageIterator(fetcher.fetch()); }
@Override public CloseableIterator<Message> fetch(String namespace, String topic, int limit, @Nullable String afterMessageId) throws IOException, TopicNotFoundException { co.cask.cdap.messaging.MessageFetcher fetcher = messagingService .prepareFetch(new NamespaceId(namespace).topic(topic)) .setLimit(limit); if (afterMessageId != null) { fetcher.setStartMessage(Bytes.fromHexString(afterMessageId), false); } if (transaction != null) { fetcher.setTransaction(transaction); } return new MessageIterator(fetcher.fetch()); }
@Override public CloseableIterator<Message> fetch(String namespace, String topic, int limit, long timestamp) throws IOException, TopicNotFoundException { co.cask.cdap.messaging.MessageFetcher fetcher = messagingService .prepareFetch(new NamespaceId(namespace).topic(topic)) .setLimit(limit) .setStartTime(timestamp); if (transaction != null) { fetcher.setTransaction(transaction); } return new MessageIterator(fetcher.fetch()); }
TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()), oldestTsMetricName, latestTsMetricName); try (CloseableIterator<RawMessage> iterator = fetcher.fetch()) { while (iterator.hasNext() && isRunning()) { RawMessage input = iterator.next();
TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()), oldestTsMetricName, latestTsMetricName); try (CloseableIterator<RawMessage> iterator = fetcher.fetch()) { while (iterator.hasNext() && isRunning()) { RawMessage input = iterator.next();
@Override public CloseableIterator<Message> fetch(String namespace, String topic, int limit, long timestamp) throws IOException, TopicNotFoundException { co.cask.cdap.messaging.MessageFetcher fetcher = messagingService .prepareFetch(new NamespaceId(namespace).topic(topic)) .setLimit(limit) .setStartTime(timestamp); if (transaction != null) { fetcher.setTransaction(transaction); } return new MessageIterator(fetcher.fetch()); }
@Test public void testReuseRequest() throws IOException, TopicAlreadyExistsException, TopicNotFoundException { // This test a StoreRequest object can be reused. // This test is to verify storing transaction messages to the payload table TopicId topicId = new NamespaceId("ns1").topic("testReuseRequest"); client.createTopic(new TopicMetadata(topicId)); StoreRequest request = StoreRequestBuilder.of(topicId).addPayload("m1").addPayload("m2").build(); // Publish the request twice client.publish(request); client.publish(request); // Expects four messages List<RawMessage> messages = new ArrayList<>(); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(10).fetch()) { Iterators.addAll(messages, iterator); } Assert.assertEquals(4, messages.size()); List<String> expected = Arrays.asList("m1", "m2", "m1", "m2"); Assert.assertEquals(expected, messages.stream() .map(RawMessage::getPayload) .map(Bytes::toString).collect(Collectors.toList())); } }
@Test public void testDeletes() throws Exception { TopicId topicId = new NamespaceId("ns1").topic("del"); TopicMetadata metadata = new TopicMetadata(topicId, "ttl", "100"); for (int j = 0; j < 10; j++) { client.createTopic(metadata); String m1 = String.format("m%d", j); String m2 = String.format("m%d", j + 1); Assert.assertNull(client.publish(StoreRequestBuilder.of(topicId).addPayload(m1).addPayload(m2).build())); // Fetch messages non-transactionally List<RawMessage> messages = new ArrayList<>(); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) { Iterators.addAll(messages, iterator); } Assert.assertEquals(2, messages.size()); Set<String> receivedMessages = new HashSet<>(); for (RawMessage message : messages) { receivedMessages.add(Bytes.toString(message.getPayload())); } Assert.assertTrue(receivedMessages.contains(m1)); Assert.assertTrue(receivedMessages.contains(m2)); client.deleteTopic(topicId); } }
@Test public void testChunkConsume() throws Exception { // This test is to verify the message fetching body producer works correctly TopicId topicId = new NamespaceId("ns1").topic("testChunkConsume"); client.createTopic(new TopicMetadata(topicId)); // Publish 10 messages, each payload is half the size of the chunk size int payloadSize = cConf.getInt(Constants.MessagingSystem.HTTP_SERVER_CONSUME_CHUNK_SIZE) / 2; for (int i = 0; i < 10; i++) { String payload = Strings.repeat(Integer.toString(i), payloadSize); client.publish(StoreRequestBuilder.of(topicId).addPayload(payload).build()); } // Fetch messages. All of them should be fetched correctly List<RawMessage> messages = new ArrayList<>(); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) { Iterators.addAll(messages, iterator); } Assert.assertEquals(10, messages.size()); for (int i = 0; i < 10; i++) { RawMessage message = messages.get(i); Assert.assertEquals(payloadSize, message.getPayload().length); String payload = Strings.repeat(Integer.toString(i), payloadSize); Assert.assertEquals(payload, Bytes.toString(message.getPayload())); } client.deleteTopic(topicId); }
try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) { Assert.assertFalse(iterator.hasNext()); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) { Iterators.addAll(messages, iterator); .setStartMessage(messages.get(10).getId(), true).fetch()) { messages.clear(); Iterators.addAll(messages, iterator); .setStartMessage(messages.get(1).getId(), false).fetch()) { messages.clear(); Iterators.addAll(messages, iterator); .setStartMessage(messages.get(messages.size() - 1).getId(), false).fetch()) { Assert.assertFalse(iterator.hasNext()); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).setLimit(6).fetch()) { Iterators.addAll(messages, iterator); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) { Iterators.addAll(messages, iterator); .setStartMessage(messages.get(19).getId(), false).fetch()) { messages.clear(); Iterators.addAll(messages, iterator);
client.prepareFetch(topicId).fetch(); Assert.fail("Expected TopicNotFoundException"); } catch (TopicNotFoundException e) { try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId).fetch()) { Iterators.addAll(messages, iterator); .setStartTime(0) .setTransaction(transaction) .fetch()) { Iterators.addAll(txMessages, iterator); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId) .setStartMessage(startMessageId, false) .fetch()) { try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId) .setStartMessage(startMessageId, false) .fetch()) { Assert.assertFalse(iterator.hasNext()); .getPublishTimestamp()) .setLimit(2) .fetch()) { messages.clear(); Iterators.addAll(messages, iterator); try (CloseableIterator<RawMessage> iterator = client.prepareFetch(topicId) .setStartMessage(startMessageId, false)
@Test public void testTxMaxLifeTime() throws Exception { NamespaceId nsId = new NamespaceId("txCheck"); TopicId topic1 = nsId.topic("t1"); // Create a topic client.createTopic(new TopicMetadata(topic1)); final RollbackDetail rollbackDetail = client.publish(StoreRequestBuilder.of(topic1).setTransaction(1L) .addPayload("a").addPayload("b").build()); try { client.publish(StoreRequestBuilder.of(topic1) .setTransaction(-Long.MAX_VALUE).addPayload("c").addPayload("d").build()); Assert.fail("Expected IOException"); } catch (IOException ex) { // expected } Set<String> msgs = new HashSet<>(); CloseableIterator<RawMessage> messages = client.prepareFetch(topic1).fetch(); while (messages.hasNext()) { RawMessage message = messages.next(); msgs.add(Bytes.toString(message.getPayload())); } Assert.assertEquals(2, msgs.size()); Assert.assertTrue(msgs.contains("a")); Assert.assertTrue(msgs.contains("b")); messages.close(); client.rollback(topic1, rollbackDetail); client.deleteTopic(topic1); }