@Override public void updateTopic(TopicMetadata topicMetadata) throws TopicNotFoundException, IOException { try (MetadataTable metadataTable = createMetadataTable()) { Map<String, String> properties = createDefaultProperties(); properties.putAll(topicMetadata.getProperties()); metadataTable.updateTopic(new TopicMetadata(topicMetadata.getTopicId(), properties, true)); topicCache.invalidate(topicMetadata.getTopicId()); } }
CacheMessageTableEntry(TopicMetadata topicMetadata, long publishTimestamp, short sequenceId) { this.lookupOnly = true; this.topicId = topicMetadata.getTopicId(); this.generation = topicMetadata.getGeneration(); this.transactional = false; this.payload = null; this.publishTimestamp = publishTimestamp; this.sequenceId = sequenceId; }
private List<TopicId> listTopics(@Nullable byte[] startKey, @Nullable byte[] stopKey) throws IOException { List<TopicId> topicList = new ArrayList<>(); try (CloseableIterator<TopicMetadata> iterator = scanTopics(startKey, stopKey)) { while (iterator.hasNext()) { TopicMetadata metadata = iterator.next(); if (metadata.exists()) { topicList.add(metadata.getTopicId()); } } } return topicList; }
/** * Validates all the required properties of the given topic. * * @throws IllegalArgumentException if any required properties is missing or having invalid values */ private void validateProperties() { validateTTL(); validateGeneration(); }
@Override public void createTopic(TopicMetadata topicMetadata) throws TopicAlreadyExistsException, IOException { try { TopicId topicId = topicMetadata.getTopicId(); byte[] key = MessagingUtils.toMetadataRowKey(topicId); TreeMap<String, String> properties = new TreeMap<>(topicMetadata.getProperties()); properties.put(TopicMetadata.GENERATION_KEY, MessagingUtils.Constants.DEFAULT_GENERATION); synchronized (this) { byte[] tableValue = levelDB.get(key); if (tableValue != null) { Map<String, String> oldProperties = GSON.fromJson(Bytes.toString(tableValue), MAP_TYPE); TopicMetadata metadata = new TopicMetadata(topicId, oldProperties); if (metadata.exists()) { throw new TopicAlreadyExistsException(topicId.getNamespace(), topicId.getTopic()); } int newGenerationId = (metadata.getGeneration() * -1) + 1; properties.put(TopicMetadata.GENERATION_KEY, Integer.toString(newGenerationId)); } byte[] value = Bytes.toBytes(GSON.toJson(properties, MAP_TYPE)); levelDB.put(key, value, WRITE_OPTIONS); } } catch (DBException e) { throw new IOException(e); } }
/** * Creates the given topic if it is not yet created. */ private void createTopicIfNotExists(TopicId topicId) throws IOException { try { createTopic(new TopicMetadata(topicId)); LOG.info("System topic created: {}", topicId); } catch (TopicAlreadyExistsException e) { // OK for the topic already created. Just log a debug as it happens on every restart. LOG.debug("System topic already exists: {}", topicId); } }
table.createTopic(new TopicMetadata(NamespaceId.DEFAULT.topic("t1"), "ttl", 10));; Assert.assertEquals(1, table.listTopics(NamespaceId.DEFAULT).size()); TopicMetadata topicMetadata = new TopicMetadata(NamespaceId.DEFAULT.topic("t2"), "ttl", 20); table.createTopic(topicMetadata); Assert.assertEquals(topicMetadata.getTopicId(), table.getMetadata(NamespaceId.DEFAULT.topic("t2")).getTopicId()); Assert.assertEquals(topicMetadata.getTTL(), table.getMetadata(NamespaceId.DEFAULT.topic("t2")).getTTL()); Assert.assertEquals(1, table.getMetadata(NamespaceId.DEFAULT.topic("t2")).getGeneration()); table.createTopic(new TopicMetadata(NamespaceId.SYSTEM.topic("t3"), "ttl", 30));
long ttlInMs = TimeUnit.SECONDS.toMillis(topicMetadata.getTTL()); byte[] startRow = MessagingUtils.toDataKeyPrefix(topicMetadata.getTopicId(), Integer.parseInt(MessagingUtils.Constants.DEFAULT_GENERATION)); byte[] stopRow = Bytes.stopKeyForPrefix(startRow); int currGeneration = topicMetadata.getGeneration(); checkTopic(topicMetadata.getTopicId(), topicMetadata.getGeneration()); if (MessagingUtils.isOlderGeneration(dataGeneration, currGeneration)) { writeBatch.delete(entry.getKey());
@Test public void testGeMetadata() throws Exception { TopicId topicId = new NamespaceId("ns2").topic("d"); TopicMetadata metadata = new TopicMetadata(topicId, "ttl", "100"); for (int i = 1; i <= 5; i++) { client.createTopic(metadata); TopicMetadata topicMetadata = client.getTopic(topicId); Assert.assertEquals(100, topicMetadata.getTTL()); Assert.assertEquals(i, topicMetadata.getGeneration()); client.deleteTopic(topicId); } }
try (MetadataTable metadataTable = createMetadataTable()) { LevelDBMetadataTable table = (LevelDBMetadataTable) metadataTable; TopicMetadata t1 = new TopicMetadata( NamespaceId.CDAP.topic("t1"), ImmutableMap.of(TopicMetadata.TTL_KEY, "10", TopicMetadata.GENERATION_KEY, "1")); TopicMetadata t2 = new TopicMetadata( NamespaceId.SYSTEM.topic("t2"), ImmutableMap.of(TopicMetadata.TTL_KEY, "20", TopicMetadata.GENERATION_KEY, "1")); Assert.assertEquals(1, allTopics.size()); metadataTable.deleteTopic(t1.getTopicId()); Assert.assertTrue(metadataTable.listTopics(NamespaceId.CDAP).isEmpty()); metadataTable.deleteTopic(t2.getTopicId()); metadatas.clear(); Iterators.addAll(metadatas, table.scanTopics()); Assert.assertEquals(-1, metadata.getGeneration());
/** * Scans the HBase table to get a list of {@link TopicId}. */ private List<TopicId> scanTopics(ScanBuilder scanBuilder) throws IOException { Scan scan = scanBuilder.setFilter(new FirstKeyOnlyFilter()).setCaching(scanCacheRows).build(); try { List<TopicId> topicIds = new ArrayList<>(); try (ResultScanner resultScanner = hTable.getScanner(scan)) { for (Result result : resultScanner) { TopicId topicId = MessagingUtils.toTopicId(result.getRow()); byte[] value = result.getValue(columnFamily, COL); Map<String, String> properties = GSON.fromJson(Bytes.toString(value), MAP_TYPE); TopicMetadata metadata = new TopicMetadata(topicId, properties); if (metadata.exists()) { topicIds.add(topicId); } } } return topicIds; } catch (IOException e) { throw exceptionHandler.handle(e); } }
@Test public void testOldGenCleanup() throws Exception { TopicId topicId = NamespaceId.DEFAULT.topic("oldGenCleanup"); TopicMetadata topic = new TopicMetadata(topicId, TopicMetadata.TTL_KEY, "100000", TopicMetadata.GENERATION_KEY, Integer.toString(GENERATION)); try (MetadataTable metadataTable = getMetadataTable()) { Map<String, String> newProperties = new HashMap<>(topic.getProperties()); newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(topic.getGeneration() + 1)); topic = new TopicMetadata(topicId, newProperties); metadataTable.createTopic(topic);
@Override public void updateTopic(TopicMetadata topicMetadata) throws TopicNotFoundException, IOException { byte[] rowKey = MessagingUtils.toMetadataRowKey(topicMetadata.getTopicId()); boolean completed = false; try { // Keep trying to update while (!completed) { TopicMetadata oldMetadata = getMetadata(topicMetadata.getTopicId()); TreeMap<String, String> newProperties = new TreeMap<>(topicMetadata.getProperties()); newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(oldMetadata.getGeneration())); Put put = tableUtil.buildPut(rowKey) .add(columnFamily, COL, Bytes.toBytes(GSON.toJson(newProperties, MAP_TYPE))) .build(); byte[] oldValue = Bytes.toBytes(GSON.toJson(new TreeMap<>(oldMetadata.getProperties()), MAP_TYPE)); completed = hTable.checkAndPut(rowKey, columnFamily, COL, oldValue, put); } } catch (IOException e) { throw exceptionHandler.handle(e); } }
@Override public void createTopic(TopicMetadata topicMetadata) throws TopicAlreadyExistsException, IOException { TopicId topicId = topicMetadata.getTopicId(); HttpRequest request = remoteClient.requestBuilder(HttpMethod.PUT, createTopicPath(topicId)) .withBody(GSON.toJson(topicMetadata.getProperties())) .build(); HttpResponse response = remoteClient.execute(request); if (response.getResponseCode() == HttpURLConnection.HTTP_CONFLICT) { throw new TopicAlreadyExistsException(topicId.getNamespace(), topicId.getTopic()); } handleError(response, "Failed to create topic " + topicId); }
table.updateTopic(new TopicMetadata(topicId, "ttl", 10)); Assert.fail("Expected TopicNotFoundException"); } catch (TopicNotFoundException e) { table.createTopic(new TopicMetadata(topicId, "ttl", 10)); Assert.assertEquals(10, table.getMetadata(topicId).getTTL()); table.updateTopic(new TopicMetadata(topicId, "ttl", 30)); Assert.assertEquals(30, table.getMetadata(topicId).getTTL()); table.createTopic(new TopicMetadata(topicId, "ttl", 10)); Assert.fail("Expected TopicAlreadyExistsException"); } catch (TopicAlreadyExistsException e) { Assert.assertEquals(30, table.getMetadata(topicId).getTTL());
@Override public void deleteTopic(TopicId topicId) throws TopicNotFoundException, IOException { byte[] rowKey = MessagingUtils.toMetadataRowKey(topicId); boolean completed = false; try { // Keep trying to delete while (!completed) { TopicMetadata oldMetadata = getMetadata(topicId); TreeMap<String, String> newProperties = new TreeMap<>(oldMetadata.getProperties()); newProperties.put(TopicMetadata.GENERATION_KEY, Integer.toString(oldMetadata.getGeneration() * -1)); Put put = tableUtil.buildPut(rowKey) .add(columnFamily, COL, Bytes.toBytes(GSON.toJson(newProperties, MAP_TYPE))) .build(); byte[] oldValue = Bytes.toBytes(GSON.toJson(new TreeMap<>(oldMetadata.getProperties()), MAP_TYPE)); completed = hTable.checkAndPut(rowKey, columnFamily, COL, oldValue, put); } } catch (IOException e) { throw exceptionHandler.handle(e); } }
MessageCloseableIterator(MessageTable messageTable) throws IOException { this.topicId = topicMetadata.getTopicId(); this.messageTable = messageTable; this.inclusive = isIncludeStart(); this.messageLimit = getLimit(); long ttl = topicMetadata.getTTL(); startOffset = getStartOffset() == null ? null : new MessageId(getStartOffset()); Long startTime = getStartTime(); // Lower bound of messages that are still valid long smallestPublishTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttl); CloseableIterator<MessageTable.Entry> messageIterator; // If there is no startOffset or if the publish time in the startOffset is smaller then TTL, // do the scanning based on time. The smallest start time should be the currentTime - TTL. if (startOffset == null || startOffset.getPublishTimestamp() < smallestPublishTime) { long fetchStartTime = Math.max(smallestPublishTime, startTime == null ? smallestPublishTime : startTime); messageIterator = messageTable.fetch(topicMetadata, fetchStartTime, messageLimit, getTransaction()); } else { // Start scanning based on the start message id if (startOffset.getPayloadWriteTimestamp() != 0L) { // This message ID refer to payload table. Scan the message table with the reference message ID inclusively. messageIterator = messageTable.fetch(topicMetadata, createMessageTableMessageId(startOffset), true, messageLimit, getTransaction()); } else { messageIterator = messageTable.fetch(topicMetadata, startOffset, isIncludeStart(), messageLimit, getTransaction()); } } this.messageIterator = messageIterator; }
@Override TestEntry getEntry(TopicMetadata metadata, boolean transactional, long transactionWritePointer, long writeTimestamp, short sequenceId, @Nullable byte[] payload) { return new TestEntry(metadata.getTopicId(), transactional, transactionWritePointer, writeTimestamp, sequenceId, payload); }
/** * Check whether the topic exists. */ public boolean exists() { return getGeneration() > 0; }
@GET @Path("/topics/{topic}") public void getTopic(HttpRequest request, HttpResponder responder, @PathParam("namespace") String namespace, @PathParam("topic") String topic) throws Exception { TopicId topicId = new NamespaceId(namespace).topic(topic); TopicMetadata metadata = messagingService.getTopic(topicId); responder.sendJson(HttpResponseStatus.OK, GSON.toJson(metadata.getProperties(), TOPIC_PROPERTY_TYPE)); }