public TopicPartition getTopicPartition(String topic) { TopicPartition topicPartition = _topicPartitionInfoMap.get(topic); if (topicPartition != null) { return new TopicPartition(topic, topicPartition.getPartition()); } else { return null; } }
public synchronized void addTopicToMirrorMaker(TopicPartition topicPartitionInfo) { this.addTopicToMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
public synchronized void expandTopicInMirrorMaker(TopicPartition topicPartitionInfo) { this.expandTopicInMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
int numPartitionsInMirrorMaker = _helixMirrorMakerManager.getIdealStateForTopic(topic).getNumPartitions(); if (numPartitionsInMirrorMaker != tp.getPartition()) { int mismatchedPartitions = Math.abs(numPartitionsInMirrorMaker - tp.getPartition()); if (_enableAutoTopicExpansion && (tp.getPartition() > numPartitionsInMirrorMaker)) { topic, numPartitionsInMirrorMaker, tp.getPartition()); _numAutoExpandedTopics.inc(); _numAutoExpandedTopicPartitions.inc(mismatchedPartitions); LOGGER.warn( "Number of partitions not matched for topic {} between mirrormaker:{} and source kafka broker: {}!", topic, numPartitionsInMirrorMaker, tp.getPartition());
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<String, CustomModeISBuilder>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<String, IdealState>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
@Test public void testAutoTopic() { for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
@Test public void testKafkaBrokerTopicObserver() { Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1); Assert.assertEquals(kafkaBrokerTopicObserver.getTopicPartition("testTopic0").getPartition(), 1); for (int i = 1; i < 10; ++i) { String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1);
public TopicPartition getTopicPartition(String topic) { TopicPartition topicPartition = _topicPartitionInfoMap.get(topic); if (topicPartition != null) { return new TopicPartition(topic, topicPartition.getPartition()); } else { return null; } }
public synchronized void expandTopicInMirrorMaker(TopicPartition topicPartitionInfo) { this.expandTopicInMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
public synchronized void addTopicToMirrorMaker(TopicPartition topicPartitionInfo) { this.addTopicToMirrorMaker(topicPartitionInfo.getTopic(), topicPartitionInfo.getPartition()); }
private TopicAndPartition toTopicAndPartition(TopicPartition topicPartition) { return new TopicAndPartition(topicPartition.getTopic(), topicPartition.getPartition()); }
private void whitelistCandiateTopics(Set<String> candidateTopicsToWhitelist) { for (String topic : candidateTopicsToWhitelist) { TopicPartition tp = _srcKafkaTopicObserver.getTopicPartition(topic); if (tp == null) { LOGGER.error("Shouldn't hit here, don't know why topic {} is not in src Kafka cluster", topic); _numErrorTopics.inc(); } else { if (_helixMirrorMakerManager.isTopicExisted(topic)) { LOGGER.info("Trying to expand topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.expandTopicInMirrorMaker(tp); _numAutoExpandedTopics.inc(); } else { LOGGER.info("Trying to whitelist topic: {} with {} partitions", tp.getTopic(), tp.getPartition()); _helixMirrorMakerManager.addTopicToMirrorMaker(tp); _numWhitelistedTopics.inc(); } } } }
@Override public int compare(TopicPartition o1, TopicPartition o2) { TopicWorkload workload1 = infoRetriever.topicWorkload(o1.getTopic()); TopicWorkload workload2 = infoRetriever.topicWorkload(o2.getTopic()); if (workload1 == workload2) { return 0; } int cmp = workload1.compareTo(workload2); if (cmp != 0) { return cmp; } // if workload is the same, compare them based on the name and partition cmp = o1.getTopic().compareTo(o2.getTopic()); if (cmp != 0) { return cmp; } return Integer.compare(o1.getPartition(), o2.getPartition()); } };
private Set<String> getPartitionMismatchedTopics() { Set<String> partitionsMismatchedTopics = new HashSet<String>(); for (String topicName : _helixMirrorMakerManager.getTopicLists()) { int numPartitionsInHelix = _helixMirrorMakerManager.getIdealStateForTopic(topicName).getNumPartitions(); if (_srcKafkaTopicObserver.getTopicPartition(topicName) != null) { int numPartitionsInSrcBroker = _srcKafkaTopicObserver.getTopicPartition(topicName).getPartition(); if (numPartitionsInHelix != numPartitionsInSrcBroker) { partitionsMismatchedTopics.add(topicName); } } } return partitionsMismatchedTopics; }
public static Map<String, IdealState> getIdealStatesFromAssignment( Set<InstanceTopicPartitionHolder> newAssignment) { Map<String, CustomModeISBuilder> idealStatesBuilderMap = new HashMap<>(); for (InstanceTopicPartitionHolder instance : newAssignment) { for (TopicPartition tpi : instance.getServingTopicPartitionSet()) { String topicName = tpi.getTopic(); String partition = Integer.toString(tpi.getPartition()); if (!idealStatesBuilderMap.containsKey(topicName)) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumReplica(1); idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder); } idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition, instance.getInstanceName(), "ONLINE"); } } Map<String, IdealState> idealStatesMap = new HashMap<>(); for (String topic : idealStatesBuilderMap.keySet()) { IdealState idealState = idealStatesBuilderMap.get(topic).build(); idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size()); idealState.setNumPartitions(idealState.getPartitionSet().size()); idealStatesMap.put(topic, idealState); } return idealStatesMap; }
@Test public void testKafkaBrokerTopicObserver() { Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1); Assert.assertEquals(kafkaBrokerTopicObserver.getTopicPartition("testTopic0").getPartition(), 1); for (int i = 1; i < 10; ++i) { String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }
@Test public void testAutoTopic() { for (int i = 0; i < 10; ++i) { Assert.assertEquals(helixMirrorMakerManager.getTopicLists().size(), i); String topicName = "testTopic" + i; // Create Kafka topic KafkaStarterUtils.createTopic(topicName, KafkaStarterUtils.DEFAULT_ZK_STR); try { Thread.sleep(5000); } catch (Exception e) { } Assert.assertEquals(kafkaBrokerTopicObserver.getNumTopics(), 1 + i); for (int j = 0; j <= i; ++j) { Assert.assertTrue(kafkaBrokerTopicObserver.getAllTopics().contains("testTopic" + j)); Assert.assertEquals( kafkaBrokerTopicObserver.getTopicPartition("testTopic" + j).getPartition(), 1); } } } }