/** * Converts a StreamSpec into a KafakStreamSpec. Special handling for coordinator and changelog stream. * @param spec a StreamSpec object * @return KafkaStreamSpec object */ public KafkaStreamSpec toKafkaSpec(StreamSpec spec) { KafkaStreamSpec kafkaSpec; if (spec.isChangeLogStream()) { String topicName = spec.getPhysicalName(); ChangelogInfo topicMeta = changelogTopicMetaInformation.get(topicName); if (topicMeta == null) { throw new StreamValidationException("Unable to find topic information for topic " + topicName); } kafkaSpec = new KafkaStreamSpec(spec.getId(), topicName, systemName, spec.getPartitionCount(), topicMeta.replicationFactor(), topicMeta.kafkaProps()); } else if (spec.isCoordinatorStream()) { kafkaSpec = new KafkaStreamSpec(spec.getId(), spec.getPhysicalName(), systemName, 1, coordinatorStreamReplicationFactor, coordinatorStreamProperties); } else if (intermediateStreamProperties.containsKey(spec.getId())) { kafkaSpec = KafkaStreamSpec.fromSpec(spec).copyWithProperties(intermediateStreamProperties.get(spec.getId())); } else { kafkaSpec = KafkaStreamSpec.fromSpec(spec); } return kafkaSpec; }
/** * Converts a StreamSpec into a KafakStreamSpec. Special handling for coordinator and changelog stream. * @param spec a StreamSpec object * @return KafkaStreamSpec object */ public KafkaStreamSpec toKafkaSpec(StreamSpec spec) { KafkaStreamSpec kafkaSpec; if (spec.isChangeLogStream()) { String topicName = spec.getPhysicalName(); ChangelogInfo topicMeta = changelogTopicMetaInformation.get(topicName); if (topicMeta == null) { throw new StreamValidationException("Unable to find topic information for topic " + topicName); } kafkaSpec = new KafkaStreamSpec(spec.getId(), topicName, systemName, spec.getPartitionCount(), topicMeta.replicationFactor(), topicMeta.kafkaProps()); } else if (spec.isCoordinatorStream()) { kafkaSpec = new KafkaStreamSpec(spec.getId(), spec.getPhysicalName(), systemName, 1, coordinatorStreamReplicationFactor, coordinatorStreamProperties); } else if (intermediateStreamProperties.containsKey(spec.getId())) { kafkaSpec = KafkaStreamSpec.fromSpec(spec).copyWithProperties(intermediateStreamProperties.get(spec.getId())); } else { kafkaSpec = KafkaStreamSpec.fromSpec(spec); } return kafkaSpec; }
/** * Converts a StreamSpec into a KafakStreamSpec. Special handling for coordinator and changelog stream. * @param spec a StreamSpec object * @return KafkaStreamSpec object */ public KafkaStreamSpec toKafkaSpec(StreamSpec spec) { KafkaStreamSpec kafkaSpec; if (spec.isChangeLogStream()) { String topicName = spec.getPhysicalName(); ChangelogInfo topicMeta = changelogTopicMetaInformation.get(topicName); if (topicMeta == null) { throw new StreamValidationException("Unable to find topic information for topic " + topicName); } kafkaSpec = new KafkaStreamSpec(spec.getId(), topicName, systemName, spec.getPartitionCount(), topicMeta.replicationFactor(), topicMeta.kafkaProps()); } else if (spec.isCoordinatorStream()) { kafkaSpec = new KafkaStreamSpec(spec.getId(), spec.getPhysicalName(), systemName, 1, coordinatorStreamReplicationFactor, coordinatorStreamProperties); } else if (intermediateStreamProperties.containsKey(spec.getId())) { kafkaSpec = KafkaStreamSpec.fromSpec(spec).copyWithProperties(intermediateStreamProperties.get(spec.getId())); } else { kafkaSpec = KafkaStreamSpec.fromSpec(spec); } return kafkaSpec; }
@Test public void testCreateCoordinatorStreamWithSpecialCharsInTopicName() { final String STREAM = "test.coordinator_test.Stream"; Map<String, String> map = new HashMap<>(); map.put("job.coordinator.segment.bytes", "123"); map.put("job.coordinator.cleanup.policy", "compact"); int coordReplicatonFactor = 2; map.put(org.apache.samza.config.KafkaConfig.JOB_COORDINATOR_REPLICATION_FACTOR(), String.valueOf(coordReplicatonFactor)); KafkaSystemAdmin admin = Mockito.spy(createSystemAdmin(SYSTEM(), map)); StreamSpec spec = StreamSpec.createCoordinatorStreamSpec(STREAM, SYSTEM()); Mockito.doAnswer(invocationOnMock -> { StreamSpec internalSpec = (StreamSpec) invocationOnMock.callRealMethod(); assertTrue(internalSpec instanceof KafkaStreamSpec); // KafkaStreamSpec is used to carry replication factor assertTrue(internalSpec.isCoordinatorStream()); assertEquals(SYSTEM(), internalSpec.getSystemName()); assertEquals(STREAM, internalSpec.getPhysicalName()); assertEquals(1, internalSpec.getPartitionCount()); Assert.assertEquals(coordReplicatonFactor, ((KafkaStreamSpec) internalSpec).getReplicationFactor()); Assert.assertEquals("123", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("segment.bytes")); // cleanup policy is overridden in the KafkaAdmin Assert.assertEquals("compact", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("cleanup.policy")); return internalSpec; }).when(admin).toKafkaSpec(Mockito.any()); admin.createStream(spec); admin.validateStream(spec); }