/** * Validates each changelog system-stream with its respective SystemAdmin. */ private void validateChangelogStreams() { LOG.info("Validating change log streams: " + changelogSystemStreams); for (SystemStream changelogSystemStream : changelogSystemStreams.values()) { SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(changelogSystemStream.getSystem()); StreamSpec changelogSpec = StreamSpec.createChangeLogStreamSpec(changelogSystemStream.getStream(), changelogSystemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.validateStream(changelogSpec); } }
StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), maxChangeLogStreamPartitions);
StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), maxChangeLogStreamPartitions);
StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), maxChangeLogStreamPartitions);
StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), maxChangeLogStreamPartitions);
StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), maxChangeLogStreamPartitions);
LOGGER.info("Clear store {} changelog {}", store, changelog); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(changelog); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), 1); systemAdmins.getSystemAdmin(spec.getSystemName()).clearStream(spec);
LOGGER.info("Clear store {} changelog {}", store, changelog); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(changelog); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), 1); systemAdmins.getSystemAdmin(spec.getSystemName()).clearStream(spec);
LOGGER.info("Clear store {} changelog {}", store, changelog); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(changelog); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), 1); systemAdmins.getSystemAdmin(spec.getSystemName()).clearStream(spec);
LOGGER.info("Clear store {} changelog {}", store, changelog); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(changelog); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), 1); systemAdmins.getSystemAdmin(spec.getSystemName()).clearStream(spec);
LOGGER.info("Clear store {} changelog {}", store, changelog); SystemStream systemStream = StreamUtil.getSystemStreamFromNames(changelog); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(systemStream.getStream(), systemStream.getSystem(), 1); systemAdmins.getSystemAdmin(spec.getSystemName()).clearStream(spec);
public void testCreateChangelogStreamHelp(final String topic) { final int PARTITIONS = 12; final int REP_FACTOR = 2; Map<String, String> map = new HashMap<>(); map.put(JobConfig.JOB_DEFAULT_SYSTEM(), SYSTEM()); map.put(String.format("stores.%s.changelog", "fakeStore"), topic); map.put(String.format("stores.%s.changelog.replication.factor", "fakeStore"), String.valueOf(REP_FACTOR)); map.put(String.format("stores.%s.changelog.kafka.segment.bytes", "fakeStore"), "139"); KafkaSystemAdmin admin = Mockito.spy(createSystemAdmin(SYSTEM(), map)); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(topic, SYSTEM(), PARTITIONS); Mockito.doAnswer(invocationOnMock -> { StreamSpec internalSpec = (StreamSpec) invocationOnMock.callRealMethod(); assertTrue(internalSpec instanceof KafkaStreamSpec); // KafkaStreamSpec is used to carry replication factor assertTrue(internalSpec.isChangeLogStream()); assertEquals(SYSTEM(), internalSpec.getSystemName()); assertEquals(topic, internalSpec.getPhysicalName()); assertEquals(REP_FACTOR, ((KafkaStreamSpec) internalSpec).getReplicationFactor()); assertEquals(PARTITIONS, internalSpec.getPartitionCount()); assertEquals("139", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("segment.bytes")); assertEquals("compact", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("cleanup.policy")); return internalSpec; }).when(admin).toKafkaSpec(Mockito.any()); admin.createStream(spec); admin.validateStream(spec); }
String.valueOf(changeLogReplicationFactor)); admin = Mockito.spy(createSystemAdmin(SYSTEM(), map)); spec = StreamSpec.createChangeLogStreamSpec(topicName, SYSTEM(), changeLogPartitionFactor); kafkaSpec = admin.toKafkaSpec(spec); Assert.assertEquals(changeLogReplicationFactor, kafkaSpec.getReplicationFactor()); spec = StreamSpec.createChangeLogStreamSpec("anotherTopic", SYSTEM(), changeLogPartitionFactor); kafkaSpec = admin.toKafkaSpec(spec); Assert.fail("toKafkaSpec should've failed for missing topic");