private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
/** * Gets the System to use for reading/writing checkpoints. Uses the following precedence. * * 1. If job.changelog.system is defined, that value is used. * 2. If job.default.system is defined, that value is used. * 3. null * * Note: Changelogs can be defined using * stores.storeName.changelog=systemName.streamName or * stores.storeName.changelog=streamName * * If the former syntax is used, that system name will still be honored. For the latter syntax, this method is used. * * @return the name of the system to use by default for all changelogs, if defined. */ public String getChangelogSystem() { return get(CHANGELOG_SYSTEM, get(JobConfig.JOB_DEFAULT_SYSTEM(), null)); }
/** * Gets the System to use for reading/writing checkpoints. Uses the following precedence. * * 1. If job.changelog.system is defined, that value is used. * 2. If job.default.system is defined, that value is used. * 3. null * * Note: Changelogs can be defined using * stores.storeName.changelog=systemName.streamName or * stores.storeName.changelog=streamName * * If the former syntax is used, that system name will still be honored. For the latter syntax, this method is used. * * @return the name of the system to use by default for all changelogs, if defined. */ public String getChangelogSystem() { return get(CHANGELOG_SYSTEM, get(JobConfig.JOB_DEFAULT_SYSTEM(), null)); }
/** * Gets the System to use for reading/writing checkpoints. Uses the following precedence. * * 1. If job.changelog.system is defined, that value is used. * 2. If job.default.system is defined, that value is used. * 3. null * * Note: Changelogs can be defined using * stores.storeName.changelog=systemName.streamName or * stores.storeName.changelog=streamName * * If the former syntax is used, that system name will still be honored. For the latter syntax, this method is used. * * @return the name of the system to use by default for all changelogs, if defined. */ public String getChangelogSystem() { return get(CHANGELOG_SYSTEM, get(JobConfig.JOB_DEFAULT_SYSTEM(), null)); }
/** * Gets the System to use for reading/writing checkpoints. Uses the following precedence. * * 1. If job.changelog.system is defined, that value is used. * 2. If job.default.system is defined, that value is used. * 3. null * * Note: Changelogs can be defined using * stores.storeName.changelog=systemName.streamName or * stores.storeName.changelog=streamName * * If the former syntax is used, that system name will still be honored. For the latter syntax, this method is used. * * @return the name of the system to use by default for all changelogs, if defined. */ public String getChangelogSystem() { return get(CHANGELOG_SYSTEM, get(JobConfig.JOB_DEFAULT_SYSTEM(), null)); }
/** * Gets the System to use for reading/writing checkpoints. Uses the following precedence. * * 1. If job.changelog.system is defined, that value is used. * 2. If job.default.system is defined, that value is used. * 3. null * * Note: Changelogs can be defined using * stores.storeName.changelog=systemName.streamName or * stores.storeName.changelog=streamName * * If the former syntax is used, that system name will still be honored. For the latter syntax, this method is used. * * @return the name of the system to use by default for all changelogs, if defined. */ public String getChangelogSystem() { return get(CHANGELOG_SYSTEM, get(JobConfig.JOB_DEFAULT_SYSTEM(), null)); }
public void testCreateChangelogStreamHelp(final String topic) { final int PARTITIONS = 12; final int REP_FACTOR = 2; Map<String, String> map = new HashMap<>(); map.put(JobConfig.JOB_DEFAULT_SYSTEM(), SYSTEM()); map.put(String.format("stores.%s.changelog", "fakeStore"), topic); map.put(String.format("stores.%s.changelog.replication.factor", "fakeStore"), String.valueOf(REP_FACTOR)); map.put(String.format("stores.%s.changelog.kafka.segment.bytes", "fakeStore"), "139"); KafkaSystemAdmin admin = Mockito.spy(createSystemAdmin(SYSTEM(), map)); StreamSpec spec = StreamSpec.createChangeLogStreamSpec(topic, SYSTEM(), PARTITIONS); Mockito.doAnswer(invocationOnMock -> { StreamSpec internalSpec = (StreamSpec) invocationOnMock.callRealMethod(); assertTrue(internalSpec instanceof KafkaStreamSpec); // KafkaStreamSpec is used to carry replication factor assertTrue(internalSpec.isChangeLogStream()); assertEquals(SYSTEM(), internalSpec.getSystemName()); assertEquals(topic, internalSpec.getPhysicalName()); assertEquals(REP_FACTOR, ((KafkaStreamSpec) internalSpec).getReplicationFactor()); assertEquals(PARTITIONS, internalSpec.getPartitionCount()); assertEquals("139", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("segment.bytes")); assertEquals("compact", ((KafkaStreamSpec) internalSpec).getProperties().getProperty("cleanup.policy")); return internalSpec; }).when(admin).toKafkaSpec(Mockito.any()); admin.createStream(spec); admin.validateStream(spec); }
configs.put(JobConfig.JOB_NAME(), "jobName"); configs.put(JobConfig.JOB_ID(), "jobId"); configs.put(JobConfig.JOB_DEFAULT_SYSTEM(), intermediateSystem); StreamTestUtils.addStreamConfigs(configs, inputStreamId, inputSystem, inputPhysicalName); StreamTestUtils.addStreamConfigs(configs, outputStreamId, outputSystem, outputPhysicalName);
Map<String, String> configMap = new HashMap<>(); configMap.put(JobConfig.JOB_NAME(), "test-app"); configMap.put(JobConfig.JOB_DEFAULT_SYSTEM(), DEFAULT_SYSTEM); StreamTestUtils.addStreamConfigs(configMap, "input1", "system1", "input1"); StreamTestUtils.addStreamConfigs(configMap, "input2", "system2", "input2");
configs.put(JobConfig.JOB_NAME(), "jobName"); configs.put(JobConfig.JOB_ID(), "jobId"); configs.put(JobConfig.JOB_DEFAULT_SYSTEM(), intermediateSystem); StreamTestUtils.addStreamConfigs(configs, inputStreamId, inputSystem, inputPhysicalName); StreamTestUtils.addStreamConfigs(configs, outputStreamId, outputSystem, outputPhysicalName);
configs.put(JobConfig.JOB_DEFAULT_SYSTEM(), "test-system"); StreamTestUtils.addStreamConfigs(configs, streamId0, system, streamId0); StreamTestUtils.addStreamConfigs(configs, streamId1, system, streamId1);
@Before public void setUp() { defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>()); inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName"); outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName"); intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName"); input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde); input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde); outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde); intermediateInputDescriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); intermediateOutputDescriptor = intermediateSystemDescriptor.getOutputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); broadcastInputDesriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-broadcast-b1", defaultSerde) .withPhysicalName("jobName-jobId-broadcast-b1"); Map<String, String> configs = new HashMap<>(); configs.put(JobConfig.JOB_NAME(), "jobName"); configs.put(JobConfig.JOB_ID(), "jobId"); configs.putAll(input1Descriptor.toConfig()); configs.putAll(input2Descriptor.toConfig()); configs.putAll(outputDescriptor.toConfig()); configs.putAll(inputSystemDescriptor.toConfig()); configs.putAll(outputSystemDescriptor.toConfig()); configs.putAll(intermediateSystemDescriptor.toConfig()); configs.put(JobConfig.JOB_DEFAULT_SYSTEM(), intermediateSystemDescriptor.getSystemName()); mockConfig = spy(new MapConfig(configs)); mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig); }
map.put(JobConfig.JOB_DEFAULT_SYSTEM(), SYSTEM());
@Test public void testGetStreamWithSystemAtDefaultScopeInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME), JobConfig.JOB_DEFAULT_SYSTEM(), TEST_DEFAULT_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_DEFAULT_SYSTEM, spec.getSystemName()); }
@Test public void testGetStreamWithSystemAtBothScopesInConfig() { Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, StreamConfig.SYSTEM(), TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM(), TEST_DEFAULT_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); }
Map<String, String> configMap = new HashMap<>(); configMap.put(JobConfig.JOB_NAME(), "test-app"); configMap.put(JobConfig.JOB_DEFAULT_SYSTEM(), "test-system"); StreamTestUtils.addStreamConfigs(configMap, "PageView", "hdfs", "hdfs:/user/dummy/PageViewEvent"); StreamTestUtils.addStreamConfigs(configMap, "PageViewCount", "kafka", "PageViewCount");