private Map<String, String> getGeneratedConfig(String runId) { Map<String, String> generatedConfig = new HashMap<>(); if (StringUtils.isNoneEmpty(runId)) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } StreamConfig streamConfig = new StreamConfig(userConfig); Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM; generatedConfig.put(ApplicationConfig.APP_MODE, mode.name()); Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc); generatedConfig.putAll(systemStreamConfigs); // adding app.class in the configuration, unless it is LegacyTaskApplication if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) { generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName()); } return generatedConfig; }
interStreamMap.put("app.mode", ApplicationConfig.ApplicationMode.BATCH.toString()); interStreamMap.put(String.format("streams.%s.samza.intermediate", interStreamId), "true"); interStreamMap.put(String.format("streams.%s.samza.system", interStreamId), "testSystem");
Map<String, String> config = new HashMap<>(); config.put(ApplicationConfig.APP_RUN_ID, "123"); config.put(ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name());
private Map<String, String> getGeneratedConfig(String runId) { Map<String, String> generatedConfig = new HashMap<>(); if (StringUtils.isNoneEmpty(runId)) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } StreamConfig streamConfig = new StreamConfig(userConfig); Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM; generatedConfig.put(ApplicationConfig.APP_MODE, mode.name()); Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc); generatedConfig.putAll(systemStreamConfigs); // adding app.class in the configuration, unless it is LegacyTaskApplication if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) { generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName()); } return generatedConfig; }
@Test public void testGetIntermediateStreamProperties() { Map<String, String> config = new HashMap<>(); KafkaSystemFactory factory = new KafkaSystemFactory(); Map<String, Properties> properties = JavaConversions.mapAsJavaMap( factory.getIntermediateStreamProperties(new MapConfig(config))); assertTrue(properties.isEmpty()); // no properties for stream config.put("streams.test.samza.intermediate", "true"); config.put("streams.test.compression.type", "lz4"); //some random config properties = JavaConversions.mapAsJavaMap( factory.getIntermediateStreamProperties(new MapConfig(config))); assertTrue(properties.isEmpty()); config.put(ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name()); properties = JavaConversions.mapAsJavaMap( factory.getIntermediateStreamProperties(new MapConfig(config))); assertTrue(!properties.isEmpty()); Properties prop = properties.get("test"); assertEquals(prop.getProperty("retention.ms"), String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH())); assertEquals(prop.getProperty("compression.type"), "lz4"); } }
private Map<String, String> getGeneratedConfig(String runId) { Map<String, String> generatedConfig = new HashMap<>(); if (StringUtils.isNoneEmpty(runId)) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } StreamConfig streamConfig = new StreamConfig(userConfig); Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM; generatedConfig.put(ApplicationConfig.APP_MODE, mode.name()); Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc); generatedConfig.putAll(systemStreamConfigs); // adding app.class in the configuration, unless it is LegacyTaskApplication if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) { generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName()); } return generatedConfig; }
private Map<String, String> getGeneratedConfig(String runId) { Map<String, String> generatedConfig = new HashMap<>(); if (StringUtils.isNoneEmpty(runId)) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } StreamConfig streamConfig = new StreamConfig(userConfig); Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM; generatedConfig.put(ApplicationConfig.APP_MODE, mode.name()); Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc); generatedConfig.putAll(systemStreamConfigs); // adding app.class in the configuration, unless it is LegacyTaskApplication if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) { generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName()); } return generatedConfig; }
private Map<String, String> getGeneratedConfig(String runId) { Map<String, String> generatedConfig = new HashMap<>(); if (StringUtils.isNoneEmpty(runId)) { generatedConfig.put(ApplicationConfig.APP_RUN_ID, runId); } StreamConfig streamConfig = new StreamConfig(userConfig); Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds()); inputStreamIds.removeAll(appDesc.getOutputStreamIds()); // exclude intermediate streams ApplicationConfig.ApplicationMode mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM; generatedConfig.put(ApplicationConfig.APP_MODE, mode.name()); Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc); generatedConfig.putAll(systemStreamConfigs); // adding app.class in the configuration, unless it is LegacyTaskApplication if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) { generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName()); } return generatedConfig; }
@Test public void testGetCheckpointTopicProperties() { Map<String, String> config = new HashMap<>(); Properties properties = new KafkaConfig(new MapConfig(config)).getCheckpointTopicProperties(); assertEquals(properties.getProperty("cleanup.policy"), "compact"); assertEquals(properties.getProperty("segment.bytes"), String.valueOf(KafkaConfig.DEFAULT_CHECKPOINT_SEGMENT_BYTES())); config.put(ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name()); properties = new KafkaConfig(new MapConfig(config)).getCheckpointTopicProperties(); assertEquals(properties.getProperty("cleanup.policy"), "compact,delete"); assertEquals(properties.getProperty("segment.bytes"), String.valueOf(KafkaConfig.DEFAULT_CHECKPOINT_SEGMENT_BYTES())); assertEquals(properties.getProperty("retention.ms"), String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH())); } }
@Test public void testGetStreamSpec_Batch() { Map<String, String> config = new HashMap<>(); config.put(ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name()); config.put(ApplicationConfig.APP_RUN_ID, "123"); StreamEdge edge = new StreamEdge(spec, true, false, new MapConfig(config)); assertEquals(edge.getStreamSpec().getPhysicalName(), spec.getPhysicalName() + "-123"); }