/** * A helper method to extract avro serialization configurations from the topology configuration and register * specific kryo serializers as necessary. A default serializer will be provided if none is specified in the * configuration. "avro.serializer" should specify the complete class name of the serializer, e.g. * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer" * * @param conf The topology configuration * @throws ClassNotFoundException If the specified serializer cannot be located. */ public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException { final Class serializerClass; if (conf.containsKey("avro.serializer")) { serializerClass = Class.forName((String) conf.get("avro.serializer")); } else { serializerClass = GenericAvroSerializer.class; } conf.registerSerialization(GenericData.Record.class, serializerClass); conf.setSkipMissingKryoRegistrations(false); } }
.assumeTrue("Check if CGroups are setup", ((boolean) config.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE)) == true); command.append(entry).append(" "); String correctCommand1 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + " -g memory,cpu:/" + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " "; String correctCommand2 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + " -g cpu,memory:/" + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " "; Assert.assertTrue("Check if cgroup launch command is correct", command.toString().equals(correctCommand1) || command.toString().equals(correctCommand2)); String pathToWorkerCgroupDir = ((String) config.get(Config.STORM_CGROUP_HIERARCHY_DIR)) + "/" + ((String) config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR)) + "/" + workerId;
public void testConfig_extra(boolean longOpt) throws IOException, ParseException { File extraFile = File.createTempFile("extra", "json"); try { FileUtils.write(extraFile, extraConfig); CommandLine cli = new CLIBuilder().with(ParserTopologyCLI.ParserOptions.BROKER_URL, "mybroker") .with(ParserTopologyCLI.ParserOptions.ZK_QUORUM, "myzk") .with(ParserTopologyCLI.ParserOptions.SENSOR_TYPES, "mysensor") .with(ParserTopologyCLI.ParserOptions.MESSAGE_TIMEOUT, "4") .with(ParserTopologyCLI.ParserOptions.EXTRA_OPTIONS, extraFile.getAbsolutePath()) .build(longOpt); Optional<Config> configOptional = ParserTopologyCLI.ParserOptions.getConfig(cli); Config config = configOptional.get(); Assert.assertEquals(4, config.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); Assert.assertEquals("foo", config.get("string")); Assert.assertEquals(1, config.get("integer")); } finally{ extraFile.deleteOnExit(); } }
@Test public void testTopologyConfig_fromConfigExplicitly() throws Exception { testConfigOption(new EnumMap<ParserTopologyCLI.ParserOptions, String>(ParserTopologyCLI.ParserOptions.class) {{ put(ParserTopologyCLI.ParserOptions.NUM_WORKERS, "10"); put(ParserTopologyCLI.ParserOptions.NUM_ACKERS, "20"); }} , input -> { Config c = input.getStormConf(); return (int)c.get(Config.TOPOLOGY_WORKERS) == 10 && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 20; } , () -> { SensorParserConfig config = getBaseConfig(); config.setNumWorkers(100); config.setNumAckers(200); return Collections.singletonList(config); } , input -> { Config c = input.getStormConf(); return (int)c.get(Config.TOPOLOGY_WORKERS) == 100 && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 200 ; } ); }
, input -> { Config c = input.getStormConf(); return (int)c.get(Config.TOPOLOGY_WORKERS) == 10 && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 20 && (boolean)c.get(Config.TOPOLOGY_DEBUG); return (int)c.get(Config.TOPOLOGY_WORKERS) == 100 && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 200 && !c.containsKey(Config.TOPOLOGY_DEBUG);
public void testConfig_noExtra(boolean longOpt) throws ParseException { CommandLine cli = new CLIBuilder().with(ParserTopologyCLI.ParserOptions.BROKER_URL, "mybroker") .with(ParserTopologyCLI.ParserOptions.ZK_QUORUM, "myzk") .with(ParserTopologyCLI.ParserOptions.SENSOR_TYPES, "mysensor") .with(ParserTopologyCLI.ParserOptions.NUM_WORKERS, "1") .with(ParserTopologyCLI.ParserOptions.NUM_ACKERS, "2") .with(ParserTopologyCLI.ParserOptions.NUM_MAX_TASK_PARALLELISM, "3") .with(ParserTopologyCLI.ParserOptions.MESSAGE_TIMEOUT, "4") .build(longOpt); Optional<Config> configOptional = ParserTopologyCLI.ParserOptions.getConfig(cli); Config config = configOptional.get(); Assert.assertEquals(1, config.get(Config.TOPOLOGY_WORKERS)); Assert.assertEquals(2, config.get(Config.TOPOLOGY_ACKER_EXECUTORS)); Assert.assertEquals(3, config.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM)); Assert.assertEquals(4, config.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); }
private void checkConfiguration(Config stormConf) { // ensure that a value has been set for the agent name and that that // agent name is the first value in the agents we advertise for robot // rules parsing String agentName = (String) stormConf.get("http.agent.name"); if (agentName == null || agentName.trim().length() == 0) { String message = "Fetcher: No agents listed in 'http.agent.name'" + " property."; LOG.error(message); throw new IllegalArgumentException(message); } }
public static JSONObject getDNs() throws FileNotFoundException { String file = LGAdapter.DEFAULT_CONFIG.get("cert_keystore").toString(); String pass = LGAdapter.DEFAULT_CONFIG.get("cert_keystore_pass").toString(); String type = LGAdapter.DEFAULT_CONFIG.get("cert_keystore_type").toString(); InputStream is = new FileInputStream(file); return getDNs(is, pass, type); }
public static void setCerts () { try { System.setProperty("javax.net.ssl.keyStore", LGAdapter.DEFAULT_CONFIG.get("cert_keystore").toString()); System.setProperty("javax.net.ssl.keyStoreType", LGAdapter.DEFAULT_CONFIG.get("cert_keystore_type").toString()); System.setProperty("javax.net.ssl.keyStorePassword", LGAdapter.DEFAULT_CONFIG.get("cert_keystore_pass").toString()); System.setProperty("javax.net.ssl.trustStore", LGAdapter.DEFAULT_CONFIG.get("cert_truststore").toString()); System.setProperty("javax.net.ssl.trustStoreType", LGAdapter.DEFAULT_CONFIG.get("cert_truststore_type").toString()); System.setProperty("javax.net.ssl.trustStorePassword", LGAdapter.DEFAULT_CONFIG.get("cert_truststore_pass").toString()); System.setProperty("jsse.enableSNIExtension", "false"); //Fixes unrecognized_name error on Adapters } catch(Exception e) { System.out.println("Error setting system properties for certificates. Ensure cert.json contains values for " + "'cert_keystore', 'cert_keystore_type', 'cert_keystore_pass', 'cert_truststore', 'cert_truststore_type', and 'cert_truststore_pass'."); throw e; } }
private void checkConfiguration() { // ensure that a value has been set for the agent name and that that // agent name is the first value in the agents we advertise for robot // rules parsing String agentName = (String) getConf().get("http.agent.name"); if (agentName == null || agentName.trim().length() == 0) { String message = "Fetcher: No agents listed in 'http.agent.name'" + " property."; LOG.error(message); throw new IllegalArgumentException(message); } }
public static JSONObject merge_config(JSONObject job_config, Config config) {//merges Config into job_config Set<String> keyring = config.keySet(); Iterator<String> keys = keyring.iterator(); while(keys.hasNext()) { String key = keys.next(); Object value = config.get(key); job_config.put(key, value); } return job_config; }
public void registerMetricsConsumer(Class klass, Object argument, long parallelismHint) { HashMap m = new HashMap<>(); m.put("class", klass.getCanonicalName()); m.put("parallelism.hint", parallelismHint); m.put("argument", argument); List l = (List) this.get(TOPOLOGY_METRICS_CONSUMER_REGISTER); if (l == null) { l = new ArrayList<>(); } l.add(m); this.put(TOPOLOGY_METRICS_CONSUMER_REGISTER, l); }
public int getTaskCount() { try { if (getConfig().containsKey("adapter.tasks")) { Object hint = config.get("adapter.tasks"); if (hint instanceof String) { return Integer.parseInt(hint.toString()); } return (int) hint; } } catch(Exception e) {} return LGProperties.getInteger("adapter.tasks", getParallelismHint());//default to 1 to 1 with Executors if none are present }//get the total number of tasks
public int getParallelismHint() { try { if (getConfig().containsKey("adapter.threads")) { Object hint = config.get("adapter.threads"); if (hint instanceof String) { return Integer.parseInt(hint.toString()); } return (int) hint; } } catch(Exception e) {} try { Object hint = config.get("topology.parallelism"); if(hint instanceof String) { return Integer.parseInt(hint.toString()); } return (int)hint; } catch(Exception e) {} if(LGProperties.has("adapter.threads")) { return LGProperties.getInteger("adapter.threads", 6); } return LGProperties.getInteger("topology.parallelism", 6);//default to 6 if neither field is present }
public void setConfig() { if(config == null) { config = DEFAULT_CONFIG; log.info("Creating adapter Config from '"+DEFAULT_CONFIG_FILENAME+"' and '"+DEFAULT_CERTS_FILENAME+"'."); String adapterConfig = getAdapterName()+"Adapter.json"; try { Config specific = readConfig(adapterConfig); Iterator iterator = specific.keySet().iterator(); while(iterator.hasNext()) {//add and override any adapter-specific config fields String key = iterator.next().toString(); Object val = specific.get(key); config.put(key, val); } log.info("Adding adapter-specific config items from '"+adapterConfig+"'."); } catch (IOException e) { log.warn("Couldn't read file '"+adapterConfig+"'. No adapter-specific config items added."); } config.put("name", this.getAdapterName());//add adapter name to config } }
@Test public void testRegister() { Config config = new Config(); Assert.assertNull(config.get(Config.TOPOLOGY_WORKER_METRICS)); Assert.assertNull(config.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER)); SigarLoggingMetricsConsumer.register(config, null); Assert.assertNotNull(config.get(Config.TOPOLOGY_WORKER_METRICS)); Assert.assertNotNull(config.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER)); Map<String, String> metrics = (Map<String, String>) config.get(Config.TOPOLOGY_WORKER_METRICS); Assert.assertEquals(metrics, SigarLoggingMetricsConsumer.METRICS); List<Object> register = (List<Object>) config.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER); Assert.assertEquals(register.size(), 1); Map<Object, Object> actual = (Map<Object, Object>) register.get(0); Map<Object, Object> expected = new HashMap<>(); expected.put("class", LoggingMetricsConsumer.class.getCanonicalName()); expected.put("parallelism.hint", 1L); expected.put("argument", null); Assert.assertEquals(actual, expected); }
/** * A helper method to extract avro serialization configurations from the topology configuration and register * specific kryo serializers as necessary. A default serializer will be provided if none is specified in the * configuration. "avro.serializer" should specify the complete class name of the serializer, e.g. * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer" * * @param conf The topology configuration * @throws ClassNotFoundException If the specified serializer cannot be located. */ public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException { final Class serializerClass; if (conf.containsKey("avro.serializer")) { serializerClass = Class.forName((String)conf.get("avro.serializer")); } else { serializerClass = GenericAvroSerializer.class; } conf.registerSerialization(GenericData.Record.class, serializerClass); conf.setSkipMissingKryoRegistrations(false); } }
@Test public void testMetricsAdditionNotReplacement() { Config config = new Config(); Map<String, String> metrics = new HashMap<>(); metrics.put("foo", "foo.bar.baz"); config.put(Config.TOPOLOGY_WORKER_METRICS, metrics); SigarLoggingMetricsConsumer.register(config, null); Assert.assertNotNull(config.get(Config.TOPOLOGY_WORKER_METRICS)); Map<String, String> actual = (Map<String, String>) config.get(Config.TOPOLOGY_WORKER_METRICS); Assert.assertTrue(actual.keySet().containsAll(SigarLoggingMetricsConsumer.METRICS.keySet())); Assert.assertTrue(actual.values().containsAll(SigarLoggingMetricsConsumer.METRICS.values())); Assert.assertEquals(actual.get("foo"), "foo.bar.baz"); } }
@Test public void testRegisteringIMetricsConsumer() { Config config = new Config(); BulletStormConfig bulletStormConfig = new BulletStormConfig(); Assert.assertNull(bulletStormConfig.get(CustomIMetricsConsumer.CUSTOM_METRICS_REGISTERED)); ReflectionUtils.registerMetricsConsumer(LoggingMetricsConsumer.class.getName(), config, bulletStormConfig); Assert.assertNull(config.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER)); ReflectionUtils.registerMetricsConsumer(CustomIMetricsConsumer.class.getName(), config, bulletStormConfig); Assert.assertNotNull(config.get(Config.TOPOLOGY_METRICS_CONSUMER_REGISTER)); Assert.assertTrue((Boolean) bulletStormConfig.get(CustomIMetricsConsumer.CUSTOM_METRICS_REGISTERED)); } }