/** * A helper method to extract avro serialization configurations from the topology configuration and register * specific kryo serializers as necessary. A default serializer will be provided if none is specified in the * configuration. "avro.serializer" should specify the complete class name of the serializer, e.g. * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer" * * @param conf The topology configuration * @throws ClassNotFoundException If the specified serializer cannot be located. */ public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException { final Class serializerClass; if (conf.containsKey("avro.serializer")) { serializerClass = Class.forName((String) conf.get("avro.serializer")); } else { serializerClass = GenericAvroSerializer.class; } conf.registerSerialization(GenericData.Record.class, serializerClass); conf.setSkipMissingKryoRegistrations(false); } }
return (int)c.get(Config.TOPOLOGY_WORKERS) == 100 && (int)c.get(Config.TOPOLOGY_ACKER_EXECUTORS) == 200 && !c.containsKey(Config.TOPOLOGY_DEBUG);
public void loadHandlers(String fileName, String field) throws Exception { Config conf = LGAdapter.readConfig(fileName); if(!conf.containsKey(field)) { throw new Exception("Config doesn't contain key:"+field); } JSONArray handlerDatas = new JSONArray(conf.get(field).toString()); Iterator iterator = handlerDatas.iterator(); while(iterator.hasNext()) { JSONObject handlerData = (JSONObject) iterator.next(); String className = handlerData.getString("class"); Class cls = Class.forName(className); Object handlerObject; if(handlerData.has("argument")) { Constructor<?> cons = cls.getConstructor(String.class); handlerObject = cons.newInstance(handlerData.getString("argument")); } else { Constructor<?> cons = cls.getConstructor(); handlerObject = cons.newInstance(); } Handler<T> tmp = (Handler) handlerObject; addHandler(tmp); } }
public int getTaskCount() { try { if (getConfig().containsKey("adapter.tasks")) { Object hint = config.get("adapter.tasks"); if (hint instanceof String) { return Integer.parseInt(hint.toString()); } return (int) hint; } } catch(Exception e) {} return LGProperties.getInteger("adapter.tasks", getParallelismHint());//default to 1 to 1 with Executors if none are present }//get the total number of tasks
/** * A helper method to extract avro serialization configurations from the topology configuration and register * specific kryo serializers as necessary. A default serializer will be provided if none is specified in the * configuration. "avro.serializer" should specify the complete class name of the serializer, e.g. * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer" * * @param conf The topology configuration * @throws ClassNotFoundException If the specified serializer cannot be located. */ public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException { final Class serializerClass; if (conf.containsKey("avro.serializer")) { serializerClass = Class.forName((String)conf.get("avro.serializer")); } else { serializerClass = GenericAvroSerializer.class; } conf.registerSerialization(GenericData.Record.class, serializerClass); conf.setSkipMissingKryoRegistrations(false); } }
public int getParallelismHint() { try { if (getConfig().containsKey("adapter.threads")) { Object hint = config.get("adapter.threads"); if (hint instanceof String) { return Integer.parseInt(hint.toString()); } return (int) hint; } } catch(Exception e) {} try { Object hint = config.get("topology.parallelism"); if(hint instanceof String) { return Integer.parseInt(hint.toString()); } return (int)hint; } catch(Exception e) {} if(LGProperties.has("adapter.threads")) { return LGProperties.getInteger("adapter.threads", 6); } return LGProperties.getInteger("topology.parallelism", 6);//default to 6 if neither field is present }
if (getConfig().containsKey("max_nodes_per_task")) { String str = getConfig().get("max_nodes_per_task").toString(); try {