public static LanguageDetectorFactory create(String subclassName) throws InvalidFormatException { if (subclassName == null) { // will create the default factory return new LanguageDetectorFactory(); } try { LanguageDetectorFactory theFactory = ExtensionLoader.instantiateExtension( LanguageDetectorFactory.class, subclassName); theFactory.init(); return theFactory; } catch (Exception e) { String msg = "Could not instantiate the " + subclassName + ". The initialization throw an exception."; throw new InvalidFormatException(msg, e); } }
/** * Initializes the current instance with a language detector model. Default feature * generation is used. * * @param model the language detector model */ public LanguageDetectorME(LanguageDetectorModel model) { this.model = model; this.mContextGenerator = model.getFactory().getContextGenerator(); }
LanguageDetectorFactory factory = LanguageDetectorFactory.create(params.getFactory()); validator = new LanguageDetectorCrossValidator(mlParams, factory, listenersArr);
public static LanguageDetectorModel trainModel() throws Exception { return trainModel(new LanguageDetectorFactory()); }
@Override public void init() { super.init(); }
LanguageDetectorFactory factory = LanguageDetectorFactory.create(params.getFactory()); model = LanguageDetectorME.train(sampleStream, mlParams, factory); } catch (IOException e) {
@Test public void evaluate() throws Exception { TrainingParameters params = new TrainingParameters(); params.put(TrainingParameters.ITERATIONS_PARAM, 100); params.put(TrainingParameters.CUTOFF_PARAM, 5); params.put("PrintMessages", false); final AtomicInteger correctCount = new AtomicInteger(); final AtomicInteger incorrectCount = new AtomicInteger(); LanguageDetectorCrossValidator cv = new LanguageDetectorCrossValidator(params, new LanguageDetectorFactory(), new LanguageDetectorEvaluationMonitor() { @Override public void correctlyClassified(LanguageSample reference, LanguageSample prediction) { correctCount.incrementAndGet(); } @Override public void missclassified(LanguageSample reference, LanguageSample prediction) { incorrectCount.incrementAndGet(); } }); LanguageDetectorSampleStream sampleStream = LanguageDetectorMETest.createSampleStream(); cv.evaluate(sampleStream, 2); Assert.assertEquals(99, cv.getDocumentCount()); Assert.assertEquals(0.98989898989899, cv.getDocumentAccuracy(), 0.01); }
public static LanguageDetectorFactory create(String subclassName) throws InvalidFormatException { if (subclassName == null) { // will create the default factory return new LanguageDetectorFactory(); } try { LanguageDetectorFactory theFactory = ExtensionLoader.instantiateExtension( LanguageDetectorFactory.class, subclassName); theFactory.init(); return theFactory; } catch (Exception e) { String msg = "Could not instantiate the " + subclassName + ". The initialization throw an exception."; throw new InvalidFormatException(msg, e); } }
public static LanguageDetectorModel train(ObjectStream<LanguageSample> samples, TrainingParameters mlParams, LanguageDetectorFactory factory) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); mlParams.putIfAbsent(AbstractEventTrainer.DATA_INDEXER_PARAM, AbstractEventTrainer.DATA_INDEXER_ONE_PASS_VALUE); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel model = trainer.train( new LanguageDetectorEventStream(samples, factory.getContextGenerator())); return new LanguageDetectorModel(model, manifestInfoEntries, factory); } }
LanguageDetectorFactory factory = LanguageDetectorFactory.create(params.getFactory()); validator = new LanguageDetectorCrossValidator(mlParams, factory, listenersArr);
public static LanguageDetectorFactory create(String subclassName) throws InvalidFormatException { if (subclassName == null) { // will create the default factory return new LanguageDetectorFactory(); } try { LanguageDetectorFactory theFactory = ExtensionLoader.instantiateExtension( LanguageDetectorFactory.class, subclassName); theFactory.init(); return theFactory; } catch (Exception e) { String msg = "Could not instantiate the " + subclassName + ". The initialization throw an exception."; throw new InvalidFormatException(msg, e); } }
@Test public void testDummyFactoryContextGenerator() throws Exception { LanguageDetectorContextGenerator cg = model.getFactory().getContextGenerator(); String[] context = cg.getContext( "a dummy text phrase to test if the context generator works!!!!!!!!!!!!"); Set<String> set = new HashSet(Arrays.asList(context)); Assert.assertTrue(set.contains("!!!!!")); // default normalizer would remove the repeated ! Assert.assertTrue(set.contains("a dum")); Assert.assertTrue(set.contains("tg=[THE,CONTEXT,GENERATOR]")); }
LanguageDetectorFactory factory = LanguageDetectorFactory.create(params.getFactory()); validator = new LanguageDetectorCrossValidator(mlParams, factory, listenersArr);
/** * Initializes the current instance with a language detector model. Default feature * generation is used. * * @param model the language detector model */ public LanguageDetectorME(LanguageDetectorModel model) { this.model = model; this.mContextGenerator = model.getFactory().getContextGenerator(); }
LanguageDetectorFactory factory = LanguageDetectorFactory.create(params.getFactory()); model = LanguageDetectorME.train(sampleStream, mlParams, factory); } catch (IOException e) {
/** * Initializes the current instance with a language detector model. Default feature * generation is used. * * @param model the language detector model */ public LanguageDetectorME(LanguageDetectorModel model) { this.model = model; this.mContextGenerator = model.getFactory().getContextGenerator(); }
LanguageDetectorFactory factory = LanguageDetectorFactory.create(params.getFactory()); model = LanguageDetectorME.train(sampleStream, mlParams, factory); } catch (IOException e) {
public static LanguageDetectorModel train(ObjectStream<LanguageSample> samples, TrainingParameters mlParams, LanguageDetectorFactory factory) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); mlParams.putIfAbsent(AbstractEventTrainer.DATA_INDEXER_PARAM, AbstractEventTrainer.DATA_INDEXER_ONE_PASS_VALUE); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel model = trainer.train( new LanguageDetectorEventStream(samples, factory.getContextGenerator())); return new LanguageDetectorModel(model, manifestInfoEntries, factory); } }
public static LanguageDetectorModel train(ObjectStream<LanguageSample> samples, TrainingParameters mlParams, LanguageDetectorFactory factory) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); mlParams.putIfAbsent(AbstractEventTrainer.DATA_INDEXER_PARAM, AbstractEventTrainer.DATA_INDEXER_ONE_PASS_VALUE); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel model = trainer.train( new LanguageDetectorEventStream(samples, factory.getContextGenerator())); return new LanguageDetectorModel(model, manifestInfoEntries, factory); } }