/** * Initializes the current instance with a language detector model. Default feature * generation is used. * * @param model the language detector model */ public LanguageDetectorME(LanguageDetectorModel model) { this.model = model; this.mContextGenerator = model.getFactory().getContextGenerator(); }
public static LanguageDetectorModel train(ObjectStream<LanguageSample> samples, TrainingParameters mlParams, LanguageDetectorFactory factory) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); mlParams.putIfAbsent(AbstractEventTrainer.DATA_INDEXER_PARAM, AbstractEventTrainer.DATA_INDEXER_ONE_PASS_VALUE); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel model = trainer.train( new LanguageDetectorEventStream(samples, factory.getContextGenerator())); return new LanguageDetectorModel(model, manifestInfoEntries, factory); } }
@Test public void testDummyFactoryContextGenerator() throws Exception { LanguageDetectorContextGenerator cg = model.getFactory().getContextGenerator(); String[] context = cg.getContext( "a dummy text phrase to test if the context generator works!!!!!!!!!!!!"); Set<String> set = new HashSet(Arrays.asList(context)); Assert.assertTrue(set.contains("!!!!!")); // default normalizer would remove the repeated ! Assert.assertTrue(set.contains("a dum")); Assert.assertTrue(set.contains("tg=[THE,CONTEXT,GENERATOR]")); }
/** * Initializes the current instance with a language detector model. Default feature * generation is used. * * @param model the language detector model */ public LanguageDetectorME(LanguageDetectorModel model) { this.model = model; this.mContextGenerator = model.getFactory().getContextGenerator(); }
/** * Initializes the current instance with a language detector model. Default feature * generation is used. * * @param model the language detector model */ public LanguageDetectorME(LanguageDetectorModel model) { this.model = model; this.mContextGenerator = model.getFactory().getContextGenerator(); }
public static LanguageDetectorModel train(ObjectStream<LanguageSample> samples, TrainingParameters mlParams, LanguageDetectorFactory factory) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); mlParams.putIfAbsent(AbstractEventTrainer.DATA_INDEXER_PARAM, AbstractEventTrainer.DATA_INDEXER_ONE_PASS_VALUE); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel model = trainer.train( new LanguageDetectorEventStream(samples, factory.getContextGenerator())); return new LanguageDetectorModel(model, manifestInfoEntries, factory); } }
public static LanguageDetectorModel train(ObjectStream<LanguageSample> samples, TrainingParameters mlParams, LanguageDetectorFactory factory) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); mlParams.putIfAbsent(AbstractEventTrainer.DATA_INDEXER_PARAM, AbstractEventTrainer.DATA_INDEXER_ONE_PASS_VALUE); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel model = trainer.train( new LanguageDetectorEventStream(samples, factory.getContextGenerator())); return new LanguageDetectorModel(model, manifestInfoEntries, factory); } }