public ObjectStream<TokenSample> create(String[] args) { Parameters params = ArgumentParser.parse(args, Parameters.class); ObjectStream<NameSample> samples = StreamFactoryRegistry.getFactory( NameSample.class, "ad").create( ArgumentParser.filter(args, ADNameSampleStreamFactory.Parameters.class)); return new NameToTokenSampleStream(createDetokenizer(params), samples); } }
private void tokenizerCrossEval(TrainingParameters params, double expectedScore) throws IOException { ObjectStream<NameSample> nameSamples = new ADNameSampleStream( getLineSample(FLORESTA_VIRGEM), true); DictionaryDetokenizer detokenizer = new DictionaryDetokenizer( new DetokenizationDictionary(new File("lang/pt/tokenizer/pt-detokenizer.xml"))); ObjectStream<TokenSample> samples = new NameToTokenSampleStream( detokenizer, nameSamples); TokenizerCrossValidator validator; TokenizerFactory tokFactory = TokenizerFactory.create(null, LANG, null, true, null); validator = new opennlp.tools.tokenize.TokenizerCrossValidator(params, tokFactory); validator.evaluate(samples, 10); System.out.println(validator.getFMeasure()); Assert.assertEquals(expectedScore, validator.getFMeasure().getFMeasure(), 0.0001d); }
public ObjectStream<TokenSample> create(String[] args) { Parameters params = ArgumentParser.parse(args, Parameters.class); ObjectStream<NameSample> nameSampleStream = StreamFactoryRegistry.getFactory( NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT).create( ArgumentParser.filter(args, NameSampleDataStreamFactory.Parameters.class)); return new NameToTokenSampleStream(createDetokenizer(params), nameSampleStream); } }
public ObjectStream<TokenSample> create(String[] args) { Parameters params = ArgumentParser.parse(args, Parameters.class); ObjectStream<NameSample> samples = StreamFactoryRegistry.getFactory( NameSample.class, "ad").create( ArgumentParser.filter(args, ADNameSampleStreamFactory.Parameters.class)); return new NameToTokenSampleStream(createDetokenizer(params), samples); } }
public ObjectStream<TokenSample> create(String[] args) { Parameters params = ArgumentParser.parse(args, Parameters.class); ObjectStream<NameSample> nameSampleStream = StreamFactoryRegistry.getFactory( NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT).create( ArgumentParser.filter(args, NameSampleDataStreamFactory.Parameters.class)); return new NameToTokenSampleStream(createDetokenizer(params), nameSampleStream); } }
public ObjectStream<TokenSample> create(String[] args) { Parameters params = ArgumentParser.parse(args, Parameters.class); ObjectStream<NameSample> samples = StreamFactoryRegistry.getFactory( NameSample.class, "ad").create( ArgumentParser.filter(args, ADNameSampleStreamFactory.Parameters.class)); return new NameToTokenSampleStream(createDetokenizer(params), samples); } }
public ObjectStream<TokenSample> create(String[] args) { Parameters params = ArgumentParser.parse(args, Parameters.class); ObjectStream<NameSample> nameSampleStream = StreamFactoryRegistry.getFactory( NameSample.class, StreamFactoryRegistry.DEFAULT_FORMAT).create( ArgumentParser.filter(args, NameSampleDataStreamFactory.Parameters.class)); return new NameToTokenSampleStream(createDetokenizer(params), nameSampleStream); } }