public String[] getContext(int index, String[] sequence, String[] priorDecisions, Object[] additionalContext) { return getContext(index,sequence,priorDecisions); }
/** * Initializes the current instance with given samples * and a {@link DefaultPOSContextGenerator}. * @param samples */ public POSSampleEventStream(ObjectStream<POSSample> samples) { this(samples, new DefaultPOSContextGenerator(null)); }
String[] suffs = getSuffixes(lex); for (int i = 0; i < suffs.length; i++) { e.add("suf=" + suffs[i]); String[] prefs = getPrefixes(lex); for (int i = 0; i < prefs.length; i++) { e.add("pre=" + prefs[i]);
public POSSampleSequenceStream(ObjectStream<POSSample> psi) throws IOException { this(psi, new DefaultPOSContextGenerator(null)); }
@Test public void dictionaryMatch() { int indexWithDictionaryMatch = 2; final String[] actual = defaultPOSContextGenerator.getContext(indexWithDictionaryMatch, tokens, tags); final String[] expected = new String[] { "default", "w=tokenC", "p=tokenB", "t=tagB", "pp=tokenA", "t2=tagA,tagB", "n=tokenD", "nn=*SE*" }; Assert.assertArrayEquals("Calling with index matching dictionary entry at: " + indexWithDictionaryMatch + "\nexpected \n" + Arrays.toString(expected) + " but actually was \n" + Arrays.toString(actual), expected, actual); }
String[] suffs = getSuffixes(lex); for (int i = 0; i < suffs.length; i++) { e.add("suf=" + suffs[i]); String[] prefs = getPrefixes(lex); for (int i = 0; i < prefs.length; i++) { e.add("pre=" + prefs[i]);
public AbstractParserEventStream(ObjectStream<Parse> d, HeadRules rules, ParserEventTypeEnum etype, Dictionary dict) { super(d); this.dict = dict; if (etype == ParserEventTypeEnum.CHUNK) { this.chunkerContextGenerator = new ChunkContextGenerator(); } else if (etype == ParserEventTypeEnum.TAG) { this.tagContextGenerator = new DefaultPOSContextGenerator(null); } this.rules = rules; punctSet = rules.getPunctuationTags(); this.etype = etype; init(); }
@Test public void noDictionaryMatch() { int index = 1; final String[] actual = defaultPOSContextGenerator.getContext(index, tokens, tags); final String[] expected = new String[] { "default", "w=tokenB", "suf=B", "suf=nB", "suf=enB", "suf=kenB", "pre=t", "pre=to", "pre=tok", "pre=toke", "c", "p=tokenA", "t=tagA", "pp=*SB*", "n=tokenC", "nn=tokenD" }; Assert.assertArrayEquals("Calling with not matching index at: " + index + "\nexpected \n" + Arrays.toString(expected) + " but actually was \n" + Arrays.toString(actual), expected, actual); }
if (dict == null || !dict.contains(new StringList(dictGram))) { String[] suffs = getSuffixes(lex); for (int i = 0; i < suffs.length; i++) { e.add("suf=" + suffs[i]); String[] prefs = getPrefixes(lex); for (int i = 0; i < prefs.length; i++) { e.add("pre=" + prefs[i]);
public POSContextGenerator getPOSContextGenerator(int cacheSize) { if (artifactProvider != null) { Properties manifest = (Properties) artifactProvider.getArtifact("manifest.properties"); String version = manifest.getProperty("OpenNLP-Version"); if (Version.parse(version).getMinor() < 8) { return new DefaultPOSContextGenerator(cacheSize, getDictionary()); } } return new ConfigurablePOSContextGenerator(cacheSize, createFeatureGenerators()); }
public String[] getContext(int index, String[] sequence, String[] priorDecisions, Object[] additionalContext) { return getContext(index,sequence,priorDecisions); }
@BeforeClass public static void setUp() { final String matchingToken = "tokenC"; tokens = new Object[] {"tokenA", "tokenB", matchingToken, "tokenD"}; final StringList stringList = new StringList(new String[] {matchingToken}); Dictionary dictionary = new Dictionary(); dictionary.put(stringList); defaultPOSContextGenerator = new DefaultPOSContextGenerator(dictionary); tags = new String[] {"tagA", "tagB", "tagC", "tagD"}; }
public String[] getContext(int index, String[] sequence, String[] priorDecisions, Object[] additionalContext) { return getContext(index,sequence,priorDecisions); }
/** * Initializes the current instance with given samples * and a {@link DefaultPOSContextGenerator}. * @param samples */ public POSSampleEventStream(ObjectStream<POSSample> samples) { this(samples, new DefaultPOSContextGenerator(null)); }
public String[] getContext(final int index, String[] sequence, String[] priorDecisions, Object[] additionalContext) { String[] context = super.getContext(index, sequence, priorDecisions, additionalContext); List<String> modContext = new ArrayList<String>(Arrays.asList(context)); getContext(index, sequence, priorDecisions, additionalContext, modContext); context = modContext.toArray(new String[modContext.size()]); return context; } }
/** * Initializes the current instance with given samples * and a {@link DefaultPOSContextGenerator}. * @param samples */ public POSSampleEventStream(ObjectStream<POSSample> samples) { this(samples, new DefaultPOSContextGenerator(null)); }
public String[] getContext(final int index, String[] sequence, String[] priorDecisions, Object[] additionalContext) { String[] context = super.getContext(index, sequence, priorDecisions, additionalContext); List<String> modContext = new ArrayList<String>(Arrays.asList(context)); getContext(index, sequence, priorDecisions, additionalContext, modContext); context = modContext.toArray(new String[modContext.size()]); return context; } }
public POSSampleSequenceStream(ObjectStream<POSSample> psi) throws IOException { this(psi, new DefaultPOSContextGenerator(null)); }
public POSSampleSequenceStream(ObjectStream<POSSample> psi) throws IOException { this(psi, new DefaultPOSContextGenerator(null)); }
/** * * @param samples * @param tagDictionary * @param ngramDictionary * @param cutoff * * @throws IOException its throws if an {@link IOException} is thrown * during IO operations on a temp file which is created during training occur. */ public static POSModel train(String languageCode, ObjectStream<POSSample> samples, POSDictionary tagDictionary, Dictionary ngramDictionary, int cutoff, int iterations) throws IOException { GISModel posModel = opennlp.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(new POSSampleEventStream(samples, new DefaultPOSContextGenerator(ngramDictionary)), cutoff)); return new POSModel(languageCode, posModel, tagDictionary, ngramDictionary); }