@Override public String writeAnnotation(JCas jCas, Token token) throws AnalysisEngineProcessException { return token.getCoveredText() + "\t" + token.getPos(); }
@Override public void setPosTag(Token token, String tag) { token.setPos(tag); }
@Override public Token createToken(JCas jCas, int begin, int end) { Token token = (new Token(jCas, begin, end)); token.addToIndexes(); return token; }
initNamedEntity(namedEntityType); Token token = new Token(jCas, documentText.length(), documentText.length() + tok.length()); token.setPos(pos); token.addToIndexes();
public UimaTokenizer(String tokens,UimaResource resource,boolean checkForLabel) { this.checkForLabel = checkForLabel; this.tokens = new ArrayList<>(); try { CAS cas = resource.process(tokens); Collection<Token> tokenList = JCasUtil.select(cas.getJCas(), Token.class); for(Token t : tokenList) { if(!checkForLabel || valid(t.getCoveredText())) if(t.getLemma() != null) this.tokens.add(t.getLemma()); else if(t.getStem() != null) this.tokens.add(t.getStem()); else this.tokens.add(t.getCoveredText()); } resource.release(cas); } catch (Exception e) { e.printStackTrace(); throw new RuntimeException(e); } }
for (Token token : tokens) { lineNo += 1; String text = token.getCoveredText(); String pos = token.getPos(); DependencyNode node; if (maltNode.getHead().getIndex() != 0) { node = new DependencyNode(jCas, token.getBegin(), token.getEnd()); } else { node = new TopDependencyNode(jCas, token.getBegin(), token.getEnd());
String pos = tokenAnn.get(PartOfSpeechAnnotation.class); String lemma = tokenAnn.get(LemmaAnnotation.class); Token token = new Token(jCas, begin, end); token.setPos(pos); token.setLemma(lemma); token.addToIndexes(); int indexBegin = stanfordNode.get(BeginIndexAnnotation.class); int indexEnd = stanfordNode.get(EndIndexAnnotation.class); int tokenBegin = tokens.get(indexBegin).getBegin(); int tokenEnd = tokens.get(indexEnd - 1).getEnd(); DependencyNode node; if (dependencies.getRoots().contains(stanfordNode)) { int begin = tokens.get(corefMention.startIndex - 1).getBegin(); int end = tokens.get(corefMention.endIndex - 2).getEnd();
@Override public String getPosTag(Token token) { return token.getPos(); }
this.write(this.baseWriter, fileName, sentIndex, tokenIndex, token.getCoveredText());
if (anchor != null) { String id = anchor.getId(); boolean isFirstToken = token.getBegin() == anchor.getBegin(); if (isFirstToken) { for (Attribute attr : attributeGetter.getAttributes(anchor)) {
@Override public void setStem(Token token, String stem) { token.setStem(stem); }
@Override public String getLemma(JCas jCas, Token token) { return token.getLemma(); }
initNamedEntity(namedEntityType); Token token = new Token(jCas, documentText.length(), documentText.length() + tok.length()); token.setPos(pos); token.addToIndexes();
public UimaTokenizer(String tokens,UimaResource resource,boolean checkForLabel) { this.checkForLabel = checkForLabel; this.tokens = new ArrayList<>(); try { CAS cas = resource.process(tokens); Collection<Token> tokenList = JCasUtil.select(cas.getJCas(), Token.class); for(Token t : tokenList) { if(!checkForLabel || valid(t.getCoveredText())) if(t.getLemma() != null) this.tokens.add(t.getLemma()); else if(t.getStem() != null) this.tokens.add(t.getStem()); else this.tokens.add(t.getCoveredText()); } resource.release(cas); } catch (Exception e) { e.printStackTrace(); throw new RuntimeException(e); } }
@Override public String getPos(JCas jCas, Token token) { return token.getPos(); }