public static String tokensToString(Word [] tokens) { StringBuilder sb = new StringBuilder(512); for(int i = 0; i < tokens.length; i ++){ if(i > 0) sb.append(" "); Word l = tokens[i]; sb.append(l.word() + "{" + l.beginPosition() + ", " + l.endPosition() + "}"); } return sb.toString(); }
public static void updateOffsets(List<Word> tokens, int offset) { for(Word l: tokens) { l.setBeginPosition(l.beginPosition() + offset); l.setEndPosition(l.endPosition() + offset); } }
public static void updateOffsets(List<Word> tokens, int offset) { for(Word l: tokens) { l.setBeginPosition(l.beginPosition() + offset); l.setEndPosition(l.endPosition() + offset); } }
@Override public StringInText[] tokenizeplus(String sentence) { Reader r = new StringReader(sentence); List<StringInText> l = new ArrayList<>(); for (String s : tokenize(sentence)) { Word w = new Word(s); l.add(new StringInText(w.word(), w.beginPosition() + startpos, w .endPosition() + startpos)); } StringInText[] tok = new StringInText[l.size()]; // tok[0]=new StringInText(is2.io.CONLLReader09.ROOT,0,0); int i = 0; for (StringInText s : l) tok[i++] = s; startpos += (1 + sentence.length()); return tok; } }
public static void updateOffsets(List<Word> tokens, int offset) { for(Word l: tokens) { l.setBeginPosition(l.beginPosition() + offset); l.setEndPosition(l.endPosition() + offset); } }
public static void updateOffsets(List<Word> tokens, int offset) { for(Word l: tokens) { l.setBeginPosition(l.beginPosition() + offset); l.setEndPosition(l.endPosition() + offset); } }
public StringInText[] tokenizeplus(String sentence) { Reader r = new StringReader(sentence); PTBTokenizer<Word> tokenizer = PTBTokenizer.newPTBTokenizer(r); List<StringInText> l = new ArrayList<>(); while (tokenizer.hasNext()) { Word w = tokenizer.next(); l.add(new StringInText(w.word(), w.beginPosition() + startpos, w .endPosition() + startpos)); } StringInText[] tok = new StringInText[l.size() + 1]; tok[0] = new StringInText(is2.io.CONLLReader09.ROOT, 0, 0); int i = 1; for (StringInText s : l) tok[i++] = s; startpos += (1 + sentence.length()); return tok; }
List<Word> words = parse.yieldWords(); span.setStart(words.get(0).beginPosition() + offset); span.setEnding(words.get(words.size() - 1).endPosition() + offset); Span rawSpan = getRawSpan(span); node.setSpan(rawSpan);