private void initializeTokens() { if (tokens == null) { tokens = textAnnotation.getTokensInSpan(this.getStartSpan(), this.getEndSpan()); } }
private void initializeTokens() { if (tokens == null) { tokens = textAnnotation.getTokensInSpan(this.getStartSpan(), this.getEndSpan()); } }
logger.error("treebank text:"); StringBuffer sb = new StringBuffer(); for (String t : treeta.getTokensInSpan(start, end)) { sb.append(t); sb.append(" ");
public String getText() { return getSpan() != null ? String.join("_", getTextAnnotation().getTokensInSpan(getSpan().getFirst(), getSpan().getSecond())).toLowerCase() : "[undefined]"; }
logger.error("treebank text:"); StringBuffer sb = new StringBuffer(); for (String t : treeta.getTokensInSpan(start, end)) { sb.append(t); sb.append(" ");
@Override protected void addView(TextAnnotation ta) throws AnnotatorException { SpanLabelView datalessView = new SpanLabelView(getViewName(), getClassName(), ta, 1d, true); List<Constituent> tokens = ta.getView(ViewNames.TOKENS).getConstituents(); int numTokens = tokens.size(); int textStart = tokens.get(0).getSpan().getFirst(); int textEnd = tokens.get(numTokens - 1).getSpan().getSecond(); StringBuilder sb = new StringBuilder(); for (String s : ta.getTokensInSpan(textStart, textEnd)) { sb.append(s); sb.append(" "); } SparseVector<Integer> docVector = embedding.getVector(sb.toString().trim()); Set<String> labelIDs = classifier.getFlatPredictions(docVector, topK); for (String labelID : labelIDs) { datalessView.addSpanLabel(textStart, textEnd, labelID, 1d); } ta.addView(getViewName(), datalessView); } }
@Override protected void addView(TextAnnotation ta) throws AnnotatorException { SpanLabelView datalessView = new SpanLabelView(getViewName(), getClassName(), ta, 1d, true); List<Constituent> tokens = ta.getView(ViewNames.TOKENS).getConstituents(); int numTokens = tokens.size(); int textStart = tokens.get(0).getSpan().getFirst(); int textEnd = tokens.get(numTokens - 1).getSpan().getSecond(); StringBuilder sb = new StringBuilder(); for (String s : ta.getTokensInSpan(textStart, textEnd)) { sb.append(s); sb.append(" "); } SparseVector<Integer> docVector = embedding.getVector(sb.toString().trim()); Set<String> labelIDs = classifier.getFlatPredictions(docVector, topK); for (String labelID : labelIDs) { datalessView.addSpanLabel(textStart, textEnd, labelID, 1d); } ta.addView(getViewName(), datalessView); } }
String text = StringUtils.join(" ", ta.getTokensInSpan(starttokint, endtokint));
String[] spantoks = ta.getTokensInSpan(tokspan.getFirst(), tokspan.getSecond()); String text = StringUtils.join(" ", spantoks);