/** Create a span, add it to this list and return it */ public Span span(int from, int length) { Span span=new Span(from,length); add(span); return span; }
/** * Returns a ListIterator that iterates over absolutely nothing. * * @return a ListIterator that iterates over absolutely nothing. */ @Override public ListIterator<SpanNode> childIteratorRecursive() { return childIterator(); }
private void correctIndexes(Span span) { if (stringPositions == null) { throw new DeserializationException("Cannot deserialize Span, no access to parent StringFieldValue."); } int fromIndex = stringPositions[span.getFrom()]; int toIndex = stringPositions[span.getTo()]; int length = toIndex - fromIndex; span.setFrom(fromIndex); span.setLength(length); }
/** * This will construct a valid span or throw {@link IllegalArgumentException} * if the span is invalid. * * @param from Start of the span. Must be >= 0. * @param length of the span. Must be >= 0. * @throws IllegalArgumentException if illegal span */ public Span(int from, int length) { setFrom(from); setLength(length); }
private static void addAnnotation(Span here, String term, String orig, TermOccurrences termOccurrences) { if (termOccurrences.termCountBelowLimit(term)) { here.annotate(lowerCaseTermAnnotation(term, orig)); } }
@Override public final CharSequence getText(CharSequence text) { return text.subSequence(from, getTo()); }
public void write(Span span) { buf.put(Span.ID); if (bytePositions != null) { int byteFrom = bytePositions[span.getFrom()]; int byteLength = bytePositions[span.getFrom() + span.getLength()] - byteFrom; buf.putInt1_2_4Bytes(byteFrom); buf.putInt1_2_4Bytes(byteLength); } else { throw new SerializationException("Cannot serialize Span " + span + ", no access to parent StringFieldValue."); } }
private Span typedSpan(int from, int length, TokenType tokenType, SpanList spanList) { return (Span)spanList.span(from, length).annotate(AnnotationTypes.TOKEN_TYPE, tokenType.getValue()); }
public String toString() { return new StringBuilder("span [").append(from).append(',').append(getTo()).append('>').toString(); }
private void correctIndexes(Span span) { if (stringPositions == null) { throw new DeserializationException("Cannot deserialize Span, no access to parent StringFieldValue."); } int fromIndex = stringPositions[span.getFrom()]; int toIndex = stringPositions[span.getTo()]; int length = toIndex - fromIndex; span.setFrom(fromIndex); span.setLength(length); }
private SpanNode copySpan(SpanNode spanTree) { if (spanTree instanceof Span) { return new Span((Span) spanTree); } else if (spanTree instanceof AlternateSpanList) { return new AlternateSpanList((AlternateSpanList) spanTree); } else if (spanTree instanceof SpanList) { return new SpanList((SpanList) spanTree); } else if (spanTree instanceof DummySpanNode) { return spanTree; //shouldn't really happen } else { throw new IllegalStateException("Cannot create copy of " + spanTree + " with class " + ((spanTree == null) ? "null" : spanTree.getClass())); } }
public void write(Span span) { buf.put(Span.ID); if (bytePositions != null) { int byteFrom = bytePositions[span.getFrom()]; int byteLength = bytePositions[span.getFrom() + span.getLength()] - byteFrom; buf.putInt1_2_4Bytes(byteFrom); buf.putInt1_2_4Bytes(byteLength); } else { throw new SerializationException("Cannot serialize Span " + span + ", no access to parent StringFieldValue."); } }
parent.span(pos, len).annotate(lowerCaseTermAnnotation(term, token.getOrig()));
/** * Deep-copies a SpanList. * * @param other the SpanList to copy. */ public SpanList(SpanList other) { this.children = new LinkedList<SpanNode>(); for (SpanNode otherNode : other.children) { if (otherNode instanceof Span) { children.add(new Span((Span) otherNode)); } else if (otherNode instanceof AlternateSpanList) { children.add(new AlternateSpanList((AlternateSpanList) otherNode)); } else if (otherNode instanceof SpanList) { children.add(new SpanList((SpanList) otherNode)); } else if (otherNode instanceof DummySpanNode) { children.add(otherNode); //shouldn't really happen } else { throw new IllegalStateException("Cannot create copy of " + otherNode + " with class " + ((otherNode == null) ? "null" : otherNode.getClass())); } } }
@Override protected void doExecute(ExecutionContext ctx) { StringFieldValue input = (StringFieldValue)ctx.getValue(); SpanList spanList = input.setSpanTree(new SpanTree(SpanTrees.LINGUISTICS)).spanList(); int lastPosition = 0; for (Iterator<GramSplitter.Gram> it = linguistics.getGramSplitter().split(input.getString(), gramSize); it.hasNext();) { GramSplitter.Gram gram = it.next(); // if there is a gap before this gram, then annotate the gram as punctuation // (technically it may be of various types, but it does not matter - we just // need to annotate it somehow (as a non-term) to make sure it is added to the summary) if (lastPosition < gram.getStart()) { typedSpan(lastPosition, gram.getStart() - lastPosition, TokenType.PUNCTUATION, spanList); } // annotate gram as a word term String gramString = gram.extractFrom(input.getString()); typedSpan(gram.getStart(), gram.getLength(), TokenType.ALPHABETIC, spanList). annotate(LinguisticsAnnotator.lowerCaseTermAnnotation(gramString, gramString)); lastPosition = gram.getStart() + gram.getLength(); } // handle punctuation at the end if (lastPosition < input.toString().length()) { typedSpan(lastPosition, input.toString().length() - lastPosition, TokenType.PUNCTUATION, spanList); } }
public AlternateSpanList(AlternateSpanList otherSpanList) { super((List<SpanNode>) null); for (Children otherSubtree : otherSpanList.childTrees) { //create our own subtree: Children children = new Children(this); //copy nodes: for (SpanNode otherNode : otherSubtree.children()) { if (otherNode instanceof Span) { children.add(new Span((Span) otherNode)); } else if (otherNode instanceof AlternateSpanList) { children.add(new AlternateSpanList((AlternateSpanList) otherNode)); } else if (otherNode instanceof SpanList) { children.add(new SpanList((SpanList) otherNode)); } else if (otherNode instanceof DummySpanNode) { children.add(otherNode); //shouldn't really happen } else { throw new IllegalStateException("Cannot create copy of " + otherNode + " with class " + ((otherNode == null) ? "null" : otherNode.getClass())); } } //add this subtree to our subtrees: childTrees.add(children); } }
private SpanNode readSpanNode() { byte type = buf.get(); buf.position(buf.position() - 1); SpanNode retval; if ((type & Span.ID) == Span.ID) { retval = new Span(); if (spanNodes != null) { spanNodes.add(retval); } read((Span) retval); } else if ((type & SpanList.ID) == SpanList.ID) { retval = new SpanList(); if (spanNodes != null) { spanNodes.add(retval); } read((SpanList) retval); } else if ((type & AlternateSpanList.ID) == AlternateSpanList.ID) { retval = new AlternateSpanList(); if (spanNodes != null) { spanNodes.add(retval); } read((AlternateSpanList) retval); } else { throw new DeserializationException("Cannot read SpanNode of type " + type); } return retval; }