public Instance pipe (Instance carrier) { Object data = carrier.getData (); if (data instanceof Tokenization) { // we're done } else if (data instanceof TokenSequence) { StringBuffer buf = new StringBuffer (); TokenSequence ts = (TokenSequence) data; StringTokenization spans = new StringTokenization (buf); // I can use a StringBuffer as the doc! Awesome! for (int i = 0; i < ts.size(); i++) { Token token = ts.get(i); int start = buf.length (); buf.append (token.getText()); int end = buf.length(); StringSpan span = new StringSpan (buf, start, end); span.setFeatures (token.getFeatures ()); span.setProperties (token.getProperties ()); spans.add (span); buf.append (" "); } carrier.setData (spans); } else { throw new IllegalArgumentException ("Can't convert "+data+" to Tokenization."); } return carrier; }
public Instance pipe (Instance carrier) { Object data = carrier.getData (); if (data instanceof Tokenization) { // we're done } else if (data instanceof TokenSequence) { StringBuffer buf = new StringBuffer (); TokenSequence ts = (TokenSequence) data; StringTokenization spans = new StringTokenization (buf); // I can use a StringBuffer as the doc! Awesome! for (int i = 0; i < ts.size(); i++) { Token token = ts.get(i); int start = buf.length (); buf.append (token.getText()); int end = buf.length(); StringSpan span = new StringSpan (buf, start, end); span.setFeatures (token.getFeatures ()); span.setProperties (token.getProperties ()); spans.add (span); buf.append (" "); } carrier.setData (spans); } else { throw new IllegalArgumentException ("Can't convert "+data+" to Tokenization."); } return carrier; }
public Instance pipe (Instance carrier) { Object data = carrier.getData (); if (data instanceof Tokenization) { // we're done } else if (data instanceof TokenSequence) { StringBuffer buf = new StringBuffer (); TokenSequence ts = (TokenSequence) data; StringTokenization spans = new StringTokenization (buf); // I can use a StringBuffer as the doc! Awesome! for (int i = 0; i < ts.size(); i++) { Token token = ts.get(i); int start = buf.length (); buf.append (token.getText()); int end = buf.length(); StringSpan span = new StringSpan (buf, start, end); span.setFeatures (token.getFeatures ()); span.setProperties (token.getProperties ()); spans.add (span); buf.append (" "); } carrier.setData (spans); } else { throw new IllegalArgumentException ("Can't convert "+data+" to Tokenization."); } return carrier; }