@Override public void annotate(Annotation annotation) { if (annotation.containsKey(CoreAnnotations.TokensAnnotation.class)) { List<CoreLabel> tokens = annotation.get(CoreAnnotations.TokensAnnotation.class); if (DEBUG) { log.info("CleanXML: starting tokens: " + tokens); } List<CoreLabel> newTokens = process(annotation, tokens); // We assume that if someone is using this annotator, they don't // want the old tokens any more and get rid of them // redo the token indexes if xml tokens have been removed setTokenBeginTokenEnd(newTokens); annotation.set(CoreAnnotations.TokensAnnotation.class, newTokens); if (DEBUG) { log.info("CleanXML: ending tokens: " + annotation.get(CoreAnnotations.TokensAnnotation.class)); } } }
@Override public void annotate(Annotation annotation) { if (annotation.containsKey(CoreAnnotations.TokensAnnotation.class)) { List<CoreLabel> tokens = annotation.get(CoreAnnotations.TokensAnnotation.class); if (DEBUG) { log.info("CleanXML: starting tokens: " + tokens); } List<CoreLabel> newTokens = process(annotation, tokens); // We assume that if someone is using this annotator, they don't // want the old tokens any more and get rid of them // redo the token indexes if xml tokens have been removed setTokenBeginTokenEnd(newTokens); annotation.set(CoreAnnotations.TokensAnnotation.class, newTokens); if (DEBUG) { log.info("CleanXML: ending tokens: " + annotation.get(CoreAnnotations.TokensAnnotation.class)); } } }