/** Make the next token. * * @param txt What the token should be * @param originalText The original String that got transformed into txt */ private Object getNext(String txt, String originalText) { if (tokenFactory == null) { throw new RuntimeException(this.getClass().getName() + ": Token factory is null."); } if (invertible) { //String str = prevWordAfter.toString(); //prevWordAfter.setLength(0); CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength()); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); //word.set(CoreAnnotations.BeforeAnnotation.class, str); //prevWord.set(CoreAnnotations.AfterAnnotation.class, str); //prevWord = word; return word; } else { return tokenFactory.makeToken(txt, yychar, yylength()); } }
throw new IllegalArgumentException("SpanishLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
throw new IllegalArgumentException("FrenchLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
private Object getNext(String txt, String originalText, String annotation) { txt = LexerUtils.removeSoftHyphens(txt); Label w = (Label) tokenFactory.makeToken(txt, yychar, yylength()); if (invertible || annotation != null) { CoreLabel word = (CoreLabel) w; if (invertible) { String str = prevWordAfter.toString(); prevWordAfter.setLength(0); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); word.set(CoreAnnotations.BeforeAnnotation.class, str); prevWord.set(CoreAnnotations.AfterAnnotation.class, str); prevWord = word; } if (annotation != null) { word.set(CoreAnnotations.ParentAnnotation.class, annotation); } } return w; }
private Object getNext(String txt, String originalText, String annotation) { txt = LexerUtils.removeSoftHyphens(txt); Label w = (Label) tokenFactory.makeToken(txt, yychar, yylength()); if (invertible || annotation != null) { CoreLabel word = (CoreLabel) w; if (invertible) { String str = prevWordAfter.toString(); prevWordAfter.setLength(0); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); word.set(CoreAnnotations.BeforeAnnotation.class, str); prevWord.set(CoreAnnotations.AfterAnnotation.class, str); prevWord = word; } if (annotation != null) { word.set(CoreAnnotations.ParentAnnotation.class, annotation); } } return w; }
/** Make the next token. * * @param txt What the token should be * @param originalText The original String that got transformed into txt */ private Object getNext(String txt, String originalText) { if (tokenFactory == null) { throw new RuntimeException(this.getClass().getName() + ": Token factory is null."); } if (invertible) { //String str = prevWordAfter.toString(); //prevWordAfter.setLength(0); CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength()); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); //word.set(CoreAnnotations.BeforeAnnotation.class, str); //prevWord.set(CoreAnnotations.AfterAnnotation.class, str); //prevWord = word; return word; } else { return tokenFactory.makeToken(txt, yychar, yylength()); } }
/** Make the next token. * * @param txt What the token should be * @param originalText The original String that got transformed into txt */ private Object getNext(String txt, String originalText) { if (tokenFactory == null) { throw new RuntimeException(this.getClass().getName() + ": Token factory is null."); } if (invertible) { //String str = prevWordAfter.toString(); //prevWordAfter.setLength(0); CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength()); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); //word.set(CoreAnnotations.BeforeAnnotation.class, str); //prevWord.set(CoreAnnotations.AfterAnnotation.class, str); //prevWord = word; return word; } else { return tokenFactory.makeToken(txt, yychar, yylength()); } }
/** Make the next token. * * @param txt What the token should be * @param originalText The original String that got transformed into txt */ private Object getNext(String txt, String originalText) { if (tokenFactory == null) { throw new RuntimeException(this.getClass().getName() + ": Token factory is null."); } if (invertible) { //String str = prevWordAfter.toString(); //prevWordAfter.setLength(0); CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength()); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); //word.set(CoreAnnotations.BeforeAnnotation.class, str); //prevWord.set(CoreAnnotations.AfterAnnotation.class, str); //prevWord = word; return word; } else { return tokenFactory.makeToken(txt, yychar, yylength()); } }
/** Make the next token. * @param txt What the token should be * @param originalText The original String that got transformed into txt */ private Object getNext(String txt, String originalText) { if (invertible) { String str = prevWordAfter.toString(); prevWordAfter.setLength(0); CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength()); word.set(OriginalTextAnnotation.class, originalText); word.set(BeforeAnnotation.class, str); prevWord.set(AfterAnnotation.class, str); prevWord = word; return word; } else { return tokenFactory.makeToken(txt, yychar, yylength()); } }
/** Make the next token. * @param txt What the token should be * @param originalText The original String that got transformed into txt */ private Object getNext(String txt, String originalText) { if (invertible) { String str = prevWordAfter.toString(); prevWordAfter.setLength(0); CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength()); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); word.set(CoreAnnotations.BeforeAnnotation.class, str); prevWord.set(CoreAnnotations.AfterAnnotation.class, str); prevWord = word; return word; } else { return tokenFactory.makeToken(txt, yychar, yylength()); } }
throw new IllegalArgumentException("FrenchLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
throw new IllegalArgumentException("FrenchLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
throw new IllegalArgumentException("SpanishLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
throw new IllegalArgumentException("FrenchLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
throw new IllegalArgumentException("SpanishLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
throw new IllegalArgumentException("PTBLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
throw new IllegalArgumentException("PTBLexer: the invertible option requires a CoreLabelTokenFactory"); prevWord = (CoreLabel) tf.makeToken("", 0, 0); prevWordAfter = new StringBuilder();
private Object getNext(String txt, String originalText, String annotation) { txt = LexerUtils.removeSoftHyphens(txt); Label w = (Label) tokenFactory.makeToken(txt, yychar, yylength()); if (invertible || annotation != null) { CoreLabel word = (CoreLabel) w; if (invertible) { String str = prevWordAfter.toString(); prevWordAfter.setLength(0); word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText); word.set(CoreAnnotations.BeforeAnnotation.class, str); prevWord.set(CoreAnnotations.AfterAnnotation.class, str); prevWord = word; } if (annotation != null) { word.set(CoreAnnotations.ParentAnnotation.class, annotation); } } return w; }