static synchronized Token getGraphemePattern() { if (Token.token_grapheme != null) return Token.token_grapheme; Token base_char = Token.createRange(); // [{ASSIGNED}]-[{M},{C}] base_char.mergeRanges(Token.getRange("ASSIGNED", true)); base_char.subtractRanges(Token.getRange("M", true)); base_char.subtractRanges(Token.getRange("C", true)); Token virama = Token.createRange(); for (int i = 0; i < Token.viramaString.length(); i++) { virama.addRange(i, i); } Token combiner_wo_virama = Token.createRange(); combiner_wo_virama.mergeRanges(Token.getRange("M", true)); combiner_wo_virama.addRange(0x1160, 0x11ff); // hangul_medial and hangul_final combiner_wo_virama.addRange(0xff9e, 0xff9f); // extras Token left = Token.createUnion(); // base_char? left.addChild(base_char); left.addChild(Token.token_empty); Token foo = Token.createUnion(); foo.addChild(Token.createConcat(virama, Token.getRange("L", true))); foo.addChild(combiner_wo_virama); foo = Token.createClosure(foo); foo = Token.createConcat(left, foo); Token.token_grapheme = foo; return Token.token_grapheme; }
static synchronized Token getGraphemePattern() { if (Token.token_grapheme != null) return Token.token_grapheme; Token base_char = Token.createRange(); // [{ASSIGNED}]-[{M},{C}] base_char.mergeRanges(Token.getRange("ASSIGNED", true)); base_char.subtractRanges(Token.getRange("M", true)); base_char.subtractRanges(Token.getRange("C", true)); Token virama = Token.createRange(); for (int i = 0; i < Token.viramaString.length(); i ++) { virama.addRange(i, i); } Token combiner_wo_virama = Token.createRange(); combiner_wo_virama.mergeRanges(Token.getRange("M", true)); combiner_wo_virama.addRange(0x1160, 0x11ff); // hangul_medial and hangul_final combiner_wo_virama.addRange(0xff9e, 0xff9f); // extras Token left = Token.createUnion(); // base_char? left.addChild(base_char); left.addChild(Token.token_empty); Token foo = Token.createUnion(); foo.addChild(Token.createConcat(virama, Token.getRange("L", true))); foo.addChild(combiner_wo_virama); foo = Token.createClosure(foo); foo = Token.createConcat(left, foo); Token.token_grapheme = foo; return Token.token_grapheme; }
/** * term ::= factor+ */ Token parseTerm() throws ParseException { int ch = this.read(); if (ch == T_OR || ch == T_RPAREN || ch == T_EOF) { return Token.createEmpty(); } else { Token tok = this.parseFactor(); Token concat = null; while ((ch = this.read()) != T_OR && ch != T_RPAREN && ch != T_EOF) { if (concat == null) { concat = Token.createConcat(); concat.addChild(tok); tok = concat; } concat.addChild(this.parseFactor()); //tok = Token.createConcat(tok, this.parseFactor()); } return tok; } }
/** * term ::= factor+ */ Token parseTerm() throws ParseException { int ch = this.read(); if (ch == T_OR || ch == T_RPAREN || ch == T_EOF) { return Token.createEmpty(); } else { Token tok = this.parseFactor(); Token concat = null; while ((ch = this.read()) != T_OR && ch != T_RPAREN && ch != T_EOF) { if (concat == null) { concat = Token.createConcat(); concat.addChild(tok); tok = concat; } concat.addChild(this.parseFactor()); //tok = Token.createConcat(tok, this.parseFactor()); } return tok; } }
@Override Token processPlus(Token tok) throws ParseException { // X+ -> XX* this.next(); return Token.createConcat(tok, Token.createClosure(tok)); }
Token processPlus(Token tok) throws ParseException { // X+ -> XX* this.next(); if (this.read() == T_QUESTION) { this.next(); return Token.createConcat(tok, Token.createNGClosure(tok)); } else return Token.createConcat(tok, Token.createClosure(tok)); } Token processQuestion(Token tok) throws ParseException {
static synchronized Token getCombiningCharacterSequence() { if (Token.token_ccs != null) return Token.token_ccs; Token foo = Token.createClosure(Token.getRange("M", true)); // \pM* foo = Token.createConcat(Token.getRange("M", false), foo); // \PM + \pM* Token.token_ccs = foo; return Token.token_ccs; }
Token processPlus(Token tok) throws ParseException { // X+ -> XX* this.next(); return Token.createConcat(tok, Token.createClosure(tok)); } Token processQuestion(Token tok) throws ParseException {
Token processPlus(Token tok) throws ParseException { // X+ -> XX* this.next(); if (this.read() == T_QUESTION) { this.next(); return Token.createConcat(tok, Token.createNGClosure(tok)); } else return Token.createConcat(tok, Token.createClosure(tok)); } Token processQuestion(Token tok) throws ParseException {
static synchronized Token getCombiningCharacterSequence() { if (Token.token_ccs != null) return Token.token_ccs; Token foo = Token.createClosure(Token.getRange("M", true)); // \pM* foo = Token.createConcat(Token.getRange("M", false), foo); // \PM + \pM* Token.token_ccs = foo; return Token.token_ccs; }