@Override public void process(char token) { // we will keep like special characters together. if (peek(-1) != token) { pop(current); // the current token is finished. if (token == '$') { Character next = peek(1); if (Character.isDigit(next) || ( next == '.' && Character.isDigit(peek(2)))) { push(new State(TokenizerState.IN_WORD), current); return; } } push(new State(TokenizerState.IN_SPECIAL), current); } } },
@Override public void process(char token) { // we will keep like special characters together. if (peek(-1) != token) { pop(current); // the current token is finished. if (token == '$') { Character next = peek(1); if (Character.isDigit(next) || ( next == '.' && Character.isDigit(peek(2)))) { push(new State(TokenizerState.IN_WORD), current); return; } } push(new State(TokenizerState.IN_SPECIAL), current); } } },
@Override public void process(char token) { // we have something, so the paragraph has mass. stack.get(stack.size()-1).hasMass = true; if (token == '$') { Character next = peek(1); if (Character.isDigit(next) || ( next == '.' && Character.isDigit(peek(2)))) { push(new State(TokenizerState.IN_WORD), current); } else { push(new State(TokenizerState.IN_SPECIAL), current); } } else { // this was just push IN_SPECIAL, added the push in_word to match the // old tokenizer push(new State(TokenizerState.IN_SPECIAL), current); } } },
@Override public void process(char token) { // we have something, so the paragraph has mass. stack.get(stack.size()-1).hasMass = true; if (token == '$') { Character next = peek(1); if (Character.isDigit(next) || ( next == '.' && Character.isDigit(peek(2)))) { push(new State(TokenizerState.IN_WORD), current); } else { push(new State(TokenizerState.IN_SPECIAL), current); } } else { // this was just push IN_SPECIAL, added the push in_word to match the // old tokenizer push(new State(TokenizerState.IN_SPECIAL), current); } } },
@Override public void process(char token) { // we will keep like special characters together. if (peek(-1) != token) { pop(current); // the current word is finished. push(new State(TokenizerState.IN_SPECIAL), current); // No matter what // we push a new // word token. } } },
char nc = peek(1); switch (nc) { case ' ':
char nc = peek(1); switch (nc) { case ' ':
char nc = peek(1); switch (nc) { case ' ':
@Override public void process(char token) { if (token == '$') { if (Character.isDigit(peek(1))) { push(new State(TokenizerState.IN_WORD), current); } else { push(new State(TokenizerState.IN_SPECIAL), current); } } else { // this was just push IN_SPECIAL, added the push in_word to match the // old tokenizer push(new State(TokenizerState.IN_SPECIAL), current); } } },
int advance = 1; while (true) { char next = peek(advance); if (Character.isDigit(next) || next == '/') { advance++; char next = peek(1); if (Character.isDigit(next) || Character.isAlphabetic(next)) return; char after = peek(1); char before = peek(-1); if (!(Character.isDigit(before) && Character.isDigit(after))) { case '`': case '’': char nc = peek(1); char nnc = peek(2); char nnnc = peek(3); char pc = peek(-1); if (token == '\'' && (pc == 'n' && peek(-2) == 'i')) { State s = getCurrent(); if (!Character.isUpperCase(text[s.start])) if (isURL()) return; char prev = peek(-1); char next = peek(1);
int advance = 1; while (true) { char next = peek(advance); if (Character.isDigit(next)) { advance++; char next = peek(1); if (Character.isDigit(next) || Character.isAlphabetic(next)) return; char after = peek(1); char before = peek(-1); if (!(Character.isDigit(before) && Character.isDigit(after))) { case '`': case '’': char nc = peek(1); char nnc = peek(2); char nnnc = peek(3); char pc = peek(-1); if (token == '\'' && (pc == 'n' && peek(-2) == 'i')) { State s = getCurrent(); if (!Character.isUpperCase(text[s.start])) if (isURL()) return; char prev = peek(-1); char next = peek(1);
int advance = 1; while (true) { char next = peek(advance); if (Character.isDigit(next) || next == '/') { advance++; char next = peek(1); if (Character.isDigit(next) || Character.isAlphabetic(next)) return; char after = peek(1); char before = peek(-1); if (!(Character.isDigit(before) && Character.isDigit(after))) { case '`': case '’': char nc = peek(1); char nnc = peek(2); char nnnc = peek(3); char pc = peek(-1); if (token == '\'' && (pc == 'n' && peek(-2) == 'i')) { State s = getCurrent(); if (!Character.isUpperCase(text[s.start])) if (isURL()) return; char prev = peek(-1); char next = peek(1);