public TokenQueue chunk(Reader reader) { CodeReader code = new CodeReader(reader); TokenQueue queue = new TokenQueue(); try { channelDispatcher.consume(code, queue); return queue; } catch (Exception e) { throw new DuplicationsException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition(), e); } }
@Override public boolean consume(CodeReader code, TokenQueue output) { if (code.popTo(matcher, tmpBuilder) > 0) { // see SONAR-2499 Cursor previousCursor = code.getPreviousCursor(); if (normalizationValue != null) { output.add(new Token(normalizationValue, previousCursor.getLine(), previousCursor.getColumn())); } else { output.add(new Token(tmpBuilder.toString(), previousCursor.getLine(), previousCursor.getColumn())); } // Godin: note that other channels use method delete in order to do the same thing tmpBuilder.setLength(0); return true; } return false; }
@Override public boolean consume(CodeReader code, MarkdownOutput output) { output.append((char)code.pop()); return true; } }
@Override public boolean consume(CodeReader code, HtmlCodeBuilder codeBuilder) { if (!isCDATAStarted(codeBuilder) && code.peek() == startToken[0] && Arrays.equals(code.peek(startToken.length), startToken)) { codeBuilder.appendWithoutTransforming(tagBefore); // Consume CDATA start code.popTo(Pattern.compile(Pattern.quote(CDATA_START)).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); setCDATAStarted(codeBuilder, true); return true; } if (isCDATAStarted(codeBuilder)) { if (code.peek() == endToken[0] && Arrays.equals(code.peek(endToken.length), endToken)) { codeBuilder.appendWithoutTransforming(tagBefore); // Consume CDATA end code.popTo(Pattern.compile(Pattern.quote(CDATA_END)).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); setCDATAStarted(codeBuilder, false); } else { // Consume everything between CDATA code.pop(codeBuilder); } return true; } return false; }
private void parseToken(Node node) { TagNode element = (TagNode) node; CodeReader codeReader = new CodeReader(node.getCode()); for (int ch = codeReader.peek(); ch != -1; ch = codeReader.peek()) { codeReader.pop(); continue; case '=': mode = ParseMode.BEFORE_ATTRIBUTE_VALUE; codeReader.pop(); continue; case '<': case '%': case '@': codeReader.pop(); continue; default:
@Override public boolean consume(CodeReader code, O output) { int nextChar = code.peek(); while (nextChar != -1) { boolean characterConsumed = false; for (Channel<O> channel : channels) { if (channel.consume(code, output)) { characterConsumed = true; break; } } if ( !characterConsumed) { if (LOG.isDebugEnabled() || failIfNoChannelToConsumeOneCharacter) { String message = "None of the channel has been able to handle character '" + (char) code.peek() + "' (decimal value " + code.peek() + ") at line " + code.getLinePosition() + ", column " + code.getColumnPosition(); if (failIfNoChannelToConsumeOneCharacter) { throw new IllegalStateException(message); } LOG.debug(message); } code.pop(); } nextChar = code.peek(); } return true; }
private String convert(String input) { CodeReader reader = new CodeReader(input); MarkdownOutput output = new MarkdownOutput(); dispatcher.consume(reader, output); return output.toString(); }
protected final void setStartPosition(CodeReader code, Node node) { node.setStartLinePosition(code.getLinePosition()); node.setStartColumnPosition(code.getColumnPosition()); }
@Override public boolean consume(CodeReader code, Tokens cpdTokens) { if (code.popTo(matcher, token) > 0) { cpdTokens.add(new TokenEntry(token.toString(), fileName, code.getLinePosition())); token = new StringBuilder(); return true; } else { return false; } } }
@Override public boolean consume(CodeReader code, HtmlCodeBuilder codeBuilder) { if (code.peek() == startToken[0] && Arrays.equals(code.peek(startToken.length), startToken)) { codeBuilder.appendWithoutTransforming(tagBefore); code.popTo(LINE_END_MATCHER, codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); return true; } else { return false; } }
/** * Read and consume the next characters according to a given regular expression * * @param matcher * the regular expression matcher * @param appendable * the consumed characters are appended to this appendable * @return number of consumed characters or -1 if the next input sequence doesn't match this matcher's pattern */ public final int popTo(Matcher matcher, Appendable appendable) { return popTo(matcher, null, appendable); }
@Override public boolean consume(CodeReader code, HtmlCodeBuilder codeBuilder) { int lineNumber = code.getLinePosition(); if (code.peek() == LF || code.peek() == CR) { code.pop(); if (lineNumber != code.getLinePosition()) { codeBuilder.appendWithoutTransforming(getTagAfter()); codeBuilder.appendWithoutTransforming(getTagBefore()); } return true; } return false; }
/** * {@inheritDoc} */ @Override public int read(char[] filteredBuffer, int offset, int length) throws IOException { if (internalCodeReader.peek() == -1) { return -1; } int initialOffset = offset; while (offset < filteredBuffer.length) { if (internalCodeReader.peek() == -1) { break; } boolean consumed = false; for (Channel<O> channel : channels) { if (channel.consume(internalCodeReader, getOutput())) { consumed = true; break; } } if ( !consumed) { int charRead = internalCodeReader.pop(); filteredBuffer[offset++] = (char) charRead; } } return offset - initialOffset; }
/** * Consume attribute value until eol or same quote */ private void consumeAttributeValueUntilQuoteOrEol(CodeReader code, HtmlCodeBuilder codeBuilder, char quote) { // consume attribute value until eol or same quote while (code.peek() != quote && code.peek() != '\r' && code.peek() != '\n') { code.pop(codeBuilder); } if (code.charAt(0) == quote) { // consume the closing quote code.pop(codeBuilder); setAttributeStarted(codeBuilder, false); setAttributeValueStarted(codeBuilder, null); } codeBuilder.appendWithoutTransforming(tagAfterAttributeValue); }
/** * @deprecated in 2.2, use {@link #popTo(Matcher matcher, Appendable appendable)} instead */ @Deprecated public final void popTo(EndMatcher matcher, Appendable appendable) { previousCursor = getCursor().clone(); try { do { appendable.append((char) pop()); } while (!matcher.match(peek()) && peek() != -1); } catch (IOException e) { throw new ChannelException(e.getMessage(), e); } }
code.popTo(Pattern.compile(Pattern.quote(DoctypeParts.DOCTYPE.getSelectorStart())).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); consumed = true; code.popTo(Pattern.compile(Pattern.quote(DoctypeParts.ELEMENT.getSelectorStart())).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); consumed = true; code.popTo(Pattern.compile(Pattern.quote(DoctypeParts.ATT_LIST.getSelectorStart())).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); consumed = true; code.popTo(Pattern.compile(Pattern.quote(DoctypeParts.ATT_LIST.getSelectorEnd())).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); consumed = true; code.popTo(Pattern.compile(Pattern.quote(DoctypeParts.ELEMENT.getSelectorEnd())).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); consumed = true; code.popTo(Pattern.compile(Pattern.quote(DoctypeParts.DOCTYPE.getSelectorEnd())).matcher(""), codeBuilder); codeBuilder.appendWithoutTransforming(tagAfter); consumed = true; code.pop(codeBuilder); consumed = true;
private boolean hasNextToken(CodeReader code) { int lastChar = code.lastChar(); if (isJavaConstantStart(code.peek()) && !Character.isJavaIdentifierPart(lastChar) && !Character.isJavaIdentifierStart(lastChar) && lastChar != DOT) { String constant = code.peekTo(endTokenMatcher); int nextCharAfterConstant = code.peek(constant.length() + 1)[constant.length()]; if (nextCharAfterConstant != 0 && Character.isJavaIdentifierPart(nextCharAfterConstant)) { return false; } return true; } return false; }
if (afterMatcher != null) { afterMatcher.reset(this); afterMatcher.region(matcher.end(), length()); if (!afterMatcher.lookingAt()) { return -1; previousCursor = getCursor().clone(); for (int i = 0; i < matcher.end(); i++) { appendable.append((char) pop()); + "' at line " + getCursor().getLine() + " and column " + getCursor().getColumn() + ", because it led to a stack overflow error." + " This error may be due to an inefficient use of alternations - see http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=5050507", e);
@Override public boolean consume(CodeReader code, MarkdownOutput output) { try { ListElementChannel currentChannel = null; if (code.getColumnPosition() == 0) { if (orderedListElement.consume(code, output)) { currentChannel = orderedListElement; } else if (unorderedListElement.consume(code, output)) { currentChannel = unorderedListElement; } if (currentChannel != null) { while (endOfLine.consume(code, output) && currentChannel.consume(code, output)) { // consume input } output.append("</" + currentChannel.listElement + ">"); return true; } } return false; } finally { pendingListConstruction = false; } }