private ChannelDispatcher<TokenQueue> getChannelDispatcher() { return channelDispatcherBuilder.build(); }
public TokenQueue chunk(Reader reader) { CodeReader code = new CodeReader(reader); TokenQueue queue = new TokenQueue(); try { channelDispatcher.consume(code, queue); return queue; } catch (Exception e) { throw new DuplicationsException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition(), e); } }
private String convert(String input) { CodeReader reader = new CodeReader(input); MarkdownOutput output = new MarkdownOutput(); dispatcher.consume(reader, output); return output.toString(); }
@Test public void shouldConsume() { BlackHoleTokenChannel channel = new BlackHoleTokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output)).isTrue(); assertThat(codeReader.getLinePosition()).isEqualTo(1); assertThat(codeReader.getColumnPosition()).isEqualTo(3); verifyZeroInteractions(output); }
@Override public boolean consume(CodeReader code, TokenQueue output) { if (code.popTo(matcher, tmpBuilder) > 0) { // see SONAR-2499 Cursor previousCursor = code.getPreviousCursor(); if (normalizationValue != null) { output.add(new Token(normalizationValue, previousCursor.getLine(), previousCursor.getColumn())); } else { output.add(new Token(tmpBuilder.toString(), previousCursor.getLine(), previousCursor.getColumn())); } // Godin: note that other channels use method delete in order to do the same thing tmpBuilder.setLength(0); return true; } return false; }
private Markdown() { dispatcher = ChannelDispatcher.builder() .addChannel(new HtmlLinkChannel()) .addChannel(new HtmlUrlChannel()) .addChannel(new HtmlEndOfLineChannel()) .addChannel(new HtmlEmphasisChannel()) .addChannel(new HtmlListChannel()) .addChannel(new HtmlBlockquoteChannel()) .addChannel(new HtmlHeadingChannel()) .addChannel(new HtmlCodeChannel()) .addChannel(new HtmlMultilineCodeChannel()) .addChannel(new IdentifierAndNumberChannel()) .addChannel(new BlackholeChannel()) .build(); }
/** * {@inheritDoc} */ @Override public void setReader(Reader reader) { super.setReader(reader); internalCodeReader = new CodeReader(reader, getConfiguration()); }
@Override public boolean consume(CodeReader code, MarkdownOutput output) { try { if (code.getColumnPosition() == 0 && quotedLineElement.consume(code, output)) { while (endOfLine.consume(code, output) && quotedLineElement.consume(code, output)) { // consume input } output.append("</blockquote>"); return true; } return false; } finally { pendingBlockConstruction = false; } }
@Override public boolean consume(CodeReader code, MarkdownOutput output) { output.append((char)code.pop()); return true; } }
/** * Defines that sequence of characters must be ignored, if it matches specified regular expression. */ public Builder ignore(String regularExpression) { channelDispatcherBuilder.addChannel(new BlackHoleTokenChannel(regularExpression)); return this; }
public Filter(Reader in, CodeReaderFilter<?> codeReaderFilter, CodeReaderConfiguration configuration) { super(in); this.codeReaderFilter = codeReaderFilter; this.codeReaderFilter.setConfiguration(configuration.cloneWithoutCodeReaderFilters()); this.codeReaderFilter.setReader(in); }
/** * Returns the character at the specified index after the cursor without consuming it * * @param index * the relative index of the character to be returned * @return the desired character * @see java.lang.CharSequence#charAt(int) */ public final char charAt(int index) { return (char)intAt(index); }
@Override public Cursor clone() { Cursor clone = new Cursor(); clone.column = column; clone.line = line; return clone; } }
@Test public void shouldNotConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("123"); assertThat(channel.consume(new CodeReader("123"), output), is(false)); verifyZeroInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(0)); }
@Override public boolean consume(CodeReader code, MarkdownOutput output) { try { ListElementChannel currentChannel = null; if (code.getColumnPosition() == 0) { if (orderedListElement.consume(code, output)) { currentChannel = orderedListElement; } else if (unorderedListElement.consume(code, output)) { currentChannel = unorderedListElement; } if (currentChannel != null) { while (endOfLine.consume(code, output) && currentChannel.consume(code, output)) { // consume input } output.append("</" + currentChannel.listElement + ">"); return true; } } return false; } finally { pendingListConstruction = false; } }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token with specified value. */ public Builder token(String regularExpression, String normalizationValue) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression, normalizationValue)); return this; }
@Test public void shouldCorrectlyDeterminePositionWhenTokenSpansMultipleLines() { TokenChannel channel = new TokenChannel("AB\nC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("AB\nCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("AB\nC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(2)); assertThat(codeReader.getColumnPosition(), is(1)); }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token. */ public Builder token(String regularExpression) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression)); return this; }
@Test public void shouldNormalize() { TokenChannel channel = new TokenChannel("ABC", "normalized"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("normalized", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }
@Test public void shouldConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("ABC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }