private String convert(String input) { CodeReader reader = new CodeReader(input); MarkdownOutput output = new MarkdownOutput(); dispatcher.consume(reader, output); return output.toString(); }
@Test public void shouldConsume() { BlackHoleTokenChannel channel = new BlackHoleTokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output)).isTrue(); assertThat(codeReader.getLinePosition()).isEqualTo(1); assertThat(codeReader.getColumnPosition()).isEqualTo(3); verifyZeroInteractions(output); }
public TokenQueue chunk(Reader reader) { CodeReader code = new CodeReader(reader); TokenQueue queue = new TokenQueue(); try { channelDispatcher.consume(code, queue); return queue; } catch (Exception e) { throw new DuplicationsException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition(), e); } }
@Test public void shouldNotConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("123"); assertThat(channel.consume(new CodeReader("123"), output), is(false)); verifyZeroInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(0)); }
@Test public void shouldCorrectlyDeterminePositionWhenTokenSpansMultipleLines() { TokenChannel channel = new TokenChannel("AB\nC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("AB\nCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("AB\nC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(2)); assertThat(codeReader.getColumnPosition(), is(1)); }
@Test public void shouldNormalize() { TokenChannel channel = new TokenChannel("ABC", "normalized"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("normalized", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }
@Test public void shouldConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("ABC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }
public class CodeAnalyzer { CodeReader cr; public static void main(String[] args) { cr = new CodeReader(); } }
/** * {@inheritDoc} */ @Override public void setReader(Reader reader) { super.setReader(reader); internalCodeReader = new CodeReader(reader, getConfiguration()); }
public void render(Reader code, List<? extends Channel<HtmlCodeBuilder>> tokenizers, NewHighlighting highlighting) { List<Channel<HtmlCodeBuilder>> allTokenizers = new ArrayList<>(); HighlightingCodeBuilder codeBuilder = new HighlightingCodeBuilder(highlighting); allTokenizers.addAll(tokenizers); new TokenizerDispatcher(allTokenizers).colorize(new CodeReader(code), codeBuilder); highlighting.save(); } }
public SyntaxHighlightingData render(Reader code, List<? extends Channel<HtmlCodeBuilder>> tokenizers) { List<Channel<HtmlCodeBuilder>> allTokenizers = new ArrayList<Channel<HtmlCodeBuilder>>(); HighlightingCodeBuilder codeBuilder = new HighlightingCodeBuilder(); allTokenizers.addAll(tokenizers); new TokenizerDispatcher(allTokenizers).colorize(new CodeReader(code), codeBuilder); return codeBuilder.getHighlightingData(); } }
/** * Parse the input into a list of tokens, with parent/child relations between the tokens. */ public List<Node> parse(Reader reader) { // CodeReader reads the file stream CodeReader codeReader = new CodeReader(reader); // ArrayList collects the nodes List<Node> nodeList = new ArrayList<Node>(); // ChannelDispatcher manages the tokenizers ChannelDispatcher<List<Node>> channelDispatcher = ChannelDispatcher.builder().addChannels((Channel[]) tokenizers.toArray(new Channel[tokenizers.size()])).build(); channelDispatcher.consume(codeReader, nodeList); createNodeHierarchy(nodeList); return nodeList; }
/** * Parse the input into a list of tokens, with parent/child relations between the tokens. */ public List<Node> parse(Reader reader) { // CodeReader reads the file stream CodeReader codeReader = new CodeReader(reader); // ArrayList collects the nodes List<Node> nodeList = new ArrayList<>(); // ChannelDispatcher manages the tokenizers ChannelDispatcher<List<Node>> channelDispatcher = ChannelDispatcher.builder().addChannels((Channel[]) tokenizers.toArray(new Channel[tokenizers.size()])).build(); channelDispatcher.consume(codeReader, nodeList); createNodeHierarchy(nodeList); return nodeList; }
public final String colorize(String code) { HtmlCodeBuilder colorizedCode = new HtmlCodeBuilder(); colorize(new CodeReader(code), colorizedCode); return colorizedCode.toString(); }
/** * Parse the input into a list of tokens, with parent/child relations between the tokens. */ public List<Node> parse(Reader reader) { // CodeReader reads the file stream CodeReader codeReader = new CodeReader(reader); // ArrayList collects the nodes List<Node> nodeList = new ArrayList<>(); // ChannelDispatcher manages the tokenizers ChannelDispatcher<List<Node>> channelDispatcher = ChannelDispatcher.builder() .addChannels((Channel[]) tokenizers.toArray(new Channel[0])) .build(); channelDispatcher.consume(codeReader, nodeList); createNodeHierarchy(nodeList); return nodeList; }
public TokenQueue chunk(Reader reader) { CodeReader code = new CodeReader(reader); TokenQueue queue = new TokenQueue(); try { channelDispatcher.consume(code, queue); return queue; } catch (Exception e) { throw new DuplicationsException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition(), e); } }
@Override public final void tokenize(SourceCode source, Tokens cpdTokens) { String fileName = source.getFileName(); ChannelDispatcher.Builder lexerBuilder = ChannelDispatcher.builder(); lexerBuilder.addChannel(CommentChannel.JSP_COMMENT); lexerBuilder.addChannel(CommentChannel.HTML_COMMENT); lexerBuilder.addChannel(CommentChannel.C_COMMENT); lexerBuilder.addChannel(CommentChannel.CPP_COMMENT); lexerBuilder.addChannel(new WordChannel(fileName)); lexerBuilder.addChannel(new LiteralChannel(fileName)); lexerBuilder.addChannel(new BlackHoleChannel()); ChannelDispatcher<Tokens> lexer = lexerBuilder.build(); try { lexer.consume(new CodeReader(new FileReader(new File(fileName))), cpdTokens); cpdTokens.add(TokenEntry.getEOF()); } catch (FileNotFoundException e) { LOG.error("Unable to open file : " + fileName, e); } }
@Override public String render(Reader code, List<? extends Channel<HtmlCodeBuilder>> tokenizers) { try { List<Channel<HtmlCodeBuilder>> allTokenizers = new ArrayList<Channel<HtmlCodeBuilder>>(); HtmlCodeBuilder codeBuilder = new HtmlCodeBuilder(); HtmlDecorator htmlDecorator = new HtmlDecorator(options); // optimization if (options != null && options.isGenerateTable()) { codeBuilder.appendWithoutTransforming(htmlDecorator.getTagBeginOfFile()); allTokenizers.add(htmlDecorator); } allTokenizers.addAll(tokenizers); new TokenizerDispatcher(allTokenizers).colorize(new CodeReader(code), codeBuilder); // optimization if (options != null && options.isGenerateTable()) { codeBuilder.appendWithoutTransforming(htmlDecorator.getTagEndOfFile()); } return codeBuilder.toString(); } catch (Exception e) { throw new SynhtaxHighlightingException("Can not render code", e); } } }
private void parseToken(Node node) { TagNode element = (TagNode) node; CodeReader codeReader = new CodeReader(node.getCode());
private void parseToken(Node node) { TagNode element = (TagNode) node; CodeReader codeReader = new CodeReader(node.getCode());