@Test public void shouldConsume() { BlackHoleTokenChannel channel = new BlackHoleTokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output)).isTrue(); assertThat(codeReader.getLinePosition()).isEqualTo(1); assertThat(codeReader.getColumnPosition()).isEqualTo(3); verifyZeroInteractions(output); }
public TokenQueue chunk(Reader reader) { CodeReader code = new CodeReader(reader); TokenQueue queue = new TokenQueue(); try { channelDispatcher.consume(code, queue); return queue; } catch (Exception e) { throw new DuplicationsException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition(), e); } }
@Test public void shouldConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("ABC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }
@Test public void shouldNormalize() { TokenChannel channel = new TokenChannel("ABC", "normalized"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("normalized", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }
@Test public void shouldCorrectlyDeterminePositionWhenTokenSpansMultipleLines() { TokenChannel channel = new TokenChannel("AB\nC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("AB\nCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("AB\nC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(2)); assertThat(codeReader.getColumnPosition(), is(1)); }
@Test public void shouldNotConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("123"); assertThat(channel.consume(new CodeReader("123"), output), is(false)); verifyZeroInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(0)); }
@Override public boolean consume(CodeReader code, Tokens cpdTokens) { if (code.popTo(matcher, token) > 0) { cpdTokens.add(new TokenEntry(token.toString(), fileName, code.getLinePosition())); token = new StringBuilder(); return true; } else { return false; } } }
@Override public boolean consume(CodeReader code, Tokens cpdTokens) { if (code.popTo(matcher, token) > 0) { cpdTokens.add(new TokenEntry(token.toString(), fileName, code.getLinePosition())); token = new StringBuilder(); return true; } else { return false; } } }
private void handleBeforeAttributeName(CodeReader codeReader, TagNode element) { Attribute attribute; StringBuilder sbQName = new StringBuilder(); codeReader.popTo(endQNameMatcher, sbQName); attribute = new Attribute(sbQName.toString().trim()); attribute.setLine(codeReader.getLinePosition() + element.getStartLinePosition() - 1); element.getAttributes().add(attribute); }
private static void handleBeforeAttributeName(CodeReader codeReader, TagNode element) { Attribute attribute; StringBuilder sbQName = new StringBuilder(); codeReader.popTo(endQNameMatcher, sbQName); attribute = new Attribute(sbQName.toString().trim()); attribute.setLine(codeReader.getLinePosition() + element.getStartLinePosition() - 1); element.getAttributes().add(attribute); }
private static void handleBeforeAttributeName(CodeReader codeReader, TagNode element) { Attribute attribute; StringBuilder sbQName = new StringBuilder(); codeReader.popTo(endQNameMatcher, sbQName); attribute = new Attribute(sbQName.toString().trim()); attribute.setLine(codeReader.getLinePosition() + element.getStartLinePosition() - 1); element.getAttributes().add(attribute); }
protected final void setStartPosition(CodeReader code, Node node) { node.setStartLinePosition(code.getLinePosition()); node.setStartColumnPosition(code.getColumnPosition()); }
protected final void setEndPosition(CodeReader code, Node node) { node.setEndLinePosition(code.getLinePosition()); node.setEndColumnPosition(code.getColumnPosition()); }
protected final void setEndPosition(CodeReader code, Node node) { node.setEndLinePosition(code.getLinePosition()); node.setEndColumnPosition(code.getColumnPosition()); }
protected final void setStartPosition(CodeReader code, Node node) { node.setStartLinePosition(code.getLinePosition()); node.setStartColumnPosition(code.getColumnPosition()); }
protected final void setEndPosition(CodeReader code, Node node) { node.setEndLinePosition(code.getLinePosition()); node.setEndColumnPosition(code.getColumnPosition()); }
protected final void setStartPosition(CodeReader code, Node node) { node.setStartLinePosition(code.getLinePosition()); node.setStartColumnPosition(code.getColumnPosition()); }
@Override public boolean consume(CodeReader code, HtmlCodeBuilder codeBuilder) { int lineNumber = code.getLinePosition(); if (code.peek() == LF || code.peek() == CR) { code.pop(); if (lineNumber != code.getLinePosition()) { codeBuilder.appendWithoutTransforming(getTagAfter()); codeBuilder.appendWithoutTransforming(getTagBefore()); } return true; } return false; }
public TokenQueue chunk(Reader reader) { CodeReader code = new CodeReader(reader); TokenQueue queue = new TokenQueue(); try { channelDispatcher.consume(code, queue); return queue; } catch (Exception e) { throw new DuplicationsException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition(), e); } }
@Override public boolean consume(CodeReader code, O output) { int nextChar = code.peek(); while (nextChar != -1) { boolean characterConsumed = false; for (Channel<O> channel : channels) { if (channel.consume(code, output)) { characterConsumed = true; break; } } if ( !characterConsumed) { if (LOG.isDebugEnabled() || failIfNoChannelToConsumeOneCharacter) { String message = "None of the channel has been able to handle character '" + (char) code.peek() + "' (decimal value " + code.peek() + ") at line " + code.getLinePosition() + ", column " + code.getColumnPosition(); if (failIfNoChannelToConsumeOneCharacter) { throw new IllegalStateException(message); } LOG.debug(message); } code.pop(); } nextChar = code.peek(); } return true; }