/** * Defines that sequence of characters, which is matched specified regular expression, is a token. */ public Builder token(String regularExpression) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression)); return this; }
@Test public void shouldCorrectlyDeterminePositionWhenTokenSpansMultipleLines() { TokenChannel channel = new TokenChannel("AB\nC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("AB\nCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("AB\nC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(2)); assertThat(codeReader.getColumnPosition(), is(1)); }
@Test public void shouldConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("ABC", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }
@Test public void shouldNormalize() { TokenChannel channel = new TokenChannel("ABC", "normalized"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("ABCD"); assertThat(channel.consume(codeReader, output), is(true)); ArgumentCaptor<Token> token = ArgumentCaptor.forClass(Token.class); verify(output).add(token.capture()); assertThat(token.getValue(), is(new Token("normalized", 1, 0))); verifyNoMoreInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(3)); }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token with specified value. */ public Builder token(String regularExpression, String normalizationValue) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression, normalizationValue)); return this; }
@Test public void shouldNotConsume() { TokenChannel channel = new TokenChannel("ABC"); TokenQueue output = mock(TokenQueue.class); CodeReader codeReader = new CodeReader("123"); assertThat(channel.consume(new CodeReader("123"), output), is(false)); verifyZeroInteractions(output); assertThat(codeReader.getLinePosition(), is(1)); assertThat(codeReader.getColumnPosition(), is(0)); }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token with specified value. */ public Builder token(String regularExpression, String normalizationValue) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression, normalizationValue)); return this; }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token. */ public Builder token(String regularExpression) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression)); return this; }