@Test public void shouldNotMatchWhenNoRight() { Token t1 = new Token("(", 1, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1))); List<Token> output = mock(List.class); BridgeTokenMatcher matcher = new BridgeTokenMatcher("(", ")"); assertThat(matcher.matchToken(tokenQueue, output), is(false)); verify(tokenQueue, times(1)).isNextTokenValue("("); verify(tokenQueue, times(1)).poll(); verify(tokenQueue, times(2)).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verifyNoMoreInteractions(output); }
@Override public boolean consume(CodeReader code, TokenQueue output) { if (code.popTo(matcher, tmpBuilder) > 0) { // see SONAR-2499 Cursor previousCursor = code.getPreviousCursor(); if (normalizationValue != null) { output.add(new Token(normalizationValue, previousCursor.getLine(), previousCursor.getColumn())); } else { output.add(new Token(tmpBuilder.toString(), previousCursor.getLine(), previousCursor.getColumn())); } // Godin: note that other channels use method delete in order to do the same thing tmpBuilder.setLength(0); return true; } return false; }
@Test public void shouldPushTokenAtBegining() { Token pushedToken = new Token("push", 1, 0); List<Token> pushedTokenList = new ArrayList<>(); pushedTokenList.add(pushedToken); tokenQueue.pushForward(pushedTokenList); assertThat(tokenQueue.peek(), is(pushedToken)); assertThat(tokenQueue.size(), is(4)); }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { if (tokenQueue.isNextTokenValue(tokenToMatch)) { matchedTokenList.add(tokenQueue.poll()); return true; } return false; }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { if (!tokenQueue.isNextTokenValue(lToken)) { return false; } int stack = 0; while (tokenQueue.peek() != null) { Token token = tokenQueue.poll(); if (lToken.equals(token.getValue())) { stack++; } else if (rToken.equals(token.getValue())) { stack--; } matchedTokenList.add(token); if (stack == 0) { return true; } } return false; }
@Test public void shouldNotMatch() { Token t1 = new Token("a", 1, 1); Token t2 = new Token("b", 2, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2))); List<Token> output = mock(List.class); UptoTokenMatcher matcher = new UptoTokenMatcher(new String[] { ";" }); assertThat(matcher.matchToken(tokenQueue, output), is(false)); verify(tokenQueue, times(2)).poll(); verify(tokenQueue, times(2)).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verify(output).add(t2); verifyNoMoreInteractions(output); }
@Test public void shouldMatch() { Token t1 = new Token("a", 1, 1); Token t2 = new Token("b", 2, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2))); List<Token> output = mock(List.class); ExactTokenMatcher matcher = new ExactTokenMatcher("a"); assertThat(matcher.matchToken(tokenQueue, output), is(true)); verify(tokenQueue).isNextTokenValue("a"); verify(tokenQueue).poll(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verifyNoMoreInteractions(output); }
@Test public void shouldNotMatchWhenNoLeft() { Token t1 = new Token("a", 1, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1))); List<Token> output = mock(List.class); BridgeTokenMatcher matcher = new BridgeTokenMatcher("(", ")"); assertThat(matcher.matchToken(tokenQueue, output), is(false)); verify(tokenQueue).isNextTokenValue("("); verifyNoMoreInteractions(tokenQueue); verifyNoMoreInteractions(output); }
@Test public void shouldMatch() { TokenQueue tokenQueue = spy(new TokenQueue()); Token token = new Token("a", 0, 0); List<Token> output = new ArrayList<>(Arrays.asList(token)); ForgetLastTokenMatcher matcher = new ForgetLastTokenMatcher(); assertThat(matcher.matchToken(tokenQueue, output), is(true)); assertThat(output.size(), is(0)); verify(tokenQueue).pushForward(eq(Collections.singletonList(token))); }
@Test public void shouldMatch() { Token t1 = new Token("a", 1, 1); Token t2 = new Token("b", 2, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2))); List<Token> output = mock(List.class); AnyTokenMatcher matcher = new AnyTokenMatcher(); assertThat(matcher.matchToken(tokenQueue, output), is(true)); verify(tokenQueue).poll(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verifyNoMoreInteractions(output); }
@Test public void shouldMatch() { TokenQueue tokenQueue = spy(new TokenQueue()); TokenMatcher delegate = mock(TokenMatcher.class); OptTokenMatcher matcher = new OptTokenMatcher(delegate); List<Token> output = mock(List.class); assertThat(matcher.matchToken(tokenQueue, output), is(true)); verify(delegate).matchToken(tokenQueue, output); verifyNoMoreInteractions(delegate); verifyNoMoreInteractions(tokenQueue); verifyNoMoreInteractions(output); }
public boolean consume(TokenQueue tokenQueue, List<Statement> statements) { Token nextToken = tokenQueue.peek(); while (nextToken != null) { boolean channelConsumed = false; for (StatementChannel channel : channels) { if (channel.consume(tokenQueue, statements)) { channelConsumed = true; break; } } if (!channelConsumed) { throw new IllegalStateException("None of the statement channel has been able to consume token: " + nextToken); } nextToken = tokenQueue.peek(); } return true; }
/** * @return always true */ @Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { int last = matchedTokenList.size() - 1; tokenQueue.pushForward(Collections.singletonList(matchedTokenList.get(last))); matchedTokenList.remove(last); return true; }
/** * @return always true */ @Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { matchedTokenList.add(tokenQueue.poll()); return true; }
private List<Token> chunk(String sourceCode) { List<Token> target = new ArrayList<>(); chunker.chunk(sourceCode).forEach(target::add); return target; }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { if (!tokenQueue.isNextTokenValue(lToken)) { return false; } int stack = 0; while (tokenQueue.peek() != null) { Token token = tokenQueue.poll(); matchedTokenList.add(token); if (lToken.equals(token.getValue())) { stack++; } else if (rToken.equals(token.getValue())) { stack--; } else if (except.equals(token.getValue())) { return false; } if (stack == 0) { return true; } } return false; }
@Test public void shouldMatch() { Token t1 = new Token("a", 1, 1); Token t2 = new Token(";", 2, 1); // should stop on this token Token t3 = new Token(";", 3, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2, t3))); List<Token> output = mock(List.class); UptoTokenMatcher matcher = new UptoTokenMatcher(new String[] { ";" }); assertThat(matcher.matchToken(tokenQueue, output), is(true)); verify(tokenQueue, times(2)).poll(); verify(tokenQueue).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verify(output).add(t2); verifyNoMoreInteractions(output); }