public boolean consume(TokenQueue tokenQueue, List<Statement> statements) { Token nextToken = tokenQueue.peek(); while (nextToken != null) { boolean channelConsumed = false; for (StatementChannel channel : channels) { if (channel.consume(tokenQueue, statements)) { channelConsumed = true; break; } } if (!channelConsumed) { throw new IllegalStateException("None of the statement channel has been able to consume token: " + nextToken); } nextToken = tokenQueue.peek(); } return true; }
public List<Statement> chunk(@Nullable TokenQueue tokenQueue) { if (tokenQueue == null) { throw new IllegalArgumentException(); } List<Statement> statements = new ArrayList<>(); try { channelDispatcher.consume(tokenQueue, statements); return statements; } catch (Exception e) { throw new DuplicationsException("Unable to build statement from token : " + tokenQueue.peek(), e); } }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { do { Token token = tokenQueue.poll(); matchedTokenList.add(token); if (uptoMatchTokens.contains(token.getValue())) { return true; } } while (tokenQueue.peek() != null); return false; }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { if (!tokenQueue.isNextTokenValue(lToken)) { return false; } int stack = 0; while (tokenQueue.peek() != null) { Token token = tokenQueue.poll(); if (lToken.equals(token.getValue())) { stack++; } else if (rToken.equals(token.getValue())) { stack--; } matchedTokenList.add(token); if (stack == 0) { return true; } } return false; }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { if (!tokenQueue.isNextTokenValue(lToken)) { return false; } int stack = 0; while (tokenQueue.peek() != null) { Token token = tokenQueue.poll(); matchedTokenList.add(token); if (lToken.equals(token.getValue())) { stack++; } else if (rToken.equals(token.getValue())) { stack--; } else if (except.equals(token.getValue())) { return false; } if (stack == 0) { return true; } } return false; }
@Test public void shouldConsume() { TokenMatcher tokenMatcher = mock(TokenMatcher.class); when(tokenMatcher.matchToken(any(TokenQueue.class), anyList())).thenReturn(true); StatementChannel channel = StatementChannel.create(tokenMatcher); StatementChannelDisptacher dispatcher = new StatementChannelDisptacher(asList(channel)); TokenQueue tokenQueue = mock(TokenQueue.class); when(tokenQueue.peek()).thenReturn(new Token("a", 1, 0)).thenReturn(null); List<Statement> statements = mock(List.class); assertThat(dispatcher.consume(tokenQueue, statements), is(true)); verify(tokenQueue, times(2)).peek(); verifyNoMoreInteractions(tokenQueue); verifyNoMoreInteractions(statements); }
@Test(expected = IllegalStateException.class) public void shouldThrowAnException() { TokenMatcher tokenMatcher = mock(TokenMatcher.class); StatementChannel channel = StatementChannel.create(tokenMatcher); StatementChannelDisptacher dispatcher = new StatementChannelDisptacher(asList(channel)); TokenQueue tokenQueue = mock(TokenQueue.class); when(tokenQueue.peek()).thenReturn(new Token("a", 1, 0)).thenReturn(null); List<Statement> statements = mock(List.class); dispatcher.consume(tokenQueue, statements); }
@Test public void shouldNotMatchWhenNoRight() { Token t1 = new Token("(", 1, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1))); List<Token> output = mock(List.class); BridgeTokenMatcher matcher = new BridgeTokenMatcher("(", ")"); assertThat(matcher.matchToken(tokenQueue, output), is(false)); verify(tokenQueue, times(1)).isNextTokenValue("("); verify(tokenQueue, times(1)).poll(); verify(tokenQueue, times(2)).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verifyNoMoreInteractions(output); }
@Test public void shouldNotMatch() { Token t1 = new Token("a", 1, 1); Token t2 = new Token("b", 2, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2))); List<Token> output = mock(List.class); UptoTokenMatcher matcher = new UptoTokenMatcher(new String[] { ";" }); assertThat(matcher.matchToken(tokenQueue, output), is(false)); verify(tokenQueue, times(2)).poll(); verify(tokenQueue, times(2)).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verify(output).add(t2); verifyNoMoreInteractions(output); }
@Test public void shouldMatch() { Token t1 = new Token("a", 1, 1); Token t2 = new Token(";", 2, 1); // should stop on this token Token t3 = new Token(";", 3, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2, t3))); List<Token> output = mock(List.class); UptoTokenMatcher matcher = new UptoTokenMatcher(new String[] { ";" }); assertThat(matcher.matchToken(tokenQueue, output), is(true)); verify(tokenQueue, times(2)).poll(); verify(tokenQueue).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verify(output).add(t2); verifyNoMoreInteractions(output); }
@Test public void shouldPeekToken() { Token token = tokenQueue.peek(); assertThat(token, is(new Token("a", 1, 0))); assertThat(tokenQueue.size(), is(3)); }
@Test public void shouldPushTokenAtBegining() { Token pushedToken = new Token("push", 1, 0); List<Token> pushedTokenList = new ArrayList<>(); pushedTokenList.add(pushedToken); tokenQueue.pushForward(pushedTokenList); assertThat(tokenQueue.peek(), is(pushedToken)); assertThat(tokenQueue.size(), is(4)); }
@Test public void shouldMatchAnyOfProvidedTokens() { Token t1 = new Token("a", 1, 1); Token t2 = new Token("{", 2, 1); Token t3 = new Token("b", 3, 1); Token t4 = new Token("}", 4, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2, t3, t4))); List<Token> output = mock(List.class); UptoTokenMatcher matcher = new UptoTokenMatcher(new String[] { "{", "}" }); assertThat(matcher.matchToken(tokenQueue, output), is(true)); assertThat(matcher.matchToken(tokenQueue, output), is(true)); verify(tokenQueue, times(4)).poll(); verify(tokenQueue, times(2)).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verify(output).add(t2); verify(output).add(t3); verify(output).add(t4); verifyNoMoreInteractions(output); }
@Test public void shouldMatch() { Token t1 = new Token("(", 1, 1); Token t2 = new Token("a", 2, 1); Token t3 = new Token("(", 3, 1); Token t4 = new Token("b", 4, 1); Token t5 = new Token(")", 5, 1); Token t6 = new Token("c", 6, 1); Token t7 = new Token(")", 7, 1); TokenQueue tokenQueue = spy(new TokenQueue(Arrays.asList(t1, t2, t3, t4, t5, t6, t7))); List<Token> output = mock(List.class); BridgeTokenMatcher matcher = new BridgeTokenMatcher("(", ")"); assertThat(matcher.matchToken(tokenQueue, output), is(true)); verify(tokenQueue, times(1)).isNextTokenValue("("); verify(tokenQueue, times(7)).poll(); verify(tokenQueue, times(7)).peek(); verifyNoMoreInteractions(tokenQueue); verify(output).add(t1); verify(output).add(t2); verify(output).add(t3); verify(output).add(t4); verify(output).add(t5); verify(output).add(t6); verify(output).add(t7); verifyNoMoreInteractions(output); }
public boolean consume(TokenQueue tokenQueue, List<Statement> statements) { Token nextToken = tokenQueue.peek(); while (nextToken != null) { boolean channelConsumed = false; for (StatementChannel channel : channels) { if (channel.consume(tokenQueue, statements)) { channelConsumed = true; break; } } if (!channelConsumed) { throw new IllegalStateException("None of the statement channel has been able to consume token: " + nextToken); } nextToken = tokenQueue.peek(); } return true; }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { do { Token token = tokenQueue.poll(); matchedTokenList.add(token); if (uptoMatchTokens.contains(token.getValue())) { return true; } } while (tokenQueue.peek() != null); return false; }
public List<Statement> chunk(TokenQueue tokenQueue) { if (tokenQueue == null) { throw new IllegalArgumentException(); } List<Statement> statements = new ArrayList<Statement>(); try { channelDispatcher.consume(tokenQueue, statements); return statements; } catch (Exception e) { throw new DuplicationsException("Unable to build statement from token : " + tokenQueue.peek(), e); } }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { if (!tokenQueue.isNextTokenValue(lToken)) { return false; } int stack = 0; while (tokenQueue.peek() != null) { Token token = tokenQueue.poll(); if (lToken.equals(token.getValue())) { stack++; } else if (rToken.equals(token.getValue())) { stack--; } matchedTokenList.add(token); if (stack == 0) { return true; } } return false; }
@Override public boolean matchToken(TokenQueue tokenQueue, List<Token> matchedTokenList) { if (!tokenQueue.isNextTokenValue(lToken)) { return false; } int stack = 0; while (tokenQueue.peek() != null) { Token token = tokenQueue.poll(); matchedTokenList.add(token); if (lToken.equals(token.getValue())) { stack++; } else if (rToken.equals(token.getValue())) { stack--; } else if (except.equals(token.getValue())) { return false; } if (stack == 0) { return true; } } return false; }