private StatementChannelDisptacher getChannelDispatcher() { return new StatementChannelDisptacher(channels); } }
@Override public String toString() { return "[" + getStartLine() + "-" + getEndLine() + "] [" + getValue() + "]"; }
public static StatementChannel createBlackHole(TokenMatcher... tokenMatchers) { return new StatementChannel(true, tokenMatchers); }
public static StatementChunker build() { return StatementChunker.builder() .ignore(from("import"), to(";")) .ignore(from("package"), to(";")) .statement(new BridgeWithExceptionTokenMatcher("{", "}", ";")) .ignore(token("}")) .ignore(token("{")) .ignore(token(";")) .statement(from("@"), anyToken(), opt(bridge("(", ")"))) .statement(from("do")) .statement(from("if"), bridge("(", ")")) .statement(from("else"), token("if"), bridge("(", ")")) .statement(from("else")) .statement(from("for"), bridge("(", ")")) .statement(from("while"), bridge("(", ")")) .statement(from("try"), bridge("(", ")")) .statement(from("case"), to(";", "{", "}"), forgetLastToken()) .statement(from("default"), to(";", "{", "}"), forgetLastToken()) .statement(to(";", "{", "}"), forgetLastToken()) .build(); }
@Test public void shouldCreateMatchers() { assertThat(TokenMatcherFactory.anyToken(), instanceOf(AnyTokenMatcher.class)); assertThat(TokenMatcherFactory.bridge("(", ")"), instanceOf(BridgeTokenMatcher.class)); assertThat(TokenMatcherFactory.forgetLastToken(), instanceOf(ForgetLastTokenMatcher.class)); assertThat(TokenMatcherFactory.from("if"), instanceOf(ExactTokenMatcher.class)); assertThat(TokenMatcherFactory.opt(mock(TokenMatcher.class)), instanceOf(OptTokenMatcher.class)); assertThat(TokenMatcherFactory.to(";"), instanceOf(UptoTokenMatcher.class)); assertThat(TokenMatcherFactory.token(";"), instanceOf(ExactTokenMatcher.class)); }
@Test(expected = IllegalStateException.class) public void shouldThrowAnException() { TokenMatcher tokenMatcher = mock(TokenMatcher.class); StatementChannel channel = StatementChannel.create(tokenMatcher); StatementChannelDisptacher dispatcher = new StatementChannelDisptacher(asList(channel)); TokenQueue tokenQueue = mock(TokenQueue.class); when(tokenQueue.peek()).thenReturn(new Token("a", 1, 0)).thenReturn(null); List<Statement> statements = mock(List.class); dispatcher.consume(tokenQueue, statements); }
private List<Statement> chunk(String sourceCode) { return statementChunker.chunk(tokenChunker.chunk(sourceCode)); }
/** * Defines that sequence of tokens, which is matched specified list of matchers, is a statement. * * @see TokenMatcherFactory */ public Builder statement(TokenMatcher... matchers) { channels.add(StatementChannel.create(matchers)); return this; }
/** * Creates list of statements from Strings, each statement on a new line starting from 0. */ protected static List<Statement> createStatementsFromStrings(String... values) { List<Statement> result = new ArrayList<>(); for (int i = 0; i < values.length; i++) { result.add(new Statement(i, i, values[i])); } return result; }
public boolean consume(TokenQueue tokenQueue, List<Statement> statements) { Token nextToken = tokenQueue.peek(); while (nextToken != null) { boolean channelConsumed = false; for (StatementChannel channel : channels) { if (channel.consume(tokenQueue, statements)) { channelConsumed = true; break; } } if (!channelConsumed) { throw new IllegalStateException("None of the statement channel has been able to consume token: " + nextToken); } nextToken = tokenQueue.peek(); } return true; }
public static Builder builder() { return new Builder(); }
public StatementChunker build() { return new StatementChunker(this); }
/** * Defines that sequence of tokens must be ignored, if it matches specified list of matchers. * * @see TokenMatcherFactory */ public Builder ignore(TokenMatcher... matchers) { channels.add(StatementChannel.createBlackHole(matchers)); return this; }
private static void addToIndex(CloneIndex index, String resourceId, String sourceCode) { List<Statement> statements = STATEMENT_CHUNKER.chunk(TOKEN_CHUNKER.chunk(sourceCode)); BlockChunker blockChunker = new BlockChunker(2); List<Block> blocks = blockChunker.chunk(resourceId, statements); for (Block block : blocks) { index.insert(block); } }
@Test(expected = IllegalArgumentException.class) public void shouldNotAcceptEmpty() { StatementChannel.create(new TokenMatcher[]{}); }
@Test(expected = IllegalArgumentException.class) public void shouldNotAcceptNull() { new Statement(null); }
public static StatementChannel create(TokenMatcher... tokenMatchers) { return new StatementChannel(false, tokenMatchers); }
@Test(expected = IllegalArgumentException.class) public void shouldNotAcceptNull() { StatementChannel.create((TokenMatcher[]) null); }
@Test(expected = IllegalArgumentException.class) public void shouldNotAcceptEmpty() { new Statement(new ArrayList<Token>()); }