/* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) { StringBuilder buffer = sourceCode.getCodeBuffer(); return CharStreams.fromString(buffer.toString()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder sb = sourceCode.getCodeBuffer(); try (BufferedReader reader = new BufferedReader(new CharArrayReader(sb.toString().toCharArray()))) { int lineNumber = 1; String line = reader.readLine(); while (line != null) { StringTokenizer tokenizer = new StringTokenizer(line, TOKENS, true); while (tokenizer.hasMoreTokens()) { String token = tokenizer.nextToken(); if (!" ".equals(token) && !"\t".equals(token)) { tokenEntries.add(new TokenEntry(token, sourceCode.getFileName(), lineNumber)); } } // advance iteration variables line = reader.readLine(); lineNumber++; } } catch (IOException ignored) { ignored.printStackTrace(); } finally { tokenEntries.add(TokenEntry.getEOF()); } } }
private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) { final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); // Note that Java version is irrelevant for tokenizing final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME) .getVersion("1.4").getLanguageVersionHandler(); final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); return new JavaTokenFilter(tokenMgr, ignoreAnnotations); }
StringBuilder sb = sourceCode.getCodeBuffer();
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { try (Tokenizer tokenizer = new Tokenizer(sourceCode.getCodeBuffer().toString())) { Token token = tokenizer.getNextToken();
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(VfLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader); Token currentToken = (Token) tokenMgr.getNextToken(); while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) { e.printStackTrace(); } tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JspLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader); Token currentToken = (Token) tokenMgr.getNextToken(); while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) { e.printStackTrace(); } tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = new StringReader(buffer.toString())) { final TokenFilter tokenFilter = new JavaCCTokenFilter(new ObjectiveCTokenManager(reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } catch (IOException e) { e.printStackTrace(); } } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { final TokenFilter tokenFilter = new JavaCCTokenFilter(new MatlabTokenManager(reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError | IOException err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) { final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader)); GenericToken currentToken = tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine())); currentToken = tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError | IOException err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode); } catch (TokenMgrError | IOException err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = new StringReader(buffer.toString())) { LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add( new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } catch (IOException e) { e.printStackTrace(); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder code = sourceCode.getCodeBuffer();
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); GroovyLexer lexer = new GroovyLexer(new StringReader(buffer.toString())); TokenStream tokenStream = lexer.plumb(); try { Token token = tokenStream.nextToken(); while (token.getType() != Token.EOF_TYPE) { TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(), token.getLine()); tokenEntries.add(tokenEntry); token = tokenStream.nextToken(); } } catch (TokenStreamException err) { // Wrap exceptions of the Groovy tokenizer in a TokenMgrError, so // they are correctly handled // when CPD is executed with the '--skipLexicalErrors' command line // option throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + lexer.getLine() + ", column " + lexer.getColumn() + ". Encountered: " + err.getMessage(), TokenMgrError.LEXICAL_ERROR); } finally { tokenEntries.add(TokenEntry.getEOF()); } } }
/* default */ static CharStream getCharStreamFromSourceCode(final SourceCode sourceCode) { StringBuilder buffer = sourceCode.getCodeBuffer(); return CharStreams.fromString(buffer.toString()); } }
private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) { final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); // Note that Java version is irrelevant for tokenizing final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME) .getVersion("1.4").getLanguageVersionHandler(); final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); return new JavaTokenFilter(tokenMgr, ignoreAnnotations); }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(VfLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader); Token currentToken = (Token) tokenMgr.getNextToken(); while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) { e.printStackTrace(); } tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { final TokenFilter tokenFilter = new JavaCCTokenFilter(new MatlabTokenManager(reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError | IOException err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(buffer.toString())))) { final TokenFilter tokenFilter = new JavaCCTokenFilter(new CppTokenManager(reader)); GenericToken currentToken = tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.getImage(), sourceCode.getFileName(), currentToken.getBeginLine())); currentToken = tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError | IOException err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = new StringReader(buffer.toString())) { final TokenFilter tokenFilter = new JavaCCTokenFilter(new ObjectiveCTokenManager(reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } catch (IOException e) { e.printStackTrace(); } } }