private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) { final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); // Note that Java version is irrelevant for tokenizing final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME) .getVersion("1.4").getLanguageVersionHandler(); final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); return new JavaTokenFilter(tokenMgr, ignoreAnnotations); }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(VfLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader); Token currentToken = (Token) tokenMgr.getNextToken(); while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) { e.printStackTrace(); } tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JspLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader); Token currentToken = (Token) tokenMgr.getNextToken(); while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) { e.printStackTrace(); } tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode); } catch (TokenMgrError | IOException err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = new StringReader(buffer.toString())) { LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(EcmascriptLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add( new TokenEntry(getTokenImage(currentToken), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode.getFileName()); } catch (TokenMgrError err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } catch (IOException e) { e.printStackTrace(); } }
private JavaTokenFilter createTokenFilter(final SourceCode sourceCode) { final StringBuilder stringBuilder = sourceCode.getCodeBuffer(); // Note that Java version is irrelevant for tokenizing final LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JavaLanguageModule.NAME) .getVersion("1.4").getLanguageVersionHandler(); final TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), new StringReader(stringBuilder.toString())); return new JavaTokenFilter(tokenMgr, ignoreAnnotations); }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(VfLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader); Token currentToken = (Token) tokenMgr.getNextToken(); while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) { e.printStackTrace(); } tokenEntries.add(TokenEntry.getEOF()); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) { LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(PythonLanguageModule.NAME) .getDefaultVersion().getLanguageVersionHandler(); TokenFilter tokenFilter = new JavaCCTokenFilter(languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(sourceCode.getFileName(), reader)); Token currentToken = (Token) tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine)); currentToken = (Token) tokenFilter.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); System.err.println("Added " + sourceCode); } catch (TokenMgrError | IOException err) { err.printStackTrace(); System.err.println("Skipping " + sourceCode + " due to parse error"); tokenEntries.add(TokenEntry.getEOF()); } } }