protected void tokenizeTest() throws IOException { Tokens tokens = new Tokens(); tokenizer.tokenize(sourceCode, tokens); List<TokenEntry> entries = tokens.getTokens(); assertEquals(expectedTokenCount, entries.size()); }
public List<TokensLine> chunk(String fileName, Reader fileReader) { SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(fileName, fileReader)); Tokens tokens = new Tokens(); TokenEntry.clearImages(); try { tokenizer.tokenize(sourceCode, tokens); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException(e); } TokenEntry.clearImages(); return convert(tokens.getTokens()); }
Tokens tokens=new Tokens();//create Tokens instance while(it1.hasNext()) { // init while (stringTokenizer.hasMoreElements()) { // set tokens fields } System.out.println(tokens);// You print the current Tokens instance }//while(it1.hasNext() return tokens; //only return one, at the last state.
public static List<Tokens> getTokens( Vector<NameBuffer> vc1 ) { List<Tokens> tokenList=new ArrayList<Tokens>();//create Tokens list to return while(it1.hasNext()) { // init Tokens tokens=new Tokens();//create Tokens instance inside loop tokenList.add(tokens); //add to list to return while (stringTokenizer.hasMoreElements()) { // set tokens fields -- unchanged } System.out.println(tokens);// You print the current Tokens instance }//while(it1.hasNext() return tokenList; //only return the last one.... }
def tokens = new Tokens()
protected void tokenizeTest() throws IOException { Tokens tokens = new Tokens(); tokenizer.tokenize(sourceCode, tokens); List<TokenEntry> entries = tokens.getTokens(); assertEquals(expectedTokenCount, entries.size()); }
public List<TokensLine> chunk(File file) { SourceCode sourceCode = new SourceCode(new FileCodeLoaderWithoutCache(file, encoding)); Tokens tokens = new Tokens(); TokenEntry.clearImages(); try { tokenizer.tokenize(sourceCode, tokens); } catch (IOException e) { throw Throwables.propagate(e); } TokenEntry.clearImages(); return convert(tokens.getTokens()); }