/** * Returns text as a List of tokens. * * @return A list of all tokens remaining in the underlying Reader */ @Override public List<T> tokenize() { ArrayList<T> result = new ArrayList<>(DEFAULT_TOKENIZE_LIST_SIZE); while (hasNext()) { result.add(next()); } // log.info("tokenize() produced " + result); // if it was tiny, reallocate small if (result.size() <= DEFAULT_TOKENIZE_LIST_SIZE / 4) { result.trimToSize(); } return result; }
/** * Returns text as a List of tokens. * * @return A list of all tokens remaining in the underlying Reader */ public List<T> tokenize() { // System.out.println("tokenize called"); List<T> result = new ArrayList<T>(); while (hasNext()) { result.add(next()); } return result; }
/** * Returns text as a List of tokens. * * @return A list of all tokens remaining in the underlying Reader */ @Override public List<T> tokenize() { // System.out.println("tokenize called"); List<T> result = new ArrayList<T>(); while (hasNext()) { result.add(next()); } return result; }
/** * Returns text as a List of tokens. * * @return A list of all tokens remaining in the underlying Reader */ @Override public List<T> tokenize() { ArrayList<T> result = new ArrayList<>(DEFAULT_TOKENIZE_LIST_SIZE); while (hasNext()) { result.add(next()); } // log.info("tokenize() produced " + result); // if it was tiny, reallocate small if (result.size() <= DEFAULT_TOKENIZE_LIST_SIZE / 4) { result.trimToSize(); } return result; }
/** * Returns text as a List of tokens. * * @return A list of all tokens remaining in the underlying Reader */ @Override public List<T> tokenize() { ArrayList<T> result = new ArrayList<>(DEFAULT_TOKENIZE_LIST_SIZE); while (hasNext()) { result.add(next()); } // log.info("tokenize() produced " + result); // if it was tiny, reallocate small if (result.size() <= DEFAULT_TOKENIZE_LIST_SIZE / 4) { result.trimToSize(); } return result; }