/** * Read a file containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param location input file location * @param lowercase if true, lowercase everything * @return a collection of unique stopwords * @throws IOException if the file cannot be read */ public static Set<String> readStopwordsFile(String location, boolean lowercase) throws IOException { return readStopwordsFile(new File(location), lowercase); }
public static Set<String> readStopwordsURL(URL url, boolean lowercase) throws IOException { return readStopwordsInputStream(url.openStream(), lowercase); }
/** * Read a file containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param file input file * @param lowercase if true, lowercase everything * @return a collection of unique stopwords * @throws IOException if the file cannot be read */ public static Set<String> readStopwordsFile(File file, boolean lowercase) throws IOException { return readStopwordsPath(file.toPath(), lowercase); }
/** * Read an {@link InputStream} containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param inputStream input stream * @param lowercase if true, lowercase everything * @return a collection of unique stopwords */ public static Set<String> readStopwordsInputStream(InputStream inputStream, boolean lowercase) { return readStream(new BufferedReader(new InputStreamReader(inputStream)).lines(), lowercase); }
private PhraseSequenceGenerator(Builder builder) throws IOException { this.lowercase = builder.lowercase; this.coveringTypeName = builder.coveringType; this.minTokenLength = builder.minTokenLength; this.featurePath = builder.featurePath; stopwords = builder.stopwordsFile.isPresent() ? TextUtils.readStopwordsURL(builder.stopwordsFile.get(), lowercase) : Collections.emptySet(); this.stopwordReplacement = builder.stopwordsReplacement; this.filterRegexes = builder.filterRegexes; this.filterRegexReplacement = builder.filterRegexReplacement; this.useCharacters = builder.characters; }
/** * Read an {@link InputStream} containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param inputStream input stream * @param lowercase if true, lowercase everything * @return a collection of unique stopwords */ public static Set<String> readStopwordsInputStream(InputStream inputStream, boolean lowercase) { return readStream(new BufferedReader(new InputStreamReader(inputStream)).lines(), lowercase); }
private PhraseSequenceGenerator(Builder builder) throws IOException { this.lowercase = builder.lowercase; this.coveringTypeName = builder.coveringType; this.minTokenLength = builder.minTokenLength; this.featurePath = builder.featurePath; stopwords = builder.stopwordsFile.isPresent() ? TextUtils.readStopwordsURL(builder.stopwordsFile.get(), lowercase) : Collections.emptySet(); this.stopwordReplacement = builder.stopwordsReplacement; this.filterRegexes = builder.filterRegexes; this.filterRegexReplacement = builder.filterRegexReplacement; this.useCharacters = builder.characters; }
public static Set<String> readStopwordsURL(URL url, boolean lowercase) throws IOException { return readStopwordsInputStream(url.openStream(), lowercase); }
/** * Read a file containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param location input file location * @param lowercase if true, lowercase everything * @return a collection of unique stopwords * @throws IOException if the file cannot be read */ public static Set<String> readStopwordsFile(String location, boolean lowercase) throws IOException { return readStopwordsFile(new File(location), lowercase); }
/** * Read a file containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param file input file * @param lowercase if true, lowercase everything * @return a collection of unique stopwords * @throws IOException if the file cannot be read */ public static Set<String> readStopwordsFile(File file, boolean lowercase) throws IOException { return readStopwordsPath(file.toPath(), lowercase); }
/** * Read a file containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param path input file {@link Path} * @param lowercase if true, lowercase everything * @return a collection of unique stopwords * @throws IOException if the file cannot be read */ public static Set<String> readStopwordsPath(Path path, boolean lowercase) throws IOException { return readStopwordsInputStream(Files.newInputStream(path), lowercase); }
/** * Read a file containing stopwords (one per line). * <p> * Empty lines and lines starting with ("#") are filtered out. * * @param path input file {@link Path} * @param lowercase if true, lowercase everything * @return a collection of unique stopwords * @throws IOException if the file cannot be read */ public static Set<String> readStopwordsPath(Path path, boolean lowercase) throws IOException { return readStopwordsInputStream(Files.newInputStream(path), lowercase); }