Logger.setGlobalLogger(new Logger.SystemLogger(System.out, System.err)); final String lmFile = fileArgs.get(0); Logger.startTrack("Reading Lm File " + lmFile + " . . . ");
private void output(final PrintStream o, final String txt) { if (o == null) return; final String[] lines = txt.split("\n"); final String prefix = getIndentPrefix(); for (final String line : lines) { o.println(prefix + line); } }
@Override public void logs(final String s, final Object... args) { logs(String.format(s, args)); }
Logger.setGlobalLogger(new Logger.SystemLogger(System.out, System.err)); final String lmFile = fileArgs.get(0); Logger.startTrack("Reading Lm File " + lmFile + " . . . ");
public static void main(final String[] argv) throws FileNotFoundException, IOException { int i = 0; if (i >= argv.length) usage(); boolean isGoogleBinary = false; if (argv[i].equals("-g")) { isGoogleBinary = true; i++; } if (i >= argv.length) usage(); String vocabFile = null; if (isGoogleBinary) { vocabFile = argv[i++]; } if (i >= argv.length) usage(); String binaryFile = argv[i++]; List<String> files = Arrays.asList(Arrays.copyOfRange(argv, i, argv.length)); if (files.isEmpty()) files = Collections.singletonList("-"); Logger.setGlobalLogger(new Logger.SystemLogger(System.err, System.err)); NgramLanguageModel<String> lm = readBinary(isGoogleBinary, vocabFile, binaryFile); double prob = computeProb(files, lm); System.err.print("Log probability of text is: "); System.out.println(prob); }
public static void main(final String[] argv) { if (argv.length < 2) { usage(); } final int lmOrder = Integer.parseInt(argv[0]); final String outputFile = argv[1]; final List<String> inputFiles = new ArrayList<String>(); for (int i = 2; i < argv.length; ++i) { inputFiles.add(argv[i]); } if (inputFiles.isEmpty()) inputFiles.add("-"); Logger.setGlobalLogger(new Logger.SystemLogger(System.out, System.err)); Logger.startTrack("Reading text files " + inputFiles + " and writing to file " + outputFile); final StringWordIndexer wordIndexer = new StringWordIndexer(); wordIndexer.setStartSymbol(ArpaLmReader.START_SYMBOL); wordIndexer.setEndSymbol(ArpaLmReader.END_SYMBOL); wordIndexer.setUnkSymbol(ArpaLmReader.UNK_SYMBOL); LmReaders.createKneserNeyLmFromTextFiles(inputFiles, wordIndexer, lmOrder, new File(outputFile), new ConfigOptions()); Logger.endTrack(); }
public static void main(final String[] argv) { if (argv.length < 3) { usage(); } int lmOrder = Integer.parseInt(argv[0]); String outputFile = argv[1]; List<File> inputFiles = new ArrayList<File>(); for (int i = 2; i < argv.length; ++i) { inputFiles.add(new File(argv[i])); } Logger.setGlobalLogger(new Logger.SystemLogger(System.out, System.err)); Logger.startTrack("Reading text files " + inputFiles + " and writing to file " + outputFile); final StringWordIndexer wordIndexer = new StringWordIndexer(); wordIndexer.setStartSymbol(ArpaLmReader.START_SYMBOL); wordIndexer.setEndSymbol(ArpaLmReader.END_SYMBOL); wordIndexer.setUnkSymbol(ArpaLmReader.UNK_SYMBOL); LmReaders.createKneserNeyLmFromTextFiles(inputFiles, wordIndexer, lmOrder, new File(outputFile), new ConfigOptions()); Logger.endTrack(); }
public void logs(final String s) { output(out, s); }
public static void main(final String[] argv) { if (argv.length != 2) usage(); Logger.setGlobalLogger(new Logger.SystemLogger(System.out, System.err)); Logger.startTrack("Reading Lm File " + argv[0] + " . . . "); final String googleDir = argv[0]; final NgramLanguageModel<String> lm = LmReaders.readLmFromGoogleNgramDir(googleDir, true); Logger.endTrack(); final String outFile = argv[1]; Logger.startTrack("Writing to file " + outFile + " . . . "); LmReaders.writeLmBinary(lm, outFile); Logger.endTrack(); } }
@Override public void logs(final String s, final Object... args) { logs(String.format(s, args)); }
@Override public void warn(final String string, final Object... args) { warn(String.format(string, args)); }
public void logs(final String s) { output(out, s); }
@Override public void warn(final String s) { output(err, "[warn] " + s); }
@Override public void dbg(final String s) { if (debug) output(out, "[dbg] " + s); }
public static void main(final String[] argv) { if (argv.length != 2) usage(); Logger.setGlobalLogger(new Logger.SystemLogger(System.out, System.err)); Logger.startTrack("Reading Lm File " + argv[0] + " . . . "); final String lmFile = argv[1]; final StupidBackoffLm<String> lm = LmReaders.readLmFromGoogleNgramDir(lmFile, true); Logger.endTrack(); final String outFile = argv[1]; Logger.startTrack("Writing to file " + outFile + " . . . "); IOUtils.writeObjFileHard(outFile, lm.getNgramMap()); Logger.endTrack(); } }
public static void main(final String[] argv) { if (argv.length != 2) usage(); Logger.setGlobalLogger(new Logger.SystemLogger(System.out, System.err)); Logger.startTrack("Reading Lm File " + argv[0] + " . . . "); final String lmFile = argv[1]; final StupidBackoffLm<String> lm = (StupidBackoffLm<String>) LmReaders.readLmFromGoogleNgramDir(lmFile, true, false); Logger.endTrack(); final String outFile = argv[1]; Logger.startTrack("Writing to file " + outFile + " . . . "); IOUtils.writeObjFileHard(outFile, lm.getNgramMap()); Logger.endTrack(); } }
@Override public void err(final String s, final Object... args) { output(err, "[err] " + String.format(s, args)); }
@Override public void logss(final String s) { output(out, s); }
@Override public void err(final String s, final Object... args) { output(err, "[err] " + String.format(s, args)); }
@Override public void dbg(final String s) { if (debug) output(out, "[dbg] " + s); }