public <N extends AbstractNLPNode<N>, S extends NLPState<N>>void reduce(String[] args) { BinUtils.initArgs(args, this); OnlineTrainer<N,S> trainer = createOnlineTrainer(); List<String> developFiles = FileUtils.getFileList(develop_path, develop_ext); GlobalLexica<N> lexica = trainer.createGlobalLexica(IOUtils.createFileInputStream(configuration_file)); BinUtils.LOG.info("Loading the model\n"); OnlineComponent<N,S> component = readComponent(IOUtils.createFileInputStream(previous_model_file), IOUtils.createFileInputStream(configuration_file)); TSVReader<N> reader = trainer.createTSVReader(component.getConfiguration().getReaderFieldMap()); trainer.reduceModel(reader, developFiles, component, lexica, previous_model_file, model_file); }
public <N extends AbstractNLPNode<N>, S extends NLPState<N>>void reduce(String[] args) { BinUtils.initArgs(args, this); OnlineTrainer<N,S> trainer = createOnlineTrainer(); List<String> developFiles = FileUtils.getFileList(develop_path, develop_ext); GlobalLexica<N> lexica = trainer.createGlobalLexica(IOUtils.createFileInputStream(configuration_file)); LOG.info("Loading the model"); OnlineComponent<N,S> component = readComponent(IOUtils.createFileInputStream(previous_model_file), IOUtils.createFileInputStream(configuration_file)); TSVReader<N> reader = trainer.createTSVReader(component.getConfiguration().getReaderFieldMap()); trainer.reduceModel(reader, developFiles, component, lexica, previous_model_file, model_file); }
public double train(NLPMode mode, List<String> trainFiles, List<String> developFiles, String configurationFile, String modelFile, String previousModelFile, int index) { InputStream previousModelStream = (previousModelFile != null) ? IOUtils.createFileInputStream(previousModelFile) : null; GlobalLexica<N> lexica = createGlobalLexica(IOUtils.createFileInputStream(configurationFile)); String name = (modelFile != null) ? FileUtils.getBaseName(modelFile) : null; OnlineComponent<N,S> component = initComponent(mode, IOUtils.createFileInputStream(configurationFile), previousModelStream, name); TSVReader<N> reader = createTSVReader(component.getConfiguration().getReaderFieldMap()); ObjectDoublePair<OnlineComponent<N,S>> p = null; try { p = train(reader, trainFiles, developFiles, component, lexica, index); if (modelFile != null) saveModel(p.o, IOUtils.createFileOutputStream(modelFile)); } catch (Exception e) {e.printStackTrace();} return p.d; }
public double train(NLPMode mode, List<String> trainFiles, List<String> developFiles, String configurationFile, String modelFile, String previousModelFile, int index) { InputStream previousModelStream = (previousModelFile != null) ? IOUtils.createFileInputStream(previousModelFile) : null; GlobalLexica<N> lexica = createGlobalLexica(IOUtils.createFileInputStream(configurationFile)); String name = (modelFile != null) ? FileUtils.getBaseName(modelFile) : null; OnlineComponent<N,S> component = initComponent(mode, IOUtils.createFileInputStream(configurationFile), previousModelStream, name); TSVReader<N> reader = createTSVReader(component.getConfiguration().getReaderFieldMap()); ObjectDoublePair<OnlineComponent<N,S>> p = null; try { p = train(reader, trainFiles, developFiles, component, lexica, index); if (modelFile != null) saveModel(p.o, IOUtils.createFileOutputStream(modelFile)); } catch (Exception e) {e.printStackTrace();} return p.d; }