public SpatialDataDownloader(Configuration config) { this.config = config; this.dir = new SpatialDataFolder(new File(config.get().getString("spatial.dir"))); }
configuration = new Configuration(confParams, pathConfs); configurator = new Configurator(configuration); if (configuration.get().hasPath("maxThreads")) { int maxThreads = configuration.get().getInt("maxThreads"); if (maxThreads > 0) { WpThreadUtils.setMaxThreads(maxThreads); if (configuration.get().hasPath("tmpDir")) { System.setProperty("java.io.tmpdir", configuration.get().getString("tmpDir")); LOG.info("using override configuration files " + Arrays.toString(pathConfs)); File baseDir = new File(configuration.get().getString("baseDir")); LOG.info("using baseDir " + baseDir.getAbsolutePath()); LOG.info("using max vm heapsize of " + (Runtime.getRuntime().maxMemory() / (1024*1024)) + "MB");
public static File getJsonLocation(Configuration conf) { return new File(conf.getFile("download.path"), "wikidata.json.bz2"); }
/** * Returns a default set of LuceneOptions. * * @return a default set of LuceneOptions */ public static LuceneOptions getDefaultOptions() { try { return new Configurator(new Configuration()).get(LuceneOptions.class, "plaintext"); } catch (ConfigurationException e) { throw new RuntimeException(e); } }
public File getFile(String ...path) { return new File(getString(path)); }
private Config getConfig(String refSys, String layerGroup, String dataset) { return env.getConfiguration().getConfig("spatial.datasets", refSys, layerGroup, dataset); }
LocalPageLiveDao testClass = (LocalPageLiveDao) new Configurator(new Configuration()).get(LocalPageDao.class, "live"); Language lang = Language.getByLangCode("en");
public File getBaseDir() { return new File(configuration.getString("baseDir")); }
private Config getConfig(String refSys, String layerGroup, String dataset) { return env.getConfiguration().getConfig("spatial.datasets", refSys, layerGroup, dataset); }
public SpatialDataDownloader(Configuration config) { this.config = config; this.dir = new SpatialDataFolder(new File(config.get().getString("spatial.dir"))); }
.withValueSeparator(',') .withLongOpt("names") .withDescription("Names of file types, separated by comma (e.g. 'articles,abstracts'). \nDefault is " + new Configuration().get().getStringList("download.matcher")) .create("f")); options.addOption( linkMatchers = FileMatcher.getListByNames(conf.getConf().get().getStringList("download.matcher")); String filePath = conf.getConf().get().getString("download.listFile"); if (cmd.hasOption('o')) { filePath = cmd.getOptionValue('o');
LocalPageLiveDao testClass = (LocalPageLiveDao) new Configurator(new Configuration()).get(LocalPageDao.class, "live"); Language lang = Language.getByLangCode("en");
/** * Hack: Better to ask the configurator for this! * @param conf * @return */ public static SpatialDataFolder get(Configuration conf) { return new SpatialDataFolder(conf.getFile("spatial.dir")); } }
public File getFile(String ...path) { return new File(getString(path)); }
public WikiBrainShapeFile download(String refSysName, String layerGroup, String datasetName, boolean mappingRequired) throws InterruptedException, IOException { if (config == null) { throw new IllegalArgumentException("To download by name, SpatialDataDownloader must have a configuration"); } Config c = config.getConfig("spatial.datasets", refSysName, layerGroup, datasetName); WikiBrainShapeFile shapeFile = download( new URL(c.getString("url")), c.getString("shp"), refSysName, layerGroup, datasetName, c.getString("encoding") ); if (mappingRequired) { downloadMapping(new URL(c.getString("mappingUrl")), shapeFile); } else { try { downloadMapping(new URL(c.getString("mappingUrl")), shapeFile); } catch (Exception e) { } } return shapeFile; }
public static void downloadDictionaryIfNecessary(Configuration conf) throws IOException, InterruptedException { String path = conf.get().getString("phrases.analyzer.stanford.path"); String url = conf.get().getString("phrases.analyzer.stanford.url"); File file = new File(path); File completed = new File(path + ".completed"); if (!completed.isFile()) { LOG.info("downloading stanford dictionary..."); FileDownloader downloader = new FileDownloader(); downloader.download(new URL(url), file); FileUtils.touch(completed); } }
/** * Hack: Better to ask the configurator for this! * @param conf * @return */ public static SpatialDataFolder get(Configuration conf) { return new SpatialDataFolder(conf.getFile("spatial.dir")); } }
getConfig().getString("sr.metric.path"), name, language.getLangCode());
public WikiBrainShapeFile download(String refSysName, String layerGroup, String datasetName, boolean mappingRequired) throws InterruptedException, IOException { if (config == null) { throw new IllegalArgumentException("To download by name, SpatialDataDownloader must have a configuration"); } Config c = config.getConfig("spatial.datasets", refSysName, layerGroup, datasetName); WikiBrainShapeFile shapeFile = download( new URL(c.getString("url")), c.getString("shp"), refSysName, layerGroup, datasetName, c.getString("encoding") ); if (mappingRequired) { downloadMapping(new URL(c.getString("mappingUrl")), shapeFile); } else { try { downloadMapping(new URL(c.getString("mappingUrl")), shapeFile); } catch (Exception e) { } } return shapeFile; }
public SpatialDataDownloader(Env env) { this.config = env.getConfiguration(); this.dir = new SpatialDataFolder(new File(config.get().getString("spatial.dir"))); }