public void init(String indexFolder) throws IOException { logger.info(String.format("Creating the thread executor (%d)", numThreads)); int blockQueueSize = DEFAULT_QUEUE_SIZE; BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(blockQueueSize); RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy(); myExecutor = new ThreadPoolExecutor(numThreads, numThreads, 1, TimeUnit.MINUTES, blockingQueue, rejectedExecutionHandler); logger.info(String.format("Reding folder %s", indexFolder)); clSchema = new BigCrossLanguageSearcher(indexFolder); }
public void end() throws IOException { clSchema.close(); }
public void interactive() throws Exception { InputStreamReader reader = null; BufferedReader myInput = null; while (true) { System.out.println("\nPlease write a key/value and type <return> to continue (CTRL C to exit):"); reader = new InputStreamReader(System.in); myInput = new BufferedReader(reader); String query = myInput.readLine().toString(); String[] s = tabPattern.split(query); if (s.length == 2) { System.out.println(search(s[0], s[1])); } } }
BigCrossLanguageSearcher clSearcher = null; if (wikidataCL != null) { clSearcher = new BigCrossLanguageSearcher(wikidataCL); clSearcher.close();
public static void main(String[] args) { String logConfig = System.getProperty("log-config"); if (logConfig == null) { logConfig = "configuration/log-config.txt"; } PropertyConfigurator.configure(logConfig); Options options = new Options(); try { Option indexNameOpt = OptionBuilder.withArgName("index").hasArg().withDescription("open an index with the specified name").isRequired().withLongOpt("index").create("i"); options.addOption("h", "help", false, "print this message"); options.addOption("v", "version", false, "output version information and exit"); options.addOption(indexNameOpt); CommandLineParser parser = new PosixParser(); CommandLine line = parser.parse(options, args); if (line.hasOption("help") || line.hasOption("version")) { throw new ParseException(""); } BigCrossLanguageSearcher crossLanguageSearcher = new BigCrossLanguageSearcher(line.getOptionValue("index")); crossLanguageSearcher.interactive(); } catch (ParseException e) { // oops, something went wrong if (e.getMessage().length() > 0) { System.out.println("Parsing failed: " + e.getMessage() + "\n"); } HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(400, "java -cp dist/thewikimachine.jar org.fbk.cit.hlt.thewikimachine.index.CrossLanguageSearcher", "\n", options, "\n", true); } catch (Exception e) { e.getMessage(); } }
public void go() throws IOException { long begin = System.currentTimeMillis(), end = 0; totalNumber = clSchema.getIndexReader().maxDoc(); for (int i = 0; i < clSchema.getIndexReader().maxDoc(); i++) { Document d = clSchema.getIndexReader().document(i); if (d == null) { continue; } myExecutor.execute(new LineProcessor(d, i)); if ((i + 1) % 1000 == 0) { System.out.print("."); } if ((i + 1) % 100000 == 0) { System.out.println(" " + (i + 1) + "/" + totalNumber); } } System.out.println(); logger.info(String.format("Elapsed time: %s", df.format(end - begin))); try { myExecutor.shutdown(); logger.debug("Wating to end"); myExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { logger.error(e); } logger.info("Ending process"); }
public Map<String, String> search(String lang, String key) { Map<String, String> result = null; try { TermDocs termDocs = indexReader.termDocs(new Term(lang, key)); if (termDocs.next()) { Document doc = indexReader.document(termDocs.doc()); result = documentToMap(doc); return result; } } catch (IOException e) { logger.error(e); } return new HashMap<String, String>(); }
public void runForAll() { for (Integer id : wikidataProperties.keySet()) { HashSet<String> res = search(id); if (res.size() > 0) { String label = null; if (clSearcher != null) { Map<String, String> langData = clSearcher.search("wikiID", id.toString()); label = langData.get("en"); } logger.debug(String.format("%s [%s] --> %s", label, id, res)); } } }
public void getBncfClass(Integer id, HashSet<String> history) { if (logger.isDebugEnabled()) { String label = null; if (clSearcher != null) { Map<String, String> langData = clSearcher.search("wikiID", id.toString()); label = langData.get("en"); } logger.debug(String.format("BNCF %s [%d]", label, id)); } HashMap<String, HashSet<String>> properties = wikidataProperties.get(id); if (properties != null) { for (String s : properties.keySet()) { if (s.equals("bncf")) { for (String val : properties.get(s)) { try { getBncfClassInside(Integer.parseInt(val), history, 1); } catch (Exception e) { logger.debug("ERROR: " + e.getMessage()); } } } } } else { logger.debug(String.format("No BNCF/WikiData properties for [%d]", id)); } }
String label = null; if (clSearcher != null) { Map<String, String> langData = clSearcher.search("wikiID", id.toString()); label = langData.get("en");