public static void outputTheWholeThing(IndexWriter writer) throws IOException { DirectoryReader reader = DirectoryReader.open(writer, true); for (int i=0; i<reader.maxDoc(); i++) { Document doc = reader.document(i); System.out.println(doc); } System.out.println("Pending deletions:" + reader.numDeletedDocs()); }
/** Returns ordinal -> label mapping, up to the provided * max ordinal or number of ordinals, whichever is * smaller. */ public String toString(int max) { ensureOpen(); StringBuilder sb = new StringBuilder(); int upperl = Math.min(max, indexReader.maxDoc()); for (int i = 0; i < upperl; i++) { try { FacetLabel category = this.getPath(i); if (category == null) { sb.append(i + ": NULL!! \n"); continue; } if (category.length == 0) { sb.append(i + ": EMPTY STRING!! \n"); continue; } sb.append(i +": "+category.toString()+"\n"); } catch (IOException e) { if (logger.isLoggable(Level.FINEST)) { logger.log(Level.FINEST, e.getMessage(), e); } } } return sb.toString(); }
/** Returns ordinal -> label mapping, up to the provided * max ordinal or number of ordinals, whichever is * smaller. */ public String toString(int max) { ensureOpen(); StringBuilder sb = new StringBuilder(); int upperl = Math.min(max, indexReader.maxDoc()); for (int i = 0; i < upperl; i++) { try { FacetLabel category = this.getPath(i); if (category == null) { sb.append(i + ": NULL!! \n"); continue; } if (category.length == 0) { sb.append(i + ": EMPTY STRING!! \n"); continue; } sb.append(i +": "+category.toString()+"\n"); } catch (IOException e) { if (log.isLoggable(Level.FINEST)) { log.log(Level.FINEST, e.getMessage(), e); } } } return sb.toString(); }
ScoreDoc[] scoreDocs; try { scoreDocs = searcher.search(new DocValuesFieldExistsQuery(LuceneDocumentGenerator.FIELD_ID), reader.maxDoc(), BREAK_SCORE_TIES_BY_DOCID).scoreDocs; } catch (IllegalStateException e) { // because this is tweets collection scoreDocs = searcher.search(new DocValuesFieldExistsQuery(TweetGenerator.StatusField.ID_LONG.name), reader.maxDoc(), BREAK_SCORE_TIES_BY_TWEETID).scoreDocs;
private void init() { try { boolean isLuceneDir = DirectoryReader.indexExists(directory); if (!isLuceneDir || getReader().maxDoc() == 0) { LOGGER.info("Metadata index is not a lucene index or is empty, trying to make a copy from shared metadata."); for (DQCategory dqCat : CategoryRegistryManager.getInstance().getSharedCategoryMetadata().values()) { createCategory(dqCat); } commitChanges(); } mgr = new SearcherManager(directory, null); } catch (IOException e) { LOGGER.error(e.getMessage(), e); } }
Bits liveDocs = MultiFields.getLiveDocs(reader); for (int i = 0; i < reader.maxDoc(); i++) { if (liveDocs != null && !liveDocs.get(i)) { continue;
/** * initialize a list of serializable BroadcastDocumentObject from existing lucene Directory */ static List<BroadcastDocumentObject> readDocumentsFromIndex(Directory indexDir) throws IOException { List<BroadcastDocumentObject> dictionaryObject = new ArrayList<>(); DirectoryReader reader = DirectoryReader.open(indexDir); Bits liveDocs = MultiFields.getLiveDocs(reader); for (int i = 0; i < reader.maxDoc(); i++) { if (liveDocs != null && !liveDocs.get(i)) { continue; } Document doc = reader.document(i); String catId = doc.getField(DictionarySearcher.F_CATID).stringValue(); Set<String> valueSet = new HashSet<String>(); // original values must be read from the F_RAW field for (IndexableField syntermField : doc.getFields(DictionarySearcher.F_RAW)) { valueSet.add(syntermField.stringValue()); } dictionaryObject.add(new BroadcastDocumentObject(catId, valueSet)); } return dictionaryObject; }
@Override public FacetLabel getPath(int ordinal) throws IOException { ensureOpen(); // Since the cache is shared with DTR instances allocated from // doOpenIfChanged, we need to ensure that the ordinal is one that this DTR // instance recognizes. Therefore we do this check up front, before we hit // the cache. if (ordinal < 0 || ordinal >= indexReader.maxDoc()) { return null; } // TODO: can we use an int-based hash impl, such as IntToObjectMap, // wrapped as LRU? Integer catIDInteger = Integer.valueOf(ordinal); synchronized (categoryCache) { FacetLabel res = categoryCache.get(catIDInteger); if (res != null) { return res; } } Document doc = indexReader.document(ordinal); FacetLabel ret = new FacetLabel(FacetsConfig.stringToPath(doc.get(Consts.FULL))); synchronized (categoryCache) { categoryCache.put(catIDInteger, ret); } return ret; }
@Override public FacetLabel getPath(int ordinal) throws IOException { ensureOpen(); // Since the cache is shared with DTR instances allocated from // doOpenIfChanged, we need to ensure that the ordinal is one that this DTR // instance recognizes. Therefore we do this check up front, before we hit // the cache. if (ordinal < 0 || ordinal >= indexReader.maxDoc()) { return null; } // TODO: can we use an int-based hash impl, such as IntToObjectMap, // wrapped as LRU? Integer catIDInteger = Integer.valueOf(ordinal); synchronized (categoryCache) { FacetLabel res = categoryCache.get(catIDInteger); if (res != null) { return res; } } Document doc = indexReader.document(ordinal); FacetLabel ret = new FacetLabel(FacetsConfig.stringToPath(doc.get(Consts.FULL))); synchronized (categoryCache) { categoryCache.put(catIDInteger, ret); } return ret; }
int docs = reader.maxDoc(); SolrClient solr = new ConcurrentUpdateSolrClient.Builder(args[1]).withQueueSize(2000).withThreadCount(2).build(); Set<SolrInputDocument> batch = new HashSet<SolrInputDocument>(1000);
int docs = reader.maxDoc(); SolrClient solr = new ConcurrentUpdateSolrClient.Builder(args[1]).withQueueSize(10000).withThreadCount(2).build(); Set<SolrInputDocument> batch = new HashSet<SolrInputDocument>(1000);
Integer res = ordinalCache.get(cp); if (res != null) { if (res.intValue() < indexReader.maxDoc()) {
Integer res = ordinalCache.get(cp); if (res != null) { if (res.intValue() < indexReader.maxDoc()) {