/** Just like {@link #open(Path)}, but allows you to * also specify a custom {@link LockFactory}. */ public static FSDirectory open(Path path, LockFactory lockFactory) throws IOException { if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { return new MMapDirectory(path, lockFactory); } else if (Constants.WINDOWS) { return new SimpleFSDirectory(path, lockFactory); } else { return new NIOFSDirectory(path, lockFactory); } }
/** Creates an IndexInput for the file with the given name. */ @Override public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); ensureCanRead(name); Path path = directory.resolve(name); try (FileChannel c = FileChannel.open(path, StandardOpenOption.READ)) { final String resourceDescription = "MMapIndexInput(path=\"" + path.toString() + "\")"; final boolean useUnmap = getUseUnmap(); return ByteBufferIndexInput.newInstance(resourceDescription, map(resourceDescription, c, 0, c.size()), c.size(), chunkSizePower, new ByteBufferGuard(resourceDescription, useUnmap ? CLEANER : null)); } }
buffer = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize); } catch (IOException ioe) { throw convertMapFailedIOException(ioe, resourceDescription, bufSize);
private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory, Set<String> preLoadExtensions) throws IOException { if (preLoadExtensions.isEmpty() == false && directory instanceof MMapDirectory && ((MMapDirectory) directory).getPreload() == false) { if (preLoadExtensions.contains("*")) { ((MMapDirectory) directory).setPreload(true); return directory; } MMapDirectory primary = new MMapDirectory(location, lockFactory); primary.setPreload(true); return new FileSwitchDirectory(preLoadExtensions, primary, directory, true) { @Override public String[] listAll() throws IOException { // avoid listing twice return primary.listAll(); } }; } return directory; } }
return; diskDir = new MMapDirectory(directorySnapshot, NoLockFactory.getNoLockFactory()); if(diskDir != null && diskDir.fileExists(INDEX_WRITE_COMPLETED)) { index = MMapDirectory.open(directoryRunning, NoLockFactory.getNoLockFactory()); } else { index = new RAMDirectory(diskDir); if(diskIndexRunning) { IOUtils.deleteDirectory(directoryRunning); index = MMapDirectory.open(directoryRunning, NoLockFactory.getNoLockFactory()); } else { index = new RAMDirectory();
public IndexWriter createWriter(File directory, boolean doUpgrade) throws Exception { Directory luceneDirectory = directory != null ? MMapDirectory.open(directory.toPath()) : new RAMDirectory(); return createWriterWithLuceneDirectory(luceneDirectory, doUpgrade); }
/** Creates an IndexInput for the file with the given name. */ @Override public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); Path path = directory.resolve(name); try (FileChannel c = FileChannel.open(path, StandardOpenOption.READ)) { final String resourceDescription = "MMapIndexInput(path=\"" + path.toString() + "\")"; final boolean useUnmap = getUseUnmap(); return ByteBufferIndexInput.newInstance(resourceDescription, map(resourceDescription, c, 0, c.size()), c.size(), chunkSizePower, useUnmap ? CLEANER : null, useUnmap); } }
MMapIndexInput(String resourceDescription, RandomAccessFile raf) throws IOException { super(resourceDescription, map(raf, 0, raf.length()), raf.length(), chunkSizePower, getUseUnmap()); this.useUnmapHack = getUseUnmap(); }
/** Creates an IndexInput for the file with the given name. */ @Override public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); File f = new File(getDirectory(), name); RandomAccessFile raf = new RandomAccessFile(f, "r"); try { return new MMapIndexInput("MMapIndexInput(path=\"" + f + "\")", raf); } finally { raf.close(); } }
@Override public IndexInput openSlice(String sliceDescription, long offset, long length) throws IOException { ensureOpen(); return full.slice(sliceDescription, offset, length); }
private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory, Set<String> preLoadExtensions) throws IOException { if (preLoadExtensions.isEmpty() == false && directory instanceof MMapDirectory && ((MMapDirectory) directory).getPreload() == false) { if (preLoadExtensions.contains("*")) { ((MMapDirectory) directory).setPreload(true); return directory; } MMapDirectory primary = new MMapDirectory(location, lockFactory); primary.setPreload(true); return new FileSwitchDirectory(preLoadExtensions, primary, directory, true) { @Override public String[] listAll() throws IOException { // avoid listing twice return primary.listAll(); } }; } return directory; } }
public SearchCollection(SearchArgs args) throws IOException { this.args = args; Path indexPath = Paths.get(args.index); if (!Files.exists(indexPath) || !Files.isDirectory(indexPath) || !Files.isReadable(indexPath)) { throw new IllegalArgumentException(args.index + " does not exist or is not a directory."); } LOG.info("Reading index at " + indexPath); if (args.inmem) { this.reader = DirectoryReader.open(MMapDirectory.open(indexPath)); } else { this.reader = DirectoryReader.open(FSDirectory.open(indexPath)); } // Are we searching tweets? if (args.searchtweets) { LOG.info("Search Tweets"); analyzer = new TweetAnalyzer(); } else { analyzer = args.keepstop ? new EnglishStemmingAnalyzer(args.stemmer, CharArraySet.EMPTY_SET) : new EnglishStemmingAnalyzer(args.stemmer); } if (args.sdm) { LOG.info("Use Sequential Dependence Model query"); qc = QueryConstructor.SequentialDependenceModel; } else { LOG.info("Use Bag of Terms query"); qc = QueryConstructor.BagOfTerms; } isRerank = args.rm3 || args.axiom; }
/** Creates an IndexInput for the file with the given name. */ @Override public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); Path path = directory.resolve(name); try (FileChannel c = FileChannel.open(path, StandardOpenOption.READ)) { final String resourceDescription = "MMapIndexInput(path=\"" + path.toString() + "\")"; final boolean useUnmap = getUseUnmap(); return ByteBufferIndexInput.newInstance(resourceDescription, map(resourceDescription, c, 0, c.size()), c.size(), chunkSizePower, useUnmap ? CLEANER : null, useUnmap); } }
@Override public IndexInput openFullSlice() throws IOException { ensureOpen(); return full.clone(); }
public LsiIndex(IndexEngineParser parser, String directoryPath, Scorer scorer, FacetingManager facetingManager) throws IOException { this.parser = parser; this.scorer = scorer; this.facetingManager = facetingManager; Preconditions.checkNotNull(directoryPath); dirLocation = new File(directoryPath); if (!dirLocation.exists() || !dirLocation.isDirectory()) { throw new IllegalArgumentException("Wrong directory path."); } directory = new MMapDirectory(dirLocation); reopenWriter(); searchObjects = new AtomicReference<Pair<BlockingDeque<IndexSearcher>, BlockingDeque<QueryMatcher>>>(); reopenSearcher(); }
private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory, Set<String> preLoadExtensions) throws IOException { if (preLoadExtensions.isEmpty() == false && directory instanceof MMapDirectory && ((MMapDirectory) directory).getPreload() == false) { if (preLoadExtensions.contains("*")) { ((MMapDirectory) directory).setPreload(true); return directory; } MMapDirectory primary = new MMapDirectory(location, lockFactory); primary.setPreload(true); return new FileSwitchDirectory(preLoadExtensions, primary, directory, true) { @Override public String[] listAll() throws IOException { // avoid listing twice return primary.listAll(); } }; } return directory; } }
public IndexWriter createWriter(File directory) throws IOException { Directory dir = MMapDirectory.open(directory.toPath()); Analyzer analyzer = new SimpleAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); if (this.indexOptions.contains(BlobIndexOption.CREATE)) { iwc.setOpenMode(OpenMode.CREATE); } else { iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); } Long totalMBs = getHost().getServiceMemoryLimitMB(getSelfLink(), MemoryLimitType.EXACT); if (totalMBs != null) { totalMBs = Math.max(1, totalMBs); iwc.setRAMBufferSizeMB(totalMBs); } IndexWriter w = new IndexWriter(dir, iwc); w.commit(); return w; }
/** Creates an IndexInput for the file with the given name. */ @Override public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); ensureCanRead(name); Path path = directory.resolve(name); try (FileChannel c = FileChannel.open(path, StandardOpenOption.READ)) { final String resourceDescription = "MMapIndexInput(path=\"" + path.toString() + "\")"; final boolean useUnmap = getUseUnmap(); return ByteBufferIndexInput.newInstance(resourceDescription, map(resourceDescription, c, 0, c.size()), c.size(), chunkSizePower, new ByteBufferGuard(resourceDescription, useUnmap ? CLEANER : null)); } }
buffer = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize); } catch (IOException ioe) { throw convertMapFailedIOException(ioe, resourceDescription, bufSize);
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException { final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey()); if (IndexModule.Type.FS.match(storeType)) { final IndexModule.Type type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings())); switch (type) { case MMAPFS: return new MMapDirectory(location, lockFactory); case SIMPLEFS: return new SimpleFSDirectory(location, lockFactory); case NIOFS: return new NIOFSDirectory(location, lockFactory); default: throw new AssertionError("unexpected built-in store type [" + type + "]"); } } else if (IndexModule.Type.SIMPLEFS.match(storeType)) { return new SimpleFSDirectory(location, lockFactory); } else if (IndexModule.Type.NIOFS.match(storeType)) { return new NIOFSDirectory(location, lockFactory); } else if (IndexModule.Type.MMAPFS.match(storeType)) { return new MMapDirectory(location, lockFactory); } throw new IllegalArgumentException("No directory found for type [" + storeType + "]"); }