@Inject LuceneGroupIndex( @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<GroupCache> groupCache, @Assisted Schema<InternalGroup> schema) throws IOException { super( schema, sitePaths, dir(schema, cfg, sitePaths), GROUPS, null, new GerritIndexWriterConfig(cfg, GROUPS), new SearcherFactory()); this.groupCache = groupCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, GROUPS); queryBuilder = new QueryBuilder<>(schema, indexWriterConfig.getAnalyzer()); }
@Inject LuceneAccountIndex( @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<AccountCache> accountCache, @Assisted Schema<AccountState> schema) throws IOException { super( schema, sitePaths, dir(schema, cfg, sitePaths), ACCOUNTS, null, new GerritIndexWriterConfig(cfg, ACCOUNTS), new SearcherFactory()); this.accountCache = accountCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, ACCOUNTS); queryBuilder = new QueryBuilder<>(schema, indexWriterConfig.getAnalyzer()); }
@Inject LuceneProjectIndex( @GerritServerConfig Config cfg, SitePaths sitePaths, Provider<ProjectCache> projectCache, @Assisted Schema<ProjectData> schema) throws IOException { super( schema, sitePaths, dir(schema, cfg, sitePaths), PROJECTS, null, new GerritIndexWriterConfig(cfg, PROJECTS), new SearcherFactory()); this.projectCache = projectCache; indexWriterConfig = new GerritIndexWriterConfig(cfg, PROJECTS); queryBuilder = new QueryBuilder<>(schema, indexWriterConfig.getAnalyzer()); }
public IndexSearcher getIndexSearcher(final ClusterSegment... clusterSegments) { final Set<KCluster> clusters; if (clusterSegments == null || clusterSegments.length == 0) { clusters = new HashSet<KCluster>(indexes.keySet()); } else { clusters = new HashSet<KCluster>(clusterSegments.length); for (final ClusterSegment clusterSegment : clusterSegments) { clusters.add(new KClusterImpl(clusterSegment.getClusterId())); } } final Collection<IndexReader> readers = new ArrayList<IndexReader>(clusters.size()); for (final KCluster cluster : clusters) { final LuceneIndex index = indexes.get(cluster); readers.add(index.nrtReader()); } try { return new SearcherFactory().newSearcher(new MultiReader(readers.toArray(new IndexReader[readers.size()])), null); } catch (IOException e) { throw new RuntimeException(e); } }
public LuceneFiler(@Nonnull Filer delegate, @Nonnull Config config) throws IOException { super(delegate); String path = config.getString("index.path"); maxAge = config.getTime("index.maxAge", "-1"); double maxMergeMb = config.getDouble("index.maxMergeMb", 4); double maxCachedMb = config.getDouble("index.maxCacheMb", 64); long targetMaxStale = config.getTime("index.targetMaxStale", "5s"); long targetMinStale = config.getTime("index.targetMinStale", "1s"); Directory dir = FSDirectory.open(new File(path).toPath()); NRTCachingDirectory cachingDir = new NRTCachingDirectory(dir, maxMergeMb, maxCachedMb); IndexWriterConfig writerConfig = new IndexWriterConfig(null); writerConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); writer = new TrackingIndexWriter(new IndexWriter(cachingDir, writerConfig)); manager = new SearcherManager(writer.getIndexWriter(), true, new SearcherFactory()); thread = new ControlledRealTimeReopenThread<>(writer, manager, targetMaxStale, targetMinStale); thread.start(); }
@Override public void setDAG(DirectedAcyclicGraph directedAcyclicGraph) { super.setDAG(directedAcyclicGraph); // Connect to the Lucene DB try { Analyzer analyser = new KeywordAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyser); config.setOpenMode(OpenMode.CREATE_OR_APPEND); Path path = DAGModule.moduleFile(directedAcyclicGraph.rootDir_, INDEX_FOLDER).toPath(); Directory directory = FSDirectory.open(path); // Directory directory = new RAMDirectory(); writer_ = new IndexWriter(directory, config); // Searching parser_ = new QueryParser(LOWERCASE_FIELD, analyser); manager_ = new SearcherManager(writer_, true, new SearcherFactory()); } catch (Exception e) { e.printStackTrace(); } }
public ShadowEngine(EngineConfig engineConfig) { super(engineConfig); if (engineConfig.getRefreshListeners() != null) { throw new IllegalArgumentException("ShadowEngine doesn't support RefreshListeners"); } SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig); final long nonexistentRetryTime = engineConfig.getIndexSettings().getSettings() .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT) .getMillis(); try { DirectoryReader reader = null; store.incRef(); boolean success = false; try { if (Lucene.waitForIndex(store.directory(), nonexistentRetryTime)) { reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(store.directory()), shardId); this.searcherManager = new SearcherManager(reader, searcherFactory); this.lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager, store); success = true; } else { throw new IllegalStateException("failed to open a shadow engine after" + nonexistentRetryTime + "ms, " + "directory is not an index"); } } catch (Exception e) { logger.warn("failed to create new reader", e); throw e; } finally { if (success == false) { IOUtils.closeWhileHandlingException(reader); store.decRef(); } } } catch (IOException ex) { throw new EngineCreationFailureException(shardId, "failed to open index reader", ex); } logger.trace("created new ShadowEngine"); }
public ShadowEngine(EngineConfig engineConfig) { super(engineConfig); SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig); final long nonexistentRetryTime = engineConfig.getIndexSettings() .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT) .getMillis(); try { DirectoryReader reader = null; store.incRef(); boolean success = false; try { if (Lucene.waitForIndex(store.directory(), nonexistentRetryTime)) { reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(store.directory()), shardId); this.searcherManager = new SearcherManager(reader, searcherFactory); this.lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager, store); success = true; } else { throw new IllegalStateException("failed to open a shadow engine after" + nonexistentRetryTime + "ms, " + "directory is not an index"); } } catch (Throwable e) { logger.warn("failed to create new reader", e); throw e; } finally { if (success == false) { IOUtils.closeWhileHandlingException(reader); store.decRef(); } } } catch (IOException ex) { throw new EngineCreationFailureException(shardId, "failed to open index reader", ex); } logger.trace("created new ShadowEngine"); }
AbstractKnowledgeBase(String uri, boolean dynamic, Similarity sim) { super(); this.indexUri = uri; this.dynamic = dynamic; File indexDir = new File(indexUri); Directory dir; try { dir = FSDirectory.open(indexDir); this.manager = new SearcherManager(dir, new SearcherFactory()); } catch (IOException e) { logger.error("IOException in "+AbstractKnowledgeBase.class.getName(), e); } }
private void doInitialize() throws ServerException { try { luceneIndexWriter = new IndexWriter(makeDirectory(), new IndexWriterConfig(makeAnalyzer())); searcherManager = new SearcherManager(luceneIndexWriter, true, true, new SearcherFactory()); closed = false; } catch (IOException e) { throw new ServerException(e); } }
/** * constructor * @param directory LuceneDirectory * @param indexFile Index File Path * @param isVolatile if true, delete file automatically * @param analyzer LuceneIndexAnalyzer * @throws IOException IOException */ public LuceneValuesDB(Directory directory, File indexFile, boolean isVolatile, Analyzer analyzer) throws IOException { this.directory = directory; this.indexFile = indexFile; IndexWriterConfig config = new IndexWriterConfig(analyzer); this.writer = new IndexWriter(directory, config); this.writer.commit(); this.manager = new SearcherManager(directory, new SearcherFactory()); this.isVolatile = isVolatile; if (isVolatile) { final LuceneValuesDB own = this; Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { close(own); } catch (IOException e) { e.printStackTrace(); } } }); } }
static List<Long> getAllNodes( DirectoryFactory dirFactory, File indexDir, Object propertyValue ) throws IOException { try ( Directory directory = dirFactory.open( indexDir ); SearcherManager manager = new SearcherManager( directory, new SearcherFactory() )) { IndexSearcher searcher = manager.acquire(); List<Long> nodes = new ArrayList<>(); LuceneDocumentStructure documentStructure = new LuceneDocumentStructure(); Query query = documentStructure.newSeekQuery( propertyValue ); searcher.search( query, new AllNodesCollector( documentStructure, nodes ) ); return nodes; } }
@Inject LuceneChangeIndex( @GerritServerConfig Config cfg, SitePaths sitePaths, @IndexExecutor(INTERACTIVE) ListeningExecutorService executor, Provider<ReviewDb> db, ChangeData.Factory changeDataFactory, @Assisted Schema<ChangeData> schema) throws IOException { this.executor = executor; this.db = db; this.changeDataFactory = changeDataFactory; this.schema = schema; GerritIndexWriterConfig openConfig = new GerritIndexWriterConfig(cfg, "changes_open"); GerritIndexWriterConfig closedConfig = new GerritIndexWriterConfig(cfg, "changes_closed"); queryBuilder = new QueryBuilder<>(schema, openConfig.getAnalyzer()); SearcherFactory searcherFactory = new SearcherFactory(); if (LuceneIndexModule.isInMemoryTest(cfg)) { openIndex = new ChangeSubIndex( schema, sitePaths, new RAMDirectory(), "ramOpen", openConfig, searcherFactory); closedIndex = new ChangeSubIndex( schema, sitePaths, new RAMDirectory(), "ramClosed", closedConfig, searcherFactory); } else { Path dir = LuceneVersionManager.getDir(sitePaths, CHANGES, schema); openIndex = new ChangeSubIndex( schema, sitePaths, dir.resolve(CHANGES_OPEN), openConfig, searcherFactory); closedIndex = new ChangeSubIndex( schema, sitePaths, dir.resolve(CHANGES_CLOSED), closedConfig, searcherFactory); } }
/** * Expert: creates a searcher from the provided {@link IndexReader} using the provided {@link * SearcherFactory}. NOTE: this decRefs incoming reader on throwing an exception. */ @SuppressWarnings("resource") public static IndexSearcher getSearcher(SearcherFactory searcherFactory, IndexReader reader) throws IOException { boolean success = false; final IndexSearcher searcher; try { searcher = searcherFactory.newSearcher(reader, null); // Modification for Gerrit: Allow searcherFactory to transitively wrap the // provided reader. IndexReader unwrapped = searcher.getIndexReader(); while (true) { if (unwrapped == reader) { break; } else if (unwrapped instanceof FilterDirectoryReader) { unwrapped = ((FilterDirectoryReader) unwrapped).getDelegate(); } else if (unwrapped instanceof FilterLeafReader) { unwrapped = ((FilterLeafReader) unwrapped).getDelegate(); } else { break; } } if (unwrapped != reader) { throw new IllegalStateException( "SearcherFactory must wrap the provided reader (got " + searcher.getIndexReader() + " but expected " + reader + ")"); } success = true; } finally { if (!success) { reader.decRef(); } } return searcher; }
ChangeSubIndex( Schema<ChangeData> schema, SitePaths sitePaths, Path path, GerritIndexWriterConfig writerConfig, SearcherFactory searcherFactory) throws IOException { this( schema, sitePaths, FSDirectory.open(path), path.getFileName().toString(), writerConfig, searcherFactory); }
ChangeSubIndex( Schema<ChangeData> schema, SitePaths sitePaths, Directory dir, String subIndex, GerritIndexWriterConfig writerConfig, SearcherFactory searcherFactory) throws IOException { super(schema, sitePaths, dir, NAME, subIndex, writerConfig, searcherFactory); }
@Override public IndexSearcher nrtSearcher() { try { return new SearcherFactory().newSearcher(nrtReader(), null); } catch (IOException e) { throw new RuntimeException(e); } }
public void init(boolean applyAllDeletes) throws IOException { this.analyzer = new CodeAnalyzer(); IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setCommitOnClose(true); config.setOpenMode(OpenMode.CREATE_OR_APPEND); try { writer = new IndexWriter(this.indexDir, config); } catch (IOException e) { config.setOpenMode(OpenMode.CREATE); writer = new IndexWriter(this.indexDir, config); } searcherFactory = new SearcherFactory(); searchManager = new SearcherManager(writer, applyAllDeletes, false, searcherFactory); }
@Test public void testSearcherManager() throws IOException { SearcherManager searcherManager = new SearcherManager(indexWriter.getDirectory(), new SearcherFactory()); IndexSearcher indexSearcher = null; try { indexSearcher = searcherManager.acquire(); } finally { if (indexSearcher == null) { searcherManager.release(indexSearcher); } } }
@Test public void testNrtManager() throws IOException { TrackingIndexWriter trackingIndexWriter = new TrackingIndexWriter(indexWriter); NRTManager nrtManager = new NRTManager(trackingIndexWriter, new SearcherFactory()); IndexSearcher indexSearcher = null; try { indexSearcher = nrtManager.acquire(); } finally { if (indexSearcher == null) { nrtManager.release(indexSearcher); } } }
/** * Builds a new {@link FSIndex}. * * @param name * the index name * @param mbeanName * the JMX MBean object name * @param path * the directory path * @param analyzer * the index writer analyzer * @param refresh * the index reader refresh frequency in seconds * @param ramBufferMB * the index writer RAM buffer size in MB * @param maxMergeMB * the directory max merge size in MB * @param maxCachedMB * the directory max cache size in MB * @param refreshTask * action to be done during refresh */ public void init(String name, String mbeanName, Path path, Analyzer analyzer, double refresh, int ramBufferMB, int maxMergeMB, int maxCachedMB, Runnable refreshTask) { try { this.path = path; this.name = name; // Open or create directory FSDirectory fsDirectory = FSDirectory.open(path); this.directory = new NRTCachingDirectory(fsDirectory, maxMergeMB, maxCachedMB); // Setup index writer IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer); indexWriterConfig.setRAMBufferSizeMB(ramBufferMB); indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); indexWriterConfig.setUseCompoundFile(true); indexWriterConfig.setMergePolicy(new TieredMergePolicy()); this.indexWriter = new IndexWriter(this.directory, indexWriterConfig); // Setup NRT search SearcherFactory searcherFactory = new SearcherFactory() { @Override public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) { if (refreshTask != null) { refreshTask.run(); } IndexSearcher searcher = new IndexSearcher(reader); searcher.setSimilarity(new NoIDFSimilarity()); return searcher; } }; TrackingIndexWriter trackingWriter = new TrackingIndexWriter(this.indexWriter); this.searcherManager = new SearcherManager(this.indexWriter, true, searcherFactory); this.searcherReopener = new ControlledRealTimeReopenThread<>(trackingWriter, this.searcherManager, refresh, refresh); this.searcherReopener.start(); // Register JMX MBean // mbean = new ObjectName(mbeanName); // ManagementFactory.getPlatformMBeanServer().registerMBean(service, // this.mbean); } catch (Exception e) { throw new FhirIndexException(e, "Error while creating index %s", name); } }
public Searcher(FullTextIndexShared index, Analyzer analyzer) throws IOException { this.index = index; this.searcherManager = new SearcherManager(index.open(), new SearcherFactory()); }
public void testTryDeleteDocument () throws IOException { Directory directory = createIndex(); IndexWriter writer = getWriter(directory); ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory()); TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); IndexSearcher searcher = mgr.acquire(); TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); assertEquals(1, topDocs.totalHits); long result; if (random().nextBoolean()) { IndexReader r = DirectoryReader.open(writer, true); result = mgrWriter.tryDeleteDocument(r, 0); r.close(); } else { result = mgrWriter.tryDeleteDocument(searcher.getIndexReader(), 0); } // The tryDeleteDocument should have succeeded: assertTrue(result != -1); assertTrue(writer.hasDeletions()); if (random().nextBoolean()) { writer.commit(); } assertTrue(writer.hasDeletions()); mgr.maybeRefresh(); searcher = mgr.acquire(); topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); assertEquals(0, topDocs.totalHits); }
public SearcherManager createSearcherManager() throws IOException { return new SearcherManager( writer, true, new SearcherFactory() ); }
public void testDeleteDocuments () throws IOException { Directory directory = createIndex(); IndexWriter writer = getWriter(directory); ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory()); IndexSearcher searcher = mgr.acquire(); TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); assertEquals(1, topDocs.totalHits); TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); long result = mgrWriter.deleteDocuments(new TermQuery(new Term("foo", "0"))); assertEquals(1, result); // writer.commit(); assertTrue(writer.hasDeletions()); mgr.maybeRefresh(); searcher = mgr.acquire(); topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); assertEquals(0, topDocs.totalHits); }
public void testTryDeleteDocumentCloseAndReopen () throws IOException { Directory directory = createIndex(); IndexWriter writer = getWriter(directory); ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory()); IndexSearcher searcher = mgr.acquire(); TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); assertEquals(1, topDocs.totalHits); TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); long result = mgrWriter.tryDeleteDocument(DirectoryReader.open(writer, true), 0); assertEquals(1, result); writer.commit(); assertTrue(writer.hasDeletions()); mgr.maybeRefresh(); searcher = mgr.acquire(); topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); assertEquals(0, topDocs.totalHits); writer.close(); searcher = new IndexSearcher(DirectoryReader.open(directory)); topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); assertEquals(0, topDocs.totalHits); }
/** * Create new NRTManager. * * @param writer TrackingIndexWriter to open near-real-time * readers * @param searcherFactory An optional {@link SearcherFactory}. Pass * <code>null</code> if you don't require the searcher to be warmed * before going live or other custom behavior. */ public NRTManager(TrackingIndexWriter writer, SearcherFactory searcherFactory) throws IOException { this(writer, searcherFactory, true); }