public static FieldSelector declaredTypesFieldSelector ( final boolean includeSource, final boolean includeSimpleName) { return includeSource ? includeSimpleName ? Queries.createFieldSelector(FIELD_PACKAGE_NAME, FIELD_BINARY_NAME, FIELD_SIMPLE_NAME, FIELD_SOURCE) : Queries.createFieldSelector(FIELD_PACKAGE_NAME, FIELD_BINARY_NAME, FIELD_SOURCE) : includeSimpleName ? Queries.createFieldSelector(FIELD_PACKAGE_NAME, FIELD_BINARY_NAME, FIELD_SIMPLE_NAME) : Queries.createFieldSelector(FIELD_PACKAGE_NAME, FIELD_BINARY_NAME); }
@Override public <T> void getDeclaredElements ( @NonNull final String simpleName, @NonNull final ClassIndex.NameKind kind, @NonNull final Set<? extends ClassIndex.SearchScopeType> scope, @NonNull final FieldSelector selector, @NonNull final Convertor<? super Document, T> convertor, @NonNull final Collection<? super T> result) throws InterruptedException, IOException { final Pair<Convertor<? super Document, T>,Index> ctu = indexPath.getPatch(convertor); try { IndexManager.priorityAccess(() -> { final Query query = QueryUtil.scopeFilter( Queries.createQuery( DocumentUtil.FIELD_SIMPLE_NAME, DocumentUtil.FIELD_CASE_INSENSITIVE_NAME, simpleName, DocumentUtil.translateQueryKind(kind)), scope); if (query != null) { index.query(result, ctu.first(), selector, cancel.get(), query); if (ctu.second() != null) { ctu.second().query(result, convertor, selector, cancel.get(), query); } } return null; }); } catch (IOException ioe) { this.<Void,IOException>handleException(null, ioe, root); } }
public abstract <T> void getDeclaredElements ( @NonNull String name, @NonNull ClassIndex.NameKind kind, @NonNull Set<? extends ClassIndex.SearchScopeType> scope, @NonNull FieldSelector selector, @NonNull Convertor<? super Document, T> convertor, @NonNull Collection<? super T> result) throws IOException, InterruptedException;
@Override public <T> void query( Collection<? super T> result, Convertor<? super org.apache.lucene.document.Document, T> convertor, FieldSelector selector, AtomicBoolean cancel, Query... queries) throws IOException, InterruptedException { await(cancel); }
@Override public <S, T> void queryDocTerms( Map<? super T, Set<S>> result, Convertor<? super org.apache.lucene.document.Document, T> convertor, Convertor<? super Term, S> termConvertor, FieldSelector selector, AtomicBoolean cancel, Query... queries) throws IOException, InterruptedException { await(cancel); }
public List<Field> get(int n, FieldSelector fieldSelector) throws IOException { Document document = ReferenceCountingReadOnlyIndexReader.super.document(n, fieldSelector); List<Field> fields = (List<Field>) document.getFields(); ArrayList<Field> cacheable = new ArrayList<Field>(fields.size()); cacheable.addAll(fields); return cacheable; }
public List<Field> get(int n, FieldSelector fieldSelector) throws IOException { Document document = ReferenceCountingReadOnlyIndexReader.super.document(n, fieldSelector); Field[] fields = document.getFields(fieldName); ArrayList<Field> cacheable = new ArrayList<Field>(fields.length); for (Field field : fields) { cacheable.add(field); } return cacheable; }
private <T> T manageCache(ConcurrentHashMap<Integer, WithUseCount<T>> cache, Accessor<T> accessor, int n, FieldSelector fieldSelector, int limit) throws IOException { Integer key = Integer.valueOf(n); WithUseCount<T> value = cache.get(key); if (value == null) { T made = accessor.get(n, fieldSelector); value = new WithUseCount<T>(made, n); cache.put(key, value); // resize if (limit >= 0) { if (cache.size() >= limit) { HashMap<Integer, WithUseCount<T>> keep = new HashMap<Integer, WithUseCount<T>>(); WithUseCount<T>[] existing = new WithUseCount[0]; synchronized (cache) { existing = cache.values().toArray(existing); cache.clear(); } Arrays.sort(existing); for (WithUseCount<T> current : existing) { keep.put(Integer.valueOf(current.doc), current); if ((current.count.get() == 0) || (keep.size() > (limit / 4))) { break; } } keep.put(key, value); cache.putAll(keep); } } } else { value.count.getAndIncrement(); } return value.object; }
private List<ResultDocument> initResultList(final Identity identity, final Roles roles, final Query query, final Analyzer analyzer, final Searcher searcher, final TopDocs docs, final int firstResult, final int maxReturns, final boolean doHighlight) throws IOException { final FieldSelector selector = new FieldSelector() { @Override public FieldSelectorResult accept(final String fieldName) { return (doHighlight || !AbstractOlatDocument.CONTENT_FIELD_NAME.equals(fieldName)) ? FieldSelectorResult.LOAD : FieldSelectorResult.NO_LOAD; } }; maxHits = SearchServiceFactory.getService().getSearchModuleConfig().getMaxHits(); totalHits = docs.totalHits; totalDocs = (docs.scoreDocs == null ? 0 : docs.scoreDocs.length); final int numOfDocs = Math.min(maxHits, docs.totalHits); final List<ResultDocument> res = new ArrayList<ResultDocument>(maxReturns + 1); for (int i = firstResult; i < numOfDocs && res.size() < maxReturns; i++) { final Document doc = searcher.doc(docs.scoreDocs[i].doc, selector); final String reservedTo = doc.get(AbstractOlatDocument.RESERVED_TO); if (StringHelper.containsNonWhitespace(reservedTo) && !"public".equals(reservedTo) && !reservedTo.contains(identity.getKey().toString())) { continue;// admin cannot see private documents } final ResultDocument rDoc = createResultDocument(doc, i, query, analyzer, doHighlight, identity, roles); if (rDoc != null) { res.add(rDoc); } if (!roles.isOLATAdmin() && i % 10 == 0) { // Do commit after certain number of documents because the transaction should not be too big DBFactory.getInstance().intermediateCommit(); } } return res; }
ReaderLocal(IndexConfig indexConfig, IndexDirectory indexDirectory) throws IOException, SearchLibException { super(indexConfig); spellCheckCache = new SpellCheckCache(100); docSetHitsCache = new DocSetHitsCache(indexConfig); this.indexDirectory = indexDirectory; references = new AtomicInteger(0); acquire(); final Directory directory = indexDirectory.getDirectory(); if (directory == null) throw new IOException("The directory is closed"); if (indexConfig.isMulti()) { final List<String> indexList = indexConfig.getIndexList(); indexDirectories = new IndexDirectory[indexList.size()]; indexReaders = new IndexReader[indexList.size()]; int i = 0; for (String indexName : indexList) { IndexDirectory indexDir = new IndexDirectory(new File(ClientCatalog.getClient(indexName).getDirectory(), "index")); indexDirectories[i] = indexDir; indexReaders[i++] = IndexReader.open(indexDir.getDirectory()); } indexReader = new MultiReader(indexReaders); } else { indexReaders = null; indexDirectories = null; indexReader = IndexReader.open(directory); } indexSearcher = new IndexSearcher(indexReader); final Similarity similarity = indexConfig.getNewSimilarityInstance(); if (similarity != null) indexSearcher.setSimilarity(similarity); // Warm final TopDocs topDocs = indexSearcher.search(new MatchAllDocsQuery(), 10); if (topDocs != null && topDocs.scoreDocs != null) for (ScoreDoc scoreDoc : topDocs.scoreDocs) indexSearcher.doc(scoreDoc.doc, (FieldSelector) fieldName -> FieldSelectorResult.LOAD); }
final public List<Document> getDocFields(final int[] docIds, final Set<String> fieldNameSet) throws IOException { if (docIds == null || docIds.length == 0) return null; List<Document> documents = new ArrayList<Document>(docIds.length); FieldSelector selector = new FieldSelectors.SetFieldSelector(fieldNameSet); for (int docId : docIds) documents.add(indexReader.document(docId, selector)); return documents; }
public AbstractRevisionFilter(final IssueManager issueManager, final PermissionManager permissionManager, final User user) { this.issueManager = issueManager; this.permissionManager = permissionManager; this.user = user; issueKeysFieldSelector = new FieldSelector() { public FieldSelectorResult accept(String s) { return StringUtils.equals(s, CommitIndexer.FIELD_ISSUEKEY) ? FieldSelectorResult.LOAD : FieldSelectorResult.NO_LOAD; } }; }
static FieldSelector sourceNameFieldSelector () { return Queries.createFieldSelector(FIELD_SOURCE); }
@Override public <T> void query( @NonNull Collection<? super T> result, @NonNull Convertor<? super Document, T> convertor, @NullAllowed FieldSelector selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException { Parameters.notNull("queries", queries); //NOI18N Parameters.notNull("convertor", convertor); //NOI18N Parameters.notNull("result", result); //NOI18N if (selector == null) { selector = AllFieldsSelector.INSTANCE; } lock.readLock().lock(); try { final IndexReader in = getReader(); if (in == null) { return; } final BitSet bs = new BitSet(in.maxDoc()); final Collector c = new BitSetCollector(bs); final Searcher searcher = new IndexSearcher(in); try { for (Query q : queries) { if (cancel != null && cancel.get()) { throw new InterruptedException (); } searcher.search(q, c); } } finally { searcher.close(); } for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) { if (cancel != null && cancel.get()) { throw new InterruptedException (); } final Document doc = in.document(docNum, selector); final T value = convertor.convert(doc); if (value != null) { result.add (value); } } } finally { lock.readLock().unlock(); } }
@Override public <S, T> void queryDocTerms( @NonNull Map<? super T, Set<S>> result, @NonNull Convertor<? super Document, T> convertor, @NonNull Convertor<? super Term, S> termConvertor, @NullAllowed FieldSelector selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException { Parameters.notNull("result", result); //NOI18N Parameters.notNull("convertor", convertor); //NOI18N Parameters.notNull("termConvertor", termConvertor); //NOI18N Parameters.notNull("queries", queries); //NOI18N if (selector == null) { selector = AllFieldsSelector.INSTANCE; } lock.readLock().lock(); try { final IndexReader in = getReader(); if (in == null) { return; } final BitSet bs = new BitSet(in.maxDoc()); final Collector c = new BitSetCollector(bs); final Searcher searcher = new IndexSearcher(in); final TermCollector termCollector = new TermCollector(c); try { for (Query q : queries) { if (cancel != null && cancel.get()) { throw new InterruptedException (); } if (q instanceof TermCollector.TermCollecting) { ((TermCollector.TermCollecting)q).attach(termCollector); } else { throw new IllegalArgumentException ( String.format("Query: %s does not implement TermCollecting", //NOI18N q.getClass().getName())); } searcher.search(q, termCollector); } } finally { searcher.close(); } for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) { if (cancel != null && cancel.get()) { throw new InterruptedException (); } final Document doc = in.document(docNum, selector); final T value = convertor.convert(doc); if (value != null) { final Set<Term> terms = termCollector.get(docNum); if (terms != null) { result.put (value, convertTerms(termConvertor, terms)); } } } } finally { lock.readLock().unlock(); } }
@Override public <T> void query ( final @NonNull Collection<? super T> result, final @NonNull Convertor<? super Document, T> convertor, @NullAllowed FieldSelector selector, final @NullAllowed AtomicBoolean cancel, final @NonNull Query... queries ) throws IOException, InterruptedException { Parameters.notNull("queries", queries); //NOI18N Parameters.notNull("convertor", convertor); //NOI18N Parameters.notNull("result", result); //NOI18N if (selector == null) { selector = AllFieldsSelector.INSTANCE; } IndexReader in = null; try { in = dirCache.acquireReader(); if (in == null) { LOGGER.log(Level.FINE, "{0} is invalid!", this); return; } final BitSet bs = new BitSet(in.maxDoc()); final Collector c = new BitSetCollector(bs); final IndexSearcher searcher = new IndexSearcher(in); try { for (Query q : queries) { if (cancel != null && cancel.get()) { throw new InterruptedException (); } searcher.search(q, c); } } finally { searcher.close(); } if (convertor instanceof IndexReaderInjection) { ((IndexReaderInjection)convertor).setIndexReader(in); } try { for (int docNum = bs.nextSetBit(0); docNum >= 0; docNum = bs.nextSetBit(docNum+1)) { if (cancel != null && cancel.get()) { throw new InterruptedException (); } final Document doc = in.document(docNum, selector); final T value = convertor.convert(doc); if (value != null) { result.add (value); } } } finally { if (convertor instanceof IndexReaderInjection) { ((IndexReaderInjection)convertor).setIndexReader(null); } } } finally { dirCache.releaseReader(in); } }
public Field get(int n, FieldSelector fieldSelector) throws IOException { return new Field(fieldName, getStringValue(n, fieldName), Store.NO, Index.UN_TOKENIZED); }
public Document document(int n, FieldSelector fieldSelector) throws IOException { Document ret = wrapped.document(n, fieldSelector); work(ret.getFields().size()); return ret; }
public final Document doc(int n, FieldSelector fs) throws IOException { return reader.document(hits.get(n).doc, fs); }
final public Document getDocFields(final int docId, final Set<String> fieldNameSet) throws IOException { FieldSelector selector = new FieldSelectors.SetFieldSelector(fieldNameSet); return indexReader.document(docId, selector); }
/** * Queries the {@link Index} by given queries. * @param result the {@link Collection} to store query results into * @param convertor the {@link Convertor} used to convert lucene documents into the user objects added into the result * @param selector the selector used to select document's fields which should be loaded, if null all fields are loaded * @param cancel the {@link AtomicBoolean} used to cancel the index iteration by the caller. When set to true the iteration * is stopped. * @param queries the queries to be performed on the {@link Index} * @throws IOException in case of IO problem * @throws InterruptedException when query was canceled */ <T> void query (Collection<? super T> result, @NonNull Convertor<? super Document, T> convertor, @NullAllowed FieldSelector selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException;
/** * Queries the {@link Index} by given queries. In addition to documents it also collects the terms which matched the queries. * @param result the {@link Collection} to store query results into * @param convertor the {@link Convertor} used to convert lucene documents into the user objects added into the result * @param termConvertor the {@link Convertor} used to convert lucene terms into the user objects added into the result * @param selector the selector used to select document's fields which should be loaded, if null all fields are loaded * @param cancel the {@link AtomicBoolean} used to cancel the index iteration by the caller. When set to true the iteration * is stopped. * @param queries the queries to be performed on the {@link Index} * @throws IOException in case of IO problem * @throws InterruptedException when query was canceled */ <S, T> void queryDocTerms(Map<? super T, Set<S>> result, @NonNull Convertor<? super Document, T> convertor, @NonNull Convertor<? super Term, S> termConvertor,@NullAllowed FieldSelector selector, @NullAllowed AtomicBoolean cancel, @NonNull Query... queries) throws IOException, InterruptedException;
/** * Creates a FieldSelector loading the given fields. * @param fieldsToLoad the fields to be loaded into the document. * @return the created FieldSelector */ public static FieldSelector createFieldSelector(final @NonNull String... fieldsToLoad) { return new FieldSelectorImpl(fieldsToLoad); }
T get(int n, FieldSelector fieldSelector) throws IOException;
