Java 类org.apache.lucene.index.DirectoryReader 实例源码

/**
 * Search all books of a given author. 
 * 
 * @throws Exception never, otherwise the test fails.
 */
@Test
public void findByAuthorSurname() throws Exception {
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));

    Query query = new QueryParser("author", new StandardAnalyzer()).parse("Gazzarini");
    TopDocs matches = searcher.search(query, 10);

    assertEquals(1, matches.totalHits);

    final String id = Arrays.stream(matches.scoreDocs)
        .map(scoreDoc -> luceneDoc(scoreDoc.doc, searcher))
        .map(doc -> doc.get("id"))
        .findFirst()
        .get();

    assertEquals("1", id);
}

/** 
 * test version lookup actually works
 */
public void testSimple() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    Document doc = new Document();
    doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE));
    doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
    writer.addDocument(doc);
    DirectoryReader reader = DirectoryReader.open(writer);
    LeafReaderContext segment = reader.leaves().get(0);
    PerThreadIDAndVersionLookup lookup = new PerThreadIDAndVersionLookup(segment.reader());
    // found doc
    DocIdAndVersion result = lookup.lookup(new BytesRef("6"), null, segment);
    assertNotNull(result);
    assertEquals(87, result.version);
    assertEquals(0, result.docId);
    // not found doc
    assertNull(lookup.lookup(new BytesRef("7"), null, segment));
    // deleted doc
    assertNull(lookup.lookup(new BytesRef("6"), new Bits.MatchNoBits(1), segment));
    reader.close();
    writer.close();
    dir.close();
}

@Override
public boolean isIndexed(Project project, ObjectId commit) {
    File indexDir = storageManager.getProjectIndexDir(project.getForkRoot().getId());
    try (Directory directory = FSDirectory.open(indexDir)) {
        if (DirectoryReader.indexExists(directory)) {
            try (IndexReader reader = DirectoryReader.open(directory)) {
                IndexSearcher searcher = new IndexSearcher(reader);
                return getCurrentCommitIndexVersion().equals(getCommitIndexVersion(searcher, commit));
            }
        } else {
            return false;
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

private DirectoryReader wrapReader(DirectoryReader reader) {
    try {
        Constructor<?>[] constructors = mockContext.wrapper.getConstructors();
        Constructor<?> nonRandom = null;
        for (Constructor<?> constructor : constructors) {
            Class<?>[] parameterTypes = constructor.getParameterTypes();
            if (parameterTypes.length > 0 && parameterTypes[0] == DirectoryReader.class) {
                if (parameterTypes.length == 1) {
                    nonRandom = constructor;
                } else if (parameterTypes.length == 2 && parameterTypes[1] == Settings.class) {

                    return (DirectoryReader) constructor.newInstance(reader, mockContext.indexSettings);
                }
            }
        }
        if (nonRandom != null) {
            return (DirectoryReader) nonRandom.newInstance(reader);
        }
    } catch (Exception e) {
        throw new ElasticsearchException("Can not wrap reader", e);
    }
    return reader;
}

BytesReference getOrCompute(CacheEntity cacheEntity, Supplier<BytesReference> loader,
        DirectoryReader reader, BytesReference cacheKey) throws Exception {
    final Key key =  new Key(cacheEntity, reader.getVersion(), cacheKey);
    Loader cacheLoader = new Loader(cacheEntity, loader);
    BytesReference value = cache.computeIfAbsent(key, cacheLoader);
    if (cacheLoader.isLoaded()) {
        key.entity.onMiss();
        // see if its the first time we see this reader, and make sure to register a cleanup key
        CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getVersion());
        if (!registeredClosedListeners.containsKey(cleanupKey)) {
            Boolean previous = registeredClosedListeners.putIfAbsent(cleanupKey, Boolean.TRUE);
            if (previous == null) {
                ElasticsearchDirectoryReader.addReaderCloseListener(reader, cleanupKey);
            }
        }
    } else {
        key.entity.onHit();
    }
    return value;
}

/**
 * Cache something calculated at the shard level.
 * @param shard the shard this item is part of
 * @param reader a reader for this shard. Used to invalidate the cache when there are changes.
 * @param cacheKey key for the thing being cached within this shard
 * @param loader loads the data into the cache if needed
 * @return the contents of the cache or the result of calling the loader
 */
private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey, Consumer<StreamOutput> loader)
        throws Exception {
    IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard);
    Supplier<BytesReference> supplier = () -> {
        /* BytesStreamOutput allows to pass the expected size but by default uses
         * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie.
         * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful
         * since we don't shrink to the actual size once we are done serializing.
         * By passing 512 as the expected size we will resize the byte array in the stream
         * slowly until we hit the page size and don't waste too much memory for small query
         * results.*/
        final int expectedSizeInBytes = 512;
        try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) {
            loader.accept(out);
            // for now, keep the paged data structure, which might have unused bytes to fill a page, but better to keep
            // the memory properly paged instead of having varied sized bytes
            return out.bytes();
        }
    };
    return indicesRequestCache.getOrCompute(cacheEntity, supplier, reader, cacheKey);
}

public void testVectorHighlighter() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, notNullValue());
    assertThat(fragment, equalTo("the big <b>bad</b> dog"));
}

public void testVectorHighlighterNoStore() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_NOT_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, nullValue());
}

public void doTestDocValueRangeQueries(NumberType type, Supplier<Number> valueSupplier) throws Exception {
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
    final int numDocs = TestUtil.nextInt(random(), 100, 500);
    for (int i = 0; i < numDocs; ++i) {
        w.addDocument(type.createFields("foo", valueSupplier.get(), true, true, false));
    }
    DirectoryReader reader = DirectoryReader.open(w);
    IndexSearcher searcher = newSearcher(reader);
    w.close();
    final int iters = 10;
    for (int iter = 0; iter < iters; ++iter) {
        Query query = type.rangeQuery("foo",
                random().nextBoolean() ? null : valueSupplier.get(),
                random().nextBoolean() ? null : valueSupplier.get(),
                randomBoolean(), randomBoolean(), true);
        assertThat(query, Matchers.instanceOf(IndexOrDocValuesQuery.class));
        IndexOrDocValuesQuery indexOrDvQuery = (IndexOrDocValuesQuery) query;
        assertEquals(
                searcher.count(indexOrDvQuery.getIndexQuery()),
                searcher.count(indexOrDvQuery.getRandomAccessQuery()));
    }
    reader.close();
    dir.close();
}

/**
 * Search sample. 
 * 
 * @param directory the index directory.
 * @throws IOException in case of I/O failure.
 * @throws ParseException in case of Query parse exception.
 */ 
public static void search(Directory directory) throws IOException, ParseException {
    IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));

    Query query = new QueryParser("title", new StandardAnalyzer()).parse("title:Solr");
    TopDocs matches = searcher.search(query, 10);

    System.out.println("Search returned " + matches.totalHits + " matches.");
    Arrays.stream(matches.scoreDocs)
        .map(scoreDoc -> luceneDoc(scoreDoc, searcher))
        .forEach(doc -> {
            System.out.println("-------------------------------------");                
            System.out.println("ID:\t" + doc.get("id"));
            System.out.println("TITLE:\t" + doc.get("title"));
            System.out.println("AUTHOR:\t" + doc.get("author"));
            System.out.println("SCORE:\t" + doc.get("score"));

        });
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
    if (getBoost() != 1.0F) {
        return super.rewrite(reader);
    }
    if (reader instanceof DirectoryReader) {
        String joinField = ParentFieldMapper.joinField(parentType);
        IndexSearcher indexSearcher = new IndexSearcher(reader);
        indexSearcher.setQueryCache(null);
        indexSearcher.setSimilarity(similarity);
        IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal((DirectoryReader) reader);
        MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType);
        return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren);
    } else {
        if (reader.leaves().isEmpty() && reader.numDocs() == 0) {
            // asserting reader passes down a MultiReader during rewrite which makes this
            // blow up since for this query to work we have to have a DirectoryReader otherwise
            // we can't load global ordinals - for this to work we simply check if the reader has no leaves
            // and rewrite to match nothing
            return new MatchNoDocsQuery();
        }
        throw new IllegalStateException("can't load global ordinals for reader of type: " + reader.getClass() + " must be a DirectoryReader");
    }
}

public SearchResult search(String index, String queryString, int page) {
    SearchResult searchResult = null;

    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(Properties.getProperties().getProperty(Values.INDEX_LOCATION, Values.DEFAULT_INDEX_LOCATION))));
        IndexSearcher searcher = new IndexSearcher(reader);
        Analyzer analyzer = new StandardAnalyzer();

        // Search over the titles only for the moment
        QueryParser parser = new QueryParser(index, analyzer);
        Query query = parser.parse(queryString);

        searchResult = this.doPagingSearch(reader, searcher, query, queryString, page);
        reader.close();
    }
    catch(Exception ex) {}

    return searchResult;
}

public void testAddingAClosedReader() throws Exception {
    LeafReader reader;
    try (Directory dir = newDirectory();
            RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
        writer.addDocument(new Document());
        try (DirectoryReader dirReader = ElasticsearchDirectoryReader.wrap(writer.getReader(), new ShardId("index1", "_na_", 1))) {
            reader = dirReader.leaves().get(0).reader();
        }
    }
    ShardCoreKeyMap map = new ShardCoreKeyMap();
    try {
        map.add(reader);
        fail("Expected AlreadyClosedException");
    } catch (AlreadyClosedException e) {
        // What we wanted
    }
    assertEquals(0, map.size());
}

public FbEntitySearcher(String indexDir, int numOfDocs, String searchingStrategy) throws IOException {

    LogInfo.begin_track("Constructing Searcher");
    if (!searchingStrategy.equals("exact") && !searchingStrategy.equals("inexact"))
      throw new RuntimeException("Bad searching strategy: " + searchingStrategy);
    this.searchStrategy = searchingStrategy;

    queryParser = new QueryParser(
        Version.LUCENE_44,
        FbIndexField.TEXT.fieldName(),
        searchingStrategy.equals("exact") ? new KeywordAnalyzer() : new StandardAnalyzer(Version.LUCENE_44));
    LogInfo.log("Opening index dir: " + indexDir);
    IndexReader indexReader = DirectoryReader.open(SimpleFSDirectory.open(new File(indexDir)));
    indexSearcher = new IndexSearcher(indexReader);
    LogInfo.log("Opened index with " + indexReader.numDocs() + " documents.");

    this.numOfDocs = numOfDocs;
    LogInfo.end_track();
  }

/** User runs a query and counts facets. */
private List<FacetResult> facetsWithSearch() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  FacetsCollector fc = new FacetsCollector();

  // MatchAllDocsQuery is for "browsing" (counts facets
  // for all non-deleted docs in the index); normally
  // you'd use a "normal" query:
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);

  // Retrieve results
  List<FacetResult> results = new ArrayList<FacetResult>();

  // Count both "Publish Date" and "Author" dimensions
  Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
  results.add(facets.getTopChildren(10, "Author"));
  results.add(facets.getTopChildren(10, "Publish Date"));

  indexReader.close();
  taxoReader.close();

  return results;
}

/** User drills down on 'Publish Date/2010', and we
 *  return facets for 'Author' */
private FacetResult drillDown() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  // Passing no baseQuery means we drill down on all
  // documents ("browse only"):
  DrillDownQuery q = new DrillDownQuery(config);

  // Now user drills down on Publish Date/2010:
  q.add("Publish Date", "2010");
  FacetsCollector fc = new FacetsCollector();
  FacetsCollector.search(searcher, q, 10, fc);

  // Retrieve results
  Facets facets = new FastTaxonomyFacetCounts(taxoReader, config, fc);
  FacetResult result = facets.getTopChildren(10, "Author");

  indexReader.close();
  taxoReader.close();

  return result;
}

private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalMax> verify)
        throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    buildIndex.accept(indexWriter);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);

    MaxAggregationBuilder aggregationBuilder = new MaxAggregationBuilder("_name").field("number");
    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    fieldType.setName("number");
    try (MaxAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
        aggregator.preCollection();
        indexSearcher.search(query, aggregator);
        aggregator.postCollection();
        verify.accept((InternalMax) aggregator.buildAggregation(0L));
    }
    indexReader.close();
    directory.close();
}

private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalAvg> verify)
        throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    buildIndex.accept(indexWriter);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);

    AvgAggregationBuilder aggregationBuilder = new AvgAggregationBuilder("_name").field("number");
    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    fieldType.setName("number");
    try (AvgAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
        aggregator.preCollection();
        indexSearcher.search(query, aggregator);
        aggregator.postCollection();
        verify.accept((InternalAvg) aggregator.buildAggregation(0L));
    }
    indexReader.close();
    directory.close();
}

public void testNoTokens() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.KEYWORD_ANALYZER));

    FieldType allFt = getAllFieldType();
    Document doc = new Document();
    doc.add(new Field("_id", "1", StoredField.TYPE));
    doc.add(new AllField("_all", "", 2.0f, allFt));
    indexWriter.addDocument(doc);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);

    TopDocs docs = searcher.search(new MatchAllDocsQuery(), 10);
    assertThat(docs.totalHits, equalTo(1));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
}

public void testMinAggregator_noDocs() throws Exception {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    indexWriter.close();

    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = newSearcher(indexReader, true, true);

    MinAggregationBuilder aggregationBuilder = new MinAggregationBuilder("_name").field("number");
    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    fieldType.setName("number");
    try (MinAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType)) {
        aggregator.preCollection();
        indexSearcher.search(new MatchAllDocsQuery(), aggregator);
        aggregator.postCollection();
        InternalMin result = (InternalMin) aggregator.buildAggregation(0L);
        assertEquals(Double.POSITIVE_INFINITY, result.getValue(), 0);
    }
    indexReader.close();
    directory.close();
}

public void testNoDocs() throws IOException {
    Directory directory = newDirectory();

    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    // intentionally not writing any docs
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);

    testCase(new MatchAllDocsQuery(), newSearcher(indexReader, false, true), parentToChild -> {
        assertEquals(0, parentToChild.getDocCount());
        assertEquals(Double.POSITIVE_INFINITY, ((InternalMin) parentToChild.getAggregations().get("in_child")).getValue(),
                Double.MIN_VALUE);
    });
    indexReader.close();
    directory.close();
}

public void testDiversifiedSampler_noDocs() throws Exception {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = new IndexSearcher(indexReader);

    MappedFieldType idFieldType = new KeywordFieldMapper.KeywordFieldType();
    idFieldType.setName("id");
    idFieldType.setHasDocValues(true);

    MappedFieldType genreFieldType = new KeywordFieldMapper.KeywordFieldType();
    genreFieldType.setName("genre");
    genreFieldType.setHasDocValues(true);

    DiversifiedAggregationBuilder builder = new DiversifiedAggregationBuilder("_name")
            .field(genreFieldType.name())
            .subAggregation(new TermsAggregationBuilder("terms", null).field("id"));

    InternalSampler result = search(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType);
    Terms terms = result.getAggregations().get("terms");
    assertEquals(0, terms.getBuckets().size());
    indexReader.close();
    directory.close();
}

/** User runs a query and aggregates facets by summing their association values. */
private List<FacetResult> sumAssociations() throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  IndexSearcher searcher = new IndexSearcher(indexReader);
  TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);

  FacetsCollector fc = new FacetsCollector();

  // MatchAllDocsQuery is for "browsing" (counts facets
  // for all non-deleted docs in the index); normally
  // you'd use a "normal" query:
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, fc);

  Facets tags = new TaxonomyFacetSumIntAssociations("$tags", taxoReader, config, fc);
  Facets genre = new TaxonomyFacetSumFloatAssociations("$genre", taxoReader, config, fc);

  // Retrieve results
  List<FacetResult> results = new ArrayList<FacetResult>();
  results.add(tags.getTopChildren(10, "tags"));
  results.add(genre.getTopChildren(10, "genre"));

  indexReader.close();
  taxoReader.close();

  return results;
}

/**
 * Test the WordScorer emitted by the smoothing model
 */
public void testBuildWordScorer() throws IOException {
    SmoothingModel testModel = createTestModel();
    Map<String, Analyzer> mapping = new HashMap<>();
    mapping.put("field", new WhitespaceAnalyzer());
    PerFieldAnalyzerWrapper wrapper = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(), mapping);
    IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(wrapper));
    Document doc = new Document();
    doc.add(new Field("field", "someText", TextField.TYPE_NOT_STORED));
    writer.addDocument(doc);
    DirectoryReader ir = DirectoryReader.open(writer);

    WordScorer wordScorer = testModel.buildWordScorerFactory().newScorer(ir, MultiFields.getTerms(ir, "field"), "field", 0.9d,
            BytesRefs.toBytesRef(" "));
    assertWordScorer(wordScorer, testModel);
}

/** Test that version map cache works, is evicted on close, etc */
public void testCache() throws Exception {
    int size = Versions.lookupStates.size();

    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    Document doc = new Document();
    doc.add(new Field(UidFieldMapper.NAME, "6", UidFieldMapper.Defaults.FIELD_TYPE));
    doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 87));
    writer.addDocument(doc);
    DirectoryReader reader = DirectoryReader.open(writer);
    // should increase cache size by 1
    assertEquals(87, Versions.loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
    assertEquals(size+1, Versions.lookupStates.size());
    // should be cache hit
    assertEquals(87, Versions.loadVersion(reader, new Term(UidFieldMapper.NAME, "6")));
    assertEquals(size+1, Versions.lookupStates.size());

    reader.close();
    writer.close();
    // core should be evicted from the map
    assertEquals(size, Versions.lookupStates.size());
    dir.close();
}

@Override
public List<RepositoryInfo> getLoaded(final List<RepositoryInfo> repos) {
    final List<RepositoryInfo> toRet = new ArrayList<RepositoryInfo>(repos.size());
    for (final RepositoryInfo repo : repos) {
        File loc = new File(getDefaultIndexLocation(), repo.getId()); // index folder
        try {
            if (loc.exists() && new File(loc, "timestamp").exists() && DirectoryReader.indexExists(new SimpleFSDirectory(loc.toPath()))) {
                toRet.add(repo);
            }
        } catch (IOException ex) {
            LOGGER.log(Level.FINER, "Index Not Available: " +repo.getId() + " at: " + loc.getAbsolutePath(), ex);
        }
    }
    return toRet;
}

public void testRedisDirectoryWithJedisPool() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    //indexWriterConfig.setInfoStream(System.out);
    //indexWriterConfig.setRAMBufferSizeMB(2048);
    //LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy();
    //logByteSizeMergePolicy.setMinMergeMB(1);
    //logByteSizeMergePolicy.setMaxMergeMB(64);
    //logByteSizeMergePolicy.setMaxCFSSegmentSizeMB(64);
    //indexWriterConfig.setRAMBufferSizeMB(1024).setMergePolicy(logByteSizeMergePolicy).setUseCompoundFile(false);
    //GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
    //获取连接等待时间
    //genericObjectPoolConfig.setMaxWaitMillis(3000);
    //10s超时时间
    JedisPool jedisPool = new JedisPool(new JedisPoolConfig(), "localhost", 6379, Constants.TIME_OUT);
    RedisDirectory redisDirectory = new RedisDirectory(new JedisPoolStream(jedisPool));
    IndexWriter indexWriter = new IndexWriter(redisDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    redisDirectory.close();
    long end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedisPool consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(new RedisDirectory(new JedisStream("localhost",
            6379))));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedisPool search consumes {}ms!", (end - start));
}

public void testRedisDirectoryWithJedis() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    //indexWriterConfig.setInfoStream(System.out);
    //indexWriterConfig.setRAMBufferSizeMB(2048);
    //LogByteSizeMergePolicy logByteSizeMergePolicy = new LogByteSizeMergePolicy();
    //logByteSizeMergePolicy.setMinMergeMB(1);
    //logByteSizeMergePolicy.setMaxMergeMB(64);
    //logByteSizeMergePolicy.setMaxCFSSegmentSizeMB(64);
    //indexWriterConfig.setRAMBufferSizeMB(1024).setMergePolicy(logByteSizeMergePolicy).setUseCompoundFile(false);
    //GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
    //获取连接等待时间
    //genericObjectPoolConfig.setMaxWaitMillis(3000);
    //10s超时时间
    RedisDirectory redisDirectory = new RedisDirectory(new JedisStream("localhost", 6379));
    IndexWriter indexWriter = new IndexWriter(redisDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    redisDirectory.close();
    long end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedis consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(new RedisDirectory(new JedisStream("localhost",
            6379))));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RedisDirectoryWithJedis search consumes {}ms!", (end - start));
}

public FieldMaskingReader(String field, DirectoryReader in) throws IOException {
    super(in, new FilterDirectoryReader.SubReaderWrapper() {
        @Override
        public LeafReader wrap(LeafReader reader) {
            return new FieldFilterLeafReader(reader, Collections.singleton(field), true);
        }
    });
    this.field = field;

}

/**
 * Can the shard request be cached at all?
 */
public boolean canCache(ShardSearchRequest request, SearchContext context) {

    // We cannot cache with DFS because results depend not only on the content of the index but also
    // on the overridden statistics. So if you ran two queries on the same index with different stats
    // (because an other shard was updated) you would get wrong results because of the scores
    // (think about top_hits aggs or scripts using the score)
    if (SearchType.QUERY_THEN_FETCH != context.searchType()) {
        return false;
    }
    IndexSettings settings = context.indexShard().indexSettings();
    // if not explicitly set in the request, use the index setting, if not, use the request
    if (request.requestCache() == null) {
        if (settings.getValue(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING) == false) {
            return false;
        } else if (context.size() != 0) {
            // If no request cache query parameter and shard request cache
            // is enabled in settings don't cache for requests with size > 0
            return false;
        }
    } else if (request.requestCache() == false) {
        return false;
    }
    // if the reader is not a directory reader, we can't get the version from it
    if ((context.searcher().getIndexReader() instanceof DirectoryReader) == false) {
        return false;
    }
    // if now in millis is used (or in the future, a more generic "isDeterministic" flag
    // then we can't cache based on "now" key within the search request, as it is not deterministic
    if (context.getQueryShardContext().isCachable() == false) {
        return false;
    }
    return true;

}

FieldMaskingReader(String field, DirectoryReader in, AtomicInteger closeCalls) throws IOException {
    super(in, new SubReaderWrapper() {
        @Override
        public LeafReader wrap(LeafReader reader) {
            return new FieldFilterLeafReader(reader, Collections.singleton(field), true);
        }
    });
    this.closeCalls = closeCalls;
    this.field = field;
}

public SearcherTracker(IndexSearcher searcher) {
  this.searcher = searcher;
  version = ((DirectoryReader) searcher.getIndexReader()).getVersion();
  searcher.getIndexReader().incRef();
  // Use nanoTime not currentTimeMillis since it [in
  // theory] reduces risk from clock shift
  recordTimeSec = System.nanoTime() / NANOS_PER_SEC;
}

private static void mergeTickets(String firstId, String secondId) throws IOException, ParseException, IndexException {

    if (firstId == null || secondId == null) {
        throw new IllegalArgumentException(ERROR_NULL_ARGUMENT);
    }

    Directory directory = IndexProvider.getInstance().getDirectory();
    try (IndexReader indexReader = DirectoryReader.open(directory);) {
        IndexSearcher searcher = new IndexSearcher(indexReader);

        BooleanQuery firstQuery = getQuery(firstId);
        BooleanQuery secondQuery = getQuery(secondId);

        ScoreDoc[] topHitsDocsIdFirst = getTopHitsDoc(firstId, firstQuery, searcher);
        ScoreDoc[] topHitsDocsIdSecond = getTopHitsDoc(secondId, secondQuery, searcher);

        if (topHitsDocsIdFirst[0].doc == topHitsDocsIdSecond[0].doc) {
            String message = MessageFormat.format(WARN_DOCUMENTS_ALREADY_MERGED, firstId, secondId);
            LOGGER.warn(message);
            return;
        }

        Document newDocument = mergeDocuments(searcher, topHitsDocsIdFirst, topHitsDocsIdSecond);

        IndexWriterConfig config = new IndexWriterConfig(analyzer);
        try (IndexWriter indexWriter = new IndexWriter(directory, config);) {
            indexWriter.deleteDocuments(firstQuery);
            indexWriter.deleteDocuments(secondQuery);
            indexWriter.addDocument(newDocument);
        }
    }       
}

public void testDefaults() throws IOException {
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
            .startObject("properties").startObject("field").field("type", "boolean").endObject().endObject()
            .endObject().endObject().string();

    DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping));

    ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
            .startObject()
            .field("field", true)
            .endObject()
            .bytes());

    try (Directory dir = new RAMDirectory();
         IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())))) {
        w.addDocuments(doc.docs());
        try (DirectoryReader reader = DirectoryReader.open(w)) {
            final LeafReader leaf = reader.leaves().get(0).reader();
            // boolean fields are indexed and have doc values by default
            assertEquals(new BytesRef("T"), leaf.terms("field").iterator().next());
            SortedNumericDocValues values = leaf.getSortedNumericDocValues("field");
            assertNotNull(values);
            values.setDocument(0);
            assertEquals(1, values.count());
            assertEquals(1, values.valueAt(0));
        }
    }
}

private static List<TicketGroup> searchForTicket(Ticket ticket, int maxCount) throws IndexException {
    if (ticket == null) {
        throw new IllegalArgumentException(ERROR_NULL_ARGUMENT);
    }

    Directory directory = IndexProvider.getInstance().getDirectory();

    try (IndexReader reader = DirectoryReader.open(directory);) {
        Query query = new QueryParser(SUBJECT, analyzer).parse(QueryParser.escape(ticket.getSubject()));
        IndexSearcher searcher = new IndexSearcher(reader);
        TopDocs docs = searcher.search(query, maxCount);
        ScoreDoc[] foundDocuments = docs.scoreDocs;

        List<TicketGroup> result = new ArrayList<>();
        for(ScoreDoc document: foundDocuments){
            String[] ids = searcher.doc(document.doc).getValues(ID);
            if (!(ids.length == 1 && ids[0].equals(ticket.getId()))){
                result.add(new TicketGroup(Arrays.asList(ids)));
            }
        }

        return result;
    } catch (IOException | ParseException e) {
        LOGGER.error(ERROR_SEARCHING_FAILED, e);
        throw new IndexException(ERROR_SEARCHING_FAILED, e);
    }
}

public ShadowEngine(EngineConfig engineConfig)  {
    super(engineConfig);
    if (engineConfig.getRefreshListeners() != null) {
        throw new IllegalArgumentException("ShadowEngine doesn't support RefreshListeners");
    }
    SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig);
    final long nonexistentRetryTime = engineConfig.getIndexSettings().getSettings()
            .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT)
            .getMillis();
    try {
        DirectoryReader reader = null;
        store.incRef();
        boolean success = false;
        try {
            if (Lucene.waitForIndex(store.directory(), nonexistentRetryTime)) {
                reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(store.directory()), shardId);
                this.searcherManager = new SearcherManager(reader, searcherFactory);
                this.lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager, store);
                success = true;
            } else {
                throw new IllegalStateException("failed to open a shadow engine after" +
                        nonexistentRetryTime + "ms, " +
                        "directory is not an index");
            }
        } catch (Exception e) {
            logger.warn("failed to create new reader", e);
            throw e;
        } finally {
            if (success == false) {
                IOUtils.closeWhileHandlingException(reader);
                store.decRef();
            }
        }
    } catch (IOException ex) {
        throw new EngineCreationFailureException(shardId, "failed to open index reader", ex);
    }
    logger.trace("created new ShadowEngine");
}

private NonClosingReaderWrapper(DirectoryReader in) throws IOException {
    super(in, new SubReaderWrapper() {
        @Override
        public LeafReader wrap(LeafReader reader) {
            return reader;
        }
    });
}

/**
 * Tries to extract the shard id from a reader if possible, when its not possible,
 * will return null.
 */
@Nullable
public static ShardId extractShardId(DirectoryReader reader) {
    final ElasticsearchDirectoryReader esReader = ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(reader);
    if (esReader != null) {
        return esReader.shardId();
    }
    throw new IllegalArgumentException("can't extract shard ID, can't unwrap ElasticsearchDirectoryReader");
}

private List<Image> getImagesOf(Collection collection) {

        List<Image> results = new ArrayList<>();

        try {
            Path path = indexPath(collection);
            if(!Files.exists(path)) return results;

            IndexReader ir = DirectoryReader.open(FSDirectory.open(path));

            int num = ir.numDocs();
            for ( int i = 0; i < num; i++)
            {
                Document d = ir.document(i);
                String imagePath = d.getField(DocumentBuilder.FIELD_NAME_IDENTIFIER).stringValue();
                String thumbnailPath = collectionUtils.getThumbnailPathFromImagePath(collection, imagePath);
                Image image = new Image(imagePath, thumbnailPath);
                image.setDocId(i);
                results.add(image);
            }
            ir.close();

        } catch (IOException e) {
            throw new LireLabException("Could not read index", e);
        }

        return results;
    }

/**
 * Adds the given listener to the provided directory reader. The reader must contain an {@link ElasticsearchDirectoryReader} in it's hierarchy
 * otherwise we can't safely install the listener.
 *
 * @throws IllegalArgumentException if the reader doesn't contain an {@link ElasticsearchDirectoryReader} in it's hierarchy
 */
@SuppressForbidden(reason = "This is the only sane way to add a ReaderClosedListener")
public static void addReaderCloseListener(DirectoryReader reader, IndexReader.ReaderClosedListener listener) {
    ElasticsearchDirectoryReader elasticsearchDirectoryReader = getElasticsearchDirectoryReader(reader);
    if (elasticsearchDirectoryReader != null) {
        assert reader.getCoreCacheKey() == elasticsearchDirectoryReader.getCoreCacheKey();
        elasticsearchDirectoryReader.addReaderClosedListener(listener);
        return;
    }
    throw new IllegalArgumentException("Can't install close listener reader is not an ElasticsearchDirectoryReader/ElasticsearchLeafReader");
}