Java 类org.apache.lucene.index.SlowCompositeReaderWrapper 实例源码

public void testSortValues() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    for (int i = 0; i < 10; i++) {
        Document document = new Document();
        String text = new String(new char[]{(char) (97 + i), (char) (97 + i)});
        document.add(new TextField("str", text, Field.Store.YES));
        document.add(new SortedDocValuesField("str", new BytesRef(text)));
        indexWriter.addDocument(document);
    }
    IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(indexWriter));
    IndexSearcher searcher = new IndexSearcher(reader);
    TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(new SortField("str", SortField.Type.STRING)));
    for (int i = 0; i < 10; i++) {
        FieldDoc fieldDoc = (FieldDoc) docs.scoreDocs[i];
        assertThat((BytesRef) fieldDoc.fields[0], equalTo(new BytesRef(new String(new char[]{(char) (97 + i), (char) (97 + i)}))));
    }
}

public void testNRTSearchOnClosedWriter() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    DirectoryReader reader = DirectoryReader.open(indexWriter);

    for (int i = 0; i < 100; i++) {
        Document document = new Document();
        TextField field = new TextField("_id", Integer.toString(i), Field.Store.YES);
        field.setBoost(i);
        document.add(field);
        indexWriter.addDocument(document);
    }
    reader = refreshReader(reader);

    indexWriter.close();

    TermsEnum termDocs = SlowCompositeReaderWrapper.wrap(reader).terms("_id").iterator();
    termDocs.next();
}

@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
  final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
  final int off = readerContext.docBase;

  final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
  final int end = sindex.getValueCount();

  return new IntDocValues(this) {
   @Override
    public int intVal(int doc) {
      return (end - sindex.getOrd(doc+off) - 1);
    }
  };
}

@Override
public List<AtomicReader> getMergeReaders() throws IOException {
  if (unsortedReaders == null) {
    unsortedReaders = super.getMergeReaders();
    final AtomicReader atomicView;
    if (unsortedReaders.size() == 1) {
      atomicView = unsortedReaders.get(0);
    } else {
      final IndexReader multiReader = new MultiReader(unsortedReaders.toArray(new AtomicReader[unsortedReaders.size()]));
      atomicView = SlowCompositeReaderWrapper.wrap(multiReader);
    }
    docMap = sorter.sort(atomicView);
    sortedView = SortingAtomicReader.wrap(atomicView, docMap);
  }
  // a null doc map means that the readers are already sorted
  return docMap == null ? unsortedReaders : Collections.singletonList(sortedView);
}

protected void checkOnlineClassification(Classifier<T> classifier, String inputDoc, T expectedResult, Analyzer analyzer, String textFieldName, String classFieldName, Query query) throws Exception {
  AtomicReader atomicReader = null;
  try {
    populateSampleIndex(analyzer);
    atomicReader = SlowCompositeReaderWrapper.wrap(indexWriter.getReader());
    classifier.train(atomicReader, textFieldName, classFieldName, analyzer, query);
    ClassificationResult<T> classificationResult = classifier.assignClass(inputDoc);
    assertNotNull(classificationResult.getAssignedClass());
    assertEquals("got an assigned class of " + classificationResult.getAssignedClass(), expectedResult, classificationResult.getAssignedClass());
    assertTrue("got a not positive score " + classificationResult.getScore(), classificationResult.getScore() > 0);
    updateSampleIndex(analyzer);
    ClassificationResult<T> secondClassificationResult = classifier.assignClass(inputDoc);
    assertEquals(classificationResult.getAssignedClass(), secondClassificationResult.getAssignedClass());
    assertEquals(Double.valueOf(classificationResult.getScore()), Double.valueOf(secondClassificationResult.getScore()));

  } finally {
    if (atomicReader != null)
      atomicReader.close();
  }
}

public void testGetFilterHandleNumericParseError() throws Exception {
  NumericRangeFilterBuilder filterBuilder = new NumericRangeFilterBuilder();
  filterBuilder.setStrictMode(false);

  String xml = "<NumericRangeFilter fieldName='AGE' type='int' lowerTerm='-1' upperTerm='NaN'/>";
  Document doc = getDocumentFromString(xml);
  Filter filter = filterBuilder.getFilter(doc.getDocumentElement());
  Directory ramDir = newDirectory();
  IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(null));
  writer.commit();
  try {
    AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(ramDir));
    try {
      assertNull(filter.getDocIdSet(reader.getContext(), reader.getLiveDocs()));
    }
    finally {
      reader.close();
    }
  }
  finally {
    writer.commit();
    writer.close();
    ramDir.close();
  }
}

public void testCachingWorks() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  writer.close();

  IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
  AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
  MockFilter filter = new MockFilter();
  CachingWrapperFilter cacher = new CachingWrapperFilter(filter);

  // first time, nested filter is called
  DocIdSet strongRef = cacher.getDocIdSet(context, context.reader().getLiveDocs());
  assertTrue("first time", filter.wasCalled());

  // make sure no exception if cache is holding the wrong docIdSet
  cacher.getDocIdSet(context, context.reader().getLiveDocs());

  // second time, nested filter should not be called
  filter.clear();
  cacher.getDocIdSet(context, context.reader().getLiveDocs());
  assertFalse("second time", filter.wasCalled());

  reader.close();
  dir.close();
}

public void testIsCacheAble() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  writer.addDocument(new Document());
  writer.close();

  IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));

  // not cacheable:
  assertDocIdSetCacheable(reader, new QueryWrapperFilter(new TermQuery(new Term("test","value"))), false);
  // returns default empty docidset, always cacheable:
  assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true);
  // is cacheable:
  assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), false);
  // a fixedbitset filter is always cacheable
  assertDocIdSetCacheable(reader, new Filter() {
    @Override
    public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
      return new FixedBitSet(context.reader().maxDoc());
    }
  }, true);

  reader.close();
  dir.close();
}

public void testBasics() throws Exception {
  // sanity check of norms writer
  // TODO: generalize
  AtomicReader slow = SlowCompositeReaderWrapper.wrap(reader);
  NumericDocValues fooNorms = slow.getNormValues("foo");
  NumericDocValues barNorms = slow.getNormValues("bar");
  for (int i = 0; i < slow.maxDoc(); i++) {
    assertFalse(fooNorms.get(i) == barNorms.get(i));
  }

  // sanity check of searching
  TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
  assertTrue(foodocs.totalHits > 0);
  TopDocs bardocs = searcher.search(new TermQuery(new Term("bar", "brown")), 10);
  assertTrue(bardocs.totalHits > 0);
  assertTrue(foodocs.scoreDocs[0].score < bardocs.scoreDocs[0].score);
}

@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();

  RandomIndexWriter writer = new RandomIndexWriter(random(), directory, 
      newIndexWriterConfig(new MockAnalyzer(random()))
      .setMergePolicy(newLogMergePolicy())
      .setSimilarity(new DefaultSimilarity()));
  for (int i = 0; i < values.length; i++) {
    Document doc = new Document();
    doc
        .add(newTextField(FIELD, values[i], Field.Store.YES));
    writer.addDocument(doc);
  }
  indexReader = SlowCompositeReaderWrapper.wrap(writer.getReader());
  writer.close();
  indexSearcher = newSearcher(indexReader);
  indexSearcher.setSimilarity(new DefaultSimilarity());
}

/**
 * @param flagConfig Contains all information necessary for configuring LuceneUtils.
 *        {@link FlagConfig#luceneindexpath()} must be non-empty. 
 */
public LuceneUtils(FlagConfig flagConfig) throws IOException {
  if (flagConfig.luceneindexpath().isEmpty()) {
    throw new IllegalArgumentException(
        "-luceneindexpath is a required argument for initializing LuceneUtils instance.");
  }

  this.compositeReader = DirectoryReader.open(
      FSDirectory.open(FileSystems.getDefault().getPath(flagConfig.luceneindexpath())));
  this.leafReader = SlowCompositeReaderWrapper.wrap(compositeReader);
  MultiFields.getFields(compositeReader);
  this.flagConfig = flagConfig;
  if (!flagConfig.stoplistfile().isEmpty())
    loadStopWords(flagConfig.stoplistfile());

  if (!flagConfig.startlistfile().isEmpty())
    loadStartWords(flagConfig.startlistfile());

  VerbatimLogger.info("Initialized LuceneUtils from Lucene index in directory: " + flagConfig.luceneindexpath() + "\n");
  VerbatimLogger.info("Fields in index are: " + String.join(", ", this.getFieldNames()) + "\n");
}

private Map<String, Integer> getTermsFromIndex() {
    Map<String, Integer> indexedTerms = Maps.newLinkedHashMap();
    try {
        DirectoryReader indexReader = indexer.getIndexReader();
        Terms terms = SlowCompositeReaderWrapper.wrap(indexReader).terms(Indexer.INDEX_CONTENT);
        if (terms == null) {
            return indexedTerms;
        }

        TermsEnum termEnum = terms.iterator(null);
        BytesRef byteRef = null;

        while ((byteRef = termEnum.next()) != null) {
            String term = byteRef.utf8ToString();
            int count = indexReader.docFreq(new Term(Indexer.INDEX_CONTENT, byteRef));
            indexedTerms.put(term, Integer.valueOf(count));
        }
        indexReader.close();
    } catch (Exception e) {
        logger.error("Failed to dump index", e);
    }
    return indexedTerms;
}

@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
  final AtomicReader r = topReader instanceof CompositeReader 
      ? new SlowCompositeReaderWrapper((CompositeReader)topReader) 
      : (AtomicReader) topReader;
  final int off = readerContext.docBase;

  final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
  final int end = sindex.getValueCount();

  return new IntDocValues(this) {
   @Override
    public int intVal(int doc) {
      return (end - sindex.getOrd(doc+off) - 1);
    }
  };
}

public void testGetFilterHandleNumericParseError() throws Exception {
  NumericRangeFilterBuilder filterBuilder = new NumericRangeFilterBuilder();
  filterBuilder.setStrictMode(false);

  String xml = "<NumericRangeFilter fieldName='AGE' type='int' lowerTerm='-1' upperTerm='NaN'/>";
  Document doc = getDocumentFromString(xml);
  Filter filter = filterBuilder.getFilter(doc.getDocumentElement());
  Directory ramDir = newDirectory();
  IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
  writer.commit();
  try {
    AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(ramDir));
    try {
      assertNull(filter.getDocIdSet(reader.getContext(), reader.getLiveDocs()));
    }
    finally {
      reader.close();
    }
  }
  finally {
    writer.commit();
    writer.close();
    ramDir.close();
  }
}

public void testCachingWorks() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  writer.close();

  IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
  AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
  MockFilter filter = new MockFilter();
  CachingWrapperFilter cacher = new CachingWrapperFilter(filter);

  // first time, nested filter is called
  DocIdSet strongRef = cacher.getDocIdSet(context, context.reader().getLiveDocs());
  assertTrue("first time", filter.wasCalled());

  // make sure no exception if cache is holding the wrong docIdSet
  cacher.getDocIdSet(context, context.reader().getLiveDocs());

  // second time, nested filter should not be called
  filter.clear();
  cacher.getDocIdSet(context, context.reader().getLiveDocs());
  assertFalse("second time", filter.wasCalled());

  reader.close();
  dir.close();
}

public void testIsCacheAble() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  writer.addDocument(new Document());
  writer.close();

  IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));

  // not cacheable:
  assertDocIdSetCacheable(reader, new QueryWrapperFilter(new TermQuery(new Term("test","value"))), false);
  // returns default empty docidset, always cacheable:
  assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true);
  // is cacheable:
  assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), true);
  // a fixedbitset filter is always cacheable
  assertDocIdSetCacheable(reader, new Filter() {
    @Override
    public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
      return new FixedBitSet(context.reader().maxDoc());
    }
  }, true);

  reader.close();
  dir.close();
}

public void testBasics() throws Exception {
  // sanity check of norms writer
  // TODO: generalize
  AtomicReader slow = new SlowCompositeReaderWrapper(reader);
  NumericDocValues fooNorms = slow.getNormValues("foo");
  NumericDocValues barNorms = slow.getNormValues("bar");
  for (int i = 0; i < slow.maxDoc(); i++) {
    assertFalse(fooNorms.get(i) == barNorms.get(i));
  }

  // sanity check of searching
  TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
  assertTrue(foodocs.totalHits > 0);
  TopDocs bardocs = searcher.search(new TermQuery(new Term("bar", "brown")), 10);
  assertTrue(bardocs.totalHits > 0);
  assertTrue(foodocs.scoreDocs[0].score < bardocs.scoreDocs[0].score);
}

@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();

  RandomIndexWriter writer = new RandomIndexWriter(random(), directory, 
      newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
      .setMergePolicy(newLogMergePolicy())
      .setSimilarity(new DefaultSimilarity()));
  for (int i = 0; i < values.length; i++) {
    Document doc = new Document();
    doc
        .add(newTextField(FIELD, values[i], Field.Store.YES));
    writer.addDocument(doc);
  }
  indexReader = SlowCompositeReaderWrapper.wrap(writer.getReader());
  writer.close();
  indexSearcher = newSearcher(indexReader);
  indexSearcher.setSimilarity(new DefaultSimilarity());
}

@Override
public TextMetrics execute(ModuleResultProvider results, ProgressListener progressListener)
    throws Exception {

  final int count;
  IndexReader reader = results.getResultFor(LuceneResult.class).getIndexReader();
  try {
    Terms terms = SlowCompositeReaderWrapper.wrap(reader)
        .terms(TextRepository.CHAPTER_TEXT_FIELD);
    count = (int) (terms.getSumTotalTermFreq());
  } finally {
    reader.close();
  }

  return new TextMetrics() {
    @Override
    public int getWordCount() {
      return count;
    }
  };
}

@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
  final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
  final int off = readerContext.docBase;

  final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
  final int end = sindex.getValueCount();

  return new IntDocValues(this) {
   @Override
    public int intVal(int doc) {
      return (end - sindex.getOrd(doc+off) - 1);
    }
  };
}

@Override
public List<AtomicReader> getMergeReaders() throws IOException {
  if (unsortedReaders == null) {
    unsortedReaders = super.getMergeReaders();
    final AtomicReader atomicView;
    if (unsortedReaders.size() == 1) {
      atomicView = unsortedReaders.get(0);
    } else {
      final IndexReader multiReader = new MultiReader(unsortedReaders.toArray(new AtomicReader[unsortedReaders.size()]));
      atomicView = SlowCompositeReaderWrapper.wrap(multiReader);
    }
    docMap = sorter.sort(atomicView);
    sortedView = SortingAtomicReader.wrap(atomicView, docMap);
  }
  // a null doc map means that the readers are already sorted
  return docMap == null ? unsortedReaders : Collections.singletonList(sortedView);
}

@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
  final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
  final int off = readerContext.docBase;

  final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
  final int end = sindex.getValueCount();

  return new IntDocValues(this) {
   @Override
    public int intVal(int doc) {
      return (end - sindex.getOrd(doc+off) - 1);
    }
  };
}

@Override
public List<AtomicReader> getMergeReaders() throws IOException {
  if (unsortedReaders == null) {
    unsortedReaders = super.getMergeReaders();
    final AtomicReader atomicView;
    if (unsortedReaders.size() == 1) {
      atomicView = unsortedReaders.get(0);
    } else {
      final IndexReader multiReader = new MultiReader(unsortedReaders.toArray(new AtomicReader[unsortedReaders.size()]));
      atomicView = SlowCompositeReaderWrapper.wrap(multiReader);
    }
    docMap = sorter.sort(atomicView);
    sortedView = SortingAtomicReader.wrap(atomicView, docMap);
  }
  // a null doc map means that the readers are already sorted
  return docMap == null ? unsortedReaders : Collections.singletonList(sortedView);
}

@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
  final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
  final int off = readerContext.docBase;

  final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
  final int end = sindex.getValueCount();

  return new IntDocValues(this) {
   @Override
    public int intVal(int doc) {
      return (end - sindex.getOrd(doc+off) - 1);
    }
  };
}

@Override
public List<AtomicReader> getMergeReaders() throws IOException {
  if (unsortedReaders == null) {
    unsortedReaders = super.getMergeReaders();
    final AtomicReader atomicView;
    if (unsortedReaders.size() == 1) {
      atomicView = unsortedReaders.get(0);
    } else {
      final IndexReader multiReader = new MultiReader(unsortedReaders.toArray(new AtomicReader[unsortedReaders.size()]));
      atomicView = SlowCompositeReaderWrapper.wrap(multiReader);
    }
    docMap = sorter.sort(atomicView);
    sortedView = SortingAtomicReader.wrap(atomicView, docMap);
  }
  // a null doc map means that the readers are already sorted
  return docMap == null ? unsortedReaders : Collections.singletonList(sortedView);
}

@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
  final IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
  final AtomicReader r = SlowCompositeReaderWrapper.wrap(topReader);
  final int off = readerContext.docBase;

  final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
  final int end = sindex.getValueCount();

  return new IntDocValues(this) {
   @Override
    public int intVal(int doc) {
      return (end - sindex.getOrd(doc+off) - 1);
    }
  };
}

@Override
public List<AtomicReader> getMergeReaders() throws IOException {
  if (unsortedReaders == null) {
    unsortedReaders = super.getMergeReaders();
    final AtomicReader atomicView;
    if (unsortedReaders.size() == 1) {
      atomicView = unsortedReaders.get(0);
    } else {
      final IndexReader multiReader = new MultiReader(unsortedReaders.toArray(new AtomicReader[unsortedReaders.size()]));
      atomicView = SlowCompositeReaderWrapper.wrap(multiReader);
    }
    docMap = sorter.sort(atomicView);
    sortedView = SortingAtomicReader.wrap(atomicView, docMap);
  }
  // a null doc map means that the readers are already sorted
  return docMap == null ? unsortedReaders : Collections.singletonList(sortedView);
}

public void testGetFilterHandleNumericParseError() throws Exception {
  NumericRangeFilterBuilder filterBuilder = new NumericRangeFilterBuilder();
  filterBuilder.setStrictMode(false);

  String xml = "<NumericRangeFilter fieldName='AGE' type='int' lowerTerm='-1' upperTerm='NaN'/>";
  Document doc = getDocumentFromString(xml);
  Filter filter = filterBuilder.getFilter(doc.getDocumentElement());
  Directory ramDir = newDirectory();
  IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
  writer.commit();
  try {
    AtomicReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(ramDir));
    try {
      assertNull(filter.getDocIdSet(reader.getContext(), reader.getLiveDocs()));
    }
    finally {
      reader.close();
    }
  }
  finally {
    writer.commit();
    writer.close();
    ramDir.close();
  }
}

public void testCachingWorks() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  writer.close();

  IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
  AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
  MockFilter filter = new MockFilter();
  CachingWrapperFilter cacher = new CachingWrapperFilter(filter);

  // first time, nested filter is called
  DocIdSet strongRef = cacher.getDocIdSet(context, context.reader().getLiveDocs());
  assertTrue("first time", filter.wasCalled());

  // make sure no exception if cache is holding the wrong docIdSet
  cacher.getDocIdSet(context, context.reader().getLiveDocs());

  // second time, nested filter should not be called
  filter.clear();
  cacher.getDocIdSet(context, context.reader().getLiveDocs());
  assertFalse("second time", filter.wasCalled());

  reader.close();
  dir.close();
}

public void testIsCacheAble() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  writer.addDocument(new Document());
  writer.close();

  IndexReader reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));

  // not cacheable:
  assertDocIdSetCacheable(reader, new QueryWrapperFilter(new TermQuery(new Term("test","value"))), false);
  // returns default empty docidset, always cacheable:
  assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true);
  // is cacheable:
  assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), true);
  // a fixedbitset filter is always cacheable
  assertDocIdSetCacheable(reader, new Filter() {
    @Override
    public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
      return new FixedBitSet(context.reader().maxDoc());
    }
  }, true);

  reader.close();
  dir.close();
}

public void testBasics() throws Exception {
  // sanity check of norms writer
  // TODO: generalize
  AtomicReader slow = SlowCompositeReaderWrapper.wrap(reader);
  NumericDocValues fooNorms = slow.getNormValues("foo");
  NumericDocValues barNorms = slow.getNormValues("bar");
  for (int i = 0; i < slow.maxDoc(); i++) {
    assertFalse(fooNorms.get(i) == barNorms.get(i));
  }

  // sanity check of searching
  TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
  assertTrue(foodocs.totalHits > 0);
  TopDocs bardocs = searcher.search(new TermQuery(new Term("bar", "brown")), 10);
  assertTrue(bardocs.totalHits > 0);
  assertTrue(foodocs.scoreDocs[0].score < bardocs.scoreDocs[0].score);
}

@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();

  RandomIndexWriter writer = new RandomIndexWriter(random(), directory, 
      newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
      .setMergePolicy(newLogMergePolicy())
      .setSimilarity(new DefaultSimilarity()));
  for (int i = 0; i < values.length; i++) {
    Document doc = new Document();
    doc
        .add(newTextField(FIELD, values[i], Field.Store.YES));
    writer.addDocument(doc);
  }
  indexReader = SlowCompositeReaderWrapper.wrap(writer.getReader());
  writer.close();
  indexSearcher = newSearcher(indexReader);
  indexSearcher.setSimilarity(new DefaultSimilarity());
}

/**
 * 
 * @return the BabelMorphAPI Version 
 */
public  String getBabelMorphVersion()throws IOException
{
    Terms terms = SlowCompositeReaderWrapper.wrap(morphologicalDictionary.getIndexReader()).terms(BabelMorphIndexField.VERSION.toString()); 
    TermsEnum termsEnum = terms.iterator();
    String version = termsEnum.next().utf8ToString();
    return "BabelMorph API v"+version;
}

/**
 * 
 * @return the available languages 
 */
public Set<Language> getBabelMorphLanguages()throws IOException
{
    Set<Language> languages = new HashSet<>();

    Terms terms = SlowCompositeReaderWrapper.wrap(morphologicalDictionary.getIndexReader()).terms(BabelMorphIndexField.LANGUAGE.toString()); 
    TermsEnum termsEnum = terms.iterator();
    BytesRef text;
    while((text = termsEnum.next()) != null) {
         languages.add(Language.valueOf(text.utf8ToString()));
    }
    return languages;
}

/**
 * 
 * @return the available part-of-speech 
 */
public Set<POS> getBabelMorphPoS()throws IOException
{
    Set<POS> pos = new HashSet<>();

    Terms terms = SlowCompositeReaderWrapper.wrap(morphologicalDictionary.getIndexReader()).terms(BabelMorphIndexField.POS.toString()); 
    TermsEnum termsEnum = terms.iterator();
    BytesRef text;
    while((text = termsEnum.next()) != null) {
        pos.add(POS.valueOf(text.utf8ToString()));
    }
    return pos;
}

/**
 * Ritorna una collezione di stringhe come valori per popolare la tabella.
 * Viene usato sia dalla parte gui per far vedere i valori, sia in
 * inizializzazione del Segmenter per popolare fisicamente la tabella
 *
 * @param tableName nome della tabella
 * @return lista di valori
 */
public Collection<? extends String> getValuesForTable(String tableName) {
    BiMap<String, String> invFields = fieldsTable.inverse();
    String field = invFields.get(tableName);
    List<String> ret = new ArrayList<>();
    if (field != null) {
        if (reader == null) {
            openIndex();
        }
        try {
            final LeafReader ar = SlowCompositeReaderWrapper.wrap(reader);
            final int maxdoc = reader.maxDoc();
            for (int i = 0; i < maxdoc; i++) {
                Document doc = ar.document(i);
                String val = doc.get(field);
                if (val != null) {
                    if (val.trim().length() > 0) {
                        ret.add(val.trim().toLowerCase());
                    }
                }
            }
        } catch (Exception e) {
            LogGui.printException(e);
        }
    }
    return ret;
}

public void testDocMakerThreadSafety() throws Exception {
  // 1. alg definition (required in every "logic" test)
  String algLines[] = {
      "# ----- properties ",
      "content.source=org.apache.lucene.benchmark.byTask.feeds.SortableSingleDocSource",
      "doc.term.vector=false",
      "log.step.AddDoc=10000",
      "content.source.forever=true",
      "directory=RAMDirectory",
      "doc.reuse.fields=false",
      "doc.stored=false",
      "doc.tokenized=false",
      "doc.index.props=true",
      "# ----- alg ",
      "CreateIndex",
      "[ { AddDoc > : 250 ] : 4",
      "CloseIndex",
  };

  // 2. we test this value later
  CountingSearchTestTask.numSearches = 0;

  // 3. execute the algorithm  (required in every "logic" test)
  Benchmark benchmark = execBenchmark(algLines);

  DirectoryReader r = DirectoryReader.open(benchmark.getRunData().getDirectory());
  SortedDocValues idx = FieldCache.DEFAULT.getTermsIndex(SlowCompositeReaderWrapper.wrap(r), "country");
  final int maxDoc = r.maxDoc();
  assertEquals(1000, maxDoc);
  for(int i=0;i<1000;i++) {
    assertTrue("doc " + i + " has null country", idx.getOrd(i) != -1);
  }
  r.close();
}

@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), directory, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));

  //Add series of docs with filterable fields : acces rights, prices, dates and "in-stock" flags
  addDoc(writer, "admin guest", "010", "20040101", "Y");
  addDoc(writer, "guest", "020", "20040101", "Y");
  addDoc(writer, "guest", "020", "20050101", "Y");
  addDoc(writer, "admin", "020", "20050101", "Maybe");
  addDoc(writer, "admin guest", "030", "20050101", "N");
  reader = SlowCompositeReaderWrapper.wrap(writer.getReader());
  writer.close();
}

public void testMissingTermAndField() throws Exception {
  String fieldName = "field1";
  Directory rd = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), rd);
  Document doc = new Document();
  doc.add(newStringField(fieldName, "value1", Field.Store.NO));
  w.addDocument(doc);
  IndexReader reader = SlowCompositeReaderWrapper.wrap(w.getReader());
  assertTrue(reader.getContext() instanceof AtomicReaderContext);
  AtomicReaderContext context = (AtomicReaderContext) reader.getContext();
  w.close();

  DocIdSet idSet = termFilter(fieldName, "value1").getDocIdSet(context, context.reader().getLiveDocs());
  assertNotNull("must not be null", idSet);
  DocIdSetIterator iter = idSet.iterator();
  assertEquals(iter.nextDoc(), 0);
  assertEquals(iter.nextDoc(), DocIdSetIterator.NO_MORE_DOCS);

  idSet = termFilter(fieldName, "value2").getDocIdSet(context, context.reader().getLiveDocs());
  assertNull("must be null", idSet);

  idSet = termFilter("field2", "value1").getDocIdSet(context, context.reader().getLiveDocs());
  assertNull("must be null", idSet);

  reader.close();
  rd.close();
}

@BeforeClass
public static void beforeClassSorterTestBase() throws Exception {
  dir = newDirectory();
  int numDocs = atLeast(20);
  createIndex(dir, numDocs, random());

  reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
}