static MergePolicy newSortingMergePolicy(Sort sort) { // usually create a MP with a low merge factor so that many merges happen MergePolicy mp; int thingToDo = random().nextInt(3); if (thingToDo == 0) { TieredMergePolicy tmp = newTieredMergePolicy(random()); final int numSegs = TestUtil.nextInt(random(), 3, 5); tmp.setSegmentsPerTier(numSegs); tmp.setMaxMergeAtOnce(TestUtil.nextInt(random(), 2, numSegs)); mp = tmp; } else if (thingToDo == 1) { LogMergePolicy lmp = newLogMergePolicy(random()); lmp.setMergeFactor(TestUtil.nextInt(random(), 3, 5)); mp = lmp; } else { // just a regular random one from LTC (could be alcoholic etc) mp = newMergePolicy(); } // wrap it with a sorting mp return new SortingMergePolicy(mp, sort); }
/** just tries to configure things to keep the open file * count lowish */ public static void reduceOpenFiles(IndexWriter w) { // keep number of open files lowish MergePolicy mp = w.getConfig().getMergePolicy(); if (mp instanceof LogMergePolicy) { LogMergePolicy lmp = (LogMergePolicy) mp; lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor())); lmp.setNoCFSRatio(1.0); } else if (mp instanceof TieredMergePolicy) { TieredMergePolicy tmp = (TieredMergePolicy) mp; tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce())); tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier())); tmp.setNoCFSRatio(1.0); } MergeScheduler ms = w.getConfig().getMergeScheduler(); if (ms instanceof ConcurrentMergeScheduler) { // wtf... shouldnt it be even lower since its 1 by default?!?! ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2); } }
public void testSubclassConcurrentMergeScheduler() throws IOException { MockDirectoryWrapper dir = newMockDirectory(); dir.failOn(new FailOnlyOnMerge()); Document doc = new Document(); Field idField = newStringField("id", "", Field.Store.YES); doc.add(idField); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) .setMergeScheduler(new MyMergeScheduler()) .setMaxBufferedDocs(2).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH) .setMergePolicy(newLogMergePolicy())); LogMergePolicy logMP = (LogMergePolicy) writer.getConfig().getMergePolicy(); logMP.setMergeFactor(10); for(int i=0;i<20;i++) writer.addDocument(doc); ((MyMergeScheduler) writer.getConfig().getMergeScheduler()).sync(); writer.close(); assertTrue(mergeThreadCreated); assertTrue(mergeCalled); assertTrue(excCalled); dir.close(); }
/** just tries to configure things to keep the open file * count lowish */ public static void reduceOpenFiles(IndexWriter w) { // keep number of open files lowish MergePolicy mp = w.getConfig().getMergePolicy(); if (mp instanceof LogMergePolicy) { LogMergePolicy lmp = (LogMergePolicy) mp; lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor())); lmp.setUseCompoundFile(true); } else if (mp instanceof TieredMergePolicy) { TieredMergePolicy tmp = (TieredMergePolicy) mp; tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce())); tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier())); tmp.setUseCompoundFile(true); } MergeScheduler ms = w.getConfig().getMergeScheduler(); if (ms instanceof ConcurrentMergeScheduler) { ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2); ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3); } }
public void testSubclassConcurrentMergeScheduler() throws IOException { MockDirectoryWrapper dir = newMockDirectory(); dir.failOn(new FailOnlyOnMerge()); Document doc = new Document(); Field idField = newStringField("id", "", Field.Store.YES); doc.add(idField); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergeScheduler(new MyMergeScheduler()) .setMaxBufferedDocs(2).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH) .setMergePolicy(newLogMergePolicy())); LogMergePolicy logMP = (LogMergePolicy) writer.getConfig().getMergePolicy(); logMP.setMergeFactor(10); for(int i=0;i<20;i++) writer.addDocument(doc); ((MyMergeScheduler) writer.getConfig().getMergeScheduler()).sync(); writer.close(); assertTrue(mergeThreadCreated); assertTrue(mergeCalled); assertTrue(excCalled); dir.close(); }
static MergePolicy newSortingMergePolicy(Sorter sorter) { // create a MP with a low merge factor so that many merges happen MergePolicy mp; if (random().nextBoolean()) { TieredMergePolicy tmp = newTieredMergePolicy(random()); final int numSegs = _TestUtil.nextInt(random(), 3, 5); tmp.setSegmentsPerTier(numSegs); tmp.setMaxMergeAtOnce(_TestUtil.nextInt(random(), 2, numSegs)); mp = tmp; } else { LogMergePolicy lmp = newLogMergePolicy(random()); lmp.setMergeFactor(_TestUtil.nextInt(random(), 3, 5)); mp = lmp; } // wrap it with a sorting mp return new SortingMergePolicy(mp, sorter); }
public static IndexWriter getIndexWriter(String indexPath, boolean create) throws IOException { Directory dir = FSDirectory.open(Paths.get(indexPath)); Analyzer analyzer = new SmartChineseAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); LogMergePolicy mergePolicy = new LogByteSizeMergePolicy(); mergePolicy.setMergeFactor(50); mergePolicy.setMaxMergeDocs(5000); if (create){ iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); } else { iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); } return new IndexWriter(dir, iwc); }
public static LogMergePolicy newLogMergePolicy(Random r) { LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); logmp.setCalibrateSizeByDeletes(r.nextBoolean()); if (rarely(r)) { logmp.setMergeFactor(TestUtil.nextInt(r, 2, 9)); } else { logmp.setMergeFactor(TestUtil.nextInt(r, 10, 50)); } configureRandom(r, logmp); return logmp; }
public void testLogMergePolicyConfig() throws Exception { final Class<? extends LogMergePolicy> mpClass = random().nextBoolean() ? LogByteSizeMergePolicy.class : LogDocMergePolicy.class; System.setProperty("solr.test.log.merge.policy", mpClass.getName()); initCore("solrconfig-logmergepolicy.xml","schema-minimal.xml"); IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore().getLatestSchema()); // verify some props set to -1 get lucene internal defaults assertEquals(-1, solrConfig.indexConfig.maxBufferedDocs); assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, iwc.getMaxBufferedDocs()); assertEquals(-1, solrConfig.indexConfig.maxIndexingThreads); assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, iwc.getMaxThreadStates()); assertEquals(-1, solrConfig.indexConfig.ramBufferSizeMB, 0.0D); assertEquals(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, iwc.getRAMBufferSizeMB(), 0.0D); LogMergePolicy logMP = assertAndCast(mpClass, iwc.getMergePolicy()); // set by legacy <mergeFactor> setting assertEquals(11, logMP.getMergeFactor()); // set by legacy <maxMergeDocs> setting assertEquals(456, logMP.getMaxMergeDocs()); }
public static void setUseCompoundFile(MergePolicy mp, boolean v) { if (mp instanceof TieredMergePolicy) { ((TieredMergePolicy) mp).setUseCompoundFile(v); } else if (mp instanceof LogMergePolicy) { ((LogMergePolicy) mp).setUseCompoundFile(v); } else { throw new IllegalArgumentException("cannot set compound file for MergePolicy " + mp); } }
@Override public synchronized boolean clearWorkflowInstances() throws InstanceRepositoryException { IndexWriter writer = null; try { IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); LogMergePolicy lmp =new LogDocMergePolicy(); lmp.setMergeFactor(mergeFactor); config.setMergePolicy(lmp); writer = new IndexWriter(indexDir, config); LOG.log(Level.FINE, "LuceneWorkflowEngine: remove all workflow instances"); writer.deleteDocuments(new Term("myfield", "myvalue")); } catch (IOException e) { LOG.log(Level.SEVERE, e.getMessage()); LOG .log(Level.WARNING, "Exception removing workflow instances from index: Message: " + e.getMessage()); throw new InstanceRepositoryException(e.getMessage()); } finally { if (writer != null){ try{ writer.close(); } catch(Exception ignore){} writer = null; } } return true; }
public JsonObject asJson() { JsonArrayBuilder strategiesJsonBuilder = Json.createArrayBuilder(); for (ClusterStrategy strategy : this.clusterConfig.strategies) { strategiesJsonBuilder.add(Json.createObjectBuilder() .add("clusteringEps", strategy.clusteringEps) .add("clusteringMinPoints", strategy.clusteringMinPoints)); } JsonObject json = Json.createObjectBuilder() .add("similarity", similarity.toString()) .add("mergePolicy", this.mergePolicy instanceof TieredMergePolicy ? Json.createObjectBuilder() .add("type", "TieredMergePolicy") .add("maxMergeAtOnce", ((TieredMergePolicy) this.mergePolicy).getMaxMergeAtOnce()) .add("segmentsPerTier", ((TieredMergePolicy) this.mergePolicy).getSegmentsPerTier()) : Json.createObjectBuilder() .add("type", "LogDocMergePolicy") .add("maxMergeDocs", ((LogMergePolicy) this.mergePolicy).getMaxMergeDocs()) .add("mergeFactor", ((LogMergePolicy) this.mergePolicy).getMergeFactor())) .add("lruTaxonomyWriterCacheSize", lruTaxonomyWriterCacheSize) .add("numberOfConcurrentTasks", numberOfConcurrentTasks) .add("commitCount", commitCount) .add("commitTimeout", commitTimeout) .add("cacheFacetOrdinals", this.cacheFacetOrdinals) .add("clustering", Json.createObjectBuilder() .add("clusterMoreRecords", clusterConfig.clusterMoreRecords) .add("strategies", strategiesJsonBuilder)) .build(); return json; }
/** * Test that IndexWriter settings stick. */ public void testIndexWriterSettings() throws Exception { // 1. alg definition (required in every "logic" test) String algLines[] = { "# ----- properties ", "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", "docs.file=" + getReuters20LinesFile(), "content.source.log.step=3", "ram.flush.mb=-1", "max.buffered=2", "compound=cmpnd:true:false", "doc.term.vector=vector:false:true", "content.source.forever=false", "directory=RAMDirectory", "doc.stored=false", "merge.factor=3", "doc.tokenized=false", "debug.level=1", "# ----- alg ", "{ \"Rounds\"", " ResetSystemErase", " CreateIndex", " { \"AddDocs\" AddDoc > : * ", " NewRound", "} : 2", }; // 2. execute the algorithm (required in every "logic" test) Benchmark benchmark = execBenchmark(algLines); final IndexWriter writer = benchmark.getRunData().getIndexWriter(); assertEquals(2, writer.getConfig().getMaxBufferedDocs()); assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, (int) writer.getConfig().getRAMBufferSizeMB()); assertEquals(3, ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor()); assertEquals(0.0d, writer.getConfig().getMergePolicy().getNoCFSRatio(), 0.0); writer.close(); Directory dir = benchmark.getRunData().getDirectory(); IndexReader reader = DirectoryReader.open(dir); Fields tfv = reader.getTermVectors(0); assertNotNull(tfv); assertTrue(tfv.size() > 0); reader.close(); }
@Test public void testOpenIfChangedManySegments() throws Exception { // test openIfChanged() when the taxonomy contains many segments Directory dir = newDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) { @Override protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) { IndexWriterConfig conf = super.createIndexWriterConfig(openMode); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setMergeFactor(2); return conf; } }; TaxonomyReader reader = new DirectoryTaxonomyReader(writer); int numRounds = random().nextInt(10) + 10; int numCategories = 1; // one for root for (int i = 0; i < numRounds; i++) { int numCats = random().nextInt(4) + 1; for (int j = 0; j < numCats; j++) { writer.addCategory(new FacetLabel(Integer.toString(i), Integer.toString(j))); } numCategories += numCats + 1 /* one for round-parent */; TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); assertNotNull(newtr); reader.close(); reader = newtr; // assert categories assertEquals(numCategories, reader.getSize()); int roundOrdinal = reader.getOrdinal(new FacetLabel(Integer.toString(i))); int[] parents = reader.getParallelTaxonomyArrays().parents(); assertEquals(0, parents[roundOrdinal]); // round's parent is root for (int j = 0; j < numCats; j++) { int ord = reader.getOrdinal(new FacetLabel(Integer.toString(i), Integer.toString(j))); assertEquals(roundOrdinal, parents[ord]); // round's parent is root } } reader.close(); writer.close(); dir.close(); }
public static LogMergePolicy newLogMergePolicy() { return newLogMergePolicy(random()); }
public static MergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) { LogMergePolicy logmp = newLogMergePolicy(); logmp.setNoCFSRatio(useCFS ? 1.0 : 0.0); logmp.setMergeFactor(mergeFactor); return logmp; }
public static MergePolicy newLogMergePolicy(int mergeFactor) { LogMergePolicy logmp = newLogMergePolicy(); logmp.setMergeFactor(mergeFactor); return logmp; }
/** we will manually instantiate preflex-rw here */ @BeforeClass public static void beforeClass() throws Exception { // NOTE: turn off compound file, this test will open some index files directly. LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true; IndexWriterConfig config = newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)) .setUseCompoundFile(false); termIndexInterval = config.getTermIndexInterval(); indexDivisor = TestUtil.nextInt(random(), 1, 10); NUMBER_OF_DOCUMENTS = atLeast(100); NUMBER_OF_FIELDS = atLeast(Math.max(10, 3*termIndexInterval*indexDivisor/NUMBER_OF_DOCUMENTS)); directory = newDirectory(); config.setCodec(new PreFlexRWCodec()); LogMergePolicy mp = newLogMergePolicy(); // NOTE: turn off compound file, this test will open some index files directly. mp.setNoCFSRatio(0.0); config.setMergePolicy(mp); populate(directory, config); DirectoryReader r0 = IndexReader.open(directory); SegmentReader r = LuceneTestCase.getOnlySegmentReader(r0); String segment = r.getSegmentName(); r.close(); FieldInfosReader infosReader = new PreFlexRWCodec().fieldInfosFormat().getFieldInfosReader(); FieldInfos fieldInfos = infosReader.read(directory, segment, "", IOContext.READONCE); String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); long tiiFileLength = directory.fileLength(segmentFileName); IndexInput input = directory.openInput(segmentFileName, newIOContext(random())); termEnum = new PreflexRWSegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random())), fieldInfos, false); int totalIndexInterval = termEnum.indexInterval * indexDivisor; SegmentTermEnum indexEnum = new PreflexRWSegmentTermEnum(input, fieldInfos, true); index = new TermInfosReaderIndex(indexEnum, indexDivisor, tiiFileLength, totalIndexInterval); indexEnum.close(); input.close(); reader = IndexReader.open(directory); sampleTerms = sample(random(),reader,1000); }
/** * Test that IndexWriter settings stick. */ public void testIndexWriterSettings() throws Exception { // 1. alg definition (required in every "logic" test) String algLines[] = { "# ----- properties ", "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource", "docs.file=" + getReuters20LinesFile(), "content.source.log.step=3", "ram.flush.mb=-1", "max.buffered=2", "compound=cmpnd:true:false", "doc.term.vector=vector:false:true", "content.source.forever=false", "directory=RAMDirectory", "doc.stored=false", "merge.factor=3", "doc.tokenized=false", "debug.level=1", "# ----- alg ", "{ \"Rounds\"", " ResetSystemErase", " CreateIndex", " { \"AddDocs\" AddDoc > : * ", " NewRound", "} : 2", }; // 2. execute the algorithm (required in every "logic" test) Benchmark benchmark = execBenchmark(algLines); final IndexWriter writer = benchmark.getRunData().getIndexWriter(); assertEquals(2, writer.getConfig().getMaxBufferedDocs()); assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, (int) writer.getConfig().getRAMBufferSizeMB()); assertEquals(3, ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor()); assertFalse(((LogMergePolicy) writer.getConfig().getMergePolicy()).getUseCompoundFile()); writer.close(); Directory dir = benchmark.getRunData().getDirectory(); IndexReader reader = DirectoryReader.open(dir); Fields tfv = reader.getTermVectors(0); assertNotNull(tfv); assertTrue(tfv.size() > 0); reader.close(); }
@Test public void testOpenIfChangedManySegments() throws Exception { // test openIfChanged() when the taxonomy contains many segments Directory dir = newDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) { @Override protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) { IndexWriterConfig conf = super.createIndexWriterConfig(openMode); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setMergeFactor(2); return conf; } }; TaxonomyReader reader = new DirectoryTaxonomyReader(writer); int numRounds = random().nextInt(10) + 10; int numCategories = 1; // one for root for (int i = 0; i < numRounds; i++) { int numCats = random().nextInt(4) + 1; for (int j = 0; j < numCats; j++) { writer.addCategory(new CategoryPath(Integer.toString(i), Integer.toString(j))); } numCategories += numCats + 1 /* one for round-parent */; TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader); assertNotNull(newtr); reader.close(); reader = newtr; // assert categories assertEquals(numCategories, reader.getSize()); int roundOrdinal = reader.getOrdinal(new CategoryPath(Integer.toString(i))); int[] parents = reader.getParallelTaxonomyArrays().parents(); assertEquals(0, parents[roundOrdinal]); // round's parent is root for (int j = 0; j < numCats; j++) { int ord = reader.getOrdinal(new CategoryPath(Integer.toString(i), Integer.toString(j))); assertEquals(roundOrdinal, parents[ord]); // round's parent is root } } reader.close(); writer.close(); dir.close(); }
/** we will manually instantiate preflex-rw here */ @BeforeClass public static void beforeClass() throws Exception { LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); termIndexInterval = config.getTermIndexInterval(); indexDivisor = _TestUtil.nextInt(random(), 1, 10); NUMBER_OF_DOCUMENTS = atLeast(100); NUMBER_OF_FIELDS = atLeast(Math.max(10, 3*termIndexInterval*indexDivisor/NUMBER_OF_DOCUMENTS)); directory = newDirectory(); config.setCodec(new PreFlexRWCodec()); LogMergePolicy mp = newLogMergePolicy(); // turn off compound file, this test will open some index files directly. mp.setUseCompoundFile(false); config.setMergePolicy(mp); populate(directory, config); DirectoryReader r0 = IndexReader.open(directory); SegmentReader r = LuceneTestCase.getOnlySegmentReader(r0); String segment = r.getSegmentName(); r.close(); FieldInfosReader infosReader = new PreFlexRWCodec().fieldInfosFormat().getFieldInfosReader(); FieldInfos fieldInfos = infosReader.read(directory, segment, IOContext.READONCE); String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); long tiiFileLength = directory.fileLength(segmentFileName); IndexInput input = directory.openInput(segmentFileName, newIOContext(random())); termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random())), fieldInfos, false); int totalIndexInterval = termEnum.indexInterval * indexDivisor; SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true); index = new TermInfosReaderIndex(indexEnum, indexDivisor, tiiFileLength, totalIndexInterval); indexEnum.close(); input.close(); reader = IndexReader.open(directory); sampleTerms = sample(random(),reader,1000); }
/** we will manually instantiate preflex-rw here */ @BeforeClass public static void beforeClass() throws Exception { // NOTE: turn off compound file, this test will open some index files directly. LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setUseCompoundFile(false); termIndexInterval = config.getTermIndexInterval(); indexDivisor = _TestUtil.nextInt(random(), 1, 10); NUMBER_OF_DOCUMENTS = atLeast(100); NUMBER_OF_FIELDS = atLeast(Math.max(10, 3*termIndexInterval*indexDivisor/NUMBER_OF_DOCUMENTS)); directory = newDirectory(); config.setCodec(new PreFlexRWCodec()); LogMergePolicy mp = newLogMergePolicy(); // NOTE: turn off compound file, this test will open some index files directly. mp.setNoCFSRatio(0.0); config.setMergePolicy(mp); populate(directory, config); DirectoryReader r0 = IndexReader.open(directory); SegmentReader r = LuceneTestCase.getOnlySegmentReader(r0); String segment = r.getSegmentName(); r.close(); FieldInfosReader infosReader = new PreFlexRWCodec().fieldInfosFormat().getFieldInfosReader(); FieldInfos fieldInfos = infosReader.read(directory, segment, "", IOContext.READONCE); String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); long tiiFileLength = directory.fileLength(segmentFileName); IndexInput input = directory.openInput(segmentFileName, newIOContext(random())); termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random())), fieldInfos, false); int totalIndexInterval = termEnum.indexInterval * indexDivisor; SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true); index = new TermInfosReaderIndex(indexEnum, indexDivisor, tiiFileLength, totalIndexInterval); indexEnum.close(); input.close(); reader = IndexReader.open(directory); sampleTerms = sample(random(),reader,1000); }