/** * Atomically deletes documents matching the provided * delTerm and adds a block of documents, analyzed using * the provided analyzer, with sequentially * assigned document IDs, such that an external reader * will see all or none of the documents. * * See {@link #addDocuments(Iterable)}. * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error * * @lucene.experimental */ public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer) throws IOException { ensureOpen(); try { boolean success = false; boolean anySegmentFlushed = false; try { anySegmentFlushed = docWriter.updateDocuments(docs, analyzer, delTerm); success = true; } finally { if (!success) { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "hit exception updating document"); } } } if (anySegmentFlushed) { maybeMerge(MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS); } } catch (OutOfMemoryError oom) { handleOOM(oom, "updateDocuments"); } }
/** * Updates a document by first deleting the document(s) * containing <code>term</code> and then adding the new * document. The delete and then add are atomic as seen * by a reader on the same index (flush may happen only after * the add). * * <p><b>NOTE</b>: if this method hits an OutOfMemoryError * you should immediately close the writer. See <a * href="#OOME">above</a> for details.</p> * * @param term the term to identify the document(s) to be * deleted * @param doc the document to be added * @param analyzer the analyzer to use when analyzing the document * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ public void updateDocument(Term term, Iterable<? extends IndexableField> doc, Analyzer analyzer) throws IOException { ensureOpen(); try { boolean success = false; boolean anySegmentFlushed = false; try { anySegmentFlushed = docWriter.updateDocument(doc, analyzer, term); success = true; } finally { if (!success) { if (infoStream.isEnabled("IW")) { infoStream.message("IW", "hit exception updating document"); } } } if (anySegmentFlushed) { maybeMerge(MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS); } } catch (OutOfMemoryError oom) { handleOOM(oom, "updateDocument"); } }
/** * Flush all in-memory buffered updates (adds and deletes) * to the Directory. * @param triggerMerge if true, we may merge segments (if * deletes or docs were flushed) if necessary * @param applyAllDeletes whether pending deletes should also */ protected final void flush(boolean triggerMerge, boolean applyAllDeletes) throws IOException { // NOTE: this method cannot be sync'd because // maybeMerge() in turn calls mergeScheduler.merge which // in turn can take a long time to run and we don't want // to hold the lock for that. In the case of // ConcurrentMergeScheduler this can lead to deadlock // when it stalls due to too many running merges. // We can be called during close, when closing==true, so we must pass false to ensureOpen: ensureOpen(false); if (doFlush(applyAllDeletes) && triggerMerge) { maybeMerge(MergeTrigger.FULL_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS); } }
@Override public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) { MergeSpecification mergeSpec = null; //System.out.println("MRMP: findMerges sis=" + segmentInfos); int numSegments = segmentInfos.size(); List<SegmentInfoPerCommit> segments = new ArrayList<SegmentInfoPerCommit>(); final Collection<SegmentInfoPerCommit> merging = writer.get().getMergingSegments(); for(SegmentInfoPerCommit sipc : segmentInfos) { if (!merging.contains(sipc)) { segments.add(sipc); } } numSegments = segments.size(); if (numSegments > 1 && (numSegments > 30 || random.nextInt(5) == 3)) { Collections.shuffle(segments, random); // TODO: sometimes make more than 1 merge? mergeSpec = new MergeSpecification(); final int segsToMerge = _TestUtil.nextInt(random, 1, numSegments); mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge))); } return mergeSpec; }
private synchronized void updatePendingMerges(MergeTrigger trigger, int maxNumSegments) throws IOException { assert maxNumSegments == -1 || maxNumSegments > 0; assert trigger != null; if (stopMerges) { return; } // Do not start new merges if we've hit OOME if (hitOOM) { return; } final MergePolicy.MergeSpecification spec; if (maxNumSegments != UNBOUNDED_MAX_MERGE_SEGMENTS) { assert trigger == MergeTrigger.EXPLICIT || trigger == MergeTrigger.MERGE_FINISHED : "Expected EXPLICT or MERGE_FINISHED as trigger even with maxNumSegments set but was: " + trigger.name(); spec = mergePolicy.findForcedMerges(segmentInfos, maxNumSegments, Collections.unmodifiableMap(segmentsToMerge)); if (spec != null) { final int numMerges = spec.merges.size(); for(int i=0;i<numMerges;i++) { final MergePolicy.OneMerge merge = spec.merges.get(i); merge.maxNumSegments = maxNumSegments; } } } else { spec = mergePolicy.findMerges(trigger, segmentInfos); } if (spec != null) { final int numMerges = spec.merges.size(); for(int i=0;i<numMerges;i++) { registerMerge(spec.merges.get(i)); } } }
@Override public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) throws IOException { MergeSpecification ms = new MergeSpecification(); if (doMerge) { OneMerge om = new OneMerge(segmentInfos.asList().subList(start, start + length)); ms.add(om); doMerge = false; return ms; } return null; }
final void doAfterSegmentFlushed(boolean triggerMerge, boolean forcePurge) throws IOException { try { purge(forcePurge); } finally { if (triggerMerge) { maybeMerge(MergeTrigger.SEGMENT_FLUSH, UNBOUNDED_MAX_MERGE_SEGMENTS); } } }
@Override public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) { MergeSpecification mergeSpec = null; //System.out.println("MRMP: findMerges sis=" + segmentInfos); int numSegments = segmentInfos.size(); List<SegmentCommitInfo> segments = new ArrayList<SegmentCommitInfo>(); final Collection<SegmentCommitInfo> merging = writer.get().getMergingSegments(); for(SegmentCommitInfo sipc : segmentInfos) { if (!merging.contains(sipc)) { segments.add(sipc); } } numSegments = segments.size(); if (numSegments > 1 && (numSegments > 30 || random.nextInt(5) == 3)) { Collections.shuffle(segments, random); // TODO: sometimes make more than 1 merge? mergeSpec = new MergeSpecification(); final int segsToMerge = _TestUtil.nextInt(random, 1, numSegments); mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge))); } return mergeSpec; }
@Override public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) { return null; }
@Override public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos) throws IOException { return base.findMerges(null, segmentInfos); }
private final void maybeMerge(MergeTrigger trigger, int maxNumSegments) throws IOException { ensureOpen(false); updatePendingMerges(trigger, maxNumSegments); mergeScheduler.merge(this); }
/** * Expert: asks the mergePolicy whether any merges are * necessary now and if so, runs the requested merges and * then iterate (test again if merges are needed) until no * more merges are returned by the mergePolicy. * * Explicit calls to maybeMerge() are usually not * necessary. The most common case is when merge policy * parameters have changed. * * This method will call the {@link MergePolicy} with * {@link MergeTrigger#EXPLICIT}. * * <p><b>NOTE</b>: if this method hits an OutOfMemoryError * you should immediately close the writer. See <a * href="#OOME">above</a> for details.</p> */ public final void maybeMerge() throws IOException { maybeMerge(MergeTrigger.EXPLICIT, UNBOUNDED_MAX_MERGE_SEGMENTS); }
