@Override public List<Segment> segments(boolean verbose) { try (ReleasableLock lock = readLock.acquire()) { Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose); // fill in the merges flag Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges(); for (OnGoingMerge onGoingMerge : onGoingMerges) { for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) { for (Segment segment : segmentsArr) { if (segment.getName().equals(segmentInfoPerCommit.info.name)) { segment.mergeId = onGoingMerge.getId(); break; } } } } return Arrays.asList(segmentsArr); } }
private boolean shouldUpgrade(SegmentCommitInfo info) { org.apache.lucene.util.Version old = info.info.getVersion(); org.apache.lucene.util.Version cur = Version.CURRENT.luceneVersion; // Something seriously wrong if this trips: assert old.major <= cur.major; if (cur.major > old.major) { // Always upgrade segment if Lucene's major version is too old return true; } if (upgradeOnlyAncientSegments == false && cur.minor > old.minor) { // If it's only a minor version difference, and we are not upgrading only ancient segments, // also upgrade: return true; } // Version matches, or segment is not ancient and we are only upgrading ancient segments: return false; }
/** * Returns an iterable that allows to iterate over all files in this segments info */ public static Iterable<String> files(SegmentInfos infos) throws IOException { final List<Collection<String>> list = new ArrayList<>(); list.add(Collections.singleton(infos.getSegmentsFileName())); for (SegmentCommitInfo info : infos) { list.add(info.files()); } return Iterables.flatten(list); }
/** * Returns the number of documents in the index referenced by this {@link SegmentInfos} */ public static int getNumDocs(SegmentInfos info) { int numDocs = 0; for (SegmentCommitInfo si : info) { numDocs += si.info.maxDoc() - si.getDelCount(); } return numDocs; }
@Override public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException { String filename = IndexFileNames.fileNameFromGeneration(info.info.name, DELETES_EXTENSION, info.getDelGen()); final BitVector liveDocs = new BitVector(dir, filename, context); if (liveDocs.length() != info.info.getDocCount()) { throw new CorruptIndexException("liveDocs.length()=" + liveDocs.length() + "info.docCount=" + info.info.getDocCount() + " (filename=" + filename + ")"); } if (liveDocs.count() != info.info.getDocCount() - info.getDelCount()) { throw new CorruptIndexException("liveDocs.count()=" + liveDocs.count() + " info.docCount=" + info.info.getDocCount() + " info.getDelCount()=" + info.getDelCount() + " (filename=" + filename + ")"); } return liveDocs; }
@Override public void writeLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException { String filename = IndexFileNames.fileNameFromGeneration(info.info.name, DELETES_EXTENSION, info.getNextDelGen()); final BitVector liveDocs = (BitVector) bits; assert liveDocs.count() == info.info.getDocCount() - info.getDelCount() - newDelCount; assert liveDocs.length() == info.info.getDocCount(); liveDocs.write(dir, filename, context); }
@Override public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException { assert info.hasDeletions(); BytesRefBuilder scratch = new BytesRefBuilder(); CharsRefBuilder scratchUTF16 = new CharsRefBuilder(); String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getDelGen()); ChecksumIndexInput in = null; boolean success = false; try { in = dir.openChecksumInput(fileName, context); SimpleTextUtil.readLine(in, scratch); assert StringHelper.startsWith(scratch.get(), SIZE); int size = parseIntAt(scratch.get(), SIZE.length, scratchUTF16); BitSet bits = new BitSet(size); SimpleTextUtil.readLine(in, scratch); while (!scratch.get().equals(END)) { assert StringHelper.startsWith(scratch.get(), DOC); int docid = parseIntAt(scratch.get(), DOC.length, scratchUTF16); bits.set(docid); SimpleTextUtil.readLine(in, scratch); } SimpleTextUtil.checkFooter(in); success = true; return new SimpleTextBits(bits, size); } finally { if (success) { IOUtils.close(in); } else { IOUtils.closeWhileHandlingException(in); } } }
@Override public void writeLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException { BitSet set = ((SimpleTextBits) bits).bits; int size = bits.length(); BytesRefBuilder scratch = new BytesRefBuilder(); String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getNextDelGen()); IndexOutput out = null; boolean success = false; try { out = dir.createOutput(fileName, context); SimpleTextUtil.write(out, SIZE); SimpleTextUtil.write(out, Integer.toString(size), scratch); SimpleTextUtil.writeNewline(out); for (int i = set.nextSetBit(0); i >= 0; i=set.nextSetBit(i + 1)) { SimpleTextUtil.write(out, DOC); SimpleTextUtil.write(out, Integer.toString(i), scratch); SimpleTextUtil.writeNewline(out); } SimpleTextUtil.write(out, END); SimpleTextUtil.writeNewline(out); SimpleTextUtil.writeChecksum(out, scratch); success = true; } finally { if (success) { IOUtils.close(out); } else { IOUtils.closeWhileHandlingException(out); } } }
@Override public void writeLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException { if (random.nextInt(100) == 0) { throw new IOException("Fake IOException from LiveDocsFormat.writeLiveDocs()"); } delegate.writeLiveDocs(bits, dir, info, newDelCount, context); }
@Override public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentCommitInfo,Boolean> segmentsToMerge, IndexWriter writer) throws IOException { return inner.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer); }
@Override public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, IndexWriter writer) throws IOException { return inner.useCompoundFile(infos, mergedInfo, writer); }
@Override public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException { String filename = IndexFileNames.fileNameFromGeneration(info.info.name, DELETES_EXTENSION, info.getDelGen()); final BitVector liveDocs = new BitVector(dir, filename, context); assert liveDocs.count() == info.info.getDocCount() - info.getDelCount(): "liveDocs.count()=" + liveDocs.count() + " info.docCount=" + info.info.getDocCount() + " info.getDelCount()=" + info.getDelCount(); assert liveDocs.length() == info.info.getDocCount(); return liveDocs; }
@Override public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException { assert info.hasDeletions(); BytesRef scratch = new BytesRef(); CharsRef scratchUTF16 = new CharsRef(); String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getDelGen()); IndexInput in = null; boolean success = false; try { in = dir.openInput(fileName, context); SimpleTextUtil.readLine(in, scratch); assert StringHelper.startsWith(scratch, SIZE); int size = parseIntAt(scratch, SIZE.length, scratchUTF16); BitSet bits = new BitSet(size); SimpleTextUtil.readLine(in, scratch); while (!scratch.equals(END)) { assert StringHelper.startsWith(scratch, DOC); int docid = parseIntAt(scratch, DOC.length, scratchUTF16); bits.set(docid); SimpleTextUtil.readLine(in, scratch); } success = true; return new SimpleTextBits(bits, size); } finally { if (success) { IOUtils.close(in); } else { IOUtils.closeWhileHandlingException(in); } } }
@Override public void writeLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException { BitSet set = ((SimpleTextBits) bits).bits; int size = bits.length(); BytesRef scratch = new BytesRef(); String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getNextDelGen()); IndexOutput out = null; boolean success = false; try { out = dir.createOutput(fileName, context); SimpleTextUtil.write(out, SIZE); SimpleTextUtil.write(out, Integer.toString(size), scratch); SimpleTextUtil.writeNewline(out); for (int i = set.nextSetBit(0); i >= 0; i=set.nextSetBit(i + 1)) { SimpleTextUtil.write(out, DOC); SimpleTextUtil.write(out, Integer.toString(i), scratch); SimpleTextUtil.writeNewline(out); } SimpleTextUtil.write(out, END); SimpleTextUtil.writeNewline(out); success = true; } finally { if (success) { IOUtils.close(out); } else { IOUtils.closeWhileHandlingException(out); } } }
/** * The list of segments that are being merged. */ public List<SegmentCommitInfo> getMergedSegments() { return mergedSegments; }
@Override public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentCommitInfo,Boolean> segmentsToMerge, IndexWriter writer) throws IOException { if (upgradeInProgress) { MergeSpecification spec = new MergeSpecification(); for (SegmentCommitInfo info : segmentInfos) { if (shouldUpgrade(info)) { // TODO: Use IndexUpgradeMergePolicy instead. We should be comparing codecs, // for now we just assume every minor upgrade has a new format. logger.debug("Adding segment {} to be upgraded", info.info.name); spec.add(new OneMerge(Collections.singletonList(info))); } // TODO: we could check IndexWriter.getMergingSegments and avoid adding merges that IW will just reject? if (spec.merges.size() == MAX_CONCURRENT_UPGRADE_MERGES) { // hit our max upgrades, so return the spec. we will get a cascaded call to continue. logger.debug("Returning {} merges for upgrade", spec.merges.size()); return spec; } } // We must have less than our max upgrade merges, so the next return will be our last in upgrading mode. if (spec.merges.isEmpty() == false) { logger.debug("Returning {} merges for end of upgrade", spec.merges.size()); return spec; } // Only set this once there are 0 segments needing upgrading, because when we return a // spec, IndexWriter may (silently!) reject that merge if some of the segments we asked // to be merged were already being (naturally) merged: upgradeInProgress = false; // fall through, so when we don't have any segments to upgrade, the delegate policy // has a chance to decide what to do (e.g. collapse the segments to satisfy maxSegmentCount) } return delegate.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer); }
@Override public boolean useCompoundFile(SegmentInfos segments, SegmentCommitInfo newSegment, IndexWriter writer) throws IOException { return delegate.useCompoundFile(segments, newSegment, writer); }
public void testAddIndices() throws IOException { Directory[] dirs = new Directory[randomIntBetween(1, 10)]; final int numDocs = randomIntBetween(50, 100); int id = 0; for (int i = 0; i < dirs.length; i++) { dirs[i] = newFSDirectory(createTempDir()); IndexWriter writer = new IndexWriter(dirs[i], newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE) .setOpenMode(IndexWriterConfig.OpenMode.CREATE)); for (int j = 0; j < numDocs; j++) { writer.addDocument(Arrays.asList(new StringField("id", Integer.toString(id++), Field.Store.YES))); } writer.commit(); writer.close(); } StoreRecovery storeRecovery = new StoreRecovery(new ShardId("foo", "bar", 1), logger); RecoveryState.Index indexStats = new RecoveryState.Index(); Directory target = newFSDirectory(createTempDir()); storeRecovery.addIndices(indexStats, target, dirs); int numFiles = 0; Predicate<String> filesFilter = (f) -> f.startsWith("segments") == false && f.equals("write.lock") == false && f.startsWith("extra") == false; for (Directory d : dirs) { numFiles += Arrays.asList(d.listAll()).stream().filter(filesFilter).count(); } final long targetNumFiles = Arrays.asList(target.listAll()).stream().filter(filesFilter).count(); assertEquals(numFiles, targetNumFiles); assertEquals(indexStats.totalFileCount(), targetNumFiles); if (hardLinksSupported(createTempDir())) { assertEquals(targetNumFiles, indexStats.reusedFileCount()); } else { assertEquals(0, indexStats.reusedFileCount(), 0); } DirectoryReader reader = DirectoryReader.open(target); SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(target); for (SegmentCommitInfo info : segmentCommitInfos) { // check that we didn't merge assertEquals("all sources must be flush", info.info.getDiagnostics().get("source"), "flush"); } assertEquals(reader.numDeletedDocs(), 0); assertEquals(reader.numDocs(), id); reader.close(); target.close(); IOUtils.close(dirs); }
/** Read live docs bits. */ public abstract Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException;
public IndexUpgraderOneMerge(List<SegmentCommitInfo> segments) { super(segments); }
@Override public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentCommitInfo,Boolean> segmentsToMerge, IndexWriter writer) throws IOException { if (upgradeInProgress) { MergeSpecification spec = new IndexUpgraderMergeSpecification(); for (SegmentCommitInfo info : segmentInfos) { if (shouldUpgrade(info)) { // TODO: Use IndexUpgradeMergePolicy instead. We should be comparing codecs, // for now we just assume every minor upgrade has a new format. logger.debug("Adding segment " + info.info.name + " to be upgraded"); spec.add(new OneMerge(Collections.singletonList(info))); } // TODO: we could check IndexWriter.getMergingSegments and avoid adding merges that IW will just reject? if (spec.merges.size() == MAX_CONCURRENT_UPGRADE_MERGES) { // hit our max upgrades, so return the spec. we will get a cascaded call to continue. logger.debug("Returning " + spec.merges.size() + " merges for upgrade"); return spec; } } // We must have less than our max upgrade merges, so the next return will be our last in upgrading mode. if (spec.merges.isEmpty() == false) { logger.debug("Returning " + spec.merges.size() + " merges for end of upgrade"); return spec; } // Only set this once there are 0 segments needing upgrading, because when we return a // spec, IndexWriter may (silently!) reject that merge if some of the segments we asked // to be merged were already being (naturally) merged: upgradeInProgress = false; // fall through, so when we don't have any segments to upgrade, the delegate policy // has a chance to decide what to do (e.g. collapse the segments to satisfy maxSegmentCount) } return upgradedMergeSpecification(delegate.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer)); }
