@Override public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexId indexId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) { SnapshotContext snapshotContext = new SnapshotContext(shard, snapshotId, indexId, snapshotStatus); snapshotStatus.startTime(System.currentTimeMillis()); try { snapshotContext.snapshot(snapshotIndexCommit); snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime()); snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE); } catch (Exception e) { snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime()); snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE); snapshotStatus.failure(ExceptionsHelper.detailedMessage(e)); if (e instanceof IndexShardSnapshotFailedException) { throw (IndexShardSnapshotFailedException) e; } else { throw new IndexShardSnapshotFailedException(shard.shardId(), e); } } }
/** * Read the last segments info from the commit pointed to by the searcher manager */ protected static SegmentInfos readLastCommittedSegmentInfos(final SearcherManager sm, final Store store) throws IOException { IndexSearcher searcher = sm.acquire(); try { IndexCommit latestCommit = ((DirectoryReader) searcher.getIndexReader()).getIndexCommit(); return Lucene.readSegmentInfos(latestCommit); } catch (IOException e) { // Fall back to reading from the store if reading from the commit fails try { return store.readLastCommittedSegmentsInfo(); } catch (IOException e2) { e2.addSuppressed(e); throw e2; } } finally { sm.release(searcher); } }
@Override public IndexCommit acquireIndexCommit(final boolean flushFirst) throws EngineException { // we have to flush outside of the readlock otherwise we might have a problem upgrading // the to a write lock when we fail the engine in this operation if (flushFirst) { logger.trace("start flush for snapshot"); flush(false, true); logger.trace("finish flush for snapshot"); } try (ReleasableLock lock = readLock.acquire()) { ensureOpen(); logger.trace("pulling snapshot"); return deletionPolicy.snapshot(); } catch (IOException e) { throw new SnapshotFailedEngineException(shardId, e); } }
/** * gets a {@link Store.MetadataSnapshot} for the current directory. This method is safe to call in all lifecycle of the index shard, * without having to worry about the current state of the engine and concurrent flushes. * * @throws org.apache.lucene.index.IndexNotFoundException if no index is found in the current directory * @throws CorruptIndexException if the lucene index is corrupted. This can be caused by a checksum mismatch or an * unexpected exception when opening the index reading the segments file. * @throws IndexFormatTooOldException if the lucene index is too old to be opened. * @throws IndexFormatTooNewException if the lucene index is too new to be opened. * @throws FileNotFoundException if one or more files referenced by a commit are not present. * @throws NoSuchFileException if one or more files referenced by a commit are not present. */ public Store.MetadataSnapshot snapshotStoreMetadata() throws IOException { IndexCommit indexCommit = null; store.incRef(); try { synchronized (mutex) { // if the engine is not running, we can access the store directly, but we need to make sure no one starts // the engine on us. If the engine is running, we can get a snapshot via the deletion policy which is initialized. // That can be done out of mutex, since the engine can be closed half way. Engine engine = getEngineOrNull(); if (engine == null) { try (Lock ignored = store.directory().obtainLock(IndexWriter.WRITE_LOCK_NAME)) { return store.getMetadata(null); } } } indexCommit = deletionPolicy.snapshot(); return store.getMetadata(indexCommit); } finally { store.decRef(); if (indexCommit != null) { deletionPolicy.release(indexCommit); } } }
public void testAcquireIndexCommit() throws IOException { final IndexShard shard = newStartedShard(); int numDocs = randomInt(20); for (int i = 0; i < numDocs; i++) { indexDoc(shard, "type", "id_" + i); } final boolean flushFirst = randomBoolean(); IndexCommit commit = shard.acquireIndexCommit(flushFirst); int moreDocs = randomInt(20); for (int i = 0; i < moreDocs; i++) { indexDoc(shard, "type", "id_" + numDocs + i); } flushShard(shard); // check that we can still read the commit that we captured try (IndexReader reader = DirectoryReader.open(commit)) { assertThat(reader.numDocs(), equalTo(flushFirst ? numDocs : 0)); } shard.releaseIndexCommit(commit); flushShard(shard, true); // check it's clean up assertThat(DirectoryReader.listCommits(shard.store().directory()), hasSize(1)); closeShards(shard); }
/** * Read the last segments info from the commit pointed to by the searcher manager */ protected static SegmentInfos readLastCommittedSegmentInfos(final SearcherManager sm, final Store store) throws IOException { IndexSearcher searcher = sm.acquire(); try { IndexCommit latestCommit = ((DirectoryReader) searcher.getIndexReader()).getIndexCommit(); return Lucene.readSegmentInfos(latestCommit); } catch (IOException e) { // Fall back to reading from the store if reading from the commit fails try { return store. readLastCommittedSegmentsInfo(); } catch (IOException e2) { e2.addSuppressed(e); throw e2; } } finally { sm.release(searcher); } }
public static IndexWriter configureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) throws IOException { IndexWriterConfig iwc = createWriterConfig(config, runData, mode, commit); String infoStreamVal = config.get("writer.info.stream", null); if (infoStreamVal != null) { if (infoStreamVal.equals("SystemOut")) { iwc.setInfoStream(System.out); } else if (infoStreamVal.equals("SystemErr")) { iwc.setInfoStream(System.err); } else { File f = new File(infoStreamVal).getAbsoluteFile(); iwc.setInfoStream(new PrintStream(new BufferedOutputStream(new FileOutputStream(f)), false, Charset.defaultCharset().name())); } } IndexWriter writer = new IndexWriter(runData.getDirectory(), iwc); return writer; }
/** * Constructor with the given index directory and callback to notify when the * indexes were updated. */ public IndexReplicationHandler(Directory indexDir, Callable<Boolean> callback) throws IOException { this.callback = callback; this.indexDir = indexDir; currentRevisionFiles = null; currentVersion = null; if (DirectoryReader.indexExists(indexDir)) { final List<IndexCommit> commits = DirectoryReader.listCommits(indexDir); final IndexCommit commit = commits.get(commits.size() - 1); currentRevisionFiles = IndexRevision.revisionFiles(commit); currentVersion = IndexRevision.revisionVersion(commit); final InfoStream infoStream = InfoStream.getDefault(); if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) { infoStream.message(INFO_STREAM_COMPONENT, "constructor(): currentVersion=" + currentVersion + " currentRevisionFiles=" + currentRevisionFiles); infoStream.message(INFO_STREAM_COMPONENT, "constructor(): commit=" + commit); } } }
private List<NamedList<Object>> getCommits() { Map<Long, IndexCommit> commits = core.getDeletionPolicy().getCommits(); List<NamedList<Object>> l = new ArrayList<>(); for (IndexCommit c : commits.values()) { try { NamedList<Object> nl = new NamedList<>(); nl.add("indexVersion", IndexDeletionPolicyWrapper.getCommitTimestamp(c)); nl.add(GENERATION, c.getGeneration()); List<String> commitList = new ArrayList<>(c.getFileNames().size()); commitList.addAll(c.getFileNames()); Collections.sort(commitList); nl.add(CMD_GET_FILE_LIST, commitList); l.add(nl); } catch (IOException e) { LOG.warn("Exception while reading files for commit " + c, e); } } return l; }
private String getId(IndexCommit commit) { StringBuilder sb = new StringBuilder(); Directory dir = commit.getDirectory(); // For anything persistent, make something that will // be the same, regardless of the Directory instance. if (dir instanceof FSDirectory) { FSDirectory fsd = (FSDirectory) dir; File fdir = fsd.getDirectory(); sb.append(fdir.getPath()); } else { sb.append(dir); } sb.append('/'); sb.append(commit.getGeneration()); return sb.toString(); }
@Test public void testCommitAge() throws InterruptedException { assumeFalse("This test is not working on Windows (or maybe machines with only 2 CPUs)", Constants.WINDOWS); IndexDeletionPolicyWrapper delPolicy = h.getCore().getDeletionPolicy(); addDocs(); Map<Long, IndexCommit> commits = delPolicy.getCommits(); IndexCommit ic = delPolicy.getLatestCommit(); String agestr = ((SolrDeletionPolicy) (delPolicy.getWrappedDeletionPolicy())).getMaxCommitAge().replaceAll("[a-zA-Z]", "").replaceAll("-", ""); long age = Long.parseLong(agestr); Thread.sleep(age); assertU(adoc("id", String.valueOf(6), "name", "name" + String.valueOf(6))); assertU(optimize()); assertQ("return all docs", req("id:[0 TO 6]"), "*[count(//doc)=6]" ); commits = delPolicy.getCommits(); assertTrue(!commits.containsKey(ic.getGeneration())); }
@Test public void shouldReturnRealSnapshotIfIndexAllowsIt() throws Exception { // Given LuceneSnapshotter snapshotter = new LuceneSnapshotter(); when(luceneSnapshot.getFileNames()).thenReturn( asList("a", "b") ); // When ResourceIterator<File> snapshot = snapshotter.snapshot( indexDir, writer ); // Then assertEquals( new File(indexDir, "a"), snapshot.next() ); assertEquals( new File(indexDir, "b"), snapshot.next() ); assertFalse( snapshot.hasNext() ); snapshot.close(); verify( snapshotPolicy ).release( any(IndexCommit.class) ); }
private long[] getIndexVersion() { long version[] = new long[2]; RefCounted<SolrIndexSearcher> searcher = core.getSearcher(); try { final IndexCommit commit = searcher.get().getIndexReader().getIndexCommit(); final Map<String,String> commitData = commit.getUserData(); String commitTime = commitData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY); if (commitTime != null) { version[0] = Long.parseLong(commitTime); } version[1] = commit.getGeneration(); } catch (IOException e) { LOG.warn("Unable to get index version : ", e); } finally { searcher.decref(); } return version; }
static String str(IndexCommit commit) { StringBuilder sb = new StringBuilder(); try { sb.append("commit{"); Directory dir = commit.getDirectory(); if (dir instanceof FSDirectory) { FSDirectory fsd = (FSDirectory) dir; sb.append("dir=").append(fsd.getDirectory()); } else { sb.append("dir=").append(dir); } sb.append(",segFN=").append(commit.getSegmentsFileName()); sb.append(",generation=").append(commit.getGeneration()); sb.append(",filenames=").append(commit.getFileNames()); } catch (Exception e) { sb.append(e); } return sb.toString(); }
@Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { _writeLock.lock(); try { int size = commits.size(); for (int i = 0; i < size - 1; i++) { IndexCommit indexCommit = commits.get(i); long generation = indexCommit.getGeneration(); if (!_generationsToNames.containsKey(generation)) { indexCommit.delete(); } } } finally { _writeLock.unlock(); } }
public void createSnapshot(String name, DirectoryReader reader, String context) throws IOException { _writeLock.lock(); try { if (_namesToGenerations.containsKey(name)) { throw new IOException("Snapshot [" + name + "] already exists."); } LOG.info("Creating snapshot [{0}] in [{1}].", name, context); IndexCommit indexCommit = reader.getIndexCommit(); long generation = indexCommit.getGeneration(); _namesToGenerations.put(name, generation); Set<String> names = _generationsToNames.get(generation); if (names == null) { names = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); _generationsToNames.put(generation, names); } names.add(name); storeGenerations(); } finally { _writeLock.unlock(); } }
private void assertFiles(Set<String> expected, int run, int commit, FastHdfsKeyValueDirectory directory) throws IOException { Set<String> actual; if (DirectoryReader.indexExists(directory)) { List<IndexCommit> listCommits = DirectoryReader.listCommits(directory); // assertEquals(1, listCommits.size()); IndexCommit indexCommit = listCommits.get(0); actual = new TreeSet<String>(indexCommit.getFileNames()); } else { actual = new TreeSet<String>(); } Set<String> missing = new TreeSet<String>(expected); missing.removeAll(actual); Set<String> extra = new TreeSet<String>(actual); extra.removeAll(expected); assertEquals("Pass [" + run + "] Missing Files " + " Extra Files " + extra + "", expected, actual); }
private List<NamedList<Object>> getCommits() { Map<Long, IndexCommit> commits = core.getDeletionPolicy().getCommits(); List<NamedList<Object>> l = new ArrayList<NamedList<Object>>(); for (IndexCommit c : commits.values()) { try { NamedList<Object> nl = new NamedList<Object>(); nl.add("indexVersion", IndexDeletionPolicyWrapper.getCommitTimestamp(c)); nl.add(GENERATION, c.getGeneration()); List<String> commitList = new ArrayList<String>(c.getFileNames().size()); commitList.addAll(c.getFileNames()); Collections.sort(commitList); nl.add(CMD_GET_FILE_LIST, commitList); l.add(nl); } catch (IOException e) { LOG.warn("Exception while reading files for commit " + c, e); } } return l; }
/** * Creates a new {@link IndexCommit} snapshot form the currently running engine. All resources referenced by this * commit won't be freed until the commit / snapshot is released via {@link #releaseIndexCommit(IndexCommit)}. * * @param flushFirst <code>true</code> if the index should first be flushed to disk / a low level lucene commit should be executed */ public IndexCommit acquireIndexCommit(boolean flushFirst) throws EngineException { IndexShardState state = this.state; // one time volatile read // we allow snapshot on closed index shard, since we want to do one after we close the shard and before we close the engine if (state == IndexShardState.STARTED || state == IndexShardState.RELOCATED || state == IndexShardState.CLOSED) { return getEngine().acquireIndexCommit(flushFirst); } else { throw new IllegalIndexShardStateException(shardId, state, "snapshot is not allowed"); } }
/** * Returns the segments info for the given commit or for the latest commit if the given commit is <code>null</code> * * @throws IOException if the index is corrupted or the segments file is not present */ private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory directory) throws IOException { assert commit == null || commit.getDirectory() == directory; try { return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit); } catch (EOFException eof) { // TODO this should be caught by lucene - EOF is almost certainly an index corruption throw new CorruptIndexException("Read past EOF while reading segment infos", "commit(" + commit + ")", eof); } catch (IOException exception) { throw exception; // IOExceptions like too many open files are not necessarily a corruption - just bubble it up } catch (Exception ex) { throw new CorruptIndexException("Hit unexpected exception while reading segment infos", "commit(" + commit + ")", ex); } }
MetadataSnapshot(IndexCommit commit, Directory directory, Logger logger) throws IOException { LoadedMetadata loadedMetadata = loadMetadata(commit, directory, logger); metadata = loadedMetadata.fileMetadata; commitUserData = loadedMetadata.userData; numDocs = loadedMetadata.numDocs; assert metadata.isEmpty() || numSegmentFiles() == 1 : "numSegmentFiles: " + numSegmentFiles(); }
/** * Reads the segments infos from the given commit, failing if it fails to load */ public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOException { // Using commit.getSegmentsFileName() does NOT work here, have to // manually create the segment filename String filename = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", commit.getGeneration()); return SegmentInfos.readCommit(commit.getDirectory(), filename); }
/** * Called by Lucene. Same as {@link #onCommit(java.util.List)}. */ @Override public void onInit(List<? extends IndexCommit> commits) throws IOException { if (!commits.isEmpty()) { // this might be empty if we create a new index. // the behavior has changed in Lucene 4.4 that calls onInit even with an empty commits list. onCommit(commits); } }
/** * Called by Lucene.. Wraps the provided commits with {@link SnapshotIndexCommit} * and delegates to the wrapped deletion policy. */ @Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { assert !commits.isEmpty() : "Commits must not be empty"; synchronized (mutex) { List<SnapshotIndexCommit> snapshotCommits = wrapCommits(commits); primary.onCommit(snapshotCommits); // clean snapshots that their respective counts are 0 (should not really happen) for (Iterator<SnapshotHolder> it = snapshots.values().iterator(); it.hasNext(); ) { SnapshotHolder holder = it.next(); if (holder.counter <= 0) { it.remove(); } } // build the current commits list (all the ones that are not deleted by the primary) List<SnapshotIndexCommit> newCommits = new ArrayList<>(); for (SnapshotIndexCommit commit : snapshotCommits) { if (!commit.isDeleted()) { newCommits.add(commit); } } this.commits = newCommits; // the last commit that is not deleted this.lastCommit = newCommits.get(newCommits.size() - 1); } }
private List<SnapshotIndexCommit> wrapCommits(List<? extends IndexCommit> commits) throws IOException { final int count = commits.size(); List<SnapshotIndexCommit> snapshotCommits = new ArrayList<>(count); for (int i = 0; i < count; i++) snapshotCommits.add(new SnapshotIndexCommit(this, commits.get(i))); return snapshotCommits; }
/** * Deletes all commits except the most recent one. */ @Override public void onCommit(List<? extends IndexCommit> commits) { // Note that commits.size() should normally be 2 (if not // called by onInit above): int size = commits.size(); for (int i = 0; i < size - 1; i++) { commits.get(i).delete(); } }
SnapshotIndexCommit(SnapshotDeletionPolicy deletionPolicy, IndexCommit cp) throws IOException { super(cp); this.deletionPolicy = deletionPolicy; ArrayList<String> tmpFiles = new ArrayList<>(); for (String o : cp.getFileNames()) { tmpFiles.add(o); } files = tmpFiles.toArray(new String[tmpFiles.size()]); }
/** * Cancella tutti i commit ad eccezione dell'ultimo * * @param commits lista dei commit * @throws java.io.IOException eccezione di input/output */ public void onCommit(List commits) throws IOException { //System.out.println("onCommit: " + commits); // Note that commits.size() should normally be 2 (if not // called by onInit above): int size = commits.size(); for (int i = 0; i < size - 1; i++) { ((IndexCommit) commits.get(i)).delete(); } }
