@Override public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) { List<Bucket> reducedBuckets = reduceBuckets(aggregations, reduceContext); // adding empty buckets if needed if (minDocCount == 0) { addEmptyBuckets(reducedBuckets, reduceContext); } if (order == InternalOrder.KEY_ASC || reduceContext.isFinalReduce() == false) { // nothing to do, data are already sorted since shards return // sorted buckets and the merge-sort performed by reduceBuckets // maintains order } else if (order == InternalOrder.KEY_DESC) { // we just need to reverse here... List<Bucket> reverse = new ArrayList<>(reducedBuckets); Collections.reverse(reverse); reducedBuckets = reverse; } else { // sorted by sub-aggregation, need to fall back to a costly n*log(n) sort CollectionUtil.introSort(reducedBuckets, order.comparator()); } return new InternalHistogram(getName(), reducedBuckets, order, minDocCount, emptyBucketInfo, format, keyed, pipelineAggregators(), getMetaData()); }
@Override public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) { List<Bucket> reducedBuckets = reduceBuckets(aggregations, reduceContext); // adding empty buckets if needed if (minDocCount == 0) { addEmptyBuckets(reducedBuckets, reduceContext); } if (order == InternalOrder.KEY_ASC || reduceContext.isFinalReduce() == false) { // nothing to do, data are already sorted since shards return // sorted buckets and the merge-sort performed by reduceBuckets // maintains order } else if (order == InternalOrder.KEY_DESC) { // we just need to reverse here... List<Bucket> reverse = new ArrayList<>(reducedBuckets); Collections.reverse(reverse); reducedBuckets = reverse; } else { // sorted by sub-aggregation, need to fall back to a costly n*log(n) sort CollectionUtil.introSort(reducedBuckets, order.comparator()); } return new InternalDateHistogram(getName(), reducedBuckets, order, minDocCount, offset, emptyBucketInfo, format, keyed, pipelineAggregators(), getMetaData()); }
/** * Validates that the setting is valid */ public final void validate(String key, Settings settings) { Setting setting = get(key); if (setting == null) { LevensteinDistance ld = new LevensteinDistance(); List<Tuple<Float, String>> scoredKeys = new ArrayList<>(); for (String k : this.keySettings.keySet()) { float distance = ld.getDistance(key, k); if (distance > 0.7f) { scoredKeys.add(new Tuple<>(distance, k)); } } CollectionUtil.timSort(scoredKeys, (a,b) -> b.v1().compareTo(a.v1())); String msgPrefix = "unknown setting"; SecureSettings secureSettings = settings.getSecureSettings(); if (secureSettings != null && settings.getSecureSettings().getSettingNames().contains(key)) { msgPrefix = "unknown secure setting"; } String msg = msgPrefix + " [" + key + "]"; List<String> keys = scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList()); if (keys.isEmpty() == false) { msg += " did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]": "any of " + keys.toString()) + "?"; } else { msg += " please check that any required plugins are installed, or check the breaking changes documentation for removed " + "settings"; } throw new IllegalArgumentException(msg); } setting.get(settings); }
public void testStoreStatusOrdering() throws Exception { DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); List<IndicesShardStoresResponse.StoreStatus> orderedStoreStatuses = new ArrayList<>(); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED, null)); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, new IOException("corrupted"))); orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, new IOException("corrupted"))); List<IndicesShardStoresResponse.StoreStatus> storeStatuses = new ArrayList<>(orderedStoreStatuses); Collections.shuffle(storeStatuses, random()); CollectionUtil.timSort(storeStatuses); assertThat(storeStatuses, equalTo(orderedStoreStatuses)); }
private List<IndexTemplateMetaData> findTemplates(BulkCreateIndicesRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) { List<IndexTemplateMetaData> templates = new ArrayList<>(); CreateIndexClusterStateUpdateRequest dummyRequest = new CreateIndexClusterStateUpdateRequest(request, "bulk-create", request.indices().iterator().next(), false); // note: only use the first index name to see if template matches. // this means for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) { IndexTemplateMetaData template = cursor.value; if (indexTemplateFilter.apply(dummyRequest, template)) { templates.add(template); } } CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() { @Override public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) { return o2.order() - o1.order(); } }); return templates; }
@Override public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) { List<B> reducedBuckets = reduceBuckets(aggregations, reduceContext); // adding empty buckets if needed if (minDocCount == 0) { addEmptyBuckets(reducedBuckets, reduceContext); } if (order == InternalOrder.KEY_ASC) { // nothing to do, data are already sorted since shards return // sorted buckets and the merge-sort performed by reduceBuckets // maintains order } else if (order == InternalOrder.KEY_DESC) { // we just need to reverse here... List<B> reverse = new ArrayList<>(reducedBuckets); Collections.reverse(reverse); reducedBuckets = reverse; } else { // sorted by sub-aggregation, need to fall back to a costly n*log(n) sort CollectionUtil.introSort(reducedBuckets, order.comparator()); } return getFactory().create(getName(), reducedBuckets, order, minDocCount, emptyBucketInfo, formatter, keyed, pipelineAggregators(), getMetaData()); }
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException { List<IndexTemplateMetaData> templates = new ArrayList<>(); for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) { IndexTemplateMetaData template = cursor.value; if (indexTemplateFilter.apply(request, template)) { templates.add(template); } } CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() { @Override public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) { return o2.order() - o1.order(); } }); return templates; }
/** * Returns a list of snapshots from repository sorted by snapshot creation date * * @param repositoryName repository name * @return list of snapshots */ public List<Snapshot> snapshots(String repositoryName, boolean ignoreUnavailable) { Set<Snapshot> snapshotSet = newHashSet(); List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null); for (SnapshotsInProgress.Entry entry : entries) { snapshotSet.add(inProgressSnapshot(entry)); } Repository repository = repositoriesService.repository(repositoryName); List<SnapshotId> snapshotIds = repository.snapshots(); for (SnapshotId snapshotId : snapshotIds) { try { snapshotSet.add(repository.readSnapshot(snapshotId)); } catch (Exception ex) { if (ignoreUnavailable) { logger.warn("failed to get snapshot [{}]", ex, snapshotId); } else { throw new SnapshotException(snapshotId, "Snapshot could not be read", ex); } } } ArrayList<Snapshot> snapshotList = new ArrayList<>(snapshotSet); CollectionUtil.timSort(snapshotList); return Collections.unmodifiableList(snapshotList); }
private List<DiscoveryNode> sortedMasterNodes(Iterable<DiscoveryNode> nodes) { List<DiscoveryNode> possibleNodes = CollectionUtils.iterableAsArrayList(nodes); if (possibleNodes.isEmpty()) { return null; } // clean non master nodes for (Iterator<DiscoveryNode> it = possibleNodes.iterator(); it.hasNext(); ) { DiscoveryNode node = it.next(); if (!node.isMasterNode()) { it.remove(); } } CollectionUtil.introSort(possibleNodes, ElectMasterService::compareNodes); return possibleNodes; }
public CommitPoints(List<CommitPoint> commitPoints) { CollectionUtil.introSort(commitPoints, new Comparator<CommitPoint>() { @Override public int compare(CommitPoint o1, CommitPoint o2) { return (o2.version() < o1.version() ? -1 : (o2.version() == o1.version() ? 0 : 1)); } }); this.commitPoints = Collections.unmodifiableList(new ArrayList<>(commitPoints)); }
protected final String unrecognized( final RestRequest request, final Set<String> invalids, final Set<String> candidates, final String detail) { String message = String.format( Locale.ROOT, "request [%s] contains unrecognized %s%s: ", request.path(), detail, invalids.size() > 1 ? "s" : ""); boolean first = true; for (final String invalid : invalids) { final LevensteinDistance ld = new LevensteinDistance(); final List<Tuple<Float, String>> scoredParams = new ArrayList<>(); for (final String candidate : candidates) { final float distance = ld.getDistance(invalid, candidate); if (distance > 0.5f) { scoredParams.add(new Tuple<>(distance, candidate)); } } CollectionUtil.timSort(scoredParams, (a, b) -> { // sort by distance in reverse order, then parameter name for equal distances int compare = a.v1().compareTo(b.v1()); if (compare != 0) return -compare; else return a.v2().compareTo(b.v2()); }); if (first == false) { message += ", "; } message += "[" + invalid + "]"; final List<String> keys = scoredParams.stream().map(Tuple::v2).collect(Collectors.toList()); if (keys.isEmpty() == false) { message += " -> did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]" : "any of " + keys.toString()) + "?"; } first = false; } return message; }
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state) throws IOException { List<IndexTemplateMetaData> templateMetadata = new ArrayList<>(); for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) { IndexTemplateMetaData metadata = cursor.value; for (String template: metadata.patterns()) { if (Regex.simpleMatch(template, request.index())) { templateMetadata.add(metadata); break; } } } CollectionUtil.timSort(templateMetadata, Comparator.comparingInt(IndexTemplateMetaData::order).reversed()); return templateMetadata; }
public List<Message> getMessages() { if (root.errors == null) { return Collections.emptyList(); } List<Message> result = new ArrayList<>(root.errors); CollectionUtil.timSort(result, new Comparator<Message>() { @Override public int compare(Message a, Message b) { return a.getSource().compareTo(b.getSource()); } }); return unmodifiableList(result); }
/** * Returns a list of snapshots from repository sorted by snapshot creation date * * @param repositoryName repository name * @param snapshotIds snapshots for which to fetch snapshot information * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning, * if false, they will throw an error * @return list of snapshots */ public List<SnapshotInfo> snapshots(final String repositoryName, List<SnapshotId> snapshotIds, final boolean ignoreUnavailable) { final Set<SnapshotInfo> snapshotSet = new HashSet<>(); final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds); // first, look at the snapshots in progress final List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList())); for (SnapshotsInProgress.Entry entry : entries) { snapshotSet.add(inProgressSnapshot(entry)); snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId()); } // then, look in the repository final Repository repository = repositoriesService.repository(repositoryName); for (SnapshotId snapshotId : snapshotIdsToIterate) { try { snapshotSet.add(repository.getSnapshotInfo(snapshotId)); } catch (Exception ex) { if (ignoreUnavailable) { logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex); } else { throw new SnapshotException(repositoryName, snapshotId, "Snapshot could not be read", ex); } } } final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet); CollectionUtil.timSort(snapshotList); return Collections.unmodifiableList(snapshotList); }
/** * Returns a list of currently running snapshots from repository sorted by snapshot creation date * * @param repositoryName repository name * @return list of snapshots */ public List<SnapshotInfo> currentSnapshots(final String repositoryName) { List<SnapshotInfo> snapshotList = new ArrayList<>(); List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Collections.emptyList()); for (SnapshotsInProgress.Entry entry : entries) { snapshotList.add(inProgressSnapshot(entry)); } CollectionUtil.timSort(snapshotList); return Collections.unmodifiableList(snapshotList); }
void finish() { ImmutableOpenMap.Builder<String, ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>>> indicesStoreStatusesBuilder = ImmutableOpenMap.builder(); java.util.List<IndicesShardStoresResponse.Failure> failureBuilder = new ArrayList<>(); for (Response fetchResponse : fetchResponses) { ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexStoreStatuses = indicesStoreStatusesBuilder.get(fetchResponse.shardId.getIndexName()); final ImmutableOpenIntMap.Builder<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexShardsBuilder; if (indexStoreStatuses == null) { indexShardsBuilder = ImmutableOpenIntMap.builder(); } else { indexShardsBuilder = ImmutableOpenIntMap.builder(indexStoreStatuses); } java.util.List<IndicesShardStoresResponse.StoreStatus> storeStatuses = indexShardsBuilder.get(fetchResponse.shardId.id()); if (storeStatuses == null) { storeStatuses = new ArrayList<>(); } for (NodeGatewayStartedShards response : fetchResponse.responses) { if (shardExistsInNode(response)) { IndicesShardStoresResponse.StoreStatus.AllocationStatus allocationStatus = getAllocationStatus(fetchResponse.shardId.getIndexName(), fetchResponse.shardId.id(), response.getNode()); storeStatuses.add(new IndicesShardStoresResponse.StoreStatus(response.getNode(), response.allocationId(), allocationStatus, response.storeException())); } } CollectionUtil.timSort(storeStatuses); indexShardsBuilder.put(fetchResponse.shardId.id(), storeStatuses); indicesStoreStatusesBuilder.put(fetchResponse.shardId.getIndexName(), indexShardsBuilder.build()); for (FailedNodeException failure : fetchResponse.failures) { failureBuilder.add(new IndicesShardStoresResponse.Failure(failure.nodeId(), fetchResponse.shardId.getIndexName(), fetchResponse.shardId.id(), failure.getCause())); } } listener.onResponse(new IndicesShardStoresResponse(indicesStoreStatusesBuilder.build(), Collections.unmodifiableList(failureBuilder))); }
/** Returns all the official plugin names that look similar to pluginId. **/ private List<String> checkMisspelledPlugin(String pluginId) { LevensteinDistance ld = new LevensteinDistance(); List<Tuple<Float, String>> scoredKeys = new ArrayList<>(); for (String officialPlugin : OFFICIAL_PLUGINS) { float distance = ld.getDistance(pluginId, officialPlugin); if (distance > 0.7f) { scoredKeys.add(new Tuple<>(distance, officialPlugin)); } } CollectionUtil.timSort(scoredKeys, (a, b) -> b.v1().compareTo(a.v1())); return scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList()); }
private List<DiscoveryNode> sortedMasterNodes(Iterable<DiscoveryNode> nodes) { List<DiscoveryNode> possibleNodes = CollectionUtils.iterableAsArrayList(nodes); if (possibleNodes.isEmpty()) { return null; } // clean non master nodes for (Iterator<DiscoveryNode> it = possibleNodes.iterator(); it.hasNext(); ) { DiscoveryNode node = it.next(); if (!node.masterNode()) { it.remove(); } } CollectionUtil.introSort(possibleNodes, nodeComparator); return possibleNodes; }
public List<Message> getMessages() { if (root.errors == null) { return Collections.emptyList(); } List<Message> result = new ArrayList<>(root.errors); CollectionUtil.timSort(result, new Comparator<Message>() { @Override public int compare(Message a, Message b) { return a.getSource().compareTo(b.getSource()); } }); return result; }
/** * Returns a list of currently running snapshots from repository sorted by snapshot creation date * * @param repositoryName repository name * @return list of snapshots */ public List<Snapshot> currentSnapshots(String repositoryName) { List<Snapshot> snapshotList = new ArrayList<>(); List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null); for (SnapshotsInProgress.Entry entry : entries) { snapshotList.add(inProgressSnapshot(entry)); } CollectionUtil.timSort(snapshotList); return Collections.unmodifiableList(snapshotList); }
void finish() { ImmutableOpenMap.Builder<String, ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>>> indicesStoreStatusesBuilder = ImmutableOpenMap.builder(); java.util.List<IndicesShardStoresResponse.Failure> failureBuilder = new ArrayList<>(); for (Response fetchResponse : fetchResponses) { ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexStoreStatuses = indicesStoreStatusesBuilder.get(fetchResponse.shardId.getIndex()); final ImmutableOpenIntMap.Builder<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexShardsBuilder; if (indexStoreStatuses == null) { indexShardsBuilder = ImmutableOpenIntMap.builder(); } else { indexShardsBuilder = ImmutableOpenIntMap.builder(indexStoreStatuses); } java.util.List<IndicesShardStoresResponse.StoreStatus> storeStatuses = indexShardsBuilder.get(fetchResponse.shardId.id()); if (storeStatuses == null) { storeStatuses = new ArrayList<>(); } for (NodeGatewayStartedShards response : fetchResponse.responses) { if (shardExistsInNode(response)) { IndicesShardStoresResponse.StoreStatus.Allocation allocation = getAllocation(fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), response.getNode()); storeStatuses.add(new IndicesShardStoresResponse.StoreStatus(response.getNode(), response.version(), allocation, response.storeException())); } } CollectionUtil.timSort(storeStatuses); indexShardsBuilder.put(fetchResponse.shardId.id(), storeStatuses); indicesStoreStatusesBuilder.put(fetchResponse.shardId.getIndex(), indexShardsBuilder.build()); for (FailedNodeException failure : fetchResponse.failures) { failureBuilder.add(new IndicesShardStoresResponse.Failure(failure.nodeId(), fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), failure.getCause())); } } listener.onResponse(new IndicesShardStoresResponse(indicesStoreStatusesBuilder.build(), Collections.unmodifiableList(failureBuilder))); }
public void testResponseHeadersArePreserved() throws InterruptedException { List<String> executors = new ArrayList<>(ThreadPool.THREAD_POOL_TYPES.keySet()); CollectionUtil.timSort(executors); // makes sure it's reproducible serviceA.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME, (request, channel) -> { threadPool.getThreadContext().putTransient("boom", new Object()); threadPool.getThreadContext().addResponseHeader("foo.bar", "baz"); if ("fail".equals(request.info)) { throw new RuntimeException("boom"); } else { channel.sendResponse(TransportResponse.Empty.INSTANCE); } }); CountDownLatch latch = new CountDownLatch(2); TransportResponseHandler<TransportResponse> transportResponseHandler = new TransportResponseHandler<TransportResponse>() { @Override public TransportResponse newInstance() { return TransportResponse.Empty.INSTANCE; } @Override public void handleResponse(TransportResponse response) { try { assertSame(response, TransportResponse.Empty.INSTANCE); assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar")); assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size()); assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0)); assertNull(threadPool.getThreadContext().getTransient("boom")); } finally { latch.countDown(); } } @Override public void handleException(TransportException exp) { try { assertTrue(threadPool.getThreadContext().getResponseHeaders().containsKey("foo.bar")); assertEquals(1, threadPool.getThreadContext().getResponseHeaders().get("foo.bar").size()); assertEquals("baz", threadPool.getThreadContext().getResponseHeaders().get("foo.bar").get(0)); assertNull(threadPool.getThreadContext().getTransient("boom")); } finally { latch.countDown(); } } @Override public String executor() { if (1 == 1) return "same"; return randomFrom(executors); } }; serviceB.sendRequest(nodeA, "action", new TestRequest(randomFrom("fail", "pass")), transportResponseHandler); serviceA.sendRequest(nodeA, "action", new TestRequest(randomFrom("fail", "pass")), transportResponseHandler); latch.await(); }
/** * Returns the given nodes sorted by likelihood of being elected as master, most likely first. * Non-master nodes are not removed but are rather put in the end */ static List<DiscoveryNode> sortByMasterLikelihood(Iterable<DiscoveryNode> nodes) { ArrayList<DiscoveryNode> sortedNodes = CollectionUtils.iterableAsArrayList(nodes); CollectionUtil.introSort(sortedNodes, ElectMasterService::compareNodes); return sortedNodes; }
protected void sort(Comparator<O> comparator) { CollectionUtil.timSort(options, comparator); }
public void sort(Comparator<ShardRouting> comparator) { nodes.ensureMutable(); CollectionUtil.timSort(unassigned, comparator); }
/** * Constructs a enw GroupShardsIterator from the given list. */ public GroupShardsIterator(List<ShardIterator> iterators) { CollectionUtil.timSort(iterators); this.iterators = iterators; }
/** * Returns the given nodes sorted by likelyhood of being elected as master, most likely first. * Non-master nodes are not removed but are rather put in the end */ public List<DiscoveryNode> sortByMasterLikelihood(Iterable<DiscoveryNode> nodes) { ArrayList<DiscoveryNode> sortedNodes = CollectionUtils.iterableAsArrayList(nodes); CollectionUtil.introSort(sortedNodes, nodeComparator); return sortedNodes; }
/** * Builds a list of nodes and version */ NodesAndVersions buildNodesAndVersions(ShardRouting shard, boolean recoveryOnAnyNode, Set<String> ignoreNodes, AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> shardState) { final Map<DiscoveryNode, Tuple<Long, Long>> nodesWithVersion = Maps.newHashMap(); int numberOfAllocationsFound = 0; long highestVersion = -1; for (TransportNodesListGatewayStartedShards.NodeGatewayStartedShards nodeShardState : shardState.getData().values()) { long version = nodeShardState.version(); long numDocs = nodeShardState.numDocs(); DiscoveryNode node = nodeShardState.getNode(); if (ignoreNodes.contains(node.id())) { continue; } // -1 version means it does not exists, which is what the API returns, and what we expect to if (nodeShardState.storeException() == null) { logger.trace("[{}] on node [{}] has version [{}] of shard, numDocs [{}]", shard, nodeShardState.getNode(), version, numDocs); } else { // when there is an store exception, we disregard the reported version and assign it as -1 (same as shard does not exist) logger.trace("[{}] on node [{}] has version [{}], numDocs [{}] but the store can not be opened, treating as version -1", nodeShardState.storeException(), shard, nodeShardState.getNode(), version, numDocs); version = -1; } if (recoveryOnAnyNode) { numberOfAllocationsFound++; if (version > highestVersion) { highestVersion = version; } // We always put the node without clearing the map nodesWithVersion.put(node, new Tuple<Long, Long>(version, numDocs)); } else if (version != -1) { numberOfAllocationsFound++; // If we've found a new "best" candidate, clear the // current candidates and add it if (version > highestVersion) { highestVersion = version; nodesWithVersion.clear(); nodesWithVersion.put(node, new Tuple<Long, Long>(version, numDocs)); } else if (version == highestVersion) { // If the candidate is the same, add it to the // list, but keep the current candidate nodesWithVersion.put(node, new Tuple<Long, Long>(version, numDocs)); } } } // Now that we have a map of nodes to versions along with the // number of allocations found (and not ignored), we need to sort // it so the node with the highest version is at the beginning List<DiscoveryNode> nodesWithHighestVersion = new ArrayList<>(); nodesWithHighestVersion.addAll(nodesWithVersion.keySet()); CollectionUtil.timSort(nodesWithHighestVersion, new Comparator<DiscoveryNode>() { @Override public int compare(DiscoveryNode o1, DiscoveryNode o2) { int compareResult = Long.compare(nodesWithVersion.get(o2).v1(), nodesWithVersion.get(o1).v1()); if (compareResult == 0) { return Long.compare(nodesWithVersion.get(o2).v2(), nodesWithVersion.get(o1).v2()); } else { return compareResult; } } }); if (logger.isTraceEnabled()) { StringBuilder sb = new StringBuilder("["); for (DiscoveryNode n : nodesWithVersion.keySet()) { sb.append("[").append(n.getName()).append("]").append(" -> ").append(nodesWithVersion.get(n)).append(", "); } sb.append("]"); logger.trace("{} candidates for allocation: {}", shard, sb.toString()); } return new NodesAndVersions(Collections.unmodifiableList(nodesWithHighestVersion), numberOfAllocationsFound, highestVersion); }
public void sort(Comparator<ShardRouting> comparator) { CollectionUtil.timSort(unassigned, comparator); }
/** * Calculates statistical measures from raw measurements. Should be called prior to accessing * individual measurements. */ public void computeSummaryStatistics() { if (computed) { return; } long totalWarmupTime = 0; final SinglePassStatistics single = new SinglePassStatistics(); for (CompetitionNodeResult nodeResult : nodeResults) { totalWarmupTime += nodeResult.warmUpTime(); totalIterations += nodeResult.totalIterations(); completedIterations += nodeResult.completedIterations(); // only calculate statistics for iterations completed thus far for (int i = 0; i < nodeResult.completedIterations(); i++) { CompetitionIteration competitionIteration = nodeResult.iterations().get(i); CompetitionIterationData iterationData = competitionIteration.competitionIterationData(); long[] data = iterationData.data(); for (long datum : data) { if (datum > -1) { // ignore unset values in the underlying array single.push(datum); } } totalQueries += competitionIteration.numQueries(); totalTime += competitionIteration.totalTime(); sumTotalHits += competitionIteration.sumTotalHits(); // keep track of slowest requests if (competitionIteration.slowRequests() != null) { for (CompetitionIteration.SlowRequest slow : competitionIteration.slowRequests()) { slowest.add(new Tuple<>(nodeResult.nodeName(), slow)); } } } } min = single.min(); max = single.max(); mean = single.mean(); stdDeviation = single.stddev(); avgWarmupTime = (nodeResults.size() > 0) ? totalWarmupTime / nodeResults.size() : 0.0; queriesPerSecond = (single.sum() > 0) ? (totalQueries * (1000.0 / (double) single.sum())) : 0.0; millisPerHit = (sumTotalHits > 0) ? (totalTime / (double) sumTotalHits) : 0.0; for (double percentile : percentiles) { percentileValues.put(percentile, single.percentile(percentile / 100.0d)); } CollectionUtil.timSort(slowest, new Comparator<Tuple<String, CompetitionIteration.SlowRequest>>() { @Override public int compare(Tuple<String, CompetitionIteration.SlowRequest> o1, Tuple<String, CompetitionIteration.SlowRequest> o2) { return Long.compare(o2.v2().maxTimeTaken(), o1.v2().maxTimeTaken()); } }); computed = true; }