@Override public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) { List<Field> fields = new ArrayList<>(); if (indexed) { fields.add(new HalfFloatPoint(name, value.floatValue())); } if (docValued) { fields.add(new SortedNumericDocValuesField(name, HalfFloatPoint.halfFloatToSortableShort(value.floatValue()))); } if (stored) { fields.add(new StoredField(name, value.floatValue())); } return fields; }
@Override public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) { List<Field> fields = new ArrayList<>(); if (indexed) { fields.add(new FloatPoint(name, value.floatValue())); } if (docValued) { fields.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(value.floatValue()))); } if (stored) { fields.add(new StoredField(name, value.floatValue())); } return fields; }
@Override public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) { List<Field> fields = new ArrayList<>(); if (indexed) { fields.add(new DoublePoint(name, value.doubleValue())); } if (docValued) { fields.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(value.doubleValue()))); } if (stored) { fields.add(new StoredField(name, value.doubleValue())); } return fields; }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored() && !fieldType().hasDocValues()) { return; } Boolean value = context.parseExternalValue(Boolean.class); if (value == null) { XContentParser.Token token = context.parser().currentToken(); if (token == XContentParser.Token.VALUE_NULL) { if (fieldType().nullValue() != null) { value = fieldType().nullValue(); } } else { if (indexCreatedVersion.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { value = context.parser().booleanValue(); } else { value = context.parser().booleanValueLenient(); if (context.parser().isBooleanValueLenient() != context.parser().isBooleanValue()) { String rawValue = context.parser().text(); deprecationLogger.deprecated("Expected a boolean for property [{}] but got [{}]", fieldType().name(), rawValue); } } } } if (value == null) { return; } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { fields.add(new Field(fieldType().name(), value ? "T" : "F", fieldType())); } if (fieldType().hasDocValues()) { fields.add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); } }
/** * without combine script, the "_aggs" map should contain a list of the size of the number of documents matched */ @SuppressWarnings("unchecked") public void testScriptedMetricWithoutCombine() throws IOException { try (Directory directory = newDirectory()) { int numDocs = randomInt(100); try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { for (int i = 0; i < numDocs; i++) { indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i))); } } try (IndexReader indexReader = DirectoryReader.open(directory)) { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT); ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); Map<String, Object> agg = (Map<String, Object>) scriptedMetric.aggregation(); assertEquals(numDocs, ((List<Integer>) agg.get("collector")).size()); } } }
/** * test that combine script sums the list produced by the "mapScript" */ public void testScriptedMetricWithCombine() throws IOException { try (Directory directory = newDirectory()) { Integer numDocs = randomInt(100); try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { for (int i = 0; i < numDocs; i++) { indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i))); } } try (IndexReader indexReader = DirectoryReader.open(directory)) { ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); aggregationBuilder.initScript(INIT_SCRIPT).mapScript(MAP_SCRIPT).combineScript(COMBINE_SCRIPT); ScriptedMetric scriptedMetric = search(newSearcher(indexReader, true, true), new MatchAllDocsQuery(), aggregationBuilder); assertEquals(AGG_NAME, scriptedMetric.getName()); assertNotNull(scriptedMetric.aggregation()); assertEquals(numDocs, scriptedMetric.aggregation()); } } }
public void testMinDocCount() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { for (long value : new long[] {7, 3, -10, -6, 5, 50}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", value)); w.addDocument(doc); } HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") .field("field") .interval(10) .minDocCount(2); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); fieldType.setName("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); Histogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertEquals(2, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); assertEquals(0d, histogram.getBuckets().get(1).getKey()); assertEquals(3, histogram.getBuckets().get(1).getDocCount()); } } }
public void testTermQuery() { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date = "2015-10-12T14:10:55"; long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis(); ft.setIndexOptions(IndexOptions.DOCS); Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant, instant + 999), SortedNumericDocValuesField.newRangeQuery("field", instant, instant + 999)); assertEquals(expected, ft.termQuery(date, context)); ft.setIndexOptions(IndexOptions.NONE); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.termQuery(date, context)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); }
public void testRangeQuery() throws IOException { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).build(); QueryShardContext context = new QueryShardContext(0, new IndexSettings(IndexMetaData.builder("foo").settings(indexSettings).build(), indexSettings), null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); MappedFieldType ft = createDefaultFieldType(); ft.setName("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; long instant1 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date1).getMillis(); long instant2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date2).getMillis() + 999; ft.setIndexOptions(IndexOptions.DOCS); Query expected = new IndexOrDocValuesQuery( LongPoint.newRangeQuery("field", instant1, instant2), SortedNumericDocValuesField.newRangeQuery("field", instant1, instant2)); assertEquals(expected, ft.rangeQuery(date1, date2, true, true, context).rewrite(new MultiReader())); ft.setIndexOptions(IndexOptions.NONE); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.rangeQuery(date1, date2, true, true, context)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { final Object value; if (context.externalValueSet()) { value = context.externalValue(); } else { value = context.parser().textOrNull(); } if (value != null) { final BytesRef bytes = new BytesRef(value.toString()); final long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()).h1; fields.add(new SortedNumericDocValuesField(fieldType().name(), hash)); if (fieldType().stored()) { fields.add(new StoredField(name(), hash)); } } }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored() && !fieldType().hasDocValues()) { return; } Boolean value = context.parseExternalValue(Boolean.class); if (value == null) { XContentParser.Token token = context.parser().currentToken(); if (token == XContentParser.Token.VALUE_NULL) { if (fieldType().nullValue() != null) { value = fieldType().nullValue(); } } else { value = context.parser().booleanValue(); } } if (value == null) { return; } fields.add(new Field(fieldType().names().indexName(), value ? "T" : "F", fieldType())); if (fieldType().hasDocValues()) { fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value ? 1 : 0)); } }
public void testSingleton() throws Exception { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(new SortedNumericDocValuesField("value", 5)); doc.add(newStringField("id", "2", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); doc.add(new SortedNumericDocValuesField("value", 3)); doc.add(newStringField("id", "1", Field.Store.YES)); writer.addDocument(doc); IndexReader ir = writer.getReader(); writer.close(); IndexSearcher searcher = newSearcher(ir); Sort sort = new Sort(new SortedNumericSortField("value", SortField.Type.INT)); TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); assertEquals(2, td.totalHits); // 3 comes before 5 assertEquals("1", searcher.doc(td.scoreDocs[0].doc).get("id")); assertEquals("2", searcher.doc(td.scoreDocs[1].doc).get("id")); ir.close(); dir.close(); }
@Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues) { float l = Float.NEGATIVE_INFINITY; float u = Float.POSITIVE_INFINITY; if (lowerTerm != null) { l = parse(lowerTerm, false); if (includeLower) { l = HalfFloatPoint.nextDown(l); } l = HalfFloatPoint.nextUp(l); } if (upperTerm != null) { u = parse(upperTerm, false); if (includeUpper) { u = HalfFloatPoint.nextUp(u); } u = HalfFloatPoint.nextDown(u); } Query query = HalfFloatPoint.newRangeQuery(field, l, u); if (hasDocValues) { Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, HalfFloatPoint.halfFloatToSortableShort(l), HalfFloatPoint.halfFloatToSortableShort(u)); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; }
@Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues) { float l = Float.NEGATIVE_INFINITY; float u = Float.POSITIVE_INFINITY; if (lowerTerm != null) { l = parse(lowerTerm, false); if (includeLower == false) { l = FloatPoint.nextUp(l); } } if (upperTerm != null) { u = parse(upperTerm, false); if (includeUpper == false) { u = FloatPoint.nextDown(u); } } Query query = FloatPoint.newRangeQuery(field, l, u); if (hasDocValues) { Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, NumericUtils.floatToSortableInt(l), NumericUtils.floatToSortableInt(u)); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; }
@Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues) { double l = Double.NEGATIVE_INFINITY; double u = Double.POSITIVE_INFINITY; if (lowerTerm != null) { l = parse(lowerTerm, false); if (includeLower == false) { l = DoublePoint.nextUp(l); } } if (upperTerm != null) { u = parse(upperTerm, false); if (includeUpper == false) { u = DoublePoint.nextDown(u); } } Query query = DoublePoint.newRangeQuery(field, l, u); if (hasDocValues) { Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, NumericUtils.doubleToSortableLong(l), NumericUtils.doubleToSortableLong(u)); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; }
@Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues) { int l = Integer.MIN_VALUE; int u = Integer.MAX_VALUE; if (lowerTerm != null) { l = parse(lowerTerm, true); // if the lower bound is decimal: // - if the bound is positive then we increment it: // if lowerTerm=1.5 then the (inclusive) bound becomes 2 // - if the bound is negative then we leave it as is: // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); if ((lowerTermHasDecimalPart == false && includeLower == false) || (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { if (l == Integer.MAX_VALUE) { return new MatchNoDocsQuery(); } ++l; } } if (upperTerm != null) { u = parse(upperTerm, true); boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); if ((upperTermHasDecimalPart == false && includeUpper == false) || (upperTermHasDecimalPart && signum(upperTerm) < 0)) { if (u == Integer.MIN_VALUE) { return new MatchNoDocsQuery(); } --u; } } Query query = IntPoint.newRangeQuery(field, l, u); if (hasDocValues) { Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, l, u); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; }
@Override public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) { List<Field> fields = new ArrayList<>(); if (indexed) { fields.add(new IntPoint(name, value.intValue())); } if (docValued) { fields.add(new SortedNumericDocValuesField(name, value.intValue())); } if (stored) { fields.add(new StoredField(name, value.intValue())); } return fields; }
@Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues) { long l = Long.MIN_VALUE; long u = Long.MAX_VALUE; if (lowerTerm != null) { l = parse(lowerTerm, true); // if the lower bound is decimal: // - if the bound is positive then we increment it: // if lowerTerm=1.5 then the (inclusive) bound becomes 2 // - if the bound is negative then we leave it as is: // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); if ((lowerTermHasDecimalPart == false && includeLower == false) || (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { if (l == Long.MAX_VALUE) { return new MatchNoDocsQuery(); } ++l; } } if (upperTerm != null) { u = parse(upperTerm, true); boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); if ((upperTermHasDecimalPart == false && includeUpper == false) || (upperTermHasDecimalPart && signum(upperTerm) < 0)) { if (u == Long.MIN_VALUE) { return new MatchNoDocsQuery(); } --u; } } Query query = LongPoint.newRangeQuery(field, l, u); if (hasDocValues) { Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, l, u); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; }
@Override public List<Field> createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored) { List<Field> fields = new ArrayList<>(); if (indexed) { fields.add(new LongPoint(name, value.longValue())); } if (docValued) { fields.add(new SortedNumericDocValuesField(name, value.longValue())); } if (stored) { fields.add(new StoredField(name, value.longValue())); } return fields; }
Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable DateTimeZone timeZone, @Nullable DateMathParser forcedDateParser, QueryShardContext context) { failIfNotIndexed(); DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; long l, u; if (lowerTerm == null) { l = Long.MIN_VALUE; } else { l = parseToMilliseconds(lowerTerm, !includeLower, timeZone, parser, context); if (includeLower == false) { ++l; } } if (upperTerm == null) { u = Long.MAX_VALUE; } else { u = parseToMilliseconds(upperTerm, includeUpper, timeZone, parser, context); if (includeUpper == false) { --u; } } Query query = LongPoint.newRangeQuery(name(), l, u); if (hasDocValues()) { Query dvQuery = SortedNumericDocValuesField.newRangeQuery(name(), l, u); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; }
@Override public void postParse(ParseContext context) throws IOException { // In the case of nested docs, let's fill nested docs with seqNo=1 and // primaryTerm=0 so that Lucene doesn't write a Bitset for documents // that don't have the field. This is consistent with the default value // for efficiency. for (int i = 1; i < context.docs().size(); i++) { final Document doc = context.docs().get(i); doc.add(new LongPoint(NAME, 1)); doc.add(new SortedNumericDocValuesField(NAME, 1L)); doc.add(new NumericDocValuesField(PRIMARY_TERM_NAME, 0L)); } }
public void testRandomDoubles() throws IOException { MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); ft.setName("field"); final ExtendedSimpleStatsAggregator expected = new ExtendedSimpleStatsAggregator(); testCase(ft, iw -> { int numDocs = randomIntBetween(10, 50); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); int numValues = randomIntBetween(1, 5); for (int j = 0; j < numValues; j++) { double value = randomDoubleBetween(-100d, 100d, true); long valueAsLong = NumericUtils.doubleToSortableLong(value); doc.add(new SortedNumericDocValuesField("field", valueAsLong)); expected.add(value); } iw.addDocument(doc); } }, stats -> { assertEquals(expected.count, stats.getCount(), 0); assertEquals(expected.sum, stats.getSum(), TOLERANCE); assertEquals(expected.min, stats.getMin(), 0); assertEquals(expected.max, stats.getMax(), 0); assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE); assertEquals(expected.sumOfSqrs, stats.getSumOfSquares(), TOLERANCE); assertEquals(expected.stdDev(), stats.getStdDeviation(), TOLERANCE); assertEquals(expected.variance(), stats.getVariance(), TOLERANCE); assertEquals(expected.stdDevBound(ExtendedStats.Bounds.LOWER, stats.getSigma()), stats.getStdDeviationBound(ExtendedStats.Bounds.LOWER), TOLERANCE); assertEquals(expected.stdDevBound(ExtendedStats.Bounds.UPPER, stats.getSigma()), stats.getStdDeviationBound(ExtendedStats.Bounds.UPPER), TOLERANCE); } ); }
public void testRandomLongs() throws IOException { MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); ft.setName("field"); final ExtendedSimpleStatsAggregator expected = new ExtendedSimpleStatsAggregator(); testCase(ft, iw -> { int numDocs = randomIntBetween(10, 50); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); int numValues = randomIntBetween(1, 5); for (int j = 0; j < numValues; j++) { long value = randomIntBetween(-100, 100); doc.add(new SortedNumericDocValuesField("field", value)); expected.add(value); } iw.addDocument(doc); } }, stats -> { assertEquals(expected.count, stats.getCount(), 0); assertEquals(expected.sum, stats.getSum(), TOLERANCE); assertEquals(expected.min, stats.getMin(), 0); assertEquals(expected.max, stats.getMax(), 0); assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE); assertEquals(expected.sumOfSqrs, stats.getSumOfSquares(), TOLERANCE); assertEquals(expected.stdDev(), stats.getStdDeviation(), TOLERANCE); assertEquals(expected.variance(), stats.getVariance(), TOLERANCE); assertEquals(expected.stdDevBound(ExtendedStats.Bounds.LOWER, stats.getSigma()), stats.getStdDeviationBound(ExtendedStats.Bounds.LOWER), TOLERANCE); assertEquals(expected.stdDevBound(ExtendedStats.Bounds.UPPER, stats.getSigma()), stats.getStdDeviationBound(ExtendedStats.Bounds.UPPER), TOLERANCE); } ); }
public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7))); iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 1))); }, max -> { assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), 0); }); }
public void testSomeMatchesSortedNumericDocValues() throws IOException { testCase(new FieldValueQuery("number"), iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("number", 7))); iw.addDocument(singleton(new SortedNumericDocValuesField("number", 1))); }, max -> { assertEquals(7, max.getValue(), 0); }); }
public void testQueryFiltering() throws IOException { testCase(IntPoint.newRangeQuery("number", 0, 5), iw -> { iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1))); }, max -> { assertEquals(1, max.getValue(), 0); }); }
public void testQueryFiltersAll() throws IOException { testCase(IntPoint.newRangeQuery("number", -1, 0), iw -> { iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 1))); }, max -> { assertEquals(Double.NEGATIVE_INFINITY, max.getValue(), 0); }); }
public void testSimple() throws IOException { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { for (double value : new double[] {3, 0.2, 10}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", NumericUtils.doubleToSortableLong(value))); w.addDocument(doc); } PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg") .field("field") .method(PercentilesMethod.HDR) .values(0.1, 0.5, 12); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator<Percentile> rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); assertThat(rank.getPercent(), Matchers.equalTo(0d)); rank = rankIterator.next(); assertEquals(0.5, rank.getValue(), 0d); assertThat(rank.getPercent(), Matchers.greaterThan(0d)); assertThat(rank.getPercent(), Matchers.lessThan(100d)); rank = rankIterator.next(); assertEquals(12, rank.getValue(), 0d); assertThat(rank.getPercent(), Matchers.equalTo(100d)); assertFalse(rankIterator.hasNext()); } } }
public void testSimple() throws IOException { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { for (double value : new double[] {3, 0.2, 10}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", NumericUtils.doubleToSortableLong(value))); w.addDocument(doc); } PercentileRanksAggregationBuilder aggBuilder = new PercentileRanksAggregationBuilder("my_agg") .field("field") .method(PercentilesMethod.TDIGEST) .values(0.1, 0.5, 12); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); fieldType.setName("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); PercentileRanks ranks = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); Iterator<Percentile> rankIterator = ranks.iterator(); Percentile rank = rankIterator.next(); assertEquals(0.1, rank.getValue(), 0d); // TODO: Fix T-Digest: this assertion should pass but we currently get ~15 // https://github.com/elastic/elasticsearch/issues/14851 // assertThat(rank.getPercent(), Matchers.equalTo(0d)); rank = rankIterator.next(); assertEquals(0.5, rank.getValue(), 0d); assertThat(rank.getPercent(), Matchers.greaterThan(0d)); assertThat(rank.getPercent(), Matchers.lessThan(100d)); rank = rankIterator.next(); assertEquals(12, rank.getValue(), 0d); // TODO: Fix T-Digest: this assertion should pass but we currently get ~59 // https://github.com/elastic/elasticsearch/issues/14851 // assertThat(rank.getPercent(), Matchers.equalTo(100d)); assertFalse(rankIterator.hasNext()); } } }
public void testNoMatchingField() throws IOException { testCase(new MatchAllDocsQuery(), iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7))); iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 3))); }, avg -> { assertEquals(Double.NaN, avg.getValue(), 0); }); }
public void testSomeMatchesSortedNumericDocValues() throws IOException { testCase(new FieldValueQuery("number"), iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("number", 7))); iw.addDocument(singleton(new SortedNumericDocValuesField("number", 2))); iw.addDocument(singleton(new SortedNumericDocValuesField("number", 3))); }, avg -> { assertEquals(4, avg.getValue(), 0); }); }
public void testQueryFiltering() throws IOException { testCase(IntPoint.newRangeQuery("number", 0, 3), iw -> { iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 2))); iw.addDocument(Arrays.asList(new IntPoint("number", 3), new SortedNumericDocValuesField("number", 3))); }, avg -> { assertEquals(2.5, avg.getValue(), 0); }); }
public void testQueryFiltersAll() throws IOException { testCase(IntPoint.newRangeQuery("number", -1, 0), iw -> { iw.addDocument(Arrays.asList(new IntPoint("number", 7), new SortedNumericDocValuesField("number", 7))); iw.addDocument(Arrays.asList(new IntPoint("number", 1), new SortedNumericDocValuesField("number", 2))); iw.addDocument(Arrays.asList(new IntPoint("number", 3), new SortedNumericDocValuesField("number", 7))); }, avg -> { assertEquals(Double.NaN, avg.getValue(), 0); }); }
public void testRandomDoubles() throws IOException { MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); ft.setName("field"); final SimpleStatsAggregator expected = new SimpleStatsAggregator(); testCase(ft, iw -> { int numDocs = randomIntBetween(10, 50); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); int numValues = randomIntBetween(1, 5); for (int j = 0; j < numValues; j++) { double value = randomDoubleBetween(-100d, 100d, true); long valueAsLong = NumericUtils.doubleToSortableLong(value); doc.add(new SortedNumericDocValuesField("field", valueAsLong)); expected.add(value); } iw.addDocument(doc); } }, stats -> { assertEquals(expected.count, stats.getCount(), 0); assertEquals(expected.sum, stats.getSum(), TOLERANCE); assertEquals(expected.min, stats.getMin(), 0); assertEquals(expected.max, stats.getMax(), 0); assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE); } ); }
public void testRandomLongs() throws IOException { MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); ft.setName("field"); final SimpleStatsAggregator expected = new SimpleStatsAggregator(); testCase(ft, iw -> { int numDocs = randomIntBetween(10, 50); for (int i = 0; i < numDocs; i++) { Document doc = new Document(); int numValues = randomIntBetween(1, 5); for (int j = 0; j < numValues; j++) { long value = randomIntBetween(-100, 100); doc.add(new SortedNumericDocValuesField("field", value)); expected.add(value); } iw.addDocument(doc); } }, stats -> { assertEquals(expected.count, stats.getCount(), 0); assertEquals(expected.sum, stats.getSum(), TOLERANCE); assertEquals(expected.min, stats.getMin(), 0); assertEquals(expected.max, stats.getMax(), 0); assertEquals(expected.sum / expected.count, stats.getAvg(), TOLERANCE); } ); }
public void testSortedNumericDocValues() throws IOException { testCase(new FieldValueQuery(FIELD_NAME), iw -> { iw.addDocument(Arrays.asList(new SortedNumericDocValuesField(FIELD_NAME, 3), new SortedNumericDocValuesField(FIELD_NAME, 4))); iw.addDocument(Arrays.asList(new SortedNumericDocValuesField(FIELD_NAME, 3), new SortedNumericDocValuesField(FIELD_NAME, 4))); iw.addDocument(singleton(new SortedNumericDocValuesField(FIELD_NAME, 1))); }, count -> assertEquals(15L, count.getValue(), 0d)); }
public void testSomeMatchesSortedNumericDocValues() throws IOException { testCase(new FieldValueQuery(FIELD_NAME), ValueType.NUMERIC, iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("wrong_number", 7))); iw.addDocument(singleton(new SortedNumericDocValuesField(FIELD_NAME, 7))); iw.addDocument(singleton(new SortedNumericDocValuesField(FIELD_NAME, 1))); }, count -> assertEquals(2L, count.getValue())); }
/** * Uses the sampler aggregation to find the minimum value of a field out of the top 3 scoring documents in a search. */ public void testSampler() throws IOException { TextFieldType textFieldType = new TextFieldType(); textFieldType.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.GLOBAL, new StandardAnalyzer())); MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); numericFieldType.setName("int"); IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment with predictable docIds try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { for (long value : new long[] {7, 3, -10, -6, 5, 50}) { Document doc = new Document(); StringBuilder text = new StringBuilder(); for (int i = 0; i < value; i++) { text.append("good "); } doc.add(new Field("text", text.toString(), textFieldType)); doc.add(new SortedNumericDocValuesField("int", value)); w.addDocument(doc); } SamplerAggregationBuilder aggBuilder = new SamplerAggregationBuilder("sampler") .shardSize(3) .subAggregation(new MinAggregationBuilder("min") .field("int")); try (IndexReader reader = DirectoryReader.open(w)) { assertEquals("test expects a single segment", 1, reader.leaves().size()); IndexSearcher searcher = new IndexSearcher(reader); Sampler sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "good")), aggBuilder, textFieldType, numericFieldType); Min min = sampler.getAggregations().get("min"); assertEquals(5.0, min.getValue(), 0); } } }
public void testSomeDocs() throws IOException { testCase(iw -> { iw.addDocument(singleton(new SortedNumericDocValuesField("number", 7))); iw.addDocument(singleton(new SortedNumericDocValuesField("number", 1))); }, (global, min) -> { assertEquals(2, global.getDocCount()); assertEquals(1, min.getValue(), 0); }); }
public void testLongs() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { for (long value : new long[] {7, 3, -10, -6, 5, 50}) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("field", value)); w.addDocument(doc); } HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") .field("field") .interval(5); MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); fieldType.setName("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); Histogram histogram = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertEquals(4, histogram.getBuckets().size()); assertEquals(-10d, histogram.getBuckets().get(0).getKey()); assertEquals(2, histogram.getBuckets().get(0).getDocCount()); assertEquals(0d, histogram.getBuckets().get(1).getKey()); assertEquals(1, histogram.getBuckets().get(1).getDocCount()); assertEquals(5d, histogram.getBuckets().get(2).getKey()); assertEquals(2, histogram.getBuckets().get(2).getDocCount()); assertEquals(50d, histogram.getBuckets().get(3).getKey()); assertEquals(1, histogram.getBuckets().get(3).getDocCount()); } } }