Java 类org.apache.lucene.document.SortedSetDocValuesField 实例源码

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
    String value;
    if (context.externalValueSet()) {
        value = context.externalValue().toString();
    } else {
        value = context.parser().textOrNull();
    }

    if (value == null) {
        return;
    }

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().name(), value, fieldType());
        fields.add(field);
    }
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(value)));
    }
}

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().nullValueAsString(), fieldType().boost());
    if (valueAndBoost.value() == null) {
        return;
    }
    if (ignoreAbove > 0 && valueAndBoost.value().length() > ignoreAbove) {
        return;
    }
    if (context.includeInAll(includeInAll, this)) {
        context.allEntries().addText(fieldType().names().fullName(), valueAndBoost.value(), valueAndBoost.boost());
    }

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().names().indexName(), valueAndBoost.value(), fieldType());
        field.setBoost(valueAndBoost.boost());
        fields.add(field);
    }
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(valueAndBoost.value())));
    }
}

private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  doc.add(new Field("textgrams", textString, ft));
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      // TODO: if we had a BinaryTermField we could fix
      // this "must be valid ut8f" limitation:
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context.utf8ToString(), Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}

private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
  //System.out.println("process SSDV: " + byField);
  for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {

    String indexFieldName = ent.getKey();
    //System.out.println("  field=" + indexFieldName);

    for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
      FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
      String fullPath = pathToString(cp.components, cp.length);
      //System.out.println("add " + fullPath);

      // For facet counts:
      doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));

      // For drill-down:
      doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));
      doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
    }
  }
}

public void testSortedSetOneValue() throws IOException {
  assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);

  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");

  dv.setDocument(0);
  assertEquals(0, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());

  BytesRef bytes = dv.lookupOrd(0);
  assertEquals(new BytesRef("hello"), bytes);

  ireader.close();
  directory.close();
}

public void testMixedTypesAfterReopenAppend2() throws IOException {
  assumeTrue("codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  doc = new Document();
  w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc.add(new StringField("foo", "bar", Field.Store.NO));
  doc.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  try {
    // NOTE: this case follows a different code path inside
    // DefaultIndexingChain/FieldInfos, because the field (foo)
    // is first added without DocValues:
    w.addDocument(doc);
    fail("did not get expected exception");
  } catch (IllegalArgumentException iae) {
    // expected
  }
  w.forceMerge(1);
  w.close();
  dir.close();
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value,
    float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = new BytesRef(value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    IndexableField storedField = createField(field, value, boost);
    fields.add(storedField);
    ByteBuffer byteBuffer = toObject(storedField);
    BytesRef bytes = new BytesRef
        (byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

public void testSortedSetOneValue() throws IOException {
  assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);

  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");

  dv.setDocument(0);
  assertEquals(0, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());

  BytesRef bytes = new BytesRef();
  dv.lookupOrd(0, bytes);
  assertEquals(new BytesRef("hello"), bytes);

  ireader.close();
  directory.close();
}

public void testTooLargeTermSortedSetBytes() throws IOException {
  assumeTrue("codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
  Analyzer analyzer = new MockAnalyzer(random());

  Directory directory = newDirectory();
  // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
  IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  IndexWriter iwriter = new IndexWriter(directory, iwc);
  Document doc = new Document();
  byte bytes[] = new byte[100000];
  BytesRef b = new BytesRef(bytes);
  random().nextBytes(bytes);
  doc.add(new SortedSetDocValuesField("dv", b));
  try {
    iwriter.addDocument(doc);
    fail("did not get expected exception");
  } catch (IllegalArgumentException expected) {
    // expected
  }
  iwriter.close();
  directory.close();
}

/**
 * populates a writer with random stuff. this must be fully reproducable with the seed!
 */
public static void createRandomIndex(int numdocs, RandomIndexWriter writer, long seed) throws IOException {
  Random random = new Random(seed);
  // primary source for our data is from linefiledocs, its realistic.
  LineFileDocs lineFileDocs = new LineFileDocs(random);

  // TODO: we should add other fields that use things like docs&freqs but omit positions,
  // because linefiledocs doesn't cover all the possibilities.
  for (int i = 0; i < numdocs; i++) {
    Document document = lineFileDocs.nextDoc();
    // grab the title and add some SortedSet instances for fun
    String title = document.get("titleTokenized");
    String split[] = title.split("\\s+");
    for (String trash : split) {
      document.add(new SortedSetDocValuesField("sortedset", new BytesRef(trash)));
    }
    writer.addDocument(document);
  }

  lineFileDocs.close();
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value,
    float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<IndexableField>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = new BytesRef(value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value,
    float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<IndexableField>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = new BytesRef(value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
  //System.out.println("process SSDV: " + byField);
  for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {

    String indexFieldName = ent.getKey();
    //System.out.println("  field=" + indexFieldName);

    for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
      FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
      String fullPath = pathToString(cp.components, cp.length);
      //System.out.println("add " + fullPath);

      // For facet counts:
      doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));

      // For drill-down:
      doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));
      doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
    }
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value,
    float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<IndexableField>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = new BytesRef(value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<IndexableField>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<IndexableField>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
  //System.out.println("process SSDV: " + byField);
  for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {

    String indexFieldName = ent.getKey();
    //System.out.println("  field=" + indexFieldName);

    for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
      FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
      String fullPath = pathToString(cp.components, cp.length);
      //System.out.println("add " + fullPath);

      // For facet counts:
      doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));

      // For drill-down:
      doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));
      doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
    }
  }
}

private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  doc.add(new Field("textgrams", textString, ft));
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      // TODO: if we had a BinaryTermField we could fix
      // this "must be valid ut8f" limitation:
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context.utf8ToString(), Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}

private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
  //System.out.println("process SSDV: " + byField);
  for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {

    String indexFieldName = ent.getKey();
    //System.out.println("  field=" + indexFieldName);

    for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
      FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
      String fullPath = pathToString(cp.components, cp.length);
      //System.out.println("add " + fullPath);

      // For facet counts:
      doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));

      // For drill-down:
      doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));
      doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
    }
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value,
    float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = new BytesRef(value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value, boost));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value, boost));
  }
}

@Override
public void addFields(Document doc, Iterable<CategoryPath> categories) throws IOException {
  if (categories == null) {
    throw new IllegalArgumentException("categories should not be null");
  }

  final Map<CategoryListParams,Iterable<CategoryPath>> categoryLists = createCategoryListMapping(categories);
  for (Entry<CategoryListParams, Iterable<CategoryPath>> e : categoryLists.entrySet()) {

    CategoryListParams clp = e.getKey();
    String dvField = clp.field + SortedSetDocValuesReaderState.FACET_FIELD_EXTENSION;

    // Add sorted-set DV fields, one per value:
    for(CategoryPath cp : e.getValue()) {
      if (cp.length != 2) {
        throw new IllegalArgumentException("only flat facets (dimension + label) are currently supported; got " + cp);
      }
      doc.add(new SortedSetDocValuesField(dvField, new BytesRef(cp.toString(indexingParams.getFacetDelimChar()))));
    }

    // add the drill-down field
    DrillDownStream drillDownStream = getDrillDownStream(e.getValue());
    Field drillDown = new Field(clp.field, drillDownStream, drillDownFieldType());
    doc.add(drillDown);
  }
}

public void testSortedSetOneValue() throws IOException {
  assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);

  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");

  dv.setDocument(0);
  assertEquals(0, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());

  BytesRef bytes = new BytesRef();
  dv.lookupOrd(0, bytes);
  assertEquals(new BytesRef("hello"), bytes);

  ireader.close();
  directory.close();
}

public void testTooLargeTermSortedSetBytes() throws IOException {
  assumeTrue("codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
  Analyzer analyzer = new MockAnalyzer(random());

  Directory directory = newDirectory();
  // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
  IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  IndexWriter iwriter = new IndexWriter(directory, iwc);
  Document doc = new Document();
  byte bytes[] = new byte[100000];
  BytesRef b = new BytesRef(bytes);
  random().nextBytes(bytes);
  doc.add(new SortedSetDocValuesField("dv", b));
  try {
    iwriter.addDocument(doc);
    fail("did not get expected exception");
  } catch (IllegalArgumentException expected) {
    // expected
  }
  iwriter.close();
  directory.close();
}

/**
 * populates a writer with random stuff. this must be fully reproducable with the seed!
 */
public static void createRandomIndex(int numdocs, RandomIndexWriter writer, long seed) throws IOException {
  Random random = new Random(seed);
  // primary source for our data is from linefiledocs, its realistic.
  LineFileDocs lineFileDocs = new LineFileDocs(random);

  // TODO: we should add other fields that use things like docs&freqs but omit positions,
  // because linefiledocs doesn't cover all the possibilities.
  for (int i = 0; i < numdocs; i++) {
    Document document = lineFileDocs.nextDoc();
    // grab the title and add some SortedSet instances for fun
    String title = document.get("titleTokenized");
    String split[] = title.split("\\s+");
    for (String trash : split) {
      document.add(new SortedSetDocValuesField("sortedset", new BytesRef(trash)));
    }
    // add a numeric dv field sometimes
    document.removeFields("sparsenumeric");
    if (random.nextInt(4) == 2) {
      document.add(new NumericDocValuesField("sparsenumeric", random.nextInt()));
    }
    writer.addDocument(document);
  }

  lineFileDocs.close();
}

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
    if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored()) {
        return;
    }
    fields.add(new Field(fieldType().name(), context.sourceToParse().type(), fieldType()));
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(context.sourceToParse().type())));
    }
}

public void testLoadGlobal_neverCacheIfFieldIsMissing() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(null);
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    IndexWriter iw = new IndexWriter(dir, iwc);
    long numDocs = scaledRandomIntBetween(32, 128);

    for (int i = 1; i <= numDocs; i++) {
        Document doc = new Document();
        doc.add(new SortedSetDocValuesField("field1", new BytesRef(String.valueOf(i))));
        doc.add(new StringField("field2", String.valueOf(i), Field.Store.NO));
        iw.addDocument(doc);
        if (i % 24 == 0) {
            iw.commit();
        }
    }
    iw.close();
    DirectoryReader ir = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId("_index", "_na_", 0));

    DummyAccountingFieldDataCache fieldDataCache = new DummyAccountingFieldDataCache();
    // Testing SortedSetDVOrdinalsIndexFieldData:
    SortedSetDVOrdinalsIndexFieldData sortedSetDVOrdinalsIndexFieldData = createSortedDV("field1", fieldDataCache);
    sortedSetDVOrdinalsIndexFieldData.loadGlobal(ir);
    assertThat(fieldDataCache.cachedGlobally, equalTo(1));
    sortedSetDVOrdinalsIndexFieldData.loadGlobal(new FieldMaskingReader("field1", ir));
    assertThat(fieldDataCache.cachedGlobally, equalTo(1));

    // Testing PagedBytesIndexFieldData
    PagedBytesIndexFieldData pagedBytesIndexFieldData = createPagedBytes("field2", fieldDataCache);
    pagedBytesIndexFieldData.loadGlobal(ir);
    assertThat(fieldDataCache.cachedGlobally, equalTo(2));
    pagedBytesIndexFieldData.loadGlobal(new FieldMaskingReader("field2", ir));
    assertThat(fieldDataCache.cachedGlobally, equalTo(2));

    ir.close();
    dir.close();
}

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored()) {
        return;
    }
    fields.add(new Field(fieldType().names().indexName(), context.type(), fieldType()));
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(context.type())));
    }
}

public void testForward() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortedSetSortField("value", false));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(2, td.totalHits);
  // 'bar' comes before 'baz'
  assertEquals("1", searcher.doc(td.scoreDocs[0].doc).get("id"));
  assertEquals("2", searcher.doc(td.scoreDocs[1].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}

public void testReverse() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);

  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortedSetSortField("value", true));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(2, td.totalHits);
  // 'bar' comes before 'baz'
  assertEquals("2", searcher.doc(td.scoreDocs[0].doc).get("id"));
  assertEquals("1", searcher.doc(td.scoreDocs[1].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}

public void testMissingFirst() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(newStringField("id", "3", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  SortField sortField = new SortedSetSortField("value", false);
  sortField.setMissingValue(SortField.STRING_FIRST);
  Sort sort = new Sort(sortField);

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits);
  // 'bar' comes before 'baz'
  // null comes first
  assertEquals("3", searcher.doc(td.scoreDocs[0].doc).get("id"));
  assertEquals("1", searcher.doc(td.scoreDocs[1].doc).get("id"));
  assertEquals("2", searcher.doc(td.scoreDocs[2].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}

public void testMissingLast() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(newStringField("id", "3", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  SortField sortField = new SortedSetSortField("value", false);
  sortField.setMissingValue(SortField.STRING_LAST);
  Sort sort = new Sort(sortField);

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits);
  // 'bar' comes before 'baz'
  assertEquals("1", searcher.doc(td.scoreDocs[0].doc).get("id"));
  assertEquals("2", searcher.doc(td.scoreDocs[1].doc).get("id"));
  // null comes last
  assertEquals("3", searcher.doc(td.scoreDocs[2].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}

public void testSingleton() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortedSetSortField("value", false));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(2, td.totalHits);
  // 'bar' comes before 'baz'
  assertEquals("1", searcher.doc(td.scoreDocs[0].doc).get("id"));
  assertEquals("2", searcher.doc(td.scoreDocs[1].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}

public void testMax() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  // slow wrapper does not support random access ordinals (there is no need for that!)
  IndexSearcher searcher = newSearcher(ir, false);

  Sort sort = new Sort(new SortedSetSortField("value", false, SortedSetSortField.Selector.MAX));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(2, td.totalHits);
  // 'baz' comes before 'foo'
  assertEquals("2", searcher.doc(td.scoreDocs[0].doc).get("id"));
  assertEquals("1", searcher.doc(td.scoreDocs[1].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}

public void testMaxReverse() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  // slow wrapper does not support random access ordinals (there is no need for that!)
  IndexSearcher searcher = newSearcher(ir, false);

  Sort sort = new Sort(new SortedSetSortField("value", true, SortedSetSortField.Selector.MAX));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(2, td.totalHits);
  // 'baz' comes before 'foo'
  assertEquals("1", searcher.doc(td.scoreDocs[0].doc).get("id"));
  assertEquals("2", searcher.doc(td.scoreDocs[1].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}

public void testMaxMissingFirst() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newStringField("id", "1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("foo")));
  doc.add(new SortedSetDocValuesField("value", new BytesRef("bar")));
  doc.add(newStringField("id", "2", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new SortedSetDocValuesField("value", new BytesRef("baz")));
  doc.add(newStringField("id", "3", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  // slow wrapper does not support random access ordinals (there is no need for that!)
  IndexSearcher searcher = newSearcher(ir, false);

  SortField sortField = new SortedSetSortField("value", false, SortedSetSortField.Selector.MAX);
  sortField.setMissingValue(SortField.STRING_FIRST);
  Sort sort = new Sort(sortField);

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits);
  // null comes first
  assertEquals("1", searcher.doc(td.scoreDocs[0].doc).get("id"));
  // 'baz' comes before 'foo'
  assertEquals("3", searcher.doc(td.scoreDocs[1].doc).get("id"));
  assertEquals("2", searcher.doc(td.scoreDocs[2].doc).get("id"));
  assertNoFieldCaches();

  ir.close();
  dir.close();
}