Java 类org.apache.lucene.document.FieldType 实例源码

public static String termVectorOptionsToString(FieldType fieldType) {
    if (!fieldType.storeTermVectors()) {
        return "no";
    } else if (!fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
        return "yes";
    } else if (fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
        return "with_offsets";
    } else {
        StringBuilder builder = new StringBuilder("with");
        if (fieldType.storeTermVectorPositions()) {
            builder.append("_positions");
        }
        if (fieldType.storeTermVectorOffsets()) {
            builder.append("_offsets");
        }
        if (fieldType.storeTermVectorPayloads()) {
            builder.append("_payloads");
        }
        return builder.toString();
    }
}

private IndexReader indexOneDoc(Directory dir, String field, String value, Analyzer analyzer) throws IOException {
    IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
    iwc.setMergePolicy(newLogMergePolicy());
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);

    FieldType ft = new FieldType(TextField.TYPE_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    Field textField = new Field(field, "", ft);
    Document doc = new Document();
    doc.add(textField);

    textField.setStringValue(value);
    iw.addDocument(doc);
    IndexReader ir = iw.getReader();
    iw.close();
    return ir;
}

public void testVectorHighlighter() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, notNullValue());
    assertThat(fragment, equalTo("the big <b>bad</b> dog"));
}

public void testVectorHighlighterNoStore() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));

    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    FieldType vectorsType = new FieldType(TextField.TYPE_NOT_STORED);
    vectorsType.setStoreTermVectors(true);
    vectorsType.setStoreTermVectorPositions(true);
    vectorsType.setStoreTermVectorOffsets(true);
    document.add(new Field("content", "the big bad dog", vectorsType));
    indexWriter.addDocument(document);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);

    assertThat(topDocs.totalHits, equalTo(1));

    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
            reader, topDocs.scoreDocs[0].doc, "content", 30);
    assertThat(fragment, nullValue());
}

public void testNoTokens() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.KEYWORD_ANALYZER));

    FieldType allFt = getAllFieldType();
    Document doc = new Document();
    doc.add(new Field("_id", "1", StoredField.TYPE));
    doc.add(new AllField("_all", "", 2.0f, allFt));
    indexWriter.addDocument(doc);

    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);

    TopDocs docs = searcher.search(new MatchAllDocsQuery(), 10);
    assertThat(docs.totalHits, equalTo(1));
    assertThat(docs.scoreDocs[0].doc, equalTo(0));
}

public void testFieldTypeToTermVectorString() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(false);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat("with_positions_payloads", equalTo(ftOpts));
    AllFieldMapper.Builder builder = new AllFieldMapper.Builder(null);
    boolean exceptiontrown = false;
    try {
        TypeParsers.parseTermVector("", ftOpts, builder);
    } catch (MapperParsingException e) {
        exceptiontrown = true;
    }
    assertThat("TypeParsers.parseTermVector should accept string with_positions_payloads but does not.", exceptiontrown, equalTo(false));
}

public static String termVectorOptionsToString(FieldType fieldType) {
    if (!fieldType.storeTermVectors()) {
        return "no";
    } else if (!fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
        return "yes";
    } else if (fieldType.storeTermVectorOffsets() && !fieldType.storeTermVectorPositions()) {
        return "with_offsets";
    } else {
        StringBuilder builder = new StringBuilder("with");
        if (fieldType.storeTermVectorPositions()) {
            builder.append("_positions");
        }
        if (fieldType.storeTermVectorOffsets()) {
            builder.append("_offsets");
        }
        if (fieldType.storeTermVectorPayloads()) {
            builder.append("_payloads");
        }
        return builder.toString();
    }
}

/**
 * Creates Lucene Document using two strings: body and title
 *
 * @return resulted document
 */
public static Document createWith(final String titleStr, final String bodyStr) {
    final Document document = new Document();

    final FieldType textIndexedType = new FieldType();
    textIndexedType.setStored(true);
    textIndexedType.setIndexOptions(IndexOptions.DOCS);
    textIndexedType.setTokenized(true);

    //index title
    Field title = new Field("title", titleStr, textIndexedType);
    //index body
    Field body = new Field("body", bodyStr, textIndexedType);

    document.add(title);
    document.add(body);
    return document;
}

public LuceneUtil(JasperReportsContext jasperReportsContext, boolean isCaseSensitive, boolean isWholeWordsOnly, boolean removeAccents) {
    this.isCaseSensitive = isCaseSensitive;
    this.isWholeWordsOnly = isWholeWordsOnly;
    this.removeAccents = removeAccents;

    this.noneSelector = JRStyledTextAttributeSelector.getNoneSelector(jasperReportsContext);
    this.styledTextUtil = JRStyledTextUtil.getInstance(jasperReportsContext);

    fieldType = new FieldType();
    fieldType.setIndexed(true);
    fieldType.setTokenized(true);
    fieldType.setStored(true);
    fieldType.setStoreTermVectors(true);
    fieldType.setStoreTermVectorPositions(true);
    fieldType.setStoreTermVectorOffsets(true);
    fieldType.freeze();
}

protected Document newGeoDocument(OIdentifiable oIdentifiable, Shape shape) {

    FieldType ft = new FieldType();
    ft.setIndexOptions(IndexOptions.DOCS);
    ft.setStored(true);

    Document doc = new Document();

    doc.add(OLuceneIndexType
        .createField(RID, oIdentifiable.getIdentity().toString(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
    for (IndexableField f : strategy.createIndexableFields(shape)) {
      doc.add(f);
    }

    doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
    return doc;
  }

public void index(Item item) throws IOException {
    String id = item.getId();
    String text = item.getText();

    long publicationTIme = item.getPublicationTime();

    Document document = new Document();

    Field idField = new StringField("id", id, Store.YES);
    document.add(idField);

    FieldType fieldType = new FieldType();
    fieldType.setStored(true);
    fieldType.setIndexed(true);
    fieldType.setStoreTermVectors(true);
    document.add(new Field("text", text, fieldType));

    document.add(new LongField("publicationTIme", publicationTIme, LongField.TYPE_STORED));
    if(iwriter != null) {
        iwriter.addDocument(document);
    }
}

public void testExcIndexingDocBeforeDocValues() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, iwc);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setDocValueType(DocValuesType.SORTED);
  ft.freeze();
  Field field = new Field("test", "value", ft);
  field.setTokenStream(new TokenStream() {
      @Override
      public boolean incrementToken() {
        throw new RuntimeException("no");
      }
    });
  doc.add(field);
  try {
    w.addDocument(doc);
    fail("did not hit exception");
  } catch (RuntimeException re) {
    // expected
  }
  w.addDocument(new Document());
  w.close();
  dir.close();
}

protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception {
  IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
      TEST_VERSION_CURRENT, analyzer).setOpenMode(OpenMode.CREATE));
  Document doc = new Document();
  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorOffsets(true);
  customType.setStoreTermVectorPositions(true);
  for( String value: values ) {
    doc.add( new Field( F, value, customType) );
  }
  writer.addDocument( doc );
  writer.close();
  if (reader != null) reader.close();
  reader = DirectoryReader.open(dir);
}

public void testMaxThreadPriority() throws IOException {
  int pri = Thread.currentThread().getPriority();
  try {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setMaxBufferedDocs(2)
                               .setMergePolicy(newLogMergePolicy());
    ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
    IndexWriter iw = new IndexWriter(dir, conf);
    Document document = new Document();
    FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
    customType.setStoreTermVectors(true);
    document.add(newField("tvtest", "a b c", customType));
    Thread.currentThread().setPriority(Thread.MAX_PRIORITY);
    for(int i=0;i<4;i++)
      iw.addDocument(document);
    iw.close();
    dir.close();
  } finally {
    Thread.currentThread().setPriority(pri);
  }
}

public void testBasic() throws Exception {   
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
  Field f = newField("foo", "this is a test test", ft);
  doc.add(f);
  for (int i = 0; i < 100; i++) {
    w.addDocument(doc);
  }

  IndexReader reader = w.getReader();
  w.close();

  assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test")));

  DocsEnum de = TestUtil.docs(random(), reader, "foo", new BytesRef("test"), null, null, DocsEnum.FLAG_FREQS);
  while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
    assertEquals(2, de.freq());
  }

  reader.close();
  dir.close();
}

@Override
public boolean filter(FullConcept concept) {
    //index
    Document doc = new Document();

    /** The customized field type for contents field */
    FieldType contentFieldType = new FieldType();
    contentFieldType.setIndexed(true);
    contentFieldType.setStored(true);
    contentFieldType.setStoreTermVectors(true);
    contentFieldType.setTokenized(true);

    doc.add(new Field("contents", concept.getTitle() + "\n" + concept.getPlainContent(), contentFieldType));
    doc.add(new StringField("id", Integer.toString(concept.getId()), Field.Store.YES));
    doc.add(new StringField("outId", concept.getOutId(), Field.Store.YES));
    doc.add(new Field("title", concept.getTitle(), contentFieldType));

    try {
        writer.addDocument(doc);
    } catch (IOException e) {
        e.printStackTrace();
    }

    return true;
}

@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
  String externalVal = value.toString();
  String[] point = parseCommaSeparatedList(externalVal, dimension);

  // TODO: this doesn't currently support polyFields as sub-field types
  List<IndexableField> f = new ArrayList<>(dimension+1);

  if (field.indexed()) {
    for (int i=0; i<dimension; i++) {
      SchemaField sf = subField(field, i, schema);
      f.add(sf.createField(point[i], sf.indexed() && !sf.omitNorms() ? boost : 1f));
    }
  }

  if (field.stored()) {
    String storedVal = externalVal;  // normalize or not?
    FieldType customType = new FieldType();
    customType.setStored(true);
    f.add(createField(field.getName(), storedVal, customType, 1f));
  }

  return f;
}

/** test that when freqs are omitted, that totalTermFreq and sumTotalTermFreq are -1 */
public void testStats() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
      newIndexWriterConfig(new MockAnalyzer(random())));
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_ONLY);
  ft.freeze();
  Field f = newField("foo", "bar", ft);
  doc.add(f);
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();
  assertEquals(-1, ir.totalTermFreq(new Term("foo", new BytesRef("bar"))));
  assertEquals(-1, ir.getSumTotalTermFreq("foo"));
  ir.close();
  dir.close();
}

public void testFlushWithNoMerging() throws IOException {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(
      dir,
      newIndexWriterConfig(new MockAnalyzer(random()))
          .setMaxBufferedDocs(2)
          .setMergePolicy(newLogMergePolicy(10))
  );
  Document doc = new Document();
  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(true);
  customType.setStoreTermVectorOffsets(true);
  doc.add(newField("field", "aaa", customType));
  for(int i=0;i<19;i++)
    writer.addDocument(doc);
  writer.flush(false, true);
  writer.close();
  SegmentInfos sis = new SegmentInfos();
  sis.read(dir);
  // Since we flushed w/o allowing merging we should now
  // have 10 segments
  assertEquals(10, sis.size());
  dir.close();
}

public void testChangeIndexOptions() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir,
                                  new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));

  FieldType docsAndFreqs = new FieldType(TextField.TYPE_NOT_STORED);
  docsAndFreqs.setIndexOptions(IndexOptions.DOCS_AND_FREQS);

  FieldType docsOnly = new FieldType(TextField.TYPE_NOT_STORED);
  docsOnly.setIndexOptions(IndexOptions.DOCS_ONLY);

  Document doc = new Document();
  doc.add(new Field("field", "a b c", docsAndFreqs));
  w.addDocument(doc);
  w.addDocument(doc);

  doc = new Document();
  doc.add(new Field("field", "a b c", docsOnly));
  w.addDocument(doc);
  w.close();
  dir.close();
}

public void testBogusTermVectors() throws IOException {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  Field field = new Field("foo", "", ft);
  field.setTokenStream(new CannedTokenStream(
      new Token("bar", 5, 10), new Token("bar", 1, 4)
  ));
  doc.add(field);
  iw.addDocument(doc);
  iw.close();
  dir.close(); // checkindex
}

@Override
protected SolrInputField mutate(SolrInputField src) {
  SchemaField sf = schema.getFieldOrNull(src.getName());
  if (sf == null) { // remove this field
    return null;
  }
  FieldType type = PreAnalyzedField.createFieldType(sf);
  if (type == null) { // neither indexed nor stored - skip
    return null;
  }
  SolrInputField res = new SolrInputField(src.getName());
  res.setBoost(src.getBoost());
  for (Object o : src) {
    if (o == null) {
      continue;
    }
    Field pre = (Field)parser.createField(sf, o, 1.0f);
    if (pre != null) {
      res.addValue(pre, 1.0f);
    } else { // restore the original value
      log.warn("Could not parse field {} - using original value as is: {}", src.getName(), o);
      res.addValue(o, 1.0f);
    }
  }
  return res;
}

private void checkTokens(Token[] field1, Token[] field2) throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
  boolean success = false;
  try {
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    // store some term vectors for the checkindex cross-check
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(true);
    ft.setStoreTermVectorOffsets(true);

    Document doc = new Document();
    doc.add(new Field("body", new CannedTokenStream(field1), ft));
    doc.add(new Field("body", new CannedTokenStream(field2), ft));
    riw.addDocument(doc);
    riw.close();
    success = true;
  } finally {
    if (success) {
      IOUtils.close(dir);
    } else {
      IOUtils.closeWhileHandlingException(riw, dir);
    }
  }
}

public static Document Document(File f)
     throws java.io.FileNotFoundException {
  Document doc = new Document();
  doc.add(new StoredField("path", f.getPath()));
  doc.add(new StoredField("modified",
                    DateTools.timeToString(f.lastModified(), DateTools.Resolution.MINUTE)));

  //create new FieldType to store term positions (TextField is not sufficiently configurable)
  FieldType ft = new FieldType();
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
  ft.setTokenized(true);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorPositions(true);
  Field contentsField = new Field("contents", new FileReader(f), ft);

  doc.add(contentsField);
  return doc;
}

public DocState(boolean reuseFields, FieldType ft, FieldType bodyFt) {

      this.reuseFields = reuseFields;

      if (reuseFields) {
        fields =  new HashMap<>();
        numericFields = new HashMap<>();

        // Initialize the map with the default fields.
        fields.put(BODY_FIELD, new Field(BODY_FIELD, "", bodyFt));
        fields.put(TITLE_FIELD, new Field(TITLE_FIELD, "", ft));
        fields.put(DATE_FIELD, new Field(DATE_FIELD, "", ft));
        fields.put(ID_FIELD, new StringField(ID_FIELD, "", Field.Store.YES));
        fields.put(NAME_FIELD, new Field(NAME_FIELD, "", ft));

        numericFields.put(DATE_MSEC_FIELD, new LongField(DATE_MSEC_FIELD, 0L, Field.Store.NO));
        numericFields.put(TIME_SEC_FIELD, new IntField(TIME_SEC_FIELD, 0, Field.Store.NO));

        doc = new Document();
      } else {
        numericFields = null;
        fields = null;
        doc = null;
      }
    }

public void testPayloadsWithoutPositions() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
  customType.setStoreTermVectors(true);
  customType.setStoreTermVectorPositions(false);
  customType.setStoreTermVectorPayloads(true);
  customType.setStoreTermVectorOffsets(random().nextBoolean());
  doc.add(new Field("field", "foo", customType));
  try {
    writer.addDocument(doc);
    fail();
  } catch (IllegalArgumentException expected) {
    // expected
  }
  writer.close();
  dir.close();
}

public void testInfiniteFreq1() throws Exception {
  String document = "drug druggy drug drug drug";

  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newField("lyrics", document, new FieldType(TextField.TYPE_NOT_STORED)));
  iw.addDocument(doc);
  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher is = newSearcher(ir);
  PhraseQuery pq = new PhraseQuery();
  // "drug the drug"~1
  pq.add(new Term("lyrics", "drug"), 1);
  pq.add(new Term("lyrics", "drug"), 3);
  pq.setSlop(1);
  assertSaneScoring(pq, is);
  ir.close();
  dir.close();
}

public void testAreaValueSource() throws IOException {
  setupGeo();
  //test we can disable indexed for this test
  BBoxStrategy bboxStrategy = (BBoxStrategy) strategy;
  if (random().nextBoolean()) {
    FieldType fieldType = new FieldType(bboxStrategy.getFieldType());
    fieldType.setIndexed(false);
    bboxStrategy.setFieldType(fieldType);
  }

  adoc("100", ctx.makeRectangle(0, 20, 40, 80));
  adoc("999", (Shape) null);
  commit();
  checkValueSource(new ShapeAreaValueSource(bboxStrategy.makeShapeValueSource(), ctx, false),
      new float[]{800f, 0f}, 0f);
  checkValueSource(new ShapeAreaValueSource(bboxStrategy.makeShapeValueSource(), ctx, true),//geo
      new float[]{391.93f, 0f}, 0.01f);
}

private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  doc.add(new Field("textgrams", textString, ft));
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      // TODO: if we had a BinaryTermField we could fix
      // this "must be valid ut8f" limitation:
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context.utf8ToString(), Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}

protected RandomDocument(int fieldCount, int maxTermCount, Options options, String[] fieldNames, String[] sampleTerms, BytesRef[] sampleTermBytes) {
  if (fieldCount > fieldNames.length) {
    throw new IllegalArgumentException();
  }
  this.fieldNames = new String[fieldCount];
  fieldTypes = new FieldType[fieldCount];
  tokenStreams = new RandomTokenStream[fieldCount];
  Arrays.fill(fieldTypes, fieldType(options));
  final Set<String> usedFileNames = new HashSet<>();
  for (int i = 0; i < fieldCount; ++i) {
    do {
      this.fieldNames[i] = RandomPicks.randomFrom(random(), fieldNames);
    } while (usedFileNames.contains(this.fieldNames[i]));
    usedFileNames.add(this.fieldNames[i]);
    tokenStreams[i] = new RandomTokenStream(TestUtil.nextInt(random(), 1, maxTermCount), sampleTerms, sampleTermBytes);
  }
}

private FieldType getAllFieldType() {
    FieldType ft = new FieldType();
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
    ft.setTokenized(true);
    ft.freeze();
    return ft;
}

private void writeEmptyTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields fields = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(fields, null, flags, fields);
        outResponse.setExists(true);
        dr.close();
        dir.close();

    }

private void writeStandardTermVector(TermVectorsResponse outResponse) throws IOException {

        Directory dir = newDirectory();
        IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());

        conf.setOpenMode(OpenMode.CREATE);
        IndexWriter writer = new IndexWriter(dir, conf);
        FieldType type = new FieldType(TextField.TYPE_STORED);
        type.setStoreTermVectorOffsets(true);
        type.setStoreTermVectorPayloads(false);
        type.setStoreTermVectorPositions(true);
        type.setStoreTermVectors(true);
        type.freeze();
        Document d = new Document();
        d.add(new Field("id", "abc", StringField.TYPE_STORED));
        d.add(new Field("title", "the1 quick brown fox jumps over  the1 lazy dog", type));
        d.add(new Field("desc", "the1 quick brown fox jumps over  the1 lazy dog", type));

        writer.updateDocument(new Term("id", "abc"), d);
        writer.commit();
        writer.close();
        DirectoryReader dr = DirectoryReader.open(dir);
        IndexSearcher s = new IndexSearcher(dr);
        TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
        ScoreDoc[] scoreDocs = search.scoreDocs;
        int doc = scoreDocs[0].doc;
        Fields termVectors = dr.getTermVectors(doc);
        EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
        outResponse.setFields(termVectors, null, flags, termVectors);
        dr.close();
        dir.close();

    }

public void testTermVectorStringGenerationWithoutPositions() throws Exception {
    FieldType ft = new FieldType();
    ft.setStoreTermVectorOffsets(true);
    ft.setStoreTermVectorPayloads(true);
    ft.setStoreTermVectors(true);
    ft.setStoreTermVectorPositions(false);
    String ftOpts = FieldMapper.termVectorOptionsToString(ft);
    assertThat(ftOpts, equalTo("with_offsets"));
}

@Override
public List<Document> getDoc(List<IfcProductRecordText> items) {

    List<Document> docs = new ArrayList<Document>();  
       FieldType storedType = new FieldType();  
       storedType.setIndexed(true);  
       storedType.setStored(true);  
       storedType.setTokenized(true); 

       FieldType unTokeType = new FieldType();
       unTokeType.setIndexed(true);  
       unTokeType.setStored(true);   
       unTokeType.setTokenized(false); 

       for (IfcProductRecordText record : items) {
        Document doc = new Document();

        Field oid = genStringFieldCheckNull(Key_Oid, record.getOid(), unTokeType);
        Field type = genStringFieldCheckNull(Key_Type, record.getType(), storedType);
        Field name = genStringFieldCheckNull(Key_Name, record.getName(), storedType);
        Field detail = genStringFieldCheckNull(Key_Detail, record.getDetail(), storedType);

        doc.add(oid);
        doc.add(type);
        doc.add(name);
        doc.add(detail);

        docs.add(doc);
       }
    return docs;


}

private Field genStringFieldCheckNull(String fieldName, String fieldValue, FieldType type) {
    if (fieldValue != null) {
         return new Field(fieldName, fieldValue, type);
    } else {
        return new Field(fieldName, "", type);
    }
}

@Override
public GeoPointFieldMapper build(BuilderContext context, String simpleName, MappedFieldType fieldType,
                                 MappedFieldType defaultFieldType, Settings indexSettings, ContentPath.Type pathType, DoubleFieldMapper latMapper,
                                 DoubleFieldMapper lonMapper, StringFieldMapper geoHashMapper, MultiFields multiFields, Explicit<Boolean> ignoreMalformed,
                                 CopyTo copyTo) {
    fieldType.setTokenized(false);
    if (context.indexCreatedVersion().before(Version.V_2_3_0)) {
        fieldType.setNumericPrecisionStep(GeoPointField.PRECISION_STEP);
        fieldType.setNumericType(FieldType.NumericType.LONG);
    }
    setupFieldType(context);
    return new GeoPointFieldMapper(simpleName, fieldType, defaultFieldType, indexSettings, pathType, latMapper, lonMapper,
            geoHashMapper, multiFields, ignoreMalformed, copyTo);
}

@Override
public GeoPointFieldMapper build(BuilderContext context) {
    if (context.indexCreatedVersion().before(Version.V_2_3_0)) {
        fieldType.setNumericPrecisionStep(GeoPointField.PRECISION_STEP);
        fieldType.setNumericType(FieldType.NumericType.LONG);
    }
    return super.build(context);
}

private static FieldType getType(int numDims) {
    FieldType type = new FieldType();
    type.setDimensions(numDims, Float.BYTES);
    type.setDocValuesType(DocValuesType.NUMERIC);
    type.freeze();
    return type;
}

private static FieldType getType(int numDims) {
    FieldType type = new FieldType();
    type.setDocValuesType(DocValuesType.NUMERIC);
    type.setDimensions(numDims, Integer.BYTES);
    type.freeze();
    return type;
}