private void addLuceneIndexFields(String indexField, List<IndexableField> list, JsonNode node, JsonSchema nodeSchema) { JsonNode.Type type = nodeSchema.getSchemaType(); if (type == JsonNode.Type.ARRAY) { for (int i = 0; i < node.getSize(); i++) { addLuceneIndexFields(indexField, list, node.get(i), nodeSchema.getItemSchema()); } } else if (type == JsonNode.Type.OBJECT) { Iterator<String> properties = node.getProperties(); while (properties.hasNext()) { String propName = properties.next(); // Index property key for object nodes list.add(new StringField(indexField, propName, Field.Store.NO)); } } else if (type == JsonNode.Type.STRING) { list.add(new StringField(indexField, node.asString(), Field.Store.NO)); } else if (type == JsonNode.Type.BOOLEAN) { list.add(new StringField(indexField, node.asString(), Field.Store.NO)); } else if (type == JsonNode.Type.INTEGER) { list.add(new LongField(indexField, node.asLong(), Field.Store.NO)); } else if (type == JsonNode.Type.NUMBER) { list.add(new DoubleField(indexField, node.asDouble(), Field.Store.NO)); } else { throw new UnsupportedOperationException("Node type " + type + " not supported for index field " + indexField); } }
/** * resolve field convertable premitive type * * premitive type * byte, short, int, long, float, double, char, boolean * * @param type field type * @return lucene field type */ private Class<? extends Field> resolveField(Type type) { if(type == String.class) { return StringField.class; } else if (type == Double.class || type == double.class) { return DoubleField.class; } else if(type == Float.class || type == float.class) { return FloatField.class; } else if(type == Integer.class || type == int.class || type == Short.class || type == short.class || type == Boolean.class || type == boolean.class || type == Byte.class || type == byte.class || type == Character.class || type == char.class) { return IntField.class; } else if(type == Long.class || type == long.class) { return LongField.class; } return null; }
@Override public void setUp() throws Exception { super.setUp(); dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); int numDocs = TestUtil.nextInt(random(), 2049, 4000); for (int i = 0; i < numDocs; i++) { Document document = new Document(); document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO)); document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO)); document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO)); document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO)); document.add(new IntField("int", random().nextInt(), Field.Store.NO)); document.add(new LongField("long", random().nextLong(), Field.Store.NO)); document.add(new FloatField("float", random().nextFloat(), Field.Store.NO)); document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO)); document.add(new NumericDocValuesField("intdocvalues", random().nextInt())); document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat())); iw.addDocument(document); } reader = iw.getReader(); iw.close(); searcher = newSearcher(reader); }
public void testDoubleFieldMinMax() throws Exception { Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir); int numDocs = atLeast(100); double minValue = Double.POSITIVE_INFINITY; double maxValue = Double.NEGATIVE_INFINITY; for(int i=0;i<numDocs;i++ ){ Document doc = new Document(); double num = random().nextDouble(); minValue = Math.min(num, minValue); maxValue = Math.max(num, maxValue); doc.add(new DoubleField("field", num, Field.Store.NO)); w.addDocument(doc); } IndexReader r = w.getReader(); Terms terms = MultiFields.getTerms(r, "field"); assertEquals(minValue, NumericUtils.sortableLongToDouble(NumericUtils.getMinLong(terms)), 0.0); assertEquals(maxValue, NumericUtils.sortableLongToDouble(NumericUtils.getMaxLong(terms)), 0.0); r.close(); w.close(); dir.close(); }
@Override public void configure(String fieldNameForThisInstance, Map<String, String> properties, Configuration configuration) { String precisionStepStr = properties.get(NUMERIC_PRECISION_STEP); if (precisionStepStr != null) { _precisionStep = Integer.parseInt(precisionStepStr); _typeStored = new FieldType(DoubleField.TYPE_STORED); _typeStored.setNumericPrecisionStep(_precisionStep); _typeStored.freeze(); _typeNotStored = new FieldType(DoubleField.TYPE_NOT_STORED); _typeNotStored.setNumericPrecisionStep(_precisionStep); _typeNotStored.freeze(); } else { _typeStored = DoubleField.TYPE_STORED; _typeNotStored = DoubleField.TYPE_NOT_STORED; } }
@Override public void setUp() throws Exception { super.setUp(); dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); int numDocs = _TestUtil.nextInt(random(), 2049, 4000); for (int i = 0; i < numDocs; i++) { Document document = new Document(); document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO)); document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO)); document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO)); document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO)); document.add(new IntField("int", random().nextInt(), Field.Store.NO)); document.add(new LongField("long", random().nextLong(), Field.Store.NO)); document.add(new FloatField("float", random().nextFloat(), Field.Store.NO)); document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO)); document.add(new NumericDocValuesField("intdocvalues", random().nextInt())); document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat())); iw.addDocument(document); } reader = iw.getReader(); iw.close(); searcher = newSearcher(reader); }
private void addCorner(IndexWriter iwriter, StreetVertex sv) throws IOException { String mainStreet = null; String crossStreet = null; // TODO score based on OSM street type, using intersection nodes instead of vertices. for (PlainStreetEdge pse : Iterables.filter(sv.getOutgoing(), PlainStreetEdge.class)) { if (mainStreet == null) { mainStreet = pse.getName(); } else { crossStreet = pse.getName(); } } if ((mainStreet == null) || (crossStreet == null)) { return; } if (mainStreet.equals(crossStreet)) { return; } Document doc = new Document(); doc.add(new TextField("name", mainStreet + " & " + crossStreet, Field.Store.YES)); doc.add(new DoubleField("lat", sv.getLat(), Field.Store.YES)); doc.add(new DoubleField("lon", sv.getLon(), Field.Store.YES)); doc.add(new StringField("category", Category.CORNER.name(), Field.Store.YES)); iwriter.addDocument(doc); }
void addToDoc(Document doc, Double value){ Preconditions.checkArgument(valueType == Double.class); if(value == null){ return; } doc.add(new DoubleField(indexFieldName, value, stored ? Store.YES : Store.NO)); if(isSorted()){ Preconditions.checkArgument(sortedValueType == SearchFieldSorting.FieldType.DOUBLE); doc.add(new DoubleDocValuesField(indexFieldName, value)); } }
public Field getField(ValueSource value) { if (value.isNull()) return null; Field.Store store = Field.Store.NO; // Only store hkey. switch (fieldType) { case INT: switch (TInstance.underlyingType(value.getType())) { case INT_8: return new IntField(name, value.getInt8(), store); case INT_16: return new IntField(name, value.getInt16(), store); case UINT_16: return new IntField(name, value.getUInt16(), store); case INT_32: default: return new IntField(name, value.getInt32(), store); } case LONG: return new LongField(name, value.getInt64(), store); case FLOAT: return new FloatField(name, value.getFloat(), store); case DOUBLE: return new DoubleField(name, value.getDouble(), store); case STRING: switch (TInstance.underlyingType(value.getType())) { case STRING: return new StringField(name, value.getString(), store); default: { StringBuilder str = new StringBuilder(); value.getType().format(value, AkibanAppender.of(str)); return new StringField(name, str.toString(), store); } } case TEXT: return new TextField(name, value.getString(), store); default: return null; } }
private Document createDocument(int rowId, Object[] row) { Document doc = new Document(); if (row == null) return doc; try { //FieldType crosscol = new FieldType(); //crosscol.setStored(true); //crosscol.setTokenized(false); //doc.add(new Field("at__rowid__", Integer.toString(rowId), //Field.Store.YES, Field.Index.UN_TOKENIZED)); // for cross column search doc.add(new StringField("at__rowid__", Integer.toString(rowId),Field.Store.YES) ); // crosscol)); //FieldType forhive = new FieldType(); //forhive.setStored(true); //forhive.setTokenized(true); for (int i = 0; i < row.length; i++) { if (row[i] != null && colName[i] != null) //doc.add(new Field(colName[i], row[i].toString(), // Field.Store.NO, Field.Index.TOKENIZED)); we have to do hive query // Field.Store.YES, Field.Index.TOKENIZED)); if(row[i] instanceof java.lang.Number ) { doc.add(new DoubleField(colName[i], new Double(row[i].toString()),Field.Store.YES)); } else if (row[i] instanceof java.util.Date) doc.add(new TextField(colName[i], DateTools.dateToString((Date) row[i],DateTools.Resolution.SECOND),Field.Store.YES)); else doc.add(new TextField(colName[i], row[i].toString(),Field.Store.YES)); //forhive)); } } catch (Exception e) { e.printStackTrace(); System.out.println("\n " + e.getMessage()); System.out.println("\n Error: Document Creation Error"); } return doc; }
/** * add lucene field in document * @param doc document * @param name fieldName * @param val value * @param type field original Type * @param store store * @param textFieldable isTextField * @return if true, added document */ private boolean addField(Document doc, String name, Object val, Type type, Store store, TextFieldable textFieldable, NoIndex noIndex) { boolean add = true; if (noIndex != null) { if (type == Character.class || type == char.class) { val = (int) val; } else if(type == Boolean.class || type == boolean.class) { val = (boolean)val ? 1 : 0; } doc.add(new StoredField(name, val.toString())); } else if (textFieldable != null) { doc.add(new TextField(name, val.toString(), store)); } else if(type == String.class) { doc.add(new StringField(name, val.toString(), store)); }else if (type == Double.class || type == double.class) { doc.add(new DoubleField(name, (double) val, store)); } else if(type == Float.class || type == float.class) { doc.add(new FloatField(name, (float) val, store)); } else if(type == Short.class || type == short.class || type == Integer.class || type == int.class || type == Byte.class || type == byte.class) { doc.add(new IntField(name, Integer.valueOf(val.toString()), store)); } else if(type == Character.class || type == char.class) { doc.add(new IntField(name, Integer.valueOf((char)val), store)); } else if(type == Boolean.class || type == boolean.class) { if ((boolean)val) { doc.add(new IntField(name, 1, store)); } else { doc.add(new IntField(name, 0, store)); } } else if(type == Long.class || type == long.class) { doc.add(new LongField(name, (long) val, store)); } else { add = false; } return add; }
@Override public int doLogic() throws Exception { List<IndexableField> fields = doc.getFields(); Analyzer analyzer = getRunData().getAnalyzer(); int tokenCount = 0; for(final IndexableField field : fields) { if (!field.fieldType().tokenized() || field instanceof IntField || field instanceof LongField || field instanceof FloatField || field instanceof DoubleField) { continue; } final TokenStream stream = field.tokenStream(analyzer, null); // reset the TokenStream to the first token stream.reset(); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); while(stream.incrementToken()) { termAtt.fillBytesRef(); tokenCount++; } stream.end(); stream.close(); } totalTokenCount += tokenCount; return tokenCount; }
/** Build the example index. */ public void index() throws IOException { IndexWriter writer = new IndexWriter(indexDir, new IndexWriterConfig(FacetExamples.EXAMPLES_VER, new WhitespaceAnalyzer())); // TODO: we could index in radians instead ... saves all the conversions in getBoundingBoxFilter // Add documents with latitude/longitude location: Document doc = new Document(); doc.add(new DoubleField("latitude", 40.759011, Field.Store.NO)); doc.add(new DoubleField("longitude", -73.9844722, Field.Store.NO)); writer.addDocument(doc); doc = new Document(); doc.add(new DoubleField("latitude", 40.718266, Field.Store.NO)); doc.add(new DoubleField("longitude", -74.007819, Field.Store.NO)); writer.addDocument(doc); doc = new Document(); doc.add(new DoubleField("latitude", 40.7051157, Field.Store.NO)); doc.add(new DoubleField("longitude", -74.0088305, Field.Store.NO)); writer.addDocument(doc); // Open near-real-time searcher searcher = new IndexSearcher(DirectoryReader.open(writer, true)); writer.close(); }
public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix) { super(ctx, fieldNamePrefix); field_bbox = fieldNamePrefix; field_minX = fieldNamePrefix + SUFFIX_MINX; field_maxX = fieldNamePrefix + SUFFIX_MAXX; field_minY = fieldNamePrefix + SUFFIX_MINY; field_maxY = fieldNamePrefix + SUFFIX_MAXY; field_xdl = fieldNamePrefix + SUFFIX_XDL; FieldType fieldType = new FieldType(DoubleField.TYPE_NOT_STORED); fieldType.setNumericPrecisionStep(8);//Solr's default fieldType.setDocValueType(FieldInfo.DocValuesType.NUMERIC); setFieldType(fieldType); }
/** @see #createIndexableFields(com.spatial4j.core.shape.Shape) */ public Field[] createIndexableFields(Point point) { FieldType doubleFieldType = new FieldType(DoubleField.TYPE_NOT_STORED); doubleFieldType.setNumericPrecisionStep(precisionStep); Field[] f = new Field[2]; f[0] = new DoubleField(fieldNameX, point.getX(), doubleFieldType); f[1] = new DoubleField(fieldNameY, point.getY(), doubleFieldType); return f; }
@Override public void setUp() throws Exception { super.setUp(); dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("id", "1", Field.Store.YES)); doc.add(newTextField("body", "some contents and more contents", Field.Store.NO)); doc.add(new NumericDocValuesField("popularity", 5)); doc.add(new DoubleField("latitude", 40.759011, Field.Store.NO)); doc.add(new DoubleField("longitude", -73.9844722, Field.Store.NO)); iw.addDocument(doc); doc = new Document(); doc.add(newStringField("id", "2", Field.Store.YES)); doc.add(newTextField("body", "another document with different contents", Field.Store.NO)); doc.add(new NumericDocValuesField("popularity", 20)); doc.add(new DoubleField("latitude", 40.718266, Field.Store.NO)); doc.add(new DoubleField("longitude", -74.007819, Field.Store.NO)); iw.addDocument(doc); doc = new Document(); doc.add(newStringField("id", "3", Field.Store.YES)); doc.add(newTextField("body", "crappy contents", Field.Store.NO)); doc.add(new NumericDocValuesField("popularity", 2)); doc.add(new DoubleField("latitude", 40.7051157, Field.Store.NO)); doc.add(new DoubleField("longitude", -74.0088305, Field.Store.NO)); iw.addDocument(doc); reader = iw.getReader(); searcher = new IndexSearcher(reader); iw.close(); }
@Override public void doubleField(FieldInfo fieldInfo, double value) { FieldType ft = new FieldType(DoubleField.TYPE_NOT_STORED); ft.setStored(true); ft.setIndexed(fieldInfo.isIndexed()); doc.add(new DoubleField(fieldInfo.name, value, ft)); }
@Override public int doLogic() throws Exception { List<IndexableField> fields = doc.getFields(); Analyzer analyzer = getRunData().getAnalyzer(); int tokenCount = 0; for(final IndexableField field : fields) { if (!field.fieldType().tokenized() || field instanceof IntField || field instanceof LongField || field instanceof FloatField || field instanceof DoubleField) { continue; } final TokenStream stream = field.tokenStream(analyzer); // reset the TokenStream to the first token stream.reset(); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); while(stream.incrementToken()) { termAtt.fillBytesRef(); tokenCount++; } } totalTokenCount += tokenCount; return tokenCount; }
@Override public void setUp() throws Exception { super.setUp(); dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir); int numDocs = atLeast(200); for (int i = 0; i < numDocs; i++) { Document document = new Document(); document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO)); document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO)); document.add(newStringField("byte", "" + ((byte) random().nextInt()), Field.Store.NO)); document.add(newStringField("short", "" + ((short) random().nextInt()), Field.Store.NO)); document.add(new IntField("int", random().nextInt(), Field.Store.NO)); document.add(new LongField("long", random().nextLong(), Field.Store.NO)); document.add(new FloatField("float", random().nextFloat(), Field.Store.NO)); document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO)); document.add(newStringField("bytes", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO)); document.add(newStringField("bytesval", _TestUtil.randomRealisticUnicodeString(random()), Field.Store.NO)); document.add(new DoubleField("double", random().nextDouble(), Field.Store.NO)); if (supportsDocValues) { document.add(new NumericDocValuesField("intdocvalues", random().nextInt())); document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat())); document.add(new SortedDocValuesField("sortedbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random())))); document.add(new SortedDocValuesField("sortedbytesdocvaluesval", new BytesRef(_TestUtil.randomRealisticUnicodeString(random())))); document.add(new BinaryDocValuesField("straightbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random())))); } iw.addDocument(document); } reader = iw.getReader(); iw.close(); searcher = newSearcher(reader); }
@Override public Iterable<? extends Field> getFieldsForColumn(String family, Column column) { String name = getName(family, column.getName()); double value = Double.parseDouble(column.getValue()); DoubleField field = new DoubleField(name, value, _typeStored); if (isSortEnable()) { return addSort(name, Double.doubleToRawLongBits(value), field); } return makeIterable(field); }
@Override public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) { String name = getName(family, column.getName(), subName); double value = Double.parseDouble(column.getValue()); DoubleField field = new DoubleField(name, value, _typeNotStored); if (isSortEnable()) { return addSort(name, Double.doubleToRawLongBits(value), field); } return makeIterable(field); }
@Override public int doLogic() throws Exception { List<IndexableField> fields = doc.getFields(); Analyzer analyzer = getRunData().getAnalyzer(); int tokenCount = 0; for(final IndexableField field : fields) { if (!field.fieldType().tokenized() || field instanceof IntField || field instanceof LongField || field instanceof FloatField || field instanceof DoubleField) { continue; } final TokenStream stream = field.tokenStream(analyzer); // reset the TokenStream to the first token stream.reset(); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); while(stream.incrementToken()) { termAtt.fillBytesRef(); tokenCount++; } stream.end(); stream.close(); } totalTokenCount += tokenCount; return tokenCount; }
public void addPatchToIndex(String productName, int patchX, int patchY, Properties featureValues) throws IOException { Document doc = new Document(); doc.add(new LongField("id", docID, storedLongType)); doc.add(new TextField("product", productName, Field.Store.YES)); doc.add(new IntField("px", patchX, storedIntType)); doc.add(new IntField("py", patchY, storedIntType)); // 'rnd' is for selecting random subsets doc.add(new DoubleField("rnd", Math.random(), unstoredDoubleType)); // todo - put useful values into 'lat', 'lon', 'time' doc.add(new FloatField("lat", -90 + 180 * (float) Math.random(), unstoredFloatType)); doc.add(new FloatField("lon", -180 + 360 * (float) Math.random(), unstoredFloatType)); doc.add(new LongField("time", docID, unstoredLongType)); for (FeatureFieldFactory featureFieldFactory : featureFieldFactories) { String fieldName = featureFieldFactory.getFieldName(); String fieldValue = featureValues.getProperty(fieldName); if (fieldValue != null) { IndexableField indexableField = featureFieldFactory.createIndexableField(fieldValue, Field.Store.YES); doc.add(indexableField); } } indexWriter.addDocument(doc); System.out.printf("[%7d]: product:\"%s\", px:%d, py:%d\n", docID, productName, patchX, patchY); docID++; }
private void addStop(IndexWriter iwriter, Stop stop) throws IOException { Document doc = new Document(); doc.add(new TextField("name", stop.getName(), Field.Store.YES)); if (stop.getCode() != null) { doc.add(new StringField("code", stop.getCode(), Field.Store.YES)); } doc.add(new DoubleField("lat", stop.getLat(), Field.Store.YES)); doc.add(new DoubleField("lon", stop.getLon(), Field.Store.YES)); doc.add(new StringField("id", stop.getId().toString(), Field.Store.YES)); doc.add(new StringField("category", Category.STOP.name(), Field.Store.YES)); iwriter.addDocument(doc); }
/** * {@inheritDoc} * @see DoubleIndexFieldConf */ @Override public Field buildField(final Double content) { return new DoubleField(getName(), content, isStored()); }
@Override public List<Field> createIndexedFields() { Field field = new DoubleField(name, this.value, Field.Store.NO); field.setBoost(0.1f); return Arrays.asList(field); }