@Override public final void visitDocument(int n, StoredFieldVisitor visitor) throws IOException { seekIndex(n); fieldsStream.seek(indexStream.readLong()); final int numFields = fieldsStream.readVInt(); for (int fieldIDX = 0; fieldIDX < numFields; fieldIDX++) { int fieldNumber = fieldsStream.readVInt(); FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); int bits = fieldsStream.readByte() & 0xFF; assert bits <= (FIELD_IS_NUMERIC_MASK | FIELD_IS_BINARY): "bits=" + Integer.toHexString(bits); switch(visitor.needsField(fieldInfo)) { case YES: readField(visitor, fieldInfo, bits); break; case NO: skipField(bits); break; case STOP: return; } } }
public final void visitDocument(int n, StoredFieldVisitor visitor) throws CorruptIndexException, IOException { seekIndex(n); fieldsStream.seek(indexStream.readLong()); final int numFields = fieldsStream.readVInt(); for (int fieldIDX = 0; fieldIDX < numFields; fieldIDX++) { int fieldNumber = fieldsStream.readVInt(); FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); int bits = fieldsStream.readByte() & 0xFF; assert bits <= (FIELD_IS_NUMERIC_MASK | FIELD_IS_BINARY): "bits=" + Integer.toHexString(bits); switch(visitor.needsField(fieldInfo)) { case YES: readField(visitor, fieldInfo, bits); break; case NO: skipField(bits); break; case STOP: return; } } }
private void readField(BytesRef type, FieldInfo fieldInfo, StoredFieldVisitor visitor) throws IOException { readLine(); assert StringHelper.startsWith(scratch.get(), VALUE); if (type == TYPE_STRING) { visitor.stringField(fieldInfo, new String(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length, StandardCharsets.UTF_8)); } else if (type == TYPE_BINARY) { byte[] copy = new byte[scratch.length()-VALUE.length]; System.arraycopy(scratch.bytes(), VALUE.length, copy, 0, copy.length); visitor.binaryField(fieldInfo, copy); } else if (type == TYPE_INT) { scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length); visitor.intField(fieldInfo, Integer.parseInt(scratchUTF16.toString())); } else if (type == TYPE_LONG) { scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length); visitor.longField(fieldInfo, Long.parseLong(scratchUTF16.toString())); } else if (type == TYPE_FLOAT) { scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length); visitor.floatField(fieldInfo, Float.parseFloat(scratchUTF16.toString())); } else if (type == TYPE_DOUBLE) { scratchUTF16.copyUTF8Bytes(scratch.bytes(), VALUE.length, scratch.length()-VALUE.length); visitor.doubleField(fieldInfo, Double.parseDouble(scratchUTF16.toString())); } }
private void readField(BytesRef type, FieldInfo fieldInfo, StoredFieldVisitor visitor) throws IOException { readLine(); assert StringHelper.startsWith(scratch, VALUE); if (type == TYPE_STRING) { visitor.stringField(fieldInfo, new String(scratch.bytes, scratch.offset+VALUE.length, scratch.length-VALUE.length, "UTF-8")); } else if (type == TYPE_BINARY) { byte[] copy = new byte[scratch.length-VALUE.length]; System.arraycopy(scratch.bytes, scratch.offset+VALUE.length, copy, 0, copy.length); visitor.binaryField(fieldInfo, copy); } else if (type == TYPE_INT) { UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+VALUE.length, scratch.length-VALUE.length, scratchUTF16); visitor.intField(fieldInfo, Integer.parseInt(scratchUTF16.toString())); } else if (type == TYPE_LONG) { UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+VALUE.length, scratch.length-VALUE.length, scratchUTF16); visitor.longField(fieldInfo, Long.parseLong(scratchUTF16.toString())); } else if (type == TYPE_FLOAT) { UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+VALUE.length, scratch.length-VALUE.length, scratchUTF16); visitor.floatField(fieldInfo, Float.parseFloat(scratchUTF16.toString())); } else if (type == TYPE_DOUBLE) { UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+VALUE.length, scratch.length-VALUE.length, scratchUTF16); visitor.doubleField(fieldInfo, Double.parseDouble(scratchUTF16.toString())); } }
/** * Here, we verify that the order that we add fields to a document counts, and not the lexi order * of the field. This means that heavily accessed fields that use field selector should be added * first (with load and break). */ public void testOrdering() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)); Document document = new Document(); document.add(new TextField("_id", "1", Field.Store.YES)); document.add(new TextField("#id", "1", Field.Store.YES)); indexWriter.addDocument(document); IndexReader reader = DirectoryReader.open(indexWriter); IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1); final ArrayList<String> fieldsOrder = new ArrayList<>(); searcher.doc(topDocs.scoreDocs[0].doc, new StoredFieldVisitor() { @Override public Status needsField(FieldInfo fieldInfo) throws IOException { fieldsOrder.add(fieldInfo.name); return Status.YES; } }); assertThat(fieldsOrder.size(), equalTo(2)); assertThat(fieldsOrder.get(0), equalTo("_id")); assertThat(fieldsOrder.get(1), equalTo("#id")); indexWriter.close(); }
private void readField(StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException { final int numeric = bits & FIELD_IS_NUMERIC_MASK; if (numeric != 0) { switch(numeric) { case FIELD_IS_NUMERIC_INT: visitor.intField(info, fieldsStream.readInt()); return; case FIELD_IS_NUMERIC_LONG: visitor.longField(info, fieldsStream.readLong()); return; case FIELD_IS_NUMERIC_FLOAT: visitor.floatField(info, Float.intBitsToFloat(fieldsStream.readInt())); return; case FIELD_IS_NUMERIC_DOUBLE: visitor.doubleField(info, Double.longBitsToDouble(fieldsStream.readLong())); return; default: throw new CorruptIndexException("Invalid numeric type: " + Integer.toHexString(numeric)); } } else { final int length = fieldsStream.readVInt(); byte bytes[] = new byte[length]; fieldsStream.readBytes(bytes, 0, length); if ((bits & FIELD_IS_BINARY) != 0) { visitor.binaryField(info, bytes); } else { visitor.stringField(info, new String(bytes, 0, bytes.length, StandardCharsets.UTF_8)); } } }
private static void readField(DataInput in, StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException { switch (bits & TYPE_MASK) { case BYTE_ARR: int length = in.readVInt(); byte[] data = new byte[length]; in.readBytes(data, 0, length); visitor.binaryField(info, data); break; case STRING: length = in.readVInt(); data = new byte[length]; in.readBytes(data, 0, length); visitor.stringField(info, new String(data, StandardCharsets.UTF_8)); break; case NUMERIC_INT: visitor.intField(info, in.readInt()); break; case NUMERIC_FLOAT: visitor.floatField(info, Float.intBitsToFloat(in.readInt())); break; case NUMERIC_LONG: visitor.longField(info, in.readLong()); break; case NUMERIC_DOUBLE: visitor.doubleField(info, Double.longBitsToDouble(in.readLong())); break; default: throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits)); } }
/** Visit a document's fields using a {@link StoredFieldVisitor} * This method does not currently add to the Solr document cache. * * @see IndexReader#document(int, StoredFieldVisitor) */ @Override public void doc(int n, StoredFieldVisitor visitor) throws IOException { if (documentCache != null) { Document cached = documentCache.get(n); if (cached != null) { visitFromCached(cached, visitor); return; } } getIndexReader().document(n, visitor); }
private void readField(StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException { final int numeric = bits & FIELD_IS_NUMERIC_MASK; if (numeric != 0) { switch(numeric) { case FIELD_IS_NUMERIC_INT: visitor.intField(info, fieldsStream.readInt()); return; case FIELD_IS_NUMERIC_LONG: visitor.longField(info, fieldsStream.readLong()); return; case FIELD_IS_NUMERIC_FLOAT: visitor.floatField(info, Float.intBitsToFloat(fieldsStream.readInt())); return; case FIELD_IS_NUMERIC_DOUBLE: visitor.doubleField(info, Double.longBitsToDouble(fieldsStream.readLong())); return; default: throw new CorruptIndexException("Invalid numeric type: " + Integer.toHexString(numeric)); } } else { final int length = fieldsStream.readVInt(); byte bytes[] = new byte[length]; fieldsStream.readBytes(bytes, 0, length); if ((bits & FIELD_IS_BINARY) != 0) { visitor.binaryField(info, bytes); } else { visitor.stringField(info, new String(bytes, 0, bytes.length, IOUtils.CHARSET_UTF_8)); } } }
private static void readField(ByteArrayDataInput in, StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException { switch (bits & TYPE_MASK) { case BYTE_ARR: int length = in.readVInt(); byte[] data = new byte[length]; in.readBytes(data, 0, length); visitor.binaryField(info, data); break; case STRING: length = in.readVInt(); data = new byte[length]; in.readBytes(data, 0, length); visitor.stringField(info, new String(data, IOUtils.CHARSET_UTF_8)); break; case NUMERIC_INT: visitor.intField(info, in.readInt()); break; case NUMERIC_FLOAT: visitor.floatField(info, Float.intBitsToFloat(in.readInt())); break; case NUMERIC_LONG: visitor.longField(info, in.readLong()); break; case NUMERIC_DOUBLE: visitor.doubleField(info, Double.longBitsToDouble(in.readLong())); break; default: throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits)); } }
/** * Only wrap with a {@link DelegatingRestrictedFieldVisitor} if necessary * * @param visitor To wrap * @return {@link RestrictedStoredFieldVisitor} */ public static RestrictedStoredFieldVisitor wrap( StoredFieldVisitor visitor) { return visitor instanceof RestrictedStoredFieldVisitor ? (RestrictedStoredFieldVisitor) visitor : new DelegatingRestrictedFieldVisitor(visitor); }
private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException, IOException { reader.document(docId, new StoredFieldVisitor() { @Override public Status needsField(FieldInfo fieldInfo) throws IOException { if (ROW_ID.equals(fieldInfo.name)) { return StoredFieldVisitor.Status.STOP; } return StoredFieldVisitor.Status.NO; } }); return reader.document(docId).get(ROW_ID); }
public static ResetableDocumentStoredFieldVisitor getFieldSelector(final Selector selector) { return new ResetableDocumentStoredFieldVisitor() { @Override public Status needsField(FieldInfo fieldInfo) throws IOException { if (ROW_ID.equals(fieldInfo.name)) { return StoredFieldVisitor.Status.YES; } if (RECORD_ID.equals(fieldInfo.name)) { return StoredFieldVisitor.Status.YES; } if (PRIME_DOC.equals(fieldInfo.name)) { return StoredFieldVisitor.Status.NO; } if (FAMILY.equals(fieldInfo.name)) { return StoredFieldVisitor.Status.YES; } if (selector.columnFamiliesToFetch == null && selector.columnsToFetch == null) { return StoredFieldVisitor.Status.YES; } String columnFamily = getColumnFamily(fieldInfo.name); if (selector.columnFamiliesToFetch != null) { if (selector.columnFamiliesToFetch.contains(columnFamily)) { return StoredFieldVisitor.Status.YES; } } String columnName = getColumnName(fieldInfo.name); if (selector.columnsToFetch != null) { Set<String> columns = selector.columnsToFetch.get(columnFamily); if (columns != null && columns.contains(columnName)) { return StoredFieldVisitor.Status.YES; } } return StoredFieldVisitor.Status.NO; } }; }
private static AtomicReader setDocSize(AtomicReader reader, final int count) { return new FilterAtomicReader(reader) { @Override public Bits getLiveDocs() { return new Bits() { @Override public boolean get(int index) { return true; } @Override public int length() { return count; } }; } @Override public int numDocs() { return count; } @Override public int maxDoc() { return count; } @Override public void document(int docID, StoredFieldVisitor visitor) throws IOException { // Do nothing } }; }
/** * Visit a document's fields using a {@link StoredFieldVisitor} This method does not currently * add to the Solr document cache. * * @see IndexReader#document(int, StoredFieldVisitor) */ @Override public void doc(int n, StoredFieldVisitor visitor) throws IOException { if(documentCache != null) { Document cached = documentCache.get(n); if(cached != null) { visitFromCached(cached, visitor); return; } } getIndexReader().document(n, visitor); }
private static void readField(DataInput in, StoredFieldVisitor visitor, FieldInfo info, int bits) throws IOException { switch (bits & TYPE_MASK) { case BYTE_ARR: int length = in.readVInt(); byte[] data = new byte[length]; in.readBytes(data, 0, length); visitor.binaryField(info, data); break; case STRING: length = in.readVInt(); data = new byte[length]; in.readBytes(data, 0, length); visitor.stringField(info, new String(data, IOUtils.CHARSET_UTF_8)); break; case NUMERIC_INT: visitor.intField(info, in.readInt()); break; case NUMERIC_FLOAT: visitor.floatField(info, Float.intBitsToFloat(in.readInt())); break; case NUMERIC_LONG: visitor.longField(info, in.readLong()); break; case NUMERIC_DOUBLE: visitor.doubleField(info, Double.longBitsToDouble(in.readLong())); break; default: throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits)); } }