@Override public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException { BinaryDocValues instance = binaryInstances.get(field.number); if (instance == null) { switch(LegacyDocValuesType.valueOf(field.getAttribute(legacyKey))) { case BYTES_FIXED_STRAIGHT: instance = loadBytesFixedStraight(field); break; case BYTES_VAR_STRAIGHT: instance = loadBytesVarStraight(field); break; case BYTES_FIXED_DEREF: instance = loadBytesFixedDeref(field); break; case BYTES_VAR_DEREF: instance = loadBytesVarDeref(field); break; default: throw new AssertionError(); } binaryInstances.put(field.number, instance); } return instance; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes); return new LongBinaryDocValues() { final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength)); @Override public BytesRef get(long id) { long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1)); long endAddress = bytes.offset + addresses.get(id); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); data.readBytes(term.bytes, 0, length); term.length = length; return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes); return new LongBinaryDocValues() { final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength)); @Override public BytesRef get(long id) { long startAddress = bytes.offset + addresses.get(id); long endAddress = bytes.offset + addresses.get(id+1); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); data.readBytes(term.bytes, 0, length); term.length = length; return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.slice("fixed-binary", bytes.offset, bytes.count * bytes.maxLength); final BytesRef term = new BytesRef(bytes.maxLength); final byte[] buffer = term.bytes; final int length = term.length = bytes.maxLength; return new LongBinaryDocValues() { @Override public BytesRef get(long id) { try { data.seek(id * length); data.readBytes(buffer, 0, buffer.length); return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final MonotonicBlockPackedReader addresses = getAddressInstance(field, bytes); final IndexInput data = this.data.slice("var-binary", bytes.offset, bytes.addressesOffset - bytes.offset); final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength)); final byte buffer[] = term.bytes; return new LongBinaryDocValues() { @Override public BytesRef get(long id) { long startAddress = addresses.get(id); long endAddress = addresses.get(id+1); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); data.readBytes(buffer, 0, length); term.length = length; return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
public BinaryDocValues getTerms(AtomicReader reader, String field, boolean setDocsWithField, float acceptableOverheadRatio) throws IOException { BinaryDocValues valuesIn = reader.getBinaryDocValues(field); if (valuesIn == null) { valuesIn = reader.getSortedDocValues(field); } if (valuesIn != null) { // Not cached here by FieldCacheImpl (cached instead // per-thread by SegmentReader): return valuesIn; } final FieldInfo info = reader.getFieldInfos().fieldInfo(field); if (info == null) { return DocValues.emptyBinary(); } else if (info.hasDocValues()) { throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); } else if (!info.isIndexed()) { return DocValues.emptyBinary(); } BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), setDocsWithField); return impl.iterator(); }
@Override public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException { BinaryRawValues instance = binaryInstances.get(field.number); if (instance == null) { // Lazy load instance = loadBinary(binaries.get(field.number)); binaryInstances.put(field.number, instance); } final byte[] bytes = instance.bytes; final int[] address = instance.address; return new BinaryDocValues() { final BytesRef term = new BytesRef(); @Override public BytesRef get(int docID) { term.bytes = bytes; term.offset = address[docID]; term.length = address[docID+1] - term.offset; return term; } }; }
private SortedDocValues newSortedInstance(final NumericDocValues docToOrd, final BinaryDocValues values, final int count) { return new SortedDocValues() { @Override public int getOrd(int docID) { return (int) docToOrd.get(docID); } @Override public BytesRef lookupOrd(int ord) { return values.get(ord); } @Override public int getValueCount() { return count; } // Leave lookupTerm to super's binary search // Leave termsEnum to super }; }
@Override public OrdinalsSegmentReader getReader(AtomicReaderContext context) throws IOException { BinaryDocValues values0 = context.reader().getBinaryDocValues(field); if (values0 == null) { values0 = DocValues.emptyBinary(); } final BinaryDocValues values = values0; return new OrdinalsSegmentReader() { @Override public void get(int docID, IntsRef ordinals) throws IOException { final BytesRef bytes = values.get(docID); decode(bytes, ordinals); } }; }
private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) { final IndexInput data = this.data.clone(); return new LongBinaryDocValues() { final BytesRef term; { term = new BytesRef(bytes.maxLength); term.offset = 0; term.length = bytes.maxLength; } @Override public BytesRef get(long id) { long address = bytes.offset + id * bytes.maxLength; try { data.seek(address); data.readBytes(term.bytes, 0, term.length); return term; } catch (IOException e) { throw new RuntimeException(e); } } }; }
private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) { final IndexInput data = this.data.clone(); return new LongBinaryDocValues() { @Override public void get(long id, BytesRef result) { long address = bytes.offset + id * bytes.maxLength; try { data.seek(address); // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource) // assume "they" own the bytes after calling this! final byte[] buffer = new byte[bytes.maxLength]; data.readBytes(buffer, 0, buffer.length); result.bytes = buffer; result.offset = 0; result.length = buffer.length; } catch (IOException e) { throw new RuntimeException(e); } } }; }
@Override public SortedDocValues getSorted(FieldInfo field) throws IOException { final int valueCount = (int) binaries.get(field.number).count; final BinaryDocValues binary = getBinary(field); final NumericDocValues ordinals = getNumeric(field, ords.get(field.number)); return new SortedDocValues() { @Override public int getOrd(int docID) { return (int) ordinals.get(docID); } @Override public void lookupOrd(int ord, BytesRef result) { binary.get(ord, result); } @Override public int getValueCount() { return valueCount; } }; }
@Override public BinaryDocValues getBinary(FieldInfo field) throws IOException { BinaryDocValues binaryDocValues = _binaryDocValuesCache.get(field.number); if (binaryDocValues != null) { return binaryDocValues; } synchronized (_binaryDocValuesCache) { binaryDocValues = _binaryDocValuesCache.get(field.number); if (binaryDocValues != null) { return binaryDocValues; } binaryDocValues = newBinary(field); if (_cache && binaryDocValues != null) { _binaryDocValuesCache.put(field.number, binaryDocValues); } return binaryDocValues; } }
@Test public void testBinaryDocValues() throws IOException { SecureAtomicReader secureReader = getSecureReader(); BinaryDocValues binaryDocValues = secureReader.getBinaryDocValues("bin"); BytesRef result = new BytesRef(); binaryDocValues.get(0, result); assertEquals(new BytesRef("0".getBytes()), result); binaryDocValues.get(1, result); assertEquals(new BytesRef(), result); binaryDocValues.get(2, result); assertEquals(new BytesRef("2".getBytes()), result); binaryDocValues.get(3, result); assertEquals(new BytesRef(), result); }
private BinaryDocValues loadBinary(BinaryEntry entry) throws IOException { data.seek(entry.offset); final byte[] bytes = new byte[entry.numBytes]; data.readBytes(bytes, 0, entry.numBytes); data.seek(entry.offset + entry.numBytes + entry.missingBytes); final int[] address = new int[entry.count+1]; for(int i=0;i<entry.count;i++) { address[i] = data.readInt(); } address[entry.count] = data.readInt(); ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address)); return new BinaryDocValues() { @Override public void get(int docID, BytesRef result) { result.bytes = bytes; result.offset = address[docID]; result.length = address[docID+1] - result.offset; }; }; }
private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException { final IndexInput data = this.data.clone(); final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes); return new LongBinaryDocValues() { @Override public void get(long id, BytesRef result) { long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1)); long endAddress = bytes.offset + addresses.get(id); int length = (int) (endAddress - startAddress); try { data.seek(startAddress); // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource) // assume "they" own the bytes after calling this! final byte[] buffer = new byte[length]; data.readBytes(buffer, 0, buffer.length); result.bytes = buffer; result.offset = 0; result.length = length; } catch (IOException e) { throw new RuntimeException(e); } } }; }
@Override public OrdinalsSegmentReader getReader(AtomicReaderContext context) throws IOException { BinaryDocValues values0 = context.reader().getBinaryDocValues(field); if (values0 == null) { values0 = BinaryDocValues.EMPTY; } final BinaryDocValues values = values0; return new OrdinalsSegmentReader() { private final BytesRef bytes = new BytesRef(32); @Override public void get(int docID, IntsRef ordinals) throws IOException { values.get(docID, bytes); decode(bytes, ordinals); } }; }
public BinaryDocValues getTerms(AtomicReader reader, String field, boolean setDocsWithField, float acceptableOverheadRatio) throws IOException { BinaryDocValues valuesIn = reader.getBinaryDocValues(field); if (valuesIn == null) { valuesIn = reader.getSortedDocValues(field); } if (valuesIn != null) { // Not cached here by FieldCacheImpl (cached instead // per-thread by SegmentReader): return valuesIn; } final FieldInfo info = reader.getFieldInfos().fieldInfo(field); if (info == null) { return BinaryDocValues.EMPTY; } else if (info.hasDocValues()) { throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); } else if (!info.isIndexed()) { return BinaryDocValues.EMPTY; } return (BinaryDocValues) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), setDocsWithField); }
private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.FieldType fieldType, QueryShardContext context, boolean mapUnmappedFieldsAsString) { return ctx -> { LeafReader leafReader = ctx.reader(); BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.queryBuilderField.name()); if (binaryDocValues == null) { return docId -> null; } Bits bits = leafReader.getDocsWithField(fieldType.queryBuilderField.name()); return docId -> { if (bits.get(docId)) { BytesRef qbSource = binaryDocValues.get(docId); if (qbSource.length > 0) { XContent xContent = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent(); try (XContentParser sourceParser = xContent.createParser(context.getXContentRegistry(), qbSource.bytes, qbSource.offset, qbSource.length)) { return parseQuery(context, mapUnmappedFieldsAsString, sourceParser); } } else { return null; } } else { return null; } }; }; }
@Override public SortedBinaryDocValues getBytesValues() { try { final BinaryDocValues values = DocValues.getBinary(reader, field); final Bits docsWithField = DocValues.getDocsWithField(reader, field); return FieldData.singleton(values, docsWithField); } catch (IOException e) { throw new IllegalStateException("Cannot load doc values", e); } }
public void testSingleValuedStrings() throws Exception { final int numDocs = scaledRandomIntBetween(1, 100); final BytesRef[] array = new BytesRef[numDocs]; final FixedBitSet docsWithValue = randomBoolean() ? null : new FixedBitSet(numDocs); for (int i = 0; i < array.length; ++i) { if (randomBoolean()) { array[i] = new BytesRef(RandomStrings.randomAsciiOfLength(random(), 8)); if (docsWithValue != null) { docsWithValue.set(i); } } else { array[i] = new BytesRef(); if (docsWithValue != null && randomBoolean()) { docsWithValue.set(i); } } } final BinaryDocValues singleValues = new BinaryDocValues() { @Override public BytesRef get(int docID) { return BytesRef.deepCopyOf(array[docID]); } }; final SortedBinaryDocValues multiValues = FieldData.singleton(singleValues, docsWithValue); verify(multiValues, numDocs); final FixedBitSet rootDocs = randomRootDocs(numDocs); final FixedBitSet innerDocs = randomInnerDocs(rootDocs); verify(multiValues, numDocs, rootDocs, innerDocs); }