@Override public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException { finishLeaf(); context = ctx; docDeltas = PackedLongValues.packedBuilder(PackedInts.DEFAULT); buckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); return new LeafBucketCollector() { int lastDoc = 0; @Override public void collect(int doc, long bucket) throws IOException { docDeltas.add(doc - lastDoc); buckets.add(bucket); lastDoc = doc; maxBucket = Math.max(maxBucket, bucket); } }; }
private long getPageMemoryUsage(PackedLongValues values, float acceptableOverheadRatio, int pageSize, long pageMinOrdinal, long pageMaxOrdinal) { int bitsRequired; long pageMemorySize = 0; PackedInts.FormatAndBits formatAndBits; if (pageMaxOrdinal == Long.MIN_VALUE) { // empty page - will use the null reader which just stores size pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); } else { long pageMinValue = values.get(pageMinOrdinal); long pageMaxValue = values.get(pageMaxOrdinal); long pageDelta = pageMaxValue - pageMinValue; if (pageDelta != 0) { bitsRequired = pageDelta < 0 ? 64 : PackedInts.bitsRequired(pageDelta); formatAndBits = PackedInts.fastestFormatAndBits(pageSize, bitsRequired, acceptableOverheadRatio); pageMemorySize += formatAndBits.format.longCount(PackedInts.VERSION_CURRENT, pageSize, formatAndBits.bitsPerValue) * RamUsageEstimator.NUM_BYTES_LONG; pageMemorySize += RamUsageEstimator.NUM_BYTES_LONG; // min value per page storage } else { // empty page pageMemorySize += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT); } } return pageMemorySize; }
private PackedLongValues getDeletes(List<AtomicReader> readers) { PackedLongValues.Builder deletes = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); int deleteCount = 0; for (AtomicReader reader : readers) { final int maxDoc = reader.maxDoc(); final Bits liveDocs = reader.getLiveDocs(); for (int i = 0; i < maxDoc; ++i) { if (liveDocs != null && !liveDocs.get(i)) { ++deleteCount; } else { deletes.add(deleteCount); } } } return deletes.build(); }
public BinaryDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) { this.fieldInfo = fieldInfo; this.bytes = new PagedBytes(BLOCK_BITS); this.bytesOut = bytes.getDataOutput(); this.lengths = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); this.iwBytesUsed = iwBytesUsed; this.docsWithField = new FixedBitSet(64); this.bytesUsed = docsWithFieldBytesUsed(); iwBytesUsed.addAndGet(bytesUsed); }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.getDocCount(); bytes.freeze(false); final PackedLongValues lengths = this.lengths.build(); dvConsumer.addBinaryField(fieldInfo, new Iterable<BytesRef>() { @Override public Iterator<BytesRef> iterator() { return new BytesIterator(maxDoc, lengths); } }); }
public SortedSetDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) { this.fieldInfo = fieldInfo; this.iwBytesUsed = iwBytesUsed; hash = new BytesRefHash( new ByteBlockPool( new ByteBlockPool.DirectTrackingAllocator(iwBytesUsed)), BytesRefHash.DEFAULT_CAPACITY, new DirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY, iwBytesUsed)); pending = PackedLongValues.packedBuilder(PackedInts.COMPACT); pendingCounts = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); bytesUsed = pending.ramBytesUsed() + pendingCounts.ramBytesUsed(); iwBytesUsed.addAndGet(bytesUsed); }
OrdsIterator(int ordMap[], int maxCount, PackedLongValues ords, PackedLongValues ordCounts) { this.currentDoc = new int[maxCount]; this.ordMap = ordMap; this.numOrds = ords.size(); this.iter = ords.iterator(); this.counts = ordCounts.iterator(); }
public SortedDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) { this.fieldInfo = fieldInfo; this.iwBytesUsed = iwBytesUsed; hash = new BytesRefHash( new ByteBlockPool( new ByteBlockPool.DirectTrackingAllocator(iwBytesUsed)), BytesRefHash.DEFAULT_CAPACITY, new DirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY, iwBytesUsed)); pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); bytesUsed = pending.ramBytesUsed(); iwBytesUsed.addAndGet(bytesUsed); }
public NumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed, boolean trackDocsWithField) { pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); docsWithField = trackDocsWithField ? new FixedBitSet(64) : null; bytesUsed = pending.ramBytesUsed() + docsWithFieldBytesUsed(); this.fieldInfo = fieldInfo; this.iwBytesUsed = iwBytesUsed; iwBytesUsed.addAndGet(bytesUsed); }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.getDocCount(); final PackedLongValues values = pending.build(); dvConsumer.addNumericField(fieldInfo, new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new NumericIterator(maxDoc, values, docsWithField); } }); }
public SortedNumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) { this.fieldInfo = fieldInfo; this.iwBytesUsed = iwBytesUsed; pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); pendingCounts = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); bytesUsed = pending.ramBytesUsed() + pendingCounts.ramBytesUsed(); iwBytesUsed.addAndGet(bytesUsed); }
WAH8DocIdSet(byte[] data, int cardinality, int indexInterval, PackedLongValues positions, PackedLongValues wordNums) { this.data = data; this.cardinality = cardinality; this.indexInterval = indexInterval; this.positions = positions; this.wordNums = wordNums; }
Iterator(byte[] data, int cardinality, int indexInterval, PackedLongValues positions, PackedLongValues wordNums) { this.in = new ByteArrayDataInput(data); this.cardinality = cardinality; this.indexInterval = indexInterval; this.positions = positions; this.wordNums = wordNums; wordNum = -1; word = 0; bitList = 0; sequenceNum = -1; docID = -1; indexThreshold = indexThreshold(cardinality, indexInterval); }
PForDeltaDocIdSet(byte[] data, int cardinality, int indexInterval, PackedLongValues docIDs, PackedLongValues offsets) { this.data = data; this.cardinality = cardinality; this.indexInterval = indexInterval; this.docIDs = docIDs; this.offsets = offsets; }
Iterator(byte[] data, int cardinality, int indexInterval, PackedLongValues docIDs, PackedLongValues offsets) { this.data = data; this.cardinality = cardinality; this.indexInterval = indexInterval; this.docIDs = docIDs; this.offsets = offsets; offset = 0; nextDocs = new int[BLOCK_SIZE]; Arrays.fill(nextDocs, -1); i = BLOCK_SIZE; nextExceptions = new int[BLOCK_SIZE]; blockIdx = -1; docID = -1; }
private static SortedNumericDocValues pagedSingles(final PackedLongValues values, final Bits docsWithValue) { return DocValues.singleton(new NumericDocValues() { // we need to wrap since NumericDocValues must return 0 when a doc has no value @Override public long get(int docID) { if (docsWithValue == null || docsWithValue.get(docID)) { return values.get(docID); } else { return 0; } } }, docsWithValue); }
