protected int store(ByteRange bytes) { int indexOfNewElement = numUniqueRanges; if (uniqueRanges.size() <= numUniqueRanges) { uniqueRanges.add(new SimpleMutableByteRange()); } ByteRange storedRange = uniqueRanges.get(numUniqueRanges); int neededBytes = numBytes + bytes.getLength(); byteAppender = ArrayUtils.growIfNecessary(byteAppender, neededBytes, 2 * neededBytes); bytes.deepCopyTo(byteAppender, numBytes); storedRange.set(byteAppender, numBytes, bytes.getLength());// this isn't valid yet numBytes += bytes.getLength(); uniqueIndexByUniqueRange.put(storedRange, indexOfNewElement); int newestUniqueIndex = numUniqueRanges; ++numUniqueRanges; return newestUniqueIndex; }
public ByteRangeSet compile() { addToSortedRanges(); for (int i = 0; i < sortedRanges.size(); ++i) { sortedIndexByUniqueIndex.add(null);// need to grow the size } // TODO move this to an invert(int[]) util method for (int i = 0; i < sortedIndexByUniqueIndex.size(); ++i) { int uniqueIndex = uniqueIndexByUniqueRange.get(sortedRanges.get(i)); sortedIndexByUniqueIndex.set(uniqueIndex, i); } sortedIndexByInsertionId = ArrayUtils.growIfNecessary(sortedIndexByInsertionId, numInputs, numInputs); for (int i = 0; i < numInputs; ++i) { int uniqueRangeIndex = uniqueRangeIndexByInsertionId[i]; int sortedIndex = sortedIndexByUniqueIndex.get(uniqueRangeIndex); sortedIndexByInsertionId[i] = sortedIndex; } return this; }
protected int store(ByteRange bytes) { int indexOfNewElement = numUniqueRanges; if (uniqueRanges.size() <= numUniqueRanges) { uniqueRanges.add(new SimpleByteRange()); } ByteRange storedRange = uniqueRanges.get(numUniqueRanges); int neededBytes = numBytes + bytes.getLength(); byteAppender = ArrayUtils.growIfNecessary(byteAppender, neededBytes, 2 * neededBytes); bytes.deepCopyTo(byteAppender, numBytes); storedRange.set(byteAppender, numBytes, bytes.getLength());// this isn't valid yet numBytes += bytes.getLength(); uniqueIndexByUniqueRange.put(storedRange, indexOfNewElement); int newestUniqueIndex = numUniqueRanges; ++numUniqueRanges; return newestUniqueIndex; }
public LongEncoder compile() { int numUnique = uniqueValues.size(); if (numUnique == 1) { min = CollectionUtils.getFirst(uniqueValues); sortedUniqueValues = new long[] { min }; return this; } sortedUniqueValues = new long[numUnique]; int lastIndex = -1; for (long value : uniqueValues) { sortedUniqueValues[++lastIndex] = value; } Arrays.sort(sortedUniqueValues); min = ArrayUtils.getFirst(sortedUniqueValues); max = ArrayUtils.getLast(sortedUniqueValues); maxDelta = max - min; if (maxDelta > 0) { bytesPerDelta = UFIntTool.numBytes(maxDelta); } else { bytesPerDelta = 0; } int maxIndex = numUnique - 1; bytesPerIndex = UFIntTool.numBytes(maxIndex); totalCompressedBytes = numUnique * bytesPerDelta; return this; }
/******************* Object methods **********************/ @Override public String toString() { if (ArrayUtils.isEmpty(sortedUniqueValues)) { return "[]"; } return "[" + Joiner.on(",").join(ArrayUtils.toList(sortedUniqueValues)) + "]"; }
/** * Check that the arrays used to hold cell fragments are large enough for the cell that is being * added. Since the PrefixTreeEncoder is cached between uses, these arrays may grow during the * first few block encodings but should stabilize quickly. */ protected void ensurePerCellCapacities() { int currentCapacity = valueOffsets.length; int neededCapacity = totalCells + 2;// some things write one index ahead. +2 to be safe if (neededCapacity < currentCapacity) { return; } int padding = neededCapacity;//this will double the array size timestamps = ArrayUtils.growIfNecessary(timestamps, neededCapacity, padding); mvccVersions = ArrayUtils.growIfNecessary(mvccVersions, neededCapacity, padding); typeBytes = ArrayUtils.growIfNecessary(typeBytes, neededCapacity, padding); valueOffsets = ArrayUtils.growIfNecessary(valueOffsets, neededCapacity, padding); }
/***************** internal add methods ************************/ private void addAfterRowFamilyQualifier(Cell cell){ // timestamps timestamps[totalCells] = cell.getTimestamp(); timestampEncoder.add(cell.getTimestamp()); // memstore timestamps if (includeMvccVersion) { mvccVersions[totalCells] = cell.getMvccVersion(); mvccVersionEncoder.add(cell.getMvccVersion()); totalUnencodedBytes += WritableUtils.getVIntSize(cell.getMvccVersion()); }else{ //must overwrite in case there was a previous version in this array slot mvccVersions[totalCells] = 0L; if(totalCells == 0){//only need to do this for the first cell added mvccVersionEncoder.add(0L); } //totalUncompressedBytes += 0;//mvccVersion takes zero bytes when disabled } // types typeBytes[totalCells] = cell.getTypeByte(); cellTypeEncoder.add(cell.getTypeByte()); // values totalValueBytes += cell.getValueLength(); // double the array each time we run out of space values = ArrayUtils.growIfNecessary(values, totalValueBytes, 2 * totalValueBytes); CellUtil.copyValueTo(cell, values, valueOffsets[totalCells]); if (cell.getValueLength() > maxValueLength) { maxValueLength = cell.getValueLength(); } valueOffsets[totalCells + 1] = totalValueBytes; // general totalUnencodedBytes += KeyValueUtil.length(cell); ++totalCells; }
protected int appendTokenAndRepointByteRange(final ByteRange token, int inputTokenOffset) { int newOffset = tokensLength; int inputTokenLength = token.getLength() - inputTokenOffset; int newMinimum = tokensLength + inputTokenLength; tokens = ArrayUtils.growIfNecessary(tokens, newMinimum, 2 * newMinimum); token.deepCopySubRangeTo(inputTokenOffset, inputTokenLength, tokens, tokensLength); tokensLength += inputTokenLength; return newOffset; }
/** * Check if the incoming byte range exists. If not, add it to the backing byteAppender[] and * insert it into the tracking Map uniqueIndexByUniqueRange. */ public void add(ByteRange bytes) { Integer index = uniqueIndexByUniqueRange.get(bytes); if (index == null) { index = store(bytes); } int minLength = numInputs + 1; uniqueRangeIndexByInsertionId = ArrayUtils.growIfNecessary(uniqueRangeIndexByInsertionId, minLength, 2 * minLength); uniqueRangeIndexByInsertionId[numInputs] = index; ++numInputs; }
