FSWALEntry(final long sequence, final WALKey key, final WALEdit edit, final HTableDescriptor htd, final HRegionInfo hri, final boolean inMemstore) { super(key, edit); this.inMemstore = inMemstore; this.htd = htd; this.hri = hri; this.sequence = sequence; if (inMemstore) { // construct familyNames here to reduce the work of log sinker. ArrayList<Cell> cells = this.getEdit().getCells(); if (CollectionUtils.isEmpty(cells)) { this.familyNames = Collections.<byte[]> emptySet(); } else { Set<byte[]> familySet = Sets.newTreeSet(Bytes.BYTES_COMPARATOR); for (Cell cell : cells) { if (!CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { familySet.add(CellUtil.cloneFamily(cell)); } } this.familyNames = Collections.unmodifiableSet(familySet); } } else { this.familyNames = Collections.<byte[]> emptySet(); } }
private static Cell reckonDelta(final Cell delta, final Cell currentCell, final byte[] columnFamily, final long now, Mutation mutation, Function<Cell, byte[]> supplier) throws IOException { // Forward any tags found on the delta. List<Tag> tags = TagUtil.carryForwardTags(delta); tags = TagUtil.carryForwardTTLTag(tags, mutation.getTTL()); if (currentCell != null) { tags = TagUtil.carryForwardTags(tags, currentCell); byte[] newValue = supplier.apply(currentCell); return ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) .setRow(mutation.getRow(), 0, mutation.getRow().length) .setFamily(columnFamily, 0, columnFamily.length) // copy the qualifier if the cell is located in shared memory. .setQualifier(CellUtil.cloneQualifier(delta)) .setTimestamp(Math.max(currentCell.getTimestamp() + 1, now)) .setType(KeyValue.Type.Put.getCode()) .setValue(newValue, 0, newValue.length) .setTags(TagUtil.fromList(tags)) .build(); } else { PrivateCellUtil.updateLatestStamp(delta, now); return CollectionUtils.isEmpty(tags) ? delta : PrivateCellUtil.createCell(delta, tags); } }
public LongEncoder compile() { int numUnique = uniqueValues.size(); if (numUnique == 1) { min = CollectionUtils.getFirst(uniqueValues); sortedUniqueValues = new long[] { min }; return this; } sortedUniqueValues = new long[numUnique]; int lastIndex = -1; for (long value : uniqueValues) { sortedUniqueValues[++lastIndex] = value; } Arrays.sort(sortedUniqueValues); min = ArrayUtils.getFirst(sortedUniqueValues); max = ArrayUtils.getLast(sortedUniqueValues); maxDelta = max - min; if (maxDelta > 0) { bytesPerDelta = UFIntTool.numBytes(maxDelta); } else { bytesPerDelta = 0; } int maxIndex = numUnique - 1; bytesPerIndex = UFIntTool.numBytes(maxIndex); totalCompressedBytes = numUnique * bytesPerDelta; return this; }
/********************* methods ****************************/ protected void calculateOffsetsAndLengths(){ tokenWidth = tokenizerNode.getTokenLength(); if(!tokenizerNode.isRoot()){ --tokenWidth;//root has no parent } fanOut = CollectionUtils.nullSafeSize(tokenizerNode.getChildren()); numCells = tokenizerNode.getNumOccurrences(); }
public List<byte[]> getArrays() { List<TokenizerNode> nodes = new ArrayList<TokenizerNode>(); root.appendNodesToExternalList(nodes, true, true); List<byte[]> byteArrays = Lists.newArrayListWithCapacity(CollectionUtils.nullSafeSize(nodes)); for (int i = 0; i < nodes.size(); ++i) { TokenizerNode node = nodes.get(i); for (int j = 0; j < node.getNumOccurrences(); ++j) { byte[] byteArray = node.getNewByteArray(); byteArrays.add(byteArray); } } return byteArrays; }
@Override public List<String> listPeerIds() throws ReplicationException { try { return CollectionUtils.nullToEmpty(ZKUtil.listChildrenNoWatch(zookeeper, peersZNode)); } catch (KeeperException e) { throw new ReplicationException("Cannot get the list of peers", e); } }
@Override public void updateReaders(List<HStoreFile> sfs, List<KeyValueScanner> memStoreScanners) throws IOException { if (CollectionUtils.isEmpty(sfs) && CollectionUtils.isEmpty(memStoreScanners)) { return; } flushLock.lock(); try { flushed = true; final boolean isCompaction = false; boolean usePread = get || scanUsePread; // SEE HBASE-19468 where the flushed files are getting compacted even before a scanner // calls next(). So its better we create scanners here rather than next() call. Ensure // these scanners are properly closed() whether or not the scan is completed successfully // Eagerly creating scanners so that we have the ref counting ticking on the newly created // store files. In case of stream scanners this eager creation does not induce performance // penalty because in scans (that uses stream scanners) the next() call is bound to happen. List<KeyValueScanner> scanners = store.getScanners(sfs, cacheBlocks, get, usePread, isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt, false); flushedstoreFileScanners.addAll(scanners); if (!CollectionUtils.isEmpty(memStoreScanners)) { clearAndClose(memStoreScannersAfterFlush); memStoreScannersAfterFlush.addAll(memStoreScanners); } } finally { flushLock.unlock(); } // Let the next() call handle re-creating and seeking }
@VisibleForTesting static Set<byte[]> collectFamilies(List<Cell> cells) { if (CollectionUtils.isEmpty(cells)) { return Collections.emptySet(); } else { return cells.stream() .filter(v -> !CellUtil.matchingFamily(v, WALEdit.METAFAMILY)) .collect(toCollection(() -> new TreeSet<>(CellComparator.getInstance()::compareFamilies))) .stream() .map(CellUtil::cloneFamily) .collect(toCollection(() -> new TreeSet<>(Bytes.BYTES_COMPARATOR))); } }
/** * Create a {@link RowMutations} with the specified mutations. * @param mutations the mutations to send * @return RowMutations * @throws IOException if any row in mutations is different to another */ public static RowMutations of(List<? extends Mutation> mutations) throws IOException { if (CollectionUtils.isEmpty(mutations)) { throw new IllegalArgumentException("Can't instantiate a RowMutations by empty list"); } return new RowMutations(mutations.get(0).getRow(), mutations.size()) .add(mutations); }
/************************* building *********************************/ /* * <li>Only public method used during the tokenization process * <li>Requires that the input ByteRange sort after the previous, and therefore after all previous * inputs * <li>Only looks at bytes of the input array that align with this node's token */ public void addSorted(final ByteRange bytes) {// recursively build the tree /* * Recurse deeper into the existing trie structure */ if (matchesToken(bytes) && CollectionUtils.notEmpty(children)) { TokenizerNode lastChild = CollectionUtils.getLast(children); if (lastChild.partiallyMatchesToken(bytes)) { lastChild.addSorted(bytes); return; } } /* * Recursion ended. We must either * <li>1: increment numOccurrences if this input was equal to the previous * <li>2: convert this node from a leaf to a nub, and add a new child leaf * <li>3: split this node into a branch and leaf, and then add a second leaf */ // add it as a child of this node int numIdenticalTokenBytes = numIdenticalBytes(bytes);// should be <= token.length int tailOffset = tokenStartOffset + numIdenticalTokenBytes; int tailLength = bytes.getLength() - tailOffset; if (numIdenticalTokenBytes == token.getLength()) { if (tailLength == 0) {// identical to this node (case 1) incrementNumOccurrences(1); } else {// identical to this node, but with a few extra tailing bytes. (leaf -> nub) (case 2) int childNodeDepth = nodeDepth + 1; int childTokenStartOffset = tokenStartOffset + numIdenticalTokenBytes; TokenizerNode newChildNode = builder.addNode(this, childNodeDepth, childTokenStartOffset, bytes, tailOffset); addChild(newChildNode); } } else {//numIdenticalBytes > 0, split into branch/leaf and then add second leaf (case 3) split(numIdenticalTokenBytes, bytes); } }
/***************** searching *********************************/ /* * Do a trie style search through the tokenizer. One option for looking up families or qualifiers * during encoding, but currently unused in favor of tracking this information as they are added. * * Keeping code pending further performance testing. */ public void getNode(TokenizerRowSearchResult resultHolder, byte[] key, int keyOffset, int keyLength) { int thisNodeDepthPlusLength = tokenStartOffset + token.getLength(); // quick check if the key is shorter than this node (may not work for binary search) if (CollectionUtils.isEmpty(children)) { if (thisNodeDepthPlusLength < keyLength) {// ran out of bytes resultHolder.set(TokenizerRowSearchPosition.NO_MATCH, null); return; } } // all token bytes must match for (int i = 0; i < token.getLength(); ++i) { if (key[tokenStartOffset + keyOffset + i] != token.get(i)) { // TODO return whether it's before or after so we can binary search resultHolder.set(TokenizerRowSearchPosition.NO_MATCH, null); return; } } if (thisNodeDepthPlusLength == keyLength && numOccurrences > 0) { resultHolder.set(TokenizerRowSearchPosition.MATCH, this);// MATCH return; } if (CollectionUtils.notEmpty(children)) { // TODO binary search the children for (int i = 0; i < children.size(); ++i) { TokenizerNode child = children.get(i); child.getNode(resultHolder, key, keyOffset, keyLength); if (resultHolder.isMatch()) { return; } else if (resultHolder.getDifference() == TokenizerRowSearchPosition.BEFORE) { // passed it, so it doesn't exist resultHolder.set(TokenizerRowSearchPosition.NO_MATCH, null); return; } // key is still AFTER the current node, so continue searching } } // checked all children (or there were no children), and didn't find it resultHolder.set(TokenizerRowSearchPosition.NO_MATCH, null); return; }
public int getNumChildren() { return CollectionUtils.nullSafeSize(children); }
public TokenizerNode getLastChild() { if (CollectionUtils.isEmpty(children)) { return null; } return CollectionUtils.getLast(children); }
public boolean isLeaf() { return CollectionUtils.isEmpty(children) && hasOccurrences(); }
public boolean isBranch() { return CollectionUtils.notEmpty(children) && !hasOccurrences(); }
public boolean isNub() { return CollectionUtils.notEmpty(children) && hasOccurrences(); }
public void incrementNumOccurrencesOfLatestValue(){ CollectionUtils.getLast(nodes).incrementNumOccurrences(1); }
@Override public void addToSortedRanges() { sortedRanges.addAll(CollectionUtils.nullSafe(uniqueIndexByUniqueRange.keySet())); }
@Override public void addToSortedRanges() { sortedRanges.addAll(CollectionUtils.nullSafe(uniqueIndexByUniqueRange.keySet())); Collections.sort(sortedRanges); }
@Test public void testRandomSeekMisses() throws IOException { CellSearcher searcher = null; List<Integer> rowStartIndexes = rows.getRowStartIndexes(); try { searcher = DecoderFactory.checkOut(block, true); //test both the positionAtOrBefore and positionAtOrAfter methods for(boolean beforeVsAfterOnMiss : new boolean[]{true, false}){ for (int i=0; i < rows.getInputs().size(); ++i) { KeyValue kv = rows.getInputs().get(i); //nextRow KeyValue inputNextRow = KeyValueUtil.createFirstKeyInNextRow(kv); CellScannerPosition position = beforeVsAfterOnMiss ? searcher.positionAtOrBefore(inputNextRow) : searcher.positionAtOrAfter(inputNextRow); boolean isFirstInRow = rowStartIndexes.contains(i); if(isFirstInRow){ int rowIndex = rowStartIndexes.indexOf(i); if(rowIndex < rowStartIndexes.size() - 1){ if(beforeVsAfterOnMiss){ Assert.assertEquals(CellScannerPosition.BEFORE, position); }else{ Assert.assertEquals(CellScannerPosition.AFTER, position); } int expectedInputIndex = beforeVsAfterOnMiss ? rowStartIndexes.get(rowIndex + 1) - 1 : rowStartIndexes.get(rowIndex + 1); Assert.assertEquals(rows.getInputs().get(expectedInputIndex), searcher.current()); } } //previous KV KeyValue inputPreviousKv = KeyValueUtil.previousKey(kv); boolean hit = searcher.positionAt(inputPreviousKv); Assert.assertFalse(hit); position = searcher.positionAtOrAfter(inputPreviousKv); if(CollectionUtils.isLastIndex(rows.getInputs(), i)){ Assert.assertTrue(CellScannerPosition.AFTER_LAST == position); }else{ Assert.assertTrue(CellScannerPosition.AFTER == position); /* * TODO: why i+1 instead of i? */ Assert.assertEquals(rows.getInputs().get(i+1), searcher.current()); } } } } finally { DecoderFactory.checkIn(searcher); } }
@Override public void individualSearcherAssertions(CellSearcher searcher) { CellScannerPosition p;// reuse searcher.resetToBeforeFirstEntry(); // test first cell try { searcher.advance(); } catch (IOException e) { throw new RuntimeException(e); } Cell first = searcher.current(); Assert.assertTrue(CellComparator.equals(d.get(0), first)); // test first cell in second row Assert.assertTrue(searcher.positionAt(d.get(3))); Assert.assertTrue(CellComparator.equals(d.get(3), searcher.current())); Cell between4And5 = new KeyValue(rowB, cf, cq1, ts - 2, v0); // test exact Assert.assertFalse(searcher.positionAt(between4And5)); // test atOrBefore p = searcher.positionAtOrBefore(between4And5); Assert.assertEquals(CellScannerPosition.BEFORE, p); Assert.assertTrue(CellComparator.equals(searcher.current(), d.get(4))); // test atOrAfter p = searcher.positionAtOrAfter(between4And5); Assert.assertEquals(CellScannerPosition.AFTER, p); Assert.assertTrue(CellComparator.equals(searcher.current(), d.get(5))); // test when key falls before first key in block Cell beforeFirst = new KeyValue(Bytes.toBytes("A"), cf, cq0, ts, v0); Assert.assertFalse(searcher.positionAt(beforeFirst)); p = searcher.positionAtOrBefore(beforeFirst); Assert.assertEquals(CellScannerPosition.BEFORE_FIRST, p); p = searcher.positionAtOrAfter(beforeFirst); Assert.assertEquals(CellScannerPosition.AFTER, p); Assert.assertTrue(CellComparator.equals(searcher.current(), d.get(0))); Assert.assertEquals(d.get(0), searcher.current()); // test when key falls after last key in block Cell afterLast = new KeyValue(Bytes.toBytes("z"), cf, cq0, ts, v0);// must be lower case z Assert.assertFalse(searcher.positionAt(afterLast)); p = searcher.positionAtOrAfter(afterLast); Assert.assertEquals(CellScannerPosition.AFTER_LAST, p); p = searcher.positionAtOrBefore(afterLast); Assert.assertEquals(CellScannerPosition.BEFORE, p); Assert.assertTrue(CellComparator.equals(searcher.current(), CollectionUtils.getLast(d))); }
