/** * Used for major compactions. * <p> * Opens a scanner across specified StoreFiles. * @param store who we scan * @param scan the spec * @param scanners ancillary scanners * @param smallestReadPoint the readPoint that we should use for tracking versions */ public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, List<? extends KeyValueScanner> scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException { this(store, false, scan, null, scanInfo.getTtl(), scanInfo.getMinVersions()); initializeMetricNames(); matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType, smallestReadPoint, earliestPutTs, oldestUnexpiredTS); // Filter the list of scanners using Bloom filters, time range, TTL, etc. scanners = selectScannersFrom(scanners); // Seek all scanners to the initial key for (KeyValueScanner scanner : scanners) { scanner.seek(matcher.getStartKey()); } // Combine all seeked scanners with a heap heap = new KeyValueHeap(scanners, store.comparator); }
/** * Used for major compactions.<p> * * Opens a scanner across specified StoreFiles. * @param store who we scan * @param scan the spec * @param scanners ancillary scanners * @param smallestReadPoint the readPoint that we should use for tracking * versions */ public StoreScanner(Store store, ScanInfo scanInfo, Scan scan, List<? extends KeyValueScanner> scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException { this(store, false, scan, null, scanInfo.getTtl(), scanInfo.getMinVersions()); initializeMetricNames(); matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType, smallestReadPoint, earliestPutTs, oldestUnexpiredTS); // Filter the list of scanners using Bloom filters, time range, TTL, etc. scanners = selectScannersFrom(scanners); // Seek all scanners to the initial key for(KeyValueScanner scanner : scanners) { scanner.seek(matcher.getStartKey()); } // Combine all seeked scanners with a heap heap = new KeyValueHeap(scanners, store.comparator); }
StoreScanner(final Scan scan, Store.ScanInfo scanInfo, ScanType scanType, final NavigableSet<byte[]> columns, final List<KeyValueScanner> scanners, long earliestPutTs) throws IOException { this(null, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(), scanInfo.getMinVersions()); this.initializeMetricNames(); this.matcher = new ScanQueryMatcher(scan, scanInfo, columns, scanType, Long.MAX_VALUE, earliestPutTs, oldestUnexpiredTS); // Seek all scanners to the initial key for (KeyValueScanner scanner : scanners) { scanner.seek(matcher.getStartKey()); } heap = new KeyValueHeap(scanners, scanInfo.getComparator()); }
protected ScanInfo getScanInfo(Store store, RegionCoprocessorEnvironment e) { byte[] data = ((ZKWatcher)e.getSharedData().get(zkkey)).getData(); if (data == null) { return null; } ScanInfo oldSI = store.getScanInfo(); if (oldSI.getTtl() == Long.MAX_VALUE) { return null; } long ttl = Math.max(EnvironmentEdgeManager.currentTimeMillis() - Bytes.toLong(data), oldSI.getTtl()); return new ScanInfo(store.getFamily(), ttl, oldSI.getTimeToPurgeDeletes(), oldSI.getComparator()); }
@Override public InternalScanner preFlushScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException { Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment()); if (scanInfo == null) { // take default action return null; } Scan scan = new Scan(); scan.setMaxVersions(scanInfo.getMaxVersions()); return new StoreScanner(store, scanInfo, scan, Collections.singletonList(memstoreScanner), ScanType.MINOR_COMPACT, store.getHRegion().getSmallestReadPoint(), HConstants.OLDEST_TIMESTAMP); }
@Override public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s) throws IOException { Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment()); if (scanInfo == null) { // take default action return null; } Scan scan = new Scan(); scan.setMaxVersions(scanInfo.getMaxVersions()); return new StoreScanner(store, scanInfo, scan, scanners, scanType, store.getHRegion() .getSmallestReadPoint(), earliestPutTs); }
@Override public KeyValueScanner preStoreScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store, final Scan scan, final NavigableSet<byte[]> targetCols, final KeyValueScanner s) throws IOException { Store.ScanInfo scanInfo = getScanInfo(store, c.getEnvironment()); if (scanInfo == null) { // take default action return null; } return new StoreScanner(store, scanInfo, scan, targetCols); }
public void testWildCardTtlScan() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs = new KeyValue[] { KeyValueTestUtil.create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scan = new Scan(); scan.setMaxVersions(1); ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, KeyValue.COMPARATOR); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners); List<KeyValue> results = new ArrayList<KeyValue>(); assertEquals(true, scanner.next(results)); assertEquals(2, results.size()); assertEquals(kvs[1], results.get(0)); assertEquals(kvs[2], results.get(1)); results.clear(); assertEquals(true, scanner.next(results)); assertEquals(3, results.size()); assertEquals(kvs[4], results.get(0)); assertEquals(kvs[5], results.get(1)); assertEquals(kvs[6], results.get(2)); results.clear(); assertEquals(false, scanner.next(results)); }
/** * Ensure that expired delete family markers don't override valid puts */ public void testExpiredDeleteFamily() throws Exception { long now = System.currentTimeMillis(); KeyValue [] kvs = new KeyValue[] { new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000, KeyValue.Type.DeleteFamily), KeyValueTestUtil.create("R1", "cf", "a", now-10, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); Scan scan = new Scan(); scan.setMaxVersions(1); // scanner with ttl equal to 500 ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, KeyValue.COMPARATOR); ScanType scanType = ScanType.USER_SCAN; StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners); List<KeyValue> results = new ArrayList<KeyValue>(); assertEquals(true, scanner.next(results)); assertEquals(1, results.size()); assertEquals(kvs[1], results.get(0)); results.clear(); assertEquals(false, scanner.next(results)); }
/** Constructor for testing. */ StoreScanner(final Scan scan, Store.ScanInfo scanInfo, ScanType scanType, final NavigableSet<byte[]> columns, final List<KeyValueScanner> scanners) throws IOException { this(scan, scanInfo, scanType, columns, scanners, HConstants.LATEST_TIMESTAMP); }
