/** * Execute the actual compaction job. * If the compact once flag is not specified, execute the compaction until * no more compactions are needed. Uses the Configuration settings provided. */ private void compactStoreFiles(final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri, final String familyName, final boolean compactOnce, final boolean major) throws IOException { HStore store = getStore(conf, fs, tableDir, htd, hri, familyName, tmpDir); LOG.info("Compact table=" + htd.getTableName() + " region=" + hri.getRegionNameAsString() + " family=" + familyName); if (major) { store.triggerMajorCompaction(); } do { CompactionContext compaction = store.requestCompaction(Store.PRIORITY_USER, null); if (compaction == null) break; List<StoreFile> storeFiles = store.compact(compaction, NoLimitCompactionThroughputController.INSTANCE); if (storeFiles != null && !storeFiles.isEmpty()) { if (keepCompactedFiles && deleteCompacted) { for (StoreFile storeFile: storeFiles) { fs.delete(storeFile.getPath(), false); } } } } while (store.needsCompaction() && !compactOnce); }
private CompactionContext selectCompaction(final Region r, final Store s, int priority, CompactionRequest request, User user) throws IOException { CompactionContext compaction = s.requestCompaction(priority, request, user); if (compaction == null) { if(LOG.isDebugEnabled() && r.getRegionInfo() != null) { LOG.debug("Not compacting " + r.getRegionInfo().getRegionNameAsString() + " because compaction request was cancelled"); } return null; } assert compaction.hasSelection(); if (priority != Store.NO_PRIORITY) { compaction.getRequest().setPriority(priority); } return compaction; }
@Override public void compact(final boolean majorCompaction) throws IOException { if (majorCompaction) { triggerMajorCompaction(); } for (Store s : getStores()) { CompactionContext compaction = s.requestCompaction(); if (compaction != null) { CompactionThroughputController controller = null; if (rsServices != null) { controller = CompactionThroughputControllerFactory.create(rsServices, conf); } if (controller == null) { controller = NoLimitCompactionThroughputController.INSTANCE; } compact(compaction, s, controller, null); } } }
/** * Execute the actual compaction job. * If the compact once flag is not specified, execute the compaction until * no more compactions are needed. Uses the Configuration settings provided. */ private void compactStoreFiles(final Path tableDir, final HTableDescriptor htd, final HRegionInfo hri, final String familyName, final boolean compactOnce, final boolean major) throws IOException { HStore store = getStore(conf, fs, tableDir, htd, hri, familyName, tmpDir); LOG.info("Compact table=" + htd.getTableName() + " region=" + hri.getRegionNameAsString() + " family=" + familyName); if (major) { store.triggerMajorCompaction(); } do { CompactionContext compaction = store.requestCompaction(Store.PRIORITY_USER, null); if (compaction == null) break; List<StoreFile> storeFiles = store.compact(compaction); if (storeFiles != null && !storeFiles.isEmpty()) { if (keepCompactedFiles && deleteCompacted) { for (StoreFile storeFile: storeFiles) { fs.delete(storeFile.getPath(), false); } } } } while (store.needsCompaction() && !compactOnce); }
private CompactionContext selectCompaction(final HRegion r, final Store s, int priority, CompactionRequest request) throws IOException { CompactionContext compaction = s.requestCompaction(priority, request); if (compaction == null) { if(LOG.isDebugEnabled()) { LOG.debug("Not compacting " + r.getRegionNameAsString() + " because compaction request was cancelled"); } return null; } assert compaction.hasSelection(); if (priority != Store.NO_PRIORITY) { compaction.getRequest().setPriority(priority); } return compaction; }
/** * Synchronously compact all stores in the region. * <p>This operation could block for a long time, so don't call it from a * time-sensitive thread. * <p>Note that no locks are taken to prevent possible conflicts between * compaction and splitting activities. The regionserver does not normally compact * and split in parallel. However by calling this method you may introduce * unexpected and unhandled concurrency. Don't do this unless you know what * you are doing. * * @param majorCompaction True to force a major compaction regardless of thresholds * @throws IOException */ public void compact(boolean majorCompaction) throws IOException { if (majorCompaction) { stores.values().forEach(HStore::triggerMajorCompaction); } for (HStore s : stores.values()) { Optional<CompactionContext> compaction = s.requestCompaction(); if (compaction.isPresent()) { ThroughputController controller = null; if (rsServices != null) { controller = CompactionThroughputControllerFactory.create(rsServices, conf); } if (controller == null) { controller = NoLimitThroughputController.INSTANCE; } compact(compaction.get(), s, controller, null); } } }
/** * @param r region store belongs to * @param s Store to request compaction on * @param why Why compaction requested -- used in debug messages * @param priority override the default priority (NO_PRIORITY == decide) * @param request custom compaction request. Can be <tt>null</tt> in which case a simple * compaction will be used. */ private synchronized CompactionRequest requestCompactionInternal(final Region r, final Store s, final String why, int priority, CompactionRequest request, boolean selectNow, User user) throws IOException { if (this.server.isStopped() || (r.getTableDesc() != null && !r.getTableDesc().isCompactionEnabled())) { return null; } CompactionContext compaction = null; if (selectNow) { compaction = selectCompaction(r, s, priority, request, user); if (compaction == null) return null; // message logged inside } // We assume that most compactions are small. So, put system compactions into small // pool; we will do selection there, and move to large pool if necessary. ThreadPoolExecutor pool = (selectNow && s.throttleCompaction(compaction.getRequest().getSize())) ? longCompactions : shortCompactions; pool.execute(new CompactionRunner(s, r, compaction, pool, user)); if (LOG.isDebugEnabled()) { String type = (pool == shortCompactions) ? "Small " : "Large "; LOG.debug(type + "Compaction requested: " + (selectNow ? compaction.toString() : "system") + (why != null && !why.isEmpty() ? "; Because: " + why : "") + "; " + this); } return selectNow ? compaction.getRequest() : null; }
public CompactionRunner(Store store, Region region, CompactionContext compaction, ThreadPoolExecutor parent, User user) { super(); this.store = store; this.region = (HRegion)region; this.compaction = compaction; this.queuedPriority = (this.compaction == null) ? store.getCompactPriority() : compaction.getRequest().getPriority(); this.parent = parent; this.user = user; }
@Override public int compareTo(CompactionRunner o) { // Only compare the underlying request (if any), for queue sorting purposes. int compareVal = queuedPriority - o.queuedPriority; // compare priority if (compareVal != 0) return compareVal; CompactionContext tc = this.compaction, oc = o.compaction; // Sort pre-selected (user?) compactions before system ones with equal priority. return (tc == null) ? ((oc == null) ? 0 : 1) : ((oc == null) ? -1 : tc.getRequest().compareTo(oc.getRequest())); }
/** * This is a helper function that compact all the stores synchronously It is used by utilities and * testing * * @throws IOException e */ public void compactStores() throws IOException { for (Store s : getStores()) { CompactionContext compaction = s.requestCompaction(); if (compaction != null) { compact(compaction, s, NoLimitCompactionThroughputController.INSTANCE, null); } } }
/** * This is a helper function that compact the given store It is used by utilities and testing * * @throws IOException e */ @VisibleForTesting void compactStore(byte[] family, CompactionThroughputController throughputController) throws IOException { Store s = getStore(family); CompactionContext compaction = s.requestCompaction(); if (compaction != null) { compact(compaction, s, throughputController, null); } }
@Override public boolean compact(CompactionContext compaction, Store store, CompactionThroughputController throughputController) throws IOException { try { return super.compact(compaction, store, throughputController); } finally { compactCount++; } }
@Override public boolean compact(CompactionContext compaction, Store store, CompactionThroughputController throughputController, User user) throws IOException { try { return super.compact(compaction, store, throughputController, user); } finally { compactCount++; } }
@Override public boolean compact(CompactionContext compaction, Store store, CompactionThroughputController throughputController) throws IOException { boolean ret = super.compact(compaction, store, throughputController); if (ret) compactionStateChangeLatch.countDown(); return ret; }
@Override public boolean compact(CompactionContext compaction, Store store, CompactionThroughputController throughputController, User user) throws IOException { boolean ret = super.compact(compaction, store, throughputController, user); if (ret) compactionStateChangeLatch.countDown(); return ret; }
@Override public synchronized CompactionContext selectCompaction() { CompactionContext ctx = new TestCompactionContext(new ArrayList<StoreFile>(notCompacting)); compacting.addAll(notCompacting); notCompacting.clear(); try { ctx.select(null, false, false, false); } catch (IOException ex) { fail("Shouldn't happen"); } return ctx; }
@Override public CompactionContext selectCompaction() { this.blocked = new BlockingCompactionContext(); try { this.blocked.select(null, false, false, false); } catch (IOException ex) { fail("Shouldn't happen"); } return this.blocked; }
@Test public void testCompactionContextForceSelect() throws Exception { Configuration conf = HBaseConfiguration.create(); int targetCount = 2; conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, targetCount); conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 2); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, TestStoreEngine.class.getName()); TestStoreEngine se = createEngine(conf); StripeCompactor mockCompactor = mock(StripeCompactor.class); se.setCompactorOverride(mockCompactor); when( mockCompactor.compact(any(CompactionRequest.class), anyInt(), anyLong(), any(byte[].class), any(byte[].class), any(byte[].class), any(byte[].class), any(CompactionThroughputController.class), any(User.class))) .thenReturn(new ArrayList<Path>()); // Produce 3 L0 files. StoreFile sf = createFile(); ArrayList<StoreFile> compactUs = al(sf, createFile(), createFile()); se.getStoreFileManager().loadFiles(compactUs); // Create a compaction that would want to split the stripe. CompactionContext compaction = se.createCompaction(); compaction.select(al(), false, false, false); assertEquals(3, compaction.getRequest().getFiles().size()); // Override the file list. Granted, overriding this compaction in this manner will // break things in real world, but we only want to verify the override. compactUs.remove(sf); CompactionRequest req = new CompactionRequest(compactUs); compaction.forceSelect(req); assertEquals(2, compaction.getRequest().getFiles().size()); assertFalse(compaction.getRequest().getFiles().contains(sf)); // Make sure the correct method it called on compactor. compaction.compact(NoLimitCompactionThroughputController.INSTANCE); verify(mockCompactor, times(1)).compact(compaction.getRequest(), targetCount, 0L, StripeStoreFileManager.OPEN_KEY, StripeStoreFileManager.OPEN_KEY, null, null, NoLimitCompactionThroughputController.INSTANCE, null); }
public Store createStoreMock(String name) throws Exception { Store store = mock(Store.class, name); when(store.requestCompaction( anyInt(), isNull(CompactionRequest.class))).then(new SelectAnswer()); when(store.requestCompaction( anyInt(), isNull(CompactionRequest.class), any(User.class))).then(new SelectAnswer()); when(store.getCompactPriority()).then(new PriorityAnswer()); doAnswer(new CancelAnswer()).when( store).cancelRequestedCompaction(any(CompactionContext.class)); return store; }
/** * @param r HRegion store belongs to * @param s Store to request compaction on * @param why Why compaction requested -- used in debug messages * @param priority override the default priority (NO_PRIORITY == decide) * @param request custom compaction request. Can be <tt>null</tt> in which case a simple * compaction will be used. */ private synchronized CompactionRequest requestCompactionInternal(final HRegion r, final Store s, final String why, int priority, CompactionRequest request, boolean selectNow) throws IOException { if (this.server.isStopped() || (r.getTableDesc() != null && !r.getTableDesc().isCompactionEnabled())) { return null; } CompactionContext compaction = null; if (selectNow) { compaction = selectCompaction(r, s, priority, request); if (compaction == null) return null; // message logged inside } // We assume that most compactions are small. So, put system compactions into small // pool; we will do selection there, and move to large pool if necessary. long size = selectNow ? compaction.getRequest().getSize() : 0; ThreadPoolExecutor pool = (!selectNow && s.throttleCompaction(size)) ? longCompactions : shortCompactions; pool.execute(new CompactionRunner(s, r, compaction, pool)); if (LOG.isDebugEnabled()) { String type = (pool == shortCompactions) ? "Small " : "Large "; LOG.debug(type + "Compaction requested: " + (selectNow ? compaction.toString() : "system") + (why != null && !why.isEmpty() ? "; Because: " + why : "") + "; " + this); } return selectNow ? compaction.getRequest() : null; }
public CompactionRunner(Store store, HRegion region, CompactionContext compaction, ThreadPoolExecutor parent) { super(); this.store = store; this.region = region; this.compaction = compaction; this.queuedPriority = (this.compaction == null) ? store.getCompactPriority() : compaction.getRequest().getPriority(); this.parent = parent; }
/** * This is a helper function that compact all the stores synchronously * It is used by utilities and testing * * @throws IOException e */ public void compactStores() throws IOException { for (Store s : getStores().values()) { CompactionContext compaction = s.requestCompaction(); if (compaction != null) { compact(compaction, s); } } }
@Override public boolean compact(CompactionContext compaction, Store store) throws IOException { try { return super.compact(compaction, store); } finally { compactCount++; } }
@Test public void testCompactionContextForceSelect() throws Exception { Configuration conf = HBaseConfiguration.create(); int targetCount = 2; conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, targetCount); conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 2); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, TestStoreEngine.class.getName()); TestStoreEngine se = createEngine(conf); StripeCompactor mockCompactor = mock(StripeCompactor.class); se.setCompactorOverride(mockCompactor); when(mockCompactor.compact(any(CompactionRequest.class), anyInt(), anyLong(), any(byte[].class), any(byte[].class), any(byte[].class), any(byte[].class))) .thenReturn(new ArrayList<Path>()); // Produce 3 L0 files. StoreFile sf = createFile(); ArrayList<StoreFile> compactUs = al(sf, createFile(), createFile()); se.getStoreFileManager().loadFiles(compactUs); // Create a compaction that would want to split the stripe. CompactionContext compaction = se.createCompaction(); compaction.select(al(), false, false, false); assertEquals(3, compaction.getRequest().getFiles().size()); // Override the file list. Granted, overriding this compaction in this manner will // break things in real world, but we only want to verify the override. compactUs.remove(sf); CompactionRequest req = new CompactionRequest(compactUs); compaction.forceSelect(req); assertEquals(2, compaction.getRequest().getFiles().size()); assertFalse(compaction.getRequest().getFiles().contains(sf)); // Make sure the correct method it called on compactor. compaction.compact(); verify(mockCompactor, times(1)).compact(compaction.getRequest(), targetCount, 0L, StripeStoreFileManager.OPEN_KEY, StripeStoreFileManager.OPEN_KEY, null, null); }
public Store createStoreMock(String name) throws Exception { Store store = mock(Store.class, name); when(store.requestCompaction( anyInt(), isNull(CompactionRequest.class))).then(new SelectAnswer()); when(store.getCompactPriority()).then(new PriorityAnswer()); doAnswer(new CancelAnswer()).when( store).cancelRequestedCompaction(any(CompactionContext.class)); return store; }
/** * @param r HRegion store belongs to * @param s Store to request compaction on * @param why Why compaction requested -- used in debug messages * @param priority override the default priority (NO_PRIORITY == decide) * @param request custom compaction request. Can be <tt>null</tt> in which case a simple * compaction will be used. */ private synchronized CompactionRequest requestCompactionInternal(final HRegion r, final Store s, final String why, int priority, CompactionRequest request, boolean selectNow) throws IOException { if (this.server.isStopped() || (r.getTableDesc() != null && !r.getTableDesc().isCompactionEnabled())) { return null; } CompactionContext compaction = null; if (selectNow) { compaction = selectCompaction(r, s, priority, request); if (compaction == null) return null; // message logged inside } // We assume that most compactions are small. So, put system compactions into small // pool; we will do selection there, and move to large pool if necessary. long size = selectNow ? compaction.getRequest().getSize() : 0; ThreadPoolExecutor pool = (!selectNow && s.throttleCompaction(size)) ? largeCompactions : smallCompactions; pool.execute(new CompactionRunner(s, r, compaction, pool)); if (LOG.isDebugEnabled()) { String type = (pool == smallCompactions) ? "Small " : "Large "; LOG.debug(type + "Compaction requested: " + (selectNow ? compaction.toString() : "system") + (why != null && !why.isEmpty() ? "; Because: " + why : "") + "; " + this); } return selectNow ? compaction.getRequest() : null; }
