public static void verifySizeCompaction(KeyValue[] input, int targetCount, long targetSize, byte[] left, byte[] right, KeyValue[][] output) throws Exception { StoreFileWritersCapture writers = new StoreFileWritersCapture(); StripeCompactor sc = createCompactor(writers, input); List<Path> paths = sc.compact(createDummyRequest(), targetCount, targetSize, left, right, null, null, NoLimitCompactionThroughputController.INSTANCE); assertEquals(output.length, paths.size()); writers.verifyKvs(output, true, true); List<byte[]> boundaries = new ArrayList<byte[]>(); boundaries.add(left); for (int i = 1; i < output.length; ++i) { boundaries.add(output[i][0].getRow()); } boundaries.add(right); writers.verifyBoundaries(boundaries.toArray(new byte[][] {})); }
public static void verifySizeCompaction(KeyValue[] input, int targetCount, long targetSize, byte[] left, byte[] right, KeyValue[][] output) throws Exception { StoreFileWritersCapture writers = new StoreFileWritersCapture(); StripeCompactor sc = createCompactor(writers, input); List<Path> paths = sc.compact( createDummyRequest(), targetCount, targetSize, left, right, null, null); assertEquals(output.length, paths.size()); writers.verifyKvs(output, true, true); List<byte[]> boundaries = new ArrayList<byte[]>(); boundaries.add(left); for (int i = 1; i < output.length; ++i) { boundaries.add(output[i][0].getRow()); } boundaries.add(right); writers.verifyBoundaries(boundaries.toArray(new byte[][] {})); }
@Override protected void createComponents( Configuration conf, Store store, KVComparator comparator) throws IOException { this.config = new StripeStoreConfig(conf, store); this.compactionPolicy = new StripeCompactionPolicy(conf, store, config); this.storeFileManager = new StripeStoreFileManager(comparator, conf, this.config); this.storeFlusher = new StripeStoreFlusher( conf, store, this.compactionPolicy, this.storeFileManager); this.compactor = new StripeCompactor(conf, store); }
public static void verifyBoundaryCompaction(KeyValue[] input, byte[][] boundaries, KeyValue[][] output, byte[] majorFrom, byte[] majorTo, boolean allFiles) throws Exception { StoreFileWritersCapture writers = new StoreFileWritersCapture(); StripeCompactor sc = createCompactor(writers, input); List<Path> paths = sc.compact(createDummyRequest(), Arrays.asList(boundaries), majorFrom, majorTo, NoLimitCompactionThroughputController.INSTANCE); writers.verifyKvs(output, allFiles, true); if (allFiles) { assertEquals(output.length, paths.size()); writers.verifyBoundaries(boundaries); } }
@Test public void testCompactionContextForceSelect() throws Exception { Configuration conf = HBaseConfiguration.create(); int targetCount = 2; conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, targetCount); conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 2); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, TestStoreEngine.class.getName()); TestStoreEngine se = createEngine(conf); StripeCompactor mockCompactor = mock(StripeCompactor.class); se.setCompactorOverride(mockCompactor); when( mockCompactor.compact(any(CompactionRequest.class), anyInt(), anyLong(), any(byte[].class), any(byte[].class), any(byte[].class), any(byte[].class), any(CompactionThroughputController.class), any(User.class))) .thenReturn(new ArrayList<Path>()); // Produce 3 L0 files. StoreFile sf = createFile(); ArrayList<StoreFile> compactUs = al(sf, createFile(), createFile()); se.getStoreFileManager().loadFiles(compactUs); // Create a compaction that would want to split the stripe. CompactionContext compaction = se.createCompaction(); compaction.select(al(), false, false, false); assertEquals(3, compaction.getRequest().getFiles().size()); // Override the file list. Granted, overriding this compaction in this manner will // break things in real world, but we only want to verify the override. compactUs.remove(sf); CompactionRequest req = new CompactionRequest(compactUs); compaction.forceSelect(req); assertEquals(2, compaction.getRequest().getFiles().size()); assertFalse(compaction.getRequest().getFiles().contains(sf)); // Make sure the correct method it called on compactor. compaction.compact(NoLimitCompactionThroughputController.INSTANCE); verify(mockCompactor, times(1)).compact(compaction.getRequest(), targetCount, 0L, StripeStoreFileManager.OPEN_KEY, StripeStoreFileManager.OPEN_KEY, null, null, NoLimitCompactionThroughputController.INSTANCE, null); }
public static void verifyBoundaryCompaction(KeyValue[] input, byte[][] boundaries, KeyValue[][] output, byte[] majorFrom, byte[] majorTo, boolean allFiles) throws Exception { StoreFileWritersCapture writers = new StoreFileWritersCapture(); StripeCompactor sc = createCompactor(writers, input); List<Path> paths = sc.compact(createDummyRequest(), Arrays.asList(boundaries), majorFrom, majorTo); writers.verifyKvs(output, allFiles, true); if (allFiles) { assertEquals(output.length, paths.size()); writers.verifyBoundaries(boundaries); } }
@Test public void testCompactionContextForceSelect() throws Exception { Configuration conf = HBaseConfiguration.create(); int targetCount = 2; conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, targetCount); conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 2); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, TestStoreEngine.class.getName()); TestStoreEngine se = createEngine(conf); StripeCompactor mockCompactor = mock(StripeCompactor.class); se.setCompactorOverride(mockCompactor); when(mockCompactor.compact(any(CompactionRequest.class), anyInt(), anyLong(), any(byte[].class), any(byte[].class), any(byte[].class), any(byte[].class))) .thenReturn(new ArrayList<Path>()); // Produce 3 L0 files. StoreFile sf = createFile(); ArrayList<StoreFile> compactUs = al(sf, createFile(), createFile()); se.getStoreFileManager().loadFiles(compactUs); // Create a compaction that would want to split the stripe. CompactionContext compaction = se.createCompaction(); compaction.select(al(), false, false, false); assertEquals(3, compaction.getRequest().getFiles().size()); // Override the file list. Granted, overriding this compaction in this manner will // break things in real world, but we only want to verify the override. compactUs.remove(sf); CompactionRequest req = new CompactionRequest(compactUs); compaction.forceSelect(req); assertEquals(2, compaction.getRequest().getFiles().size()); assertFalse(compaction.getRequest().getFiles().contains(sf)); // Make sure the correct method it called on compactor. compaction.compact(); verify(mockCompactor, times(1)).compact(compaction.getRequest(), targetCount, 0L, StripeStoreFileManager.OPEN_KEY, StripeStoreFileManager.OPEN_KEY, null, null); }
@Override protected void createComponents( Configuration conf, HStore store, CellComparator comparator) throws IOException { this.config = new StripeStoreConfig(conf, store); this.compactionPolicy = new StripeCompactionPolicy(conf, store, config); this.storeFileManager = new StripeStoreFileManager(comparator, conf, this.config); this.storeFlusher = new StripeStoreFlusher( conf, store, this.compactionPolicy, this.storeFileManager); this.compactor = new StripeCompactor(conf, store); }
@Test public void testCompactionContextForceSelect() throws Exception { Configuration conf = HBaseConfiguration.create(); int targetCount = 2; conf.setInt(StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, targetCount); conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 2); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, TestStoreEngine.class.getName()); TestStoreEngine se = createEngine(conf); StripeCompactor mockCompactor = mock(StripeCompactor.class); se.setCompactorOverride(mockCompactor); when( mockCompactor.compact(any(), anyInt(), anyLong(), any(), any(), any(), any(), any(), any())) .thenReturn(new ArrayList<>()); // Produce 3 L0 files. HStoreFile sf = createFile(); ArrayList<HStoreFile> compactUs = al(sf, createFile(), createFile()); se.getStoreFileManager().loadFiles(compactUs); // Create a compaction that would want to split the stripe. CompactionContext compaction = se.createCompaction(); compaction.select(al(), false, false, false); assertEquals(3, compaction.getRequest().getFiles().size()); // Override the file list. Granted, overriding this compaction in this manner will // break things in real world, but we only want to verify the override. compactUs.remove(sf); CompactionRequestImpl req = new CompactionRequestImpl(compactUs); compaction.forceSelect(req); assertEquals(2, compaction.getRequest().getFiles().size()); assertFalse(compaction.getRequest().getFiles().contains(sf)); // Make sure the correct method it called on compactor. compaction.compact(NoLimitThroughputController.INSTANCE, null); verify(mockCompactor, times(1)).compact(compaction.getRequest(), targetCount, 0L, StripeStoreFileManager.OPEN_KEY, StripeStoreFileManager.OPEN_KEY, null, null, NoLimitThroughputController.INSTANCE, null); }
public void setCompactorOverride(StripeCompactor compactorOverride) { this.compactor = compactorOverride; }
