@Test (timeout=180000) public void testMove() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); final ServerName server = location.getServerName(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preMove(ObserverContext.createAndPrepare(CP_ENV, null), hri, server, server); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
@Test (timeout=180000) public void testAssign() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preAssign(ObserverContext.createAndPrepare(CP_ENV, null), hri); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
@Test (timeout=180000) public void testUnassign() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preUnassign(ObserverContext.createAndPrepare(CP_ENV, null), hri, false); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
@Test (timeout=180000) public void testRegionOffline() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preRegionOffline(ObserverContext.createAndPrepare(CP_ENV, null), hri); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
@BeforeClass public static void beforeAllTests() throws Exception { Configuration c = TEST_UTIL.getConfiguration(); c.setBoolean("hbase.assignment.usezk", true); c.setBoolean("dfs.support.append", true); c.setInt("hbase.regionserver.info.port", 0); TEST_UTIL.startMiniCluster(2); TEST_UTIL.createMultiRegionTable(TABLENAME, FAMILIES); HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); countOfRegions = -1; try (RegionLocator r = t.getRegionLocator()) { countOfRegions = r.getStartKeys().length; } waitUntilAllRegionsAssigned(); addToEachStartKey(countOfRegions); t.close(); TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.initializeHandlerTrackers(); }
private void runIncrementalPELoad(Configuration conf, HTableDescriptor tableDescriptor, RegionLocator regionLocator, Path outDir) throws IOException, UnsupportedEncodingException, InterruptedException, ClassNotFoundException { Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("runIncrementalPELoad")); job.getConfiguration().setStrings("io.serializations", conf.get("io.serializations"), MutationSerialization.class.getName(), ResultSerialization.class.getName(), KeyValueSerialization.class.getName()); setupRandomGeneratorMapper(job); HFileOutputFormat2.configureIncrementalLoad(job, tableDescriptor, regionLocator); FileOutputFormat.setOutputPath(job, outDir); assertFalse(util.getTestFileSystem().exists(outDir)) ; assertEquals(regionLocator.getAllRegionLocations().size(), job.getNumReduceTasks()); assertTrue(job.waitForCompletion(true)); }
@Test public void testSimpleTestCase() throws Exception { RegionLocator regionLocator = mockRegionLocator("region1", "region2", "region3"); Admin admin = mockAdmin( mockServer( mockRegion("region1", 123), mockRegion("region3", 1232) ), mockServer( mockRegion("region2", 54321), mockRegion("otherTableRegion", 110) ) ); RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin); assertEquals(123 * megabyte, calculator.getRegionSize("region1".getBytes())); assertEquals(54321 * megabyte, calculator.getRegionSize("region2".getBytes())); assertEquals(1232 * megabyte, calculator.getRegionSize("region3".getBytes())); // if region is not inside our table, it should return 0 assertEquals(0 * megabyte, calculator.getRegionSize("otherTableRegion".getBytes())); assertEquals(3, calculator.getRegionSizeMap().size()); }
/** * When size of region in megabytes is larger than largest possible integer there could be * error caused by lost of precision. * */ @Test public void testLargeRegion() throws Exception { RegionLocator regionLocator = mockRegionLocator("largeRegion"); Admin admin = mockAdmin( mockServer( mockRegion("largeRegion", Integer.MAX_VALUE) ) ); RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin); assertEquals(((long) Integer.MAX_VALUE) * megabyte, calculator.getRegionSize("largeRegion".getBytes())); }
/** When calculator is disabled, it should return 0 for each request.*/ @Test public void testDisabled() throws Exception { String regionName = "cz.goout:/index.html"; RegionLocator table = mockRegionLocator(regionName); Admin admin = mockAdmin( mockServer( mockRegion(regionName, 999) ) ); //first request on enabled calculator RegionSizeCalculator calculator = new RegionSizeCalculator(table, admin); assertEquals(999 * megabyte, calculator.getRegionSize(regionName.getBytes())); //then disabled calculator. configuration.setBoolean(RegionSizeCalculator.ENABLE_REGIONSIZECALCULATOR, false); RegionSizeCalculator disabledCalculator = new RegionSizeCalculator(table, admin); assertEquals(0 * megabyte, disabledCalculator.getRegionSize(regionName.getBytes())); assertEquals(0, disabledCalculator.getRegionSizeMap().size()); }
private void verifyRegionResults(RegionLocator regionLocator, Map<byte[], String> results, String expected, byte[] row) throws Exception { for (Map.Entry<byte [], String> e: results.entrySet()) { LOG.info("row=" + Bytes.toString(row) + ", expected=" + expected + ", result key=" + Bytes.toString(e.getKey()) + ", value=" + e.getValue()); } HRegionLocation loc = regionLocator.getRegionLocation(row, true); byte[] region = loc.getRegionInfo().getRegionName(); assertTrue("Results should contain region " + Bytes.toStringBinary(region) + " for row '" + Bytes.toStringBinary(row)+ "'", results.containsKey(region)); assertEquals("Invalid result for row '"+Bytes.toStringBinary(row)+"'", expected, results.get(region)); }
@Override public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws IOException { HRegionLocation regionLoc = null; try (RegionLocator locator = connection.getRegionLocator(tn)) { regionLoc = locator.getRegionLocation(regionName); } if (regionLoc == null) { LOG.warn("Cannot find region server holding region " + Bytes.toString(regionName) + ", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName)) + "]"); return null; } AdminProtos.AdminService.BlockingInterface client = ((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName()); ServerInfo info = ProtobufUtil.getServerInfo(null, client); return ProtobufUtil.toServerName(info.getServerName()); }
private List<HRegionLocation> getRegionLocations(Connection connection) throws Exception { final Scan scan = read.serializableScan.get(); byte[] startRow = scan.getStartRow(); byte[] stopRow = scan.getStopRow(); final List<HRegionLocation> regionLocations = new ArrayList<>(); final boolean scanWithNoLowerBound = startRow.length == 0; final boolean scanWithNoUpperBound = stopRow.length == 0; TableName tableName = TableName.valueOf(read.tableId); RegionLocator regionLocator = connection.getRegionLocator(tableName); List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations(); for (HRegionLocation regionLocation : tableRegionInfos) { final byte[] startKey = regionLocation.getRegionInfo().getStartKey(); final byte[] endKey = regionLocation.getRegionInfo().getEndKey(); boolean isLastRegion = endKey.length == 0; // filters regions who are part of the scan if ((scanWithNoLowerBound || isLastRegion || Bytes.compareTo(startRow, endKey) < 0) && (scanWithNoUpperBound || Bytes.compareTo(stopRow, startKey) > 0)) { regionLocations.add(regionLocation); } } return regionLocations; }
@Test public void testMove() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(TEST_TABLE.getTableName())) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); final ServerName server = location.getServerName(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preMove(ObserverContext.createAndPrepare(CP_ENV, null), hri, server, server); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); }
@Test public void testAssign() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(TEST_TABLE.getTableName())) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preAssign(ObserverContext.createAndPrepare(CP_ENV, null), hri); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); }
@Test public void testUnassign() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(TEST_TABLE.getTableName())) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preUnassign(ObserverContext.createAndPrepare(CP_ENV, null), hri, false); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); }
@Test public void testRegionOffline() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(TEST_TABLE.getTableName())) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preRegionOffline(ObserverContext.createAndPrepare(CP_ENV, null), hri); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); }
private void verifyRegionResults(RegionLocator table, Map<byte[], String> results, String expected, byte[] row) throws Exception { for (Map.Entry<byte [], String> e: results.entrySet()) { LOG.info("row=" + Bytes.toString(row) + ", expected=" + expected + ", result key=" + Bytes.toString(e.getKey()) + ", value=" + e.getValue()); } HRegionLocation loc = table.getRegionLocation(row, true); byte[] region = loc.getRegionInfo().getRegionName(); assertTrue("Results should contain region " + Bytes.toStringBinary(region) + " for row '" + Bytes.toStringBinary(row)+ "'", results.containsKey(region)); assertEquals("Invalid result for row '"+Bytes.toStringBinary(row)+"'", expected, results.get(region)); }
private static void printTableRegions(String tableName) throws IOException { // co CreateTableWithRegionsExample-1-PrintTable Helper method to print the regions of a table. System.out.println("Printing regions of table: " + tableName); TableName tn = TableName.valueOf(tableName); RegionLocator locator = connection.getRegionLocator(tn); Pair<byte[][], byte[][]> pair = locator.getStartEndKeys(); // co CreateTableWithRegionsExample-2-GetKeys Retrieve the start and end keys from the newly created table. for (int n = 0; n < pair.getFirst().length; n++) { byte[] sk = pair.getFirst()[n]; byte[] ek = pair.getSecond()[n]; System.out.println("[" + (n + 1) + "]" + " start key: " + (sk.length == 8 ? Bytes.toLong(sk) : Bytes.toStringBinary(sk)) + // co CreateTableWithRegionsExample-3-Print Print the key, but guarding against the empty start (and end) key. ", end key: " + (ek.length == 8 ? Bytes.toLong(ek) : Bytes.toStringBinary(ek))); } locator.close(); }
private void doBulkLoad(LoadIncrementalHFiles loadHFiles, Table table, Deque<LoadQueueItem> queue, RegionLocator locator, int maxRetries) throws IOException { int count = 0; Pair<byte[][], byte[][]> startEndKeys; while (!queue.isEmpty()) { // need to reload split keys each iteration. startEndKeys = locator.getStartEndKeys(); if (count != 0) { LOG.warn("Error occurred while replicating HFiles, retry attempt " + count + " with " + queue.size() + " files still remaining to replicate."); } if (maxRetries != 0 && count >= maxRetries) { throw new IOException("Retry attempted " + count + " times without completing, bailing out."); } count++; // Try bulk load loadHFiles.loadHFileQueue(table, connection, queue, startEndKeys); } }
/** * Perform a bulk load of the given directory into the given pre-existing table. This method is * not threadsafe. * @param map map of family to List of hfiles * @param admin the Admin * @param table the table to load into * @param regionLocator region locator * @param silence true to ignore unmatched column families * @param copyFile always copy hfiles if true * @throws TableNotFoundException if table does not yet exist */ public Map<LoadQueueItem, ByteBuffer> doBulkLoad(Map<byte[], List<Path>> map, final Admin admin, Table table, RegionLocator regionLocator, boolean silence, boolean copyFile) throws TableNotFoundException, IOException { if (!admin.isTableAvailable(regionLocator.getName())) { throw new TableNotFoundException("Table " + table.getName() + " is not currently available."); } // LQI queue does not need to be threadsafe -- all operations on this queue // happen in this thread Deque<LoadQueueItem> queue = new ArrayDeque<>(); ExecutorService pool = null; SecureBulkLoadClient secureClient = null; try { prepareHFileQueue(map, table, queue, silence); if (queue.isEmpty()) { LOG.warn("Bulk load operation did not get any files to load"); return Collections.emptyMap(); } pool = createExecutorService(); secureClient = new SecureBulkLoadClient(table.getConfiguration(), table); return performBulkLoad(admin, table, regionLocator, queue, pool, secureClient, copyFile); } finally { cleanup(admin, queue, pool, secureClient); } }
/** * Perform bulk load on the given table. * @param hfofDir the directory that was provided as the output path of a job using * HFileOutputFormat * @param tableName the table to load into */ public Map<LoadQueueItem, ByteBuffer> run(String hfofDir, TableName tableName) throws IOException { try (Connection connection = ConnectionFactory.createConnection(getConf()); Admin admin = connection.getAdmin()) { if (!admin.tableExists(tableName)) { if (isCreateTable()) { createTable(tableName, hfofDir, admin); } else { String errorMsg = format("Table '%s' does not exist.", tableName); LOG.error(errorMsg); throw new TableNotFoundException(errorMsg); } } try (Table table = connection.getTable(tableName); RegionLocator locator = connection.getRegionLocator(tableName)) { return doBulkLoad(new Path(hfofDir), admin, table, locator, isSilence(), isAlwaysCopyFiles()); } } }
/** * Perform bulk load on the given table. * @param family2Files map of family to List of hfiles * @param tableName the table to load into */ public Map<LoadQueueItem, ByteBuffer> run(Map<byte[], List<Path>> family2Files, TableName tableName) throws IOException { try (Connection connection = ConnectionFactory.createConnection(getConf()); Admin admin = connection.getAdmin()) { if (!admin.tableExists(tableName)) { String errorMsg = format("Table '%s' does not exist.", tableName); LOG.error(errorMsg); throw new TableNotFoundException(errorMsg); } try (Table table = connection.getTable(tableName); RegionLocator locator = connection.getRegionLocator(tableName)) { return doBulkLoad(family2Files, admin, table, locator, isSilence(), isAlwaysCopyFiles()); } } }
@Test (timeout=180000) public void testMove() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); final ServerName server = location.getServerName(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preMove(ObserverContextImpl.createAndPrepare(CP_ENV), hri, server, server); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
@Test (timeout=180000) public void testAssign() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preAssign(ObserverContextImpl.createAndPrepare(CP_ENV), hri); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
@Test (timeout=180000) public void testUnassign() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preUnassign(ObserverContextImpl.createAndPrepare(CP_ENV), hri, false); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
@Test (timeout=180000) public void testRegionOffline() throws Exception { List<HRegionLocation> regions; try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); final HRegionInfo hri = location.getRegionInfo(); AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preRegionOffline(ObserverContextImpl.createAndPrepare(CP_ENV), hri); return null; } }; verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER, USER_GROUP_ADMIN); verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); }
/** * Create a table with specified table name and region number. * @param tablename * @param regionNum * @return * @throws IOException */ private static void createTable(TableName tableName, int regionNum) throws IOException { int expectedRegions = regionNum; byte[][] splitKeys = new byte[expectedRegions - 1][]; for (int i = 1; i < expectedRegions; i++) { byte splitKey = (byte) i; splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey }; } HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, splitKeys); try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) { List<HRegionLocation> regions = r.getAllRegionLocations(); assertEquals("Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); } }
private List<HRegionLocation> splitRegion(final RegionInfo r) throws IOException, InterruptedException { List<HRegionLocation> locations = new ArrayList<>(); // Split this table in two. Admin admin = TEST_UTIL.getAdmin(); Connection connection = TEST_UTIL.getConnection(); admin.splitRegion(r.getEncodedNameAsBytes()); admin.close(); PairOfSameType<RegionInfo> regions = waitOnDaughters(r); if (regions != null) { try (RegionLocator rl = connection.getRegionLocator(r.getTable())) { locations.add(rl.getRegionLocation(regions.getFirst().getEncodedNameAsBytes())); locations.add(rl.getRegionLocation(regions.getSecond().getEncodedNameAsBytes())); } return locations; } return locations; }
@Test public void testRegionObserverMultiRegion() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName) .addFamily(new HColumnDescriptor(foo)) // add the coprocessor for the region .addCoprocessor(CustomRegionObserver.class.getName()) , new byte[][]{foo}); // create with 2 regions try (Table table = connection.getTable(tableName); RegionLocator locator = connection.getRegionLocator(tableName)) { table.get(new Get(bar)); table.get(new Get(foo)); // 2 gets to 2 separate regions assertEquals(2, locator.getAllRegionLocations().size()); assertNotEquals(locator.getRegionLocation(bar).getRegionInfo(), locator.getRegionLocation(foo).getRegionInfo()); } } assertPreGetRequestsCounter(CustomRegionObserver.class); }
private void verifyBounds(List<byte[]> expectedBounds, TableName tableName) throws Exception { // Get region boundaries from the cluster and verify their endpoints final int numRegions = expectedBounds.size()-1; try (Table table = UTIL.getConnection().getTable(tableName); RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) { final List<HRegionLocation> regionInfoMap = locator.getAllRegionLocations(); assertEquals(numRegions, regionInfoMap.size()); for (HRegionLocation entry : regionInfoMap) { final HRegionInfo regionInfo = entry.getRegionInfo(); byte[] regionStart = regionInfo.getStartKey(); byte[] regionEnd = regionInfo.getEndKey(); // This region's start key should be one of the region boundaries int startBoundaryIndex = indexOfBytes(expectedBounds, regionStart); assertNotSame(-1, startBoundaryIndex); // This region's end key should be the region boundary that comes // after the starting boundary. byte[] expectedRegionEnd = expectedBounds.get(startBoundaryIndex + 1); assertEquals(0, Bytes.compareTo(regionEnd, expectedRegionEnd)); } } }
@BeforeClass public static void before() throws Exception { HTU.startMiniCluster(NB_SERVERS); final TableName tableName = TableName.valueOf(TestRegionServerNoMaster.class.getSimpleName()); // Create table then get the single region for our new table. table = HTU.createTable(tableName,HConstants.CATALOG_FAMILY); Put p = new Put(row); p.addColumn(HConstants.CATALOG_FAMILY, row, row); table.put(p); try (RegionLocator locator = HTU.getConnection().getRegionLocator(tableName)) { hri = locator.getRegionLocation(row, false).getRegionInfo(); } regionName = hri.getRegionName(); stopMasterAndAssignMeta(HTU); }
/** verify region boundaries obtained from HTable.getStartEndKeys() */ void verifyRegionsUsingHTable() throws IOException { Table table = null; try { //HTable.getStartEndKeys() table = connection.getTable(tableName); try(RegionLocator rl = connection.getRegionLocator(tableName)) { Pair<byte[][], byte[][]> keys = rl.getStartEndKeys(); verifyStartEndKeys(keys); Set<RegionInfo> regions = new TreeSet<>(RegionInfo.COMPARATOR); for (HRegionLocation loc : rl.getAllRegionLocations()) { regions.add(loc.getRegionInfo()); } verifyTableRegions(regions); } } finally { IOUtils.closeQuietly(table); } }
@BeforeClass public static void before() throws Exception { // Reduce the hdfs block size and prefetch to trigger the file-link reopen // when the file is moved to archive (e.g. compaction) HTU.getConfiguration().setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 8192); HTU.getConfiguration().setInt(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 1); HTU.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 128 * 1024 * 1024); HTU.startMiniCluster(NB_SERVERS); final TableName tableName = TableName.valueOf(TestRegionReplicas.class.getSimpleName()); // Create table then get the single region for our new table. table = HTU.createTable(tableName, f); try (RegionLocator locator = HTU.getConnection().getRegionLocator(tableName)) { hriPrimary = locator.getRegionLocation(row, false).getRegionInfo(); } // mock a secondary region info to open hriSecondary = new HRegionInfo(hriPrimary.getTable(), hriPrimary.getStartKey(), hriPrimary.getEndKey(), hriPrimary.isSplit(), hriPrimary.getRegionId(), 1); // No master TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU); }
@Override public void setup(Context context) throws IOException { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); int reduceNum = context.getNumReduceTasks(); Configuration conf = context.getConfiguration(); TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME)); try (Connection conn = ConnectionFactory.createConnection(conf); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { byte[][] startKeys = regionLocator.getStartKeys(); if (startKeys.length != reduceNum) { throw new IOException("Region split after job initialization"); } CellWritableComparable[] startKeyWraps = new CellWritableComparable[startKeys.length - 1]; for (int i = 1; i < startKeys.length; ++i) { startKeyWraps[i - 1] = new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i])); } CellWritableComparablePartitioner.START_KEYS = startKeyWraps; } }
@Test public void testSimpleTestCase() throws Exception { RegionLocator regionLocator = mockRegionLocator("region1", "region2", "region3"); Admin admin = mockAdmin( mockRegion("region1", 123), mockRegion("region3", 1232), mockRegion("region2", 54321) ); RegionSizeCalculator calculator = new RegionSizeCalculator(regionLocator, admin); assertEquals(123 * megabyte, calculator.getRegionSize("region1".getBytes())); assertEquals(54321 * megabyte, calculator.getRegionSize("region2".getBytes())); assertEquals(1232 * megabyte, calculator.getRegionSize("region3".getBytes())); // if regionCalculator does not know about a region, it should return 0 assertEquals(0 * megabyte, calculator.getRegionSize("otherTableRegion".getBytes())); assertEquals(3, calculator.getRegionSizeMap().size()); }
/** When calculator is disabled, it should return 0 for each request.*/ @Test public void testDisabled() throws Exception { String regionName = "cz.goout:/index.html"; RegionLocator table = mockRegionLocator(regionName); Admin admin = mockAdmin( mockRegion(regionName, 999) ); //first request on enabled calculator RegionSizeCalculator calculator = new RegionSizeCalculator(table, admin); assertEquals(999 * megabyte, calculator.getRegionSize(regionName.getBytes())); //then disabled calculator. configuration.setBoolean(RegionSizeCalculator.ENABLE_REGIONSIZECALCULATOR, false); RegionSizeCalculator disabledCalculator = new RegionSizeCalculator(table, admin); assertEquals(0 * megabyte, disabledCalculator.getRegionSize(regionName.getBytes())); assertEquals(0, disabledCalculator.getRegionSizeMap().size()); }