private static void bootstrap(final Path rd, final Configuration c) throws IOException { LOG.info("BOOTSTRAP: creating hbase:meta region"); try { // Bootstrapping, make sure blockcache is off. Else, one will be // created here in bootstrap and it'll need to be cleaned up. Better to // not make it in first place. Turn off block caching for bootstrap. // Enable after. HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); setInfoFamilyCachingForMeta(metaDescriptor, false); HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor, null, true, true); setInfoFamilyCachingForMeta(metaDescriptor, true); HRegion.closeHRegion(meta); } catch (IOException e) { e = RemoteExceptionHandler.checkIOException(e); LOG.error("bootstrap", e); throw e; } }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getHBaseAdmin(); // Verify descriptor from master HTableDescriptor htd = admin.getTableDescriptor(tableName); HColumnDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); htd = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }
@Test public void testReadAndWriteHRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); // Create a region. That'll write the .regioninfo file. HRegion r = HRegion.createHRegion(hri, basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HRegion.closeHRegion(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, hri, fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. HRegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); assertTrue(hri.equals(deserializedHri)); }
@Override public Void call() throws Exception { LOG.debug("Running table info copy."); this.rethrowException(); LOG.debug("Attempting to copy table info for snapshot:" + SnapshotDescriptionUtils.toString(this.snapshot)); // get the HTable descriptor HTableDescriptor orig = FSTableDescriptors.getTableDescriptor(fs, rootDir, Bytes.toBytes(this.snapshot.getTable())); this.rethrowException(); // write a copy of descriptor to the snapshot directory Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); FSTableDescriptors.createTableDescriptorForTableDirectory(fs, snapshotDir, orig, false); LOG.debug("Finished copying tableinfo."); return null; }
private static void bootstrap(final Path rd, final Configuration c) throws IOException { LOG.info("BOOTSTRAP: creating hbase:meta region"); try { // Bootstrapping, make sure blockcache is off. Else, one will be // created here in bootstrap and it'll need to be cleaned up. Better to // not make it in first place. Turn off block caching for bootstrap. // Enable after. HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); setInfoFamilyCachingForMeta(metaDescriptor, false); HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor); setInfoFamilyCachingForMeta(metaDescriptor, true); HRegion.closeHRegion(meta); } catch (IOException e) { e = RemoteExceptionHandler.checkIOException(e); LOG.error("bootstrap", e); throw e; } }
@Override public Void call() throws Exception { LOG.debug("Running table info copy."); this.rethrowException(); LOG.debug("Attempting to copy table info for snapshot:" + ClientSnapshotDescriptionUtils.toString(this.snapshot)); // get the HTable descriptor HTableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, TableName.valueOf(this.snapshot.getTable())); this.rethrowException(); // write a copy of descriptor to the snapshot directory Path snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir); new FSTableDescriptors(fs, rootDir) .createTableDescriptorForTableDirectory(snapshotDir, orig, false); LOG.debug("Finished copying tableinfo."); return null; }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); // Verify descriptor from master HTableDescriptor htd = admin.getTableDescriptor(tableName); HColumnDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); htd = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }
private static void bootstrap(final Path rd, final Configuration c) throws IOException { LOG.info("BOOTSTRAP: creating hbase:meta region"); try { // Bootstrapping, make sure blockcache is off. Else, one will be // created here in bootstrap and it'll need to be cleaned up. Better to // not make it in first place. Turn off block caching for bootstrap. // Enable after. TableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME); HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, rd, c, setInfoFamilyCachingForMeta(metaDescriptor, false), null); meta.close(); } catch (IOException e) { e = e instanceof RemoteException ? ((RemoteException)e).unwrapRemoteException() : e; LOG.error("bootstrap", e); throw e; } }
protected static List<RegionInfo> createFsLayout(final MasterProcedureEnv env, final TableDescriptor tableDescriptor, List<RegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) .createTableDescriptorForTableDirectory( tempTableDir, tableDescriptor, false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir, tableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); return newRegions; }
/** * Create region layout in file system. * @param env MasterProcedureEnv * @throws IOException */ private List<RegionInfo> createFsLayout( final MasterProcedureEnv env, final TableDescriptor tableDescriptor, List<RegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, tableDescriptor.getTableName()); ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors())) .createTableDescriptorForTableDirectory(tempTableDir, TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions( env, tempdir, tableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir); return newRegions; }
private void verifyHColumnDescriptor(int expected, final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getAdmin(); // Verify descriptor from master TableDescriptor htd = admin.getDescriptor(tableName); ColumnFamilyDescriptor[] hcds = htd.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); TableDescriptor td = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); hcds = td.getColumnFamilies(); verifyHColumnDescriptor(expected, hcds, tableName, families); }
@Test public void testReadAndWriteHRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); HRegionInfo hri = HRegionInfo.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(hri, basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, hri, fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. org.apache.hadoop.hbase.client.RegionInfo deserializedHri = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); assertTrue(org.apache.hadoop.hbase.client.RegionInfo.COMPARATOR.compare(hri, deserializedHri) == 0); HBaseTestingUtility.closeRegionAndWAL(r); }
@Test public void testReadAndWriteRegionInfoFile() throws IOException, InterruptedException { HBaseTestingUtility htu = new HBaseTestingUtility(); RegionInfo ri = RegionInfoBuilder.FIRST_META_REGIONINFO; Path basedir = htu.getDataTestDir(); // Create a region. That'll write the .regioninfo file. FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(htu.getConfiguration()); HRegion r = HBaseTestingUtility.createRegionAndWAL(convert(ri), basedir, htu.getConfiguration(), fsTableDescriptors.get(TableName.META_TABLE_NAME)); // Get modtime on the file. long modtime = getModTime(r); HBaseTestingUtility.closeRegionAndWAL(r); Thread.sleep(1001); r = HRegion.openHRegion(basedir, convert(ri), fsTableDescriptors.get(TableName.META_TABLE_NAME), null, htu.getConfiguration()); // Ensure the file is not written for a second time. long modtime2 = getModTime(r); assertEquals(modtime, modtime2); // Now load the file. RegionInfo deserializedRi = HRegionFileSystem.loadRegionInfoFileContent( r.getRegionFileSystem().getFileSystem(), r.getRegionFileSystem().getRegionDir()); HBaseTestingUtility.closeRegionAndWAL(r); }
@Test public void testGetSetOfHTD() throws IOException { HBaseTestingUtility HTU = new HBaseTestingUtility(); final String tablename = "testGetSetOfHTD"; // Delete the temporary table directory that might still be there from the // previous test run. FSTableDescriptors.deleteTableDescriptorIfExists(tablename, HTU.getConfiguration()); HTableDescriptor htd = new HTableDescriptor(tablename); FSTableDescriptors.createTableDescriptor(htd, HTU.getConfiguration()); HRegionInfo hri = new HRegionInfo(Bytes.toBytes("testGetSetOfHTD"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); HTableDescriptor htd2 = hri.getTableDesc(); assertTrue(htd.equals(htd2)); final String key = "SOME_KEY"; assertNull(htd.getValue(key)); final String value = "VALUE"; htd.setValue(key, value); hri.setTableDesc(htd); HTableDescriptor htd3 = hri.getTableDesc(); assertTrue(htd.equals(htd3)); }
protected static List<HRegionInfo> createFsLayout(final MasterProcedureEnv env, final HTableDescriptor hTableDescriptor, List<HRegionInfo> newRegions, final CreateHdfsRegions hdfsRegionHandler) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tempdir = mfs.getTempDir(); // 1. Create Table Descriptor // using a copy of descriptor, table will be created enabling first final Path tempTableDir = FSUtils.getTableDir(tempdir, hTableDescriptor.getTableName()); new FSTableDescriptors(env.getMasterConfiguration()).createTableDescriptorForTableDirectory( tempTableDir, hTableDescriptor, false); // 2. Create Regions newRegions = hdfsRegionHandler.createHdfsRegions(env, tempdir, hTableDescriptor.getTableName(), newRegions); // 3. Move Table temp directory to the hbase root location final Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), hTableDescriptor.getTableName()); FileSystem fs = mfs.getFileSystem(); if (!fs.delete(tableDir, true) && fs.exists(tableDir)) { throw new IOException("Couldn't delete " + tableDir); } if (!fs.rename(tempTableDir, tableDir)) { throw new IOException("Unable to move table from temp=" + tempTableDir + " to hbase root=" + tableDir); } return newRegions; }
public void consolidate() throws IOException { if (getSnapshotFormat(desc) == SnapshotManifestV1.DESCRIPTOR_VERSION) { Path rootDir = FSUtils.getRootDir(conf); LOG.info("Using old Snapshot Format"); // write a copy of descriptor to the snapshot directory new FSTableDescriptors(conf, fs, rootDir) .createTableDescriptorForTableDirectory(workingDir, htd, false); } else { LOG.debug("Convert to Single Snapshot Manifest"); convertToV2SingleManifest(); } }
private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major) throws IOException { HTableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { compactRegion(tableDir, htd, regionDir, compactOnce, major); } }
private static void processTable(final FileSystem fs, final Path p, final WALFactory walFactory, final Configuration c, final boolean majorCompact) throws IOException { HRegion region; FSTableDescriptors fst = new FSTableDescriptors(c); // Currently expects tables have one region only. if (FSUtils.getTableName(p).equals(TableName.META_TABLE_NAME)) { final WAL wal = walFactory.getMetaWAL(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes()); region = HRegion.newHRegion(p, wal, fs, c, HRegionInfo.FIRST_META_REGIONINFO, fst.get(TableName.META_TABLE_NAME), null); } else { throw new IOException("Not a known catalog table: " + p.toString()); } try { region.mvcc.advanceTo(region.initialize(null)); if (majorCompact) { region.compact(true); } else { // Default behavior Scan scan = new Scan(); // scan.addFamily(HConstants.CATALOG_FAMILY); RegionScanner scanner = region.getScanner(scan); try { List<Cell> kvs = new ArrayList<Cell>(); boolean done; do { kvs.clear(); done = scanner.next(kvs); if (kvs.size() > 0) LOG.info(kvs); } while (done); } finally { scanner.close(); } } } finally { region.close(); } }
/** * @return META table descriptor */ public HTableDescriptor getMetaTableDescriptor() { try { return new FSTableDescriptors(conf).get(TableName.META_TABLE_NAME); } catch (IOException e) { throw new RuntimeException("Unable to create META table descriptor", e); } }
@Test public void testShouldCreateNewTableDescriptorIfForcefulCreationIsFalse() throws IOException { final String name = "newTable2"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); assertTrue("Should create new table descriptor", fstd.createTableDescriptor(htd, false)); }
@Test public void testShouldNotCreateTheSameTableDescriptorIfForcefulCreationIsFalse() throws IOException { final String name = "testAlreadyExists"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detritus laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); HTableDescriptor htd = new HTableDescriptor(name); fstd.add(htd); assertFalse("Should not create new table descriptor", fstd.createTableDescriptor(htd, false)); }
@Test public void testShouldAllowForcefulCreationOfAlreadyExistingTableDescriptor() throws Exception { final String name = "createNewTableNew2"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name)); fstd.createTableDescriptor(htd, false); assertTrue("Should create new table descriptor", fstd.createTableDescriptor(htd, true)); }
private void verifyTableDescriptor(final TableName tableName, final byte[]... families) throws IOException { Admin admin = TEST_UTIL.getHBaseAdmin(); // Verify descriptor from master HTableDescriptor htd = admin.getTableDescriptor(tableName); verifyTableDescriptor(htd, tableName, families); // Verify descriptor from HDFS MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName); htd = FSTableDescriptors.getTableDescriptorFromFs(mfs.getFileSystem(), tableDir); verifyTableDescriptor(htd, tableName, families); }
public SnapshotBuilder(final Configuration conf, final FileSystem fs, final Path rootDir, final HTableDescriptor htd, final SnapshotDescription desc, final RegionData[] tableRegions) throws IOException { this.fs = fs; this.conf = conf; this.rootDir = rootDir; this.htd = htd; this.desc = desc; this.tableRegions = tableRegions; this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir); new FSTableDescriptors(conf) .createTableDescriptorForTableDirectory(snapshotDir, htd, false); }
/** * Execute the restore operation * @param snapshotDir The snapshot directory to use as "restore source" * @param sd The snapshot descriptor * @param htdClone The HTableDescriptor of the table to restore/clone. */ public void testRestore(final Path snapshotDir, final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException { LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG); new FSTableDescriptors(conf).createTableDescriptor(htdClone); RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone); helper.restoreHdfsRegions(); LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG); }
private void createRootTableInfo(Path rd) throws IOException { // Create ROOT tableInfo if required. if (!FSTableDescriptors.isTableInfoExists(fs, rd, Bytes.toString(HRegionInfo.ROOT_REGIONINFO.getTableName()))) { FSTableDescriptors.createTableDescriptor(HTableDescriptor.ROOT_TABLEDESC, this.conf); } }
/** * Load snapshot info and table descriptor for the specified snapshot * @param snapshotName name of the snapshot to load * @return false if snapshot is not found */ private boolean loadSnapshotInfo(final String snapshotName) throws IOException { snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); if (!fs.exists(snapshotDir)) { LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir); return false; } snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); snapshotTableDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir); return true; }
private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major) throws IOException { HTableDescriptor htd = FSTableDescriptors.getTableDescriptor(fs, tableDir); LOG.info("Compact table=" + htd.getNameAsString()); for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { compactRegion(htd, regionDir, compactOnce, major); } }