/********************************* Master related hooks **********************************/ @Override public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException { // Need to create the new system table for labels here MasterServices master = ctx.getEnvironment().getMasterServices(); if (!MetaTableAccessor.tableExists(master.getConnection(), LABELS_TABLE_NAME)) { HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME); HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY); labelsColumn.setBloomFilterType(BloomType.NONE); labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal // table block cache. labelsTable.addFamily(labelsColumn); // Let the "labels" table having only one region always. We are not expecting too many labels in // the system. labelsTable.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); labelsTable.setValue(Bytes.toBytes(HConstants.DISALLOW_WRITES_IN_RECOVERING), Bytes.toBytes(true)); master.createTable(labelsTable, null, HConstants.NO_NONCE, HConstants.NO_NONCE); } }
private HTableDescriptor createHtd(boolean isStripe) throws Exception { HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); htd.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); String noSplitsPolicy = DisabledRegionSplitPolicy.class.getName(); htd.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, noSplitsPolicy); if (isStripe) { htd.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY, StripeStoreEngine.class.getName()); if (initialStripeCount != null) { htd.setConfiguration( StripeStoreConfig.INITIAL_STRIPE_COUNT_KEY, initialStripeCount.toString()); htd.setConfiguration( HStore.BLOCKING_STOREFILES_KEY, Long.toString(10 * initialStripeCount)); } else { htd.setConfiguration(HStore.BLOCKING_STOREFILES_KEY, "500"); } if (splitSize != null) { htd.setConfiguration(StripeStoreConfig.SIZE_TO_SPLIT_KEY, splitSize.toString()); } if (splitParts != null) { htd.setConfiguration(StripeStoreConfig.SPLIT_PARTS_KEY, splitParts.toString()); } } else { htd.setConfiguration(HStore.BLOCKING_STOREFILES_KEY, "10"); // default } return htd; }
/********************************* Master related hooks **********************************/ @Override public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException { // Need to create the new system table for labels here MasterServices master = ctx.getEnvironment().getMasterServices(); if (!MetaTableAccessor.tableExists(master.getConnection(), LABELS_TABLE_NAME)) { HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME); HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY); labelsColumn.setBloomFilterType(BloomType.NONE); labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal // table block cache. labelsTable.addFamily(labelsColumn); // Let the "labels" table having only one region always. We are not expecting too many labels in // the system. labelsTable.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); labelsTable.setValue(Bytes.toBytes(HConstants.DISALLOW_WRITES_IN_RECOVERING), Bytes.toBytes(true)); master.createTable(labelsTable, null); } }
/********************************* Master related hooks **********************************/ @Override public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException { // Need to create the new system table for labels here MasterServices master = ctx.getEnvironment().getMasterServices(); if (!MetaReader.tableExists(master.getCatalogTracker(), LABELS_TABLE_NAME)) { HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME); HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY); labelsColumn.setBloomFilterType(BloomType.NONE); labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal // table block cache. labelsTable.addFamily(labelsColumn); // Let the "labels" table having only one region always. We are not expecting too many labels in // the system. labelsTable.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); labelsTable.setValue(Bytes.toBytes(HConstants.DISALLOW_WRITES_IN_RECOVERING), Bytes.toBytes(true)); master.createTable(labelsTable, null); } }
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */ public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException { Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin(); try { if (admin.tableExists(tableName)) { logger.info("disabling hbase table " + tableName); admin.disableTable(tableName); logger.info("deleting hbase table " + tableName); admin.deleteTable(tableName); } HTableDescriptor tableDesc = new HTableDescriptor(tableName); tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); tableDesc.addFamily(createColumnFamily(kylinConfig, cfName, false)); logger.info("creating hbase table " + tableName); admin.createTable(tableDesc, null); Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons"); logger.info("create hbase table " + tableName + " done."); } finally { IOUtils.closeQuietly(admin); } }
/********************************* Master related hooks **********************************/ @Override public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException { // Need to create the new system table for labels here if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getConnection(), LABELS_TABLE_NAME)) { HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME); HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY); labelsColumn.setBloomFilterType(BloomType.NONE); labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal // table block cache. labelsTable.addFamily(labelsColumn); // Let the "labels" table having only one region always. We are not expecting too many labels in // the system. labelsTable.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) { admin.createTable(labelsTable); } } }
private HStore prepareData() throws IOException { Admin admin = TEST_UTIL.getAdmin(); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build()) .build(); admin.createTable(desc); Table table = TEST_UTIL.getConnection().getTable(tableName); TimeOffsetEnvironmentEdge edge = (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate(); for (int i = 0; i < 10; i++) { for (int j = 0; j < 10; j++) { byte[] value = new byte[128 * 1024]; ThreadLocalRandom.current().nextBytes(value); table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value)); } admin.flush(tableName); edge.increment(1001); } return getStoreWithName(tableName); }
@Test public void testSanityCheckBlockingStoreFiles() throws IOException { error.expect(DoNotRetryIOException.class); error.expectMessage("Blocking file count 'hbase.hstore.blockingStoreFiles'"); error.expectMessage("is below recommended minimum of 1000 for column family"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-BlockingStoreFiles"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .setValue(HStore.BLOCKING_STOREFILES_KEY, "10") .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build()) .build(); TEST_UTIL.getAdmin().createTable(desc); }
private Store prepareData() throws IOException { HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()); desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()); HColumnDescriptor colDesc = new HColumnDescriptor(family); colDesc.setTimeToLive(1); // 1 sec desc.addFamily(colDesc); admin.createTable(desc); Table table = TEST_UTIL.getConnection().getTable(tableName); Random rand = new Random(); TimeOffsetEnvironmentEdge edge = (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate(); for (int i = 0; i < 10; i++) { for (int j = 0; j < 10; j++) { byte[] value = new byte[128 * 1024]; rand.nextBytes(value); table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value)); } admin.flush(tableName); edge.increment(1001); } return getStoreWithName(tableName); }
@Test public void testSanityCheckTTL() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); TEST_UTIL.startMiniCluster(1); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); String tableName = this.tableName.getNameAsString()+"-TTL"; if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()); desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()); HColumnDescriptor colDesc = new HColumnDescriptor(family); desc.addFamily(colDesc); try{ admin.createTable(desc); Assert.fail(); }catch(Exception e){ }finally{ TEST_UTIL.shutdownMiniCluster(); } }
@Test public void testSanityCheckMinVersion() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); TEST_UTIL.startMiniCluster(1); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); String tableName = this.tableName.getNameAsString()+"-MinVersion"; if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()); desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()); HColumnDescriptor colDesc = new HColumnDescriptor(family); colDesc.setTimeToLive(1); // 1 sec colDesc.setMinVersions(1); desc.addFamily(colDesc); try{ admin.createTable(desc); Assert.fail(); }catch(Exception e){ }finally{ TEST_UTIL.shutdownMiniCluster(); } }
@Test public void testSanityCheckBlockingStoreFiles() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10); TEST_UTIL.startMiniCluster(1); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); String tableName = this.tableName.getNameAsString()+"-MinVersion"; if (admin.tableExists(tableName)) { admin.disableTable(tableName); admin.deleteTable(tableName); } HTableDescriptor desc = new HTableDescriptor(tableName); desc.setConfiguration(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()); desc.setConfiguration(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()); HColumnDescriptor colDesc = new HColumnDescriptor(family); colDesc.setTimeToLive(1); // 1 sec desc.addFamily(colDesc); try{ admin.createTable(desc); Assert.fail(); }catch(Exception e){ }finally{ TEST_UTIL.shutdownMiniCluster(); } }
public ChangeSplitPolicyAction(TableName tableName) { this.tableName = tableName; possiblePolicies = new String[] { IncreasingToUpperBoundRegionSplitPolicy.class.getName(), ConstantSizeRegionSplitPolicy.class.getName(), DisabledRegionSplitPolicy.class.getName() }; this.random = new Random(); }
private static void createValidationRuleTable(HBaseAdmin admin) throws IOException { HTableDescriptor tableDescriptor = new HTableDescriptor(HBaseTableMetaModel.validationRulesTableName); HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(HBaseTableMetaModel.validationRulesColumnFamily); hColumnDescriptor.setMaxVersions(1); tableDescriptor.addFamily(hColumnDescriptor); tableDescriptor.setValue(tableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); admin.createTable(tableDescriptor); }
public static void createTable(String tableName) throws IOException { Connection conn = getConnection(); Admin hadmin = conn.getAdmin(); try { boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName)); if (tableExist) { logger.info("HTable '" + tableName + "' already exists"); return; } logger.info("Creating HTable '" + tableName + "'"); HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); desc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());//disable region split desc.setMemStoreFlushSize(512 << 20);//512M HColumnDescriptor fd = new HColumnDescriptor(CF); fd.setBlocksize(CELL_SIZE); desc.addFamily(fd); hadmin.createTable(desc); logger.info("HTable '" + tableName + "' created"); } finally { IOUtils.closeQuietly(conn); IOUtils.closeQuietly(hadmin); } }
@Test public void testSanityCheckTTL() throws IOException { error.expect(DoNotRetryIOException.class); error.expectMessage("Default TTL is not supported"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-TTL"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .addColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); TEST_UTIL.getAdmin().createTable(desc); }
@Test public void testSanityCheckMinVersion() throws IOException { error.expect(DoNotRetryIOException.class); error.expectMessage("MIN_VERSION > 0 is not supported for FIFO compaction"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1) .setMinVersions(1).build()) .build(); TEST_UTIL.getAdmin().createTable(desc); }
public void test() throws Exception { int maxIters = 3; String replicas = "--replicas=" + replicaCount; // TODO: splits disabled until "phase 2" is complete. String splitPolicy = "--splitPolicy=" + DisabledRegionSplitPolicy.class.getName(); String writeOpts = format("%s --nomapred --table=%s --presplit=16 sequentialWrite 4", splitPolicy, tableName); String readOpts = format("--nomapred --table=%s --latency --sampleRate=0.1 randomRead 4", tableName); String replicaReadOpts = format("%s %s", replicas, readOpts); ArrayList<TimingResult> resultsWithoutReplicas = new ArrayList<TimingResult>(maxIters); ArrayList<TimingResult> resultsWithReplicas = new ArrayList<TimingResult>(maxIters); // create/populate the table, replicas disabled LOG.debug("Populating table."); new PerfEvalCallable(util.getHBaseAdmin(), writeOpts).call(); // one last sanity check, then send in the clowns! assertEquals("Table must be created with DisabledRegionSplitPolicy. Broken test.", DisabledRegionSplitPolicy.class.getName(), util.getHBaseAdmin().getTableDescriptor(tableName).getRegionSplitPolicyClassName()); startMonkey(); // collect a baseline without region replicas. for (int i = 0; i < maxIters; i++) { LOG.debug("Launching non-replica job " + (i + 1) + "/" + maxIters); resultsWithoutReplicas.add(new PerfEvalCallable(util.getHBaseAdmin(), readOpts).call()); // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary? Thread.sleep(5000l); } // disable monkey, enable region replicas, enable monkey cleanUpMonkey("Altering table."); LOG.debug("Altering " + tableName + " replica count to " + replicaCount); IntegrationTestingUtility.setReplicas(util.getHBaseAdmin(), tableName, replicaCount); setUpMonkey(); startMonkey(); // run test with region replicas. for (int i = 0; i < maxIters; i++) { LOG.debug("Launching replica job " + (i + 1) + "/" + maxIters); resultsWithReplicas.add(new PerfEvalCallable(util.getHBaseAdmin(), replicaReadOpts).call()); // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary? Thread.sleep(5000l); } // compare the average of the stdev and 99.99pct across runs to determine if region replicas // are having an overall improvement on response variance experienced by clients. double withoutReplicasStdevMean = calcMean("withoutReplicas", Stat.STDEV, resultsWithoutReplicas); double withoutReplicas9999Mean = calcMean("withoutReplicas", Stat.FOUR_9S, resultsWithoutReplicas); double withReplicasStdevMean = calcMean("withReplicas", Stat.STDEV, resultsWithReplicas); double withReplicas9999Mean = calcMean("withReplicas", Stat.FOUR_9S, resultsWithReplicas); LOG.info(Objects.toStringHelper(this) .add("withoutReplicas", resultsWithoutReplicas) .add("withReplicas", resultsWithReplicas) .add("withoutReplicasStdevMean", withoutReplicasStdevMean) .add("withoutReplicas99.99Mean", withoutReplicas9999Mean) .add("withReplicasStdevMean", withReplicasStdevMean) .add("withReplicas99.99Mean", withReplicas9999Mean) .toString()); assertTrue( "Running with region replicas under chaos should have less request variance than without. " + "withReplicas.stdev.mean: " + withReplicasStdevMean + "ms " + "withoutReplicas.stdev.mean: " + withoutReplicasStdevMean + "ms.", withReplicasStdevMean <= withoutReplicasStdevMean); assertTrue( "Running with region replicas under chaos should improve 99.99pct latency. " + "withReplicas.99.99.mean: " + withReplicas9999Mean + "ms " + "withoutReplicas.99.99.mean: " + withoutReplicas9999Mean + "ms.", withReplicas9999Mean <= withoutReplicas9999Mean); }
private static void createProfileCacheTable(HBaseAdmin admin) throws IOException { HTableDescriptor tableDescriptor = new HTableDescriptor(HBaseTableMetaModel.profileCacheTableName); HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(HBaseTableMetaModel.profileCacheColumnFamily); hColumnDescriptor.setMaxVersions(1); tableDescriptor.addFamily(hColumnDescriptor); tableDescriptor.setValue(tableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); byte[][] splitKeys = new byte[HBaseTableMetaModel.profileCacheNumberOfProfileCacheSalts][]; for (int i = 0; i < HBaseTableMetaModel.profileCacheNumberOfProfileCacheSalts; i++) { char salt = (char)('A' + i); splitKeys[i] = Bytes.toBytes(salt); } admin.createTable(tableDescriptor, splitKeys); }
public static void createHTable(CubeSegment cubeSegment, byte[][] splitKeys) throws IOException { String tableName = cubeSegment.getStorageLocationIdentifier(); CubeInstance cubeInstance = cubeSegment.getCubeInstance(); CubeDesc cubeDesc = cubeInstance.getDescriptor(); KylinConfig kylinConfig = cubeDesc.getConfig(); HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(cubeSegment.getStorageLocationIdentifier())); tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName()); tableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis())); if (!StringUtils.isEmpty(kylinConfig.getKylinOwner())) { //HTableOwner is the team that provides kylin service tableDesc.setValue(IRealizationConstants.HTableOwner, kylinConfig.getKylinOwner()); } String commitInfo = KylinVersion.getGitCommitInfo(); if (!StringUtils.isEmpty(commitInfo)) { tableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo); } //HTableUser is the cube owner, which will be the "user" tableDesc.setValue(IRealizationConstants.HTableUser, cubeInstance.getOwner()); tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString()); Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl()); Admin admin = conn.getAdmin(); try { if (User.isHBaseSecurityEnabled(conf)) { // add coprocessor for bulk load tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint"); } for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) { HColumnDescriptor cf = createColumnFamily(kylinConfig, cfDesc.getName(), cfDesc.isMemoryHungry()); tableDesc.addFamily(cf); } if (admin.tableExists(TableName.valueOf(tableName))) { // admin.disableTable(tableName); // admin.deleteTable(tableName); throw new RuntimeException("HBase table " + tableName + " exists!"); } DeployCoprocessorCLI.deployCoprocessor(tableDesc); admin.createTable(tableDesc, splitKeys); Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons"); logger.info("create hbase table " + tableName + " done."); } finally { IOUtils.closeQuietly(admin); } }
public void test() throws Exception { int maxIters = 3; String replicas = "--replicas=" + replicaCount; // TODO: splits disabled until "phase 2" is complete. String splitPolicy = "--splitPolicy=" + DisabledRegionSplitPolicy.class.getName(); String writeOpts = format("%s --nomapred --table=%s --presplit=16 sequentialWrite 4", splitPolicy, tableName); String readOpts = format("--nomapred --table=%s --latency --sampleRate=0.1 randomRead 4", tableName); String replicaReadOpts = format("%s %s", replicas, readOpts); ArrayList<TimingResult> resultsWithoutReplicas = new ArrayList<>(maxIters); ArrayList<TimingResult> resultsWithReplicas = new ArrayList<>(maxIters); // create/populate the table, replicas disabled LOG.debug("Populating table."); new PerfEvalCallable(util.getAdmin(), writeOpts).call(); // one last sanity check, then send in the clowns! assertEquals("Table must be created with DisabledRegionSplitPolicy. Broken test.", DisabledRegionSplitPolicy.class.getName(), util.getAdmin().getTableDescriptor(tableName).getRegionSplitPolicyClassName()); startMonkey(); // collect a baseline without region replicas. for (int i = 0; i < maxIters; i++) { LOG.debug("Launching non-replica job " + (i + 1) + "/" + maxIters); resultsWithoutReplicas.add(new PerfEvalCallable(util.getAdmin(), readOpts).call()); // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary? Thread.sleep(5000l); } // disable monkey, enable region replicas, enable monkey cleanUpMonkey("Altering table."); LOG.debug("Altering " + tableName + " replica count to " + replicaCount); IntegrationTestingUtility.setReplicas(util.getAdmin(), tableName, replicaCount); setUpMonkey(); startMonkey(); // run test with region replicas. for (int i = 0; i < maxIters; i++) { LOG.debug("Launching replica job " + (i + 1) + "/" + maxIters); resultsWithReplicas.add(new PerfEvalCallable(util.getAdmin(), replicaReadOpts).call()); // TODO: sleep to let cluster stabilize, though monkey continues. is it necessary? Thread.sleep(5000l); } // compare the average of the stdev and 99.99pct across runs to determine if region replicas // are having an overall improvement on response variance experienced by clients. double withoutReplicasStdevMean = calcMean("withoutReplicas", Stat.STDEV, resultsWithoutReplicas); double withoutReplicas9999Mean = calcMean("withoutReplicas", Stat.FOUR_9S, resultsWithoutReplicas); double withReplicasStdevMean = calcMean("withReplicas", Stat.STDEV, resultsWithReplicas); double withReplicas9999Mean = calcMean("withReplicas", Stat.FOUR_9S, resultsWithReplicas); LOG.info(MoreObjects.toStringHelper(this) .add("withoutReplicas", resultsWithoutReplicas) .add("withReplicas", resultsWithReplicas) .add("withoutReplicasStdevMean", withoutReplicasStdevMean) .add("withoutReplicas99.99Mean", withoutReplicas9999Mean) .add("withReplicasStdevMean", withReplicasStdevMean) .add("withReplicas99.99Mean", withReplicas9999Mean) .toString()); assertTrue( "Running with region replicas under chaos should have less request variance than without. " + "withReplicas.stdev.mean: " + withReplicasStdevMean + "ms " + "withoutReplicas.stdev.mean: " + withoutReplicasStdevMean + "ms.", withReplicasStdevMean <= withoutReplicasStdevMean); assertTrue( "Running with region replicas under chaos should improve 99.99pct latency. " + "withReplicas.99.99.mean: " + withReplicas9999Mean + "ms " + "withoutReplicas.99.99.mean: " + withoutReplicas9999Mean + "ms.", withReplicas9999Mean <= withoutReplicas9999Mean); }
