Java 类org.apache.hadoop.hbase.client.backoff.ServerStatistics 实例源码

/**
 * Make sure that we get results in the order that we expect - backoff for a load of 1 should
 * less than backoff for 10, which should be less than that for 50.
 */
@Test
public void testResultOrdering() {
  Configuration conf = new Configuration(false);
  // make the max timeout really high so we get differentiation between load factors
  conf.setLong(ExponentialClientBackoffPolicy.MAX_BACKOFF_KEY, Integer.MAX_VALUE);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long previous = backoff.getBackoffTime(server, regionname, stats);
  for (int i = 1; i <= 100; i++) {
    update(stats, i);
    long next = backoff.getBackoffTime(server, regionname, stats);
    assertTrue(
        "Previous backoff time" + previous + " >= " + next + ", the next backoff time for " +
            "load " + i, previous < next);
    previous = next;
  }
}

@Test
public void testHeapOccupancyPolicy() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long backoffTime;

  update(stats, 0, 95, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Heap occupancy at low watermark had no effect", backoffTime > 0);

  long previous = backoffTime;
  update(stats, 0, 96, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Increase above low watermark should have increased backoff",
    backoffTime > previous);

  update(stats, 0, 98, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertEquals("We should be using max backoff when at high watermark", backoffTime,
    ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF);
}

@Test
public void testCompactionPressurePolicy() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long backoffTime;

  update(stats, 0, 0, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Compaction pressure has no effect", backoffTime == 0);

      long previous = backoffTime;
  update(stats, 0, 0, 50);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Compaction pressure should be bigger",
          backoffTime > previous);

  update(stats, 0, 0, 100);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertEquals("under heavy compaction pressure", backoffTime,
          ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF);
}

/**
 * Make sure that we get results in the order that we expect - backoff for a load of 1 should
 * less than backoff for 10, which should be less than that for 50.
 */
@Test
public void testResultOrdering() {
  Configuration conf = new Configuration(false);
  // make the max timeout really high so we get differentiation between load factors
  conf.setLong(ExponentialClientBackoffPolicy.MAX_BACKOFF_KEY, Integer.MAX_VALUE);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long previous = backoff.getBackoffTime(server, regionname, stats);
  for (int i = 1; i <= 100; i++) {
    update(stats, i);
    long next = backoff.getBackoffTime(server, regionname, stats);
    assertTrue(
        "Previous backoff time" + previous + " >= " + next + ", the next backoff time for " +
            "load " + i, previous < next);
    previous = next;
  }
}

@Test
public void testHeapOccupancyPolicy() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long backoffTime;

  update(stats, 0, 95);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Heap occupancy at low watermark had no effect", backoffTime > 0);

  long previous = backoffTime;
  update(stats, 0, 96);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Increase above low watermark should have increased backoff",
    backoffTime > previous);

  update(stats, 0, 98);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertEquals("We should be using max backoff when at high watermark", backoffTime,
    ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF);
}

/**
 * Make sure that we get results in the order that we expect - backoff for a load of 1 should
 * less than backoff for 10, which should be less than that for 50.
 */
@Test
public void testResultOrdering() {
  Configuration conf = new Configuration(false);
  // make the max timeout really high so we get differentiation between load factors
  conf.setLong(ExponentialClientBackoffPolicy.MAX_BACKOFF_KEY, Integer.MAX_VALUE);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long previous = backoff.getBackoffTime(server, regionname, stats);
  for (int i = 1; i <= 100; i++) {
    update(stats, i);
    long next = backoff.getBackoffTime(server, regionname, stats);
    assertTrue(
        "Previous backoff time" + previous + " >= " + next + ", the next backoff time for " +
            "load " + i, previous < next);
    previous = next;
  }
}

@Test
public void testHeapOccupancyPolicy() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long backoffTime;

  update(stats, 0, 95, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Heap occupancy at low watermark had no effect", backoffTime > 0);

  long previous = backoffTime;
  update(stats, 0, 96, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Increase above low watermark should have increased backoff",
    backoffTime > previous);

  update(stats, 0, 98, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertEquals("We should be using max backoff when at high watermark",
    ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoffTime);
}

@Test
public void testCompactionPressurePolicy() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  long backoffTime;

  update(stats, 0, 0, 0);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Compaction pressure has no effect", backoffTime == 0);

      long previous = backoffTime;
  update(stats, 0, 0, 50);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertTrue("Compaction pressure should be bigger",
          backoffTime > previous);

  update(stats, 0, 0, 100);
  backoffTime = backoff.getBackoffTime(server, regionname, stats);
  assertEquals("under heavy compaction pressure",
    ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoffTime);
}

/**
 * @param server server location where the target region is hosted
 * @param regionName name of the region which we are going to write some data
 * @return the amount of time the client should wait until it submit a request to the
 * specified server and region
 */
private Long getBackoff(ServerName server, byte[] regionName) {
  ServerStatisticTracker tracker = AsyncProcess.this.connection.getStatisticsTracker();
  ServerStatistics stats = tracker.getStats(server);
  return AsyncProcess.this.connection.getBackoffPolicy()
      .getBackoffTime(server, regionName, stats);
}

@Test
public void testNulls() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);
  assertEquals(0, backoff.getBackoffTime(null, null, null));

  // server name doesn't matter to calculation, but check it now anyways
  assertEquals(0, backoff.getBackoffTime(server, null, null));
  assertEquals(0, backoff.getBackoffTime(server, regionname, null));

  // check when no stats for the region yet
  ServerStatistics stats = new ServerStatistics();
  assertEquals(0, backoff.getBackoffTime(server, regionname, stats));
}

@Test
public void testMaxLoad() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  update(stats, 100);
  assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoff.getBackoffTime(server,
      regionname, stats));

  // another policy with a different max timeout
  long max = 100;
  conf.setLong(ExponentialClientBackoffPolicy.MAX_BACKOFF_KEY, max);
  ExponentialClientBackoffPolicy backoffShortTimeout = new ExponentialClientBackoffPolicy(conf);
  assertEquals(max, backoffShortTimeout.getBackoffTime(server, regionname, stats));

  // test beyond 100 still doesn't exceed the max
  update(stats, 101);
  assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoff.getBackoffTime(server,
      regionname, stats));
  assertEquals(max, backoffShortTimeout.getBackoffTime(server, regionname, stats));

  // and that when we are below 100, its less than the max timeout
  update(stats, 99);
  assertTrue(backoff.getBackoffTime(server,
      regionname, stats) < ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF);
  assertTrue(backoffShortTimeout.getBackoffTime(server, regionname, stats) < max);
}

private void update(ServerStatistics stats, int memstoreLoad, int heapOccupancy,
                    int compactionPressure) {
  ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder()
      .setMemstoreLoad(memstoreLoad)
      .setHeapOccupancy(heapOccupancy)
      .setCompactionPressure(compactionPressure)
          .build();
  stats.update(regionname, stat);
}

/**
 * @param server server location where the target region is hosted
 * @param regionName name of the region which we are going to write some data
 * @return the amount of time the client should wait until it submit a request to the
 * specified server and region
 */
private Long getBackoff(ServerName server, byte[] regionName) {
  ServerStatisticTracker tracker = AsyncProcess.this.connection.getStatisticsTracker();
  ServerStatistics stats = tracker.getStats(server);
  return AsyncProcess.this.connection.getBackoffPolicy()
      .getBackoffTime(server, regionName, stats);
}

@Test
public void testNulls() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);
  assertEquals(0, backoff.getBackoffTime(null, null, null));

  // server name doesn't matter to calculation, but check it now anyways
  assertEquals(0, backoff.getBackoffTime(server, null, null));
  assertEquals(0, backoff.getBackoffTime(server, regionname, null));

  // check when no stats for the region yet
  ServerStatistics stats = new ServerStatistics();
  assertEquals(0, backoff.getBackoffTime(server, regionname, stats));
}

@Test
public void testMaxLoad() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  update(stats, 100);
  assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoff.getBackoffTime(server,
      regionname, stats));

  // another policy with a different max timeout
  long max = 100;
  conf.setLong(ExponentialClientBackoffPolicy.MAX_BACKOFF_KEY, max);
  ExponentialClientBackoffPolicy backoffShortTimeout = new ExponentialClientBackoffPolicy(conf);
  assertEquals(max, backoffShortTimeout.getBackoffTime(server, regionname, stats));

  // test beyond 100 still doesn't exceed the max
  update(stats, 101);
  assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoff.getBackoffTime(server,
      regionname, stats));
  assertEquals(max, backoffShortTimeout.getBackoffTime(server, regionname, stats));

  // and that when we are below 100, its less than the max timeout
  update(stats, 99);
  assertTrue(backoff.getBackoffTime(server,
      regionname, stats) < ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF);
  assertTrue(backoffShortTimeout.getBackoffTime(server, regionname, stats) < max);
}

private void update(ServerStatistics stats, int memstoreLoad, int heapOccupancy) {
  ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder()
      .setMemstoreLoad(memstoreLoad)
      .setHeapOccupancy(heapOccupancy)
          .build();
  stats.update(regionname, stat);
}

@Test
public void testMutateRowStats() throws IOException {
  Configuration conf = UTIL.getConfiguration();
  ClusterConnection conn = (ClusterConnection) ConnectionFactory.createConnection(conf);
  Table table = conn.getTable(tableName);
  HRegionServer rs = UTIL.getHBaseCluster().getRegionServer(0);
  Region region = rs.getRegions(tableName).get(0);

  RowMutations mutations = new RowMutations(Bytes.toBytes("row"));
  Put p = new Put(Bytes.toBytes("row"));
  p.addColumn(family, qualifier, Bytes.toBytes("value2"));
  mutations.add(p);
  table.mutateRow(mutations);

  ServerStatisticTracker stats = conn.getStatisticsTracker();
  assertNotNull( "No stats configured for the client!", stats);
  // get the names so we can query the stats
  ServerName server = rs.getServerName();
  byte[] regionName = region.getRegionInfo().getRegionName();

  // check to see we found some load on the memstore
  ServerStatistics serverStats = stats.getServerStatsForTesting(server);
  ServerStatistics.RegionStatistics regionStats = serverStats.getStatsForRegion(regionName);

  assertNotNull(regionStats);
  assertTrue(regionStats.getMemStoreLoadPercent() > 0);
  }

/**
 * @param server server location where the target region is hosted
 * @param regionName name of the region which we are going to write some data
 * @return the amount of time the client should wait until it submit a request to the
 * specified server and region
 */
private Long getBackoff(ServerName server, byte[] regionName) {
  ServerStatisticTracker tracker = asyncProcess.connection.getStatisticsTracker();
  ServerStatistics stats = tracker.getStats(server);
  return asyncProcess.connection.getBackoffPolicy()
      .getBackoffTime(server, regionName, stats);
}

@Override
public long getBackoffTime(ServerName serverName, byte[] region, ServerStatistics stats) {
  AtomicInteger inc = count.get(serverName);
  if (inc == null) {
    inc = new AtomicInteger(0);
    count.put(serverName, inc);
  }
  return inc.getAndIncrement();
}

@Test
public void testNulls() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);
  assertEquals(0, backoff.getBackoffTime(null, null, null));

  // server name doesn't matter to calculation, but check it now anyways
  assertEquals(0, backoff.getBackoffTime(server, null, null));
  assertEquals(0, backoff.getBackoffTime(server, regionname, null));

  // check when no stats for the region yet
  ServerStatistics stats = new ServerStatistics();
  assertEquals(0, backoff.getBackoffTime(server, regionname, stats));
}

@Test
public void testMaxLoad() {
  Configuration conf = new Configuration(false);
  ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf);

  ServerStatistics stats = new ServerStatistics();
  update(stats, 100);
  assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoff.getBackoffTime(server,
      regionname, stats));

  // another policy with a different max timeout
  long max = 100;
  conf.setLong(ExponentialClientBackoffPolicy.MAX_BACKOFF_KEY, max);
  ExponentialClientBackoffPolicy backoffShortTimeout = new ExponentialClientBackoffPolicy(conf);
  assertEquals(max, backoffShortTimeout.getBackoffTime(server, regionname, stats));

  // test beyond 100 still doesn't exceed the max
  update(stats, 101);
  assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, backoff.getBackoffTime(server,
      regionname, stats));
  assertEquals(max, backoffShortTimeout.getBackoffTime(server, regionname, stats));

  // and that when we are below 100, its less than the max timeout
  update(stats, 99);
  assertTrue(backoff.getBackoffTime(server,
      regionname, stats) < ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF);
  assertTrue(backoffShortTimeout.getBackoffTime(server, regionname, stats) < max);
}

private void update(ServerStatistics stats, int memstoreLoad, int heapOccupancy,
                    int compactionPressure) {
  ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder()
      .setMemStoreLoad(memstoreLoad)
      .setHeapOccupancy(heapOccupancy)
      .setCompactionPressure(compactionPressure)
          .build();
  stats.update(regionname, ProtobufUtil.createRegionLoadStats(stat));
}

public ServerStatistics getStats(ServerName server) {
  return this.stats.get(server);
}

@VisibleForTesting
ServerStatistics getServerStatsForTesting(ServerName server) {
  return stats.get(server);
}

private void update(ServerStatistics stats, int load) {
  ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder()
      .setMemstoreLoad
          (load).build();
  stats.update(regionname, stat);
}

public ServerStatistics getStats(ServerName server) {
  return this.stats.get(server);
}

@VisibleForTesting
ServerStatistics getServerStatsForTesting(ServerName server) {
  return stats.get(server);
}

private void update(ServerStatistics stats, int load) {
  ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder()
      .setMemstoreLoad
          (load).build();
  stats.update(regionname, stat);
}

@Override
public void updateRegionStats(ServerName server, byte[] region, RegionLoadStats currentStats) {
  computeIfAbsent(stats, server, ServerStatistics::new).update(region, currentStats);
}

public ServerStatistics getStats(ServerName server) {
  return this.stats.get(server);
}

@VisibleForTesting
ServerStatistics getServerStatsForTesting(ServerName server) {
  return stats.get(server);
}

private void update(ServerStatistics stats, int load) {
  ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder()
      .setMemStoreLoad
          (load).build();
  stats.update(regionname, ProtobufUtil.createRegionLoadStats(stat));
}