Java 类org.apache.hadoop.hbase.client.HConnectionTestingUtility 实例源码

/**
 * @param implementation An {@link HRegionInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link HRegionInterface} instance when
 * {@link HConnection#getHRegionConnection(String, int)}
 * is called (Be sure call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final HRegionInterface implementation)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
      SN.getPort());
  Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (implementation != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(implementation);
  }
  return connection;
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(HRegionInterface.class),
      new ServerName("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
  this.c.set(HConstants.HBASE_DIR, rootdir.toString());
  this.ct = Mockito.mock(CatalogTracker.class);
  HRegionInterface hri = Mockito.mock(HRegionInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

/**
 * @param implementation An {@link HRegionInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link HRegionInterface} instance when
 * {@link HConnection#getHRegionConnection(String, int)}
 * is called (Be sure call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final HRegionInterface implementation)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
      SN.getPort());
  Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (implementation != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(implementation);
  }
  return connection;
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(HRegionInterface.class),
      new ServerName("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
  this.c.set(HConstants.HBASE_DIR, rootdir.toString());
  this.ct = Mockito.mock(CatalogTracker.class);
  HRegionInterface hri = Mockito.mock(HRegionInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

/**
 * @param implementation An {@link HRegionInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link HRegionInterface} instance when
 * {@link HConnection#getHRegionConnection(String, int)}
 * is called (Be sure call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final HRegionInterface implementation)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
      SN.getPort());
  Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (implementation != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(implementation);
  }
  return connection;
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(HRegionInterface.class),
      new ServerName("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
  this.c.set(HConstants.HBASE_DIR, rootdir.toString());
  this.ct = Mockito.mock(CatalogTracker.class);
  HRegionInterface hri = Mockito.mock(HRegionInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

/**
 * @param implementation An {@link HRegionInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link HRegionInterface} instance when
 * {@link HConnection#getHRegionConnection(String, int)}
 * is called (Be sure call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final HRegionInterface implementation)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
      SN.getPort());
  Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (implementation != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(implementation);
  }
  return connection;
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(HRegionInterface.class),
      new ServerName("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
  this.c.set(HConstants.HBASE_DIR, rootdir.toString());
  this.ct = Mockito.mock(CatalogTracker.class);
  HRegionInterface hri = Mockito.mock(HRegionInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

/**
 * @param implementation An {@link HRegionInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link HRegionInterface} instance when
 * {@link HConnection#getHRegionConnection(String, int)}
 * is called (Be sure call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final HRegionInterface implementation)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
      SN.getPort());
  Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (implementation != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(implementation);
  }
  return connection;
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(HRegionInterface.class),
      new ServerName("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
  this.c.set(HConstants.HBASE_DIR, rootdir.toString());
  this.ct = Mockito.mock(CatalogTracker.class);
  HRegionInterface hri = Mockito.mock(HRegionInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

/**
 * @param implementation An {@link HRegionInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link HRegionInterface} instance when
 * {@link HConnection#getHRegionConnection(String, int)}
 * is called (Be sure call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final HRegionInterface implementation)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
      SN.getPort());
  Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (implementation != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(implementation);
  }
  return connection;
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  // Mock an HConnection and a HRegionInterface implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(HRegionInterface.class),
      new ServerName("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
  this.c.set(HConstants.HBASE_DIR, rootdir.toString());
  this.ct = Mockito.mock(CatalogTracker.class);
  HRegionInterface hri = Mockito.mock(HRegionInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

/**
 * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @param client A mocked ClientProtocol instance, can be null
 * @return Mock up a connection that returns a {@link Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
 * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
 * {@link ClientProtos.ClientService.BlockingInterface} instance when
 * {@link HConnection#getClient(ServerName)} is called.
 * @throws IOException
 */
private ClusterConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
    final ClientProtos.ClientService.BlockingInterface client)
throws IOException {
  ClusterConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
  Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((TableName) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (admin != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
      thenReturn(admin);
  }
  if (client != null) {
    // If a call to getClient, return this implementation.
    Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
      thenReturn(client);
  }
  return connection;
}

@Test (expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  final CatalogTracker ct = constructAndStartCatalogTracker(connection);
  ct.waitForMeta(100);
}

/**
 * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @param client A mocked ClientProtocol instance, can be null
 * @return Mock up a connection that returns a {@link Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
 * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
 * {@link ClientProtos.ClientService.BlockingInterface} instance when
 * {@link HConnection#getClient(ServerName)} is called.
 * @throws IOException
 */
private ClusterConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
    final ClientProtos.ClientService.BlockingInterface client)
throws IOException {
  ClusterConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
  Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((TableName) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (admin != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
      thenReturn(admin);
  }
  if (client != null) {
    // If a call to getClient, return this implementation.
    Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
      thenReturn(client);
  }
  return connection;
}

/**
 * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @param client A mocked ClientProtocol instance, can be null
 * @return Mock up a connection that returns a {@link Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
 * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
 * {@link ClientProtos.ClientService.BlockingInterface} instance when
 * {@link HConnection#getClient(ServerName)} is called (Be sure to call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
    final ClientProtos.ClientService.BlockingInterface client)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
  Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((TableName) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (admin != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
      thenReturn(admin);
  }
  if (client != null) {
    // If a call to getClient, return this implementation.
    Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
      thenReturn(client);
  }
  return connection;
}

/**
 * Verify that PleaseHoldException gets retried.
 * HBASE-8764
 * @throws IOException 
 * @throws ZooKeeperConnectionException 
 * @throws MasterNotRunningException 
 * @throws ServiceException 
 */
@Test
public void testMasterMonitorCollableRetries()
throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException {
  Configuration configuration = HBaseConfiguration.create();
  // Set the pause and retry count way down.
  configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1);
  final int count = 10;
  configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count);
  // Get mocked connection.   Getting the connection will register it so when HBaseAdmin is
  // constructed with same configuration, it will find this mocked connection.
  HConnection connection = HConnectionTestingUtility.getMockedConnection(configuration);
  // Mock so we get back the master interface.  Make it so when createTable is called, we throw
  // the PleaseHoldException.
  MasterKeepAliveConnection masterAdmin =
    Mockito.mock(MasterKeepAliveConnection.class);
  Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(),
    (CreateTableRequest)Mockito.any())).
      thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test")));
  Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin);
  // Mock up our admin Interfaces
  HBaseAdmin admin = new HBaseAdmin(configuration);
  try {
    HTableDescriptor htd =
        new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries"));
    // Pass any old htable descriptor; not important
    try {
      admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
      fail();
    } catch (RetriesExhaustedException e) {
      Log.info("Expected fail", e);
    }
    // Assert we were called 'count' times.
    Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(),
      (CreateTableRequest)Mockito.any());
  } finally {
    admin.close();
    if (connection != null)HConnectionManager.deleteConnection(configuration);
  }
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  ClientProtos.ClientService.BlockingInterface ri =
    Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
  MutateResponse.Builder builder = MutateResponse.newBuilder();
  builder.setProcessed(true);
  try {
    Mockito.when(ri.mutate(
      (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
        thenReturn(builder.build());
  } catch (ServiceException se) {
    throw ProtobufUtil.getRemoteException(se);
  }
  // Mock an HConnection and a AdminProtocol implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
        ServerName.valueOf("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = FSUtils.getRootDir(this.c);
  FSUtils.setRootDir(this.c, rootdir);
  this.ct = Mockito.mock(CatalogTracker.class);
  AdminProtos.AdminService.BlockingInterface hri =
    Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

@Test (expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  final CatalogTracker ct = constructAndStartCatalogTracker(connection);
  ct.waitForMeta(100);
}

/**
 * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @param client A mocked ClientProtocol instance, can be null
 * @return Mock up a connection that returns a {@link Configuration} when
 * {@link org.apache.hadoop.hbase.client.ClusterConnection#getConfiguration()} is called, a 'location' when
 * {@link org.apache.hadoop.hbase.client.RegionLocator#getRegionLocation(byte[], boolean)} is called,
 * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
 * {@link org.apache.hadoop.hbase.client.ClusterConnection#getAdmin(ServerName)} is called, returns the passed
 * {@link ClientProtos.ClientService.BlockingInterface} instance when
 * {@link org.apache.hadoop.hbase.client.ClusterConnection#getClient(ServerName)} is called.
 * @throws IOException
 */
private ClusterConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
    final ClientProtos.ClientService.BlockingInterface client)
throws IOException {
  ClusterConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
  Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((TableName) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (admin != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getAdmin(Mockito.any())).
      thenReturn(admin);
  }
  if (client != null) {
    // If a call to getClient, return this implementation.
    Mockito.when(connection.getClient(Mockito.any())).
      thenReturn(client);
  }
  return connection;
}

/**
 * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @param client A mocked ClientProtocol instance, can be null
 * @return Mock up a connection that returns a {@link Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
 * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
 * {@link ClientProtos.ClientService.BlockingInterface} instance when
 * {@link HConnection#getClient(ServerName)} is called (Be sure to call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
    final ClientProtos.ClientService.BlockingInterface client)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
  Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((TableName) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (admin != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
      thenReturn(admin);
  }
  if (client != null) {
    // If a call to getClient, return this implementation.
    Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
      thenReturn(client);
  }
  return connection;
}

/**
 * Verify that PleaseHoldException gets retried.
 * HBASE-8764
 * @throws IOException 
 * @throws ZooKeeperConnectionException 
 * @throws MasterNotRunningException 
 * @throws ServiceException 
 */
@Test
public void testMasterMonitorCollableRetries()
throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException {
  Configuration configuration = HBaseConfiguration.create();
  // Set the pause and retry count way down.
  configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1);
  final int count = 10;
  configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count);
  // Get mocked connection.   Getting the connection will register it so when HBaseAdmin is
  // constructed with same configuration, it will find this mocked connection.
  HConnection connection = HConnectionTestingUtility.getMockedConnection(configuration);
  // Mock so we get back the master interface.  Make it so when createTable is called, we throw
  // the PleaseHoldException.
  MasterKeepAliveConnection masterAdmin =
    Mockito.mock(MasterKeepAliveConnection.class);
  Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(),
    (CreateTableRequest)Mockito.any())).
      thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test")));
  Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin);
  // Mock up our admin Interfaces
  HBaseAdmin admin = new HBaseAdmin(configuration);
  try {
    HTableDescriptor htd =
        new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries"));
    // Pass any old htable descriptor; not important
    try {
      admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
      fail();
    } catch (RetriesExhaustedException e) {
      Log.info("Expected fail", e);
    }
    // Assert we were called 'count' times.
    Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(),
      (CreateTableRequest)Mockito.any());
  } finally {
    admin.close();
    if (connection != null)HConnectionManager.deleteConnection(configuration);
  }
}

/**
 * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @param client A mocked ClientProtocol instance, can be null
 * @return Mock up a connection that returns a {@link Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
 * {@link HConnection#getAdmin(ServerName)} is called, returns the passed
 * {@link ClientProtos.ClientService.BlockingInterface} instance when
 * {@link HConnection#getClient(ServerName)} is called (Be sure to call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
    final ClientProtos.ClientService.BlockingInterface client)
throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
  Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((TableName) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (admin != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getAdmin(Mockito.any(ServerName.class))).
      thenReturn(admin);
  }
  if (client != null) {
    // If a call to getClient, return this implementation.
    Mockito.when(connection.getClient(Mockito.any(ServerName.class))).
      thenReturn(client);
  }
  return connection;
}

/**
 * Verify that PleaseHoldException gets retried.
 * HBASE-8764
 * @throws IOException 
 * @throws ZooKeeperConnectionException 
 * @throws MasterNotRunningException 
 * @throws ServiceException 
 */
@Test
public void testMasterMonitorCollableRetries()
throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException {
  Configuration configuration = HBaseConfiguration.create();
  // Set the pause and retry count way down.
  configuration.setLong(HConstants.HBASE_CLIENT_PAUSE, 1);
  final int count = 10;
  configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count);
  // Get mocked connection.   Getting the connection will register it so when HBaseAdmin is
  // constructed with same configuration, it will find this mocked connection.
  HConnection connection = HConnectionTestingUtility.getMockedConnection(configuration);
  // Mock so we get back the master interface.  Make it so when createTable is called, we throw
  // the PleaseHoldException.
  MasterKeepAliveConnection masterAdmin =
    Mockito.mock(MasterKeepAliveConnection.class);
  Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(),
    (CreateTableRequest)Mockito.any())).
      thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test")));
  Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin);
  // Mock up our admin Interfaces
  HBaseAdmin admin = new HBaseAdmin(configuration);
  try {
    HTableDescriptor htd =
        new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries"));
    // Pass any old htable descriptor; not important
    try {
      admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
      fail();
    } catch (RetriesExhaustedException e) {
      Log.info("Expected fail", e);
    }
    // Assert we were called 'count' times.
    Mockito.verify(masterAdmin, Mockito.atLeast(count)).createTable((RpcController)Mockito.any(),
      (CreateTableRequest)Mockito.any());
  } finally {
    admin.close();
    if (connection != null)HConnectionManager.deleteConnection(configuration);
  }
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  ClientProtos.ClientService.BlockingInterface ri =
    Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
  MutateResponse.Builder builder = MutateResponse.newBuilder();
  builder.setProcessed(true);
  try {
    Mockito.when(ri.mutate(
      (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
        thenReturn(builder.build());
  } catch (ServiceException se) {
    throw ProtobufUtil.getRemoteException(se);
  }
  // Mock an HConnection and a AdminProtocol implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
        ServerName.valueOf("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = FSUtils.getRootDir(this.c);
  FSUtils.setRootDir(this.c, rootdir);
  this.ct = Mockito.mock(CatalogTracker.class);
  AdminProtos.AdminService.BlockingInterface hri =
    Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
}

@Test (expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  try {
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ct.waitForMeta(100);
  } finally {
    HConnectionManager.deleteConnection(UTIL.getConfiguration());
  }
}

@Test (expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  try {
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ct.waitForMeta(100);
  } finally {
    HConnectionManager.deleteConnection(UTIL.getConfiguration());
  }
}

@Test (expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  try {
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ct.waitForMeta(100);
  } finally {
    HConnectionManager.deleteConnection(UTIL.getConfiguration());
  }
}

@Test (expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  try {
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ct.waitForMeta(100);
  } finally {
    HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
  }
}

/**
 * @param admin An {@link AdminProtocol} instance; you'll likely
 * want to pass a mocked HRS; can be null.
 * @param client A mocked ClientProtocol instance, can be null
 * @return Mock up a connection that returns a {@link Configuration} when
 * {@link HConnection#getConfiguration()} is called, a 'location' when
 * {@link HConnection#getRegionLocation(byte[], byte[], boolean)} is called,
 * and that returns the passed {@link AdminProtocol} instance when
 * {@link HConnection#getAdmin(String, int)} is called, returns the passed
 * {@link ClientProtocol} instance when {@link HConnection#getClient(String, int)}
 * is called (Be sure call
 * {@link HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration, boolean)}
 * when done with this mocked Connection.
 * @throws IOException
 */
private HConnection mockConnection(final AdminProtocol admin,
    final ClientProtocol client) throws IOException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  Mockito.doNothing().when(connection).close();
  // Make it so we return any old location when asked.
  final HRegionLocation anyLocation =
    new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(),
      SN.getPort());
  Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
      (byte[]) Mockito.any(), Mockito.anyBoolean())).
    thenReturn(anyLocation);
  Mockito.when(connection.locateRegion((byte[]) Mockito.any(),
      (byte[]) Mockito.any())).
    thenReturn(anyLocation);
  if (admin != null) {
    // If a call to getHRegionConnection, return this implementation.
    Mockito.when(connection.getAdmin(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(admin);
  }
  if (client != null) {
    // If a call to getClient, return this implementation.
    Mockito.when(connection.getClient(Mockito.anyString(), Mockito.anyInt())).
      thenReturn(client);
  }
  return connection;
}

MockServer(final HBaseTestingUtility htu)
throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
  this.c = htu.getConfiguration();
  ClientProtocol ri = Mockito.mock(ClientProtocol.class);
  MutateResponse.Builder builder = MutateResponse.newBuilder();
  builder.setProcessed(true);
  try {
    Mockito.when(ri.mutate(
      (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
        thenReturn(builder.build());
  } catch (ServiceException se) {
    throw ProtobufUtil.getRemoteException(se);
  }
  // Mock an HConnection and a AdminProtocol implementation.  Have the
  // HConnection return the HRI.  Have the HRI return a few mocked up responses
  // to make our test work.
  this.connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
      Mockito.mock(AdminProtocol.class), ri,
      new ServerName("example.org,12345,6789"),
      HRegionInfo.FIRST_META_REGIONINFO);
  // Set hbase.rootdir into test dir.
  FileSystem fs = FileSystem.get(this.c);
  Path rootdir = fs.makeQualified(new Path(this.c.get(HConstants.HBASE_DIR)));
  this.c.set(HConstants.HBASE_DIR, rootdir.toString());
  this.ct = Mockito.mock(CatalogTracker.class);
  AdminProtocol hri = Mockito.mock(AdminProtocol.class);
  Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
  Mockito.when(ct.waitForMetaServerConnectionDefault()).thenReturn(hri);
}

@Test (expected = RetriesExhaustedException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
  HConnection connection =
    HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
  try {
    final CatalogTracker ct = constructAndStartCatalogTracker(connection);
    ct.waitForMeta(100);
  } finally {
    HConnectionManager.deleteConnection(UTIL.getConfiguration());
  }
}

@Override
public int getVal(Phase phase) {
  return HConnectionTestingUtility.getConnectionCount();
}

@Test
public void testNotPullingDeadRegionServerFromZK()
    throws IOException, KeeperException, InterruptedException {
  final Configuration conf = TESTUTIL.getConfiguration();
  final ServerName newServer = ServerName.valueOf("test.sample", 1, 101);
  final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100);
  final MockRegionServer rs0 = new MockRegionServer(conf, newServer);

  CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
    TESTUTIL.getConfiguration());
  HMaster master = new HMaster(conf, cp) {
    @Override
    void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMeatRSs, int replicaId)
    { }

    @Override
    void initializeZKBasedSystemTrackers() throws IOException,
    InterruptedException, KeeperException, CoordinatedStateException {
      super.initializeZKBasedSystemTrackers();
      // Record a newer server in server manager at first
      serverManager.recordNewServerWithLock(newServer, ServerLoad.EMPTY_SERVERLOAD);

      List<ServerName> onlineServers = new ArrayList<ServerName>();
      onlineServers.add(deadServer);
      onlineServers.add(newServer);
      // Mock the region server tracker to pull the dead server from zk
      regionServerTracker = Mockito.spy(regionServerTracker);
      Mockito.doReturn(onlineServers).when(
        regionServerTracker).getOnlineServers();
    }

    @Override
    public ClusterConnection getConnection() {
      // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
      // the conf from the master; the conf will already have an HConnection
      // associate so the below mocking of a connection will fail.
      try {
        return HConnectionTestingUtility.getMockedConnectionAndDecorate(
          TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
          HRegionInfo.FIRST_META_REGIONINFO);
      } catch (IOException e) {
        return null;
      }
    }

    @Override
    void initNamespace() {
    }
  };
  master.start();

  try {
    // Wait till master is initialized.
    while (!master.isInitialized()) Threads.sleep(10);
    LOG.info("Master is initialized");

    assertFalse("The dead server should not be pulled in",
      master.serverManager.isServerOnline(deadServer));
  } finally {
    master.stopMaster();
    master.join();
  }
}

/**
 * Create an {@link AssignmentManagerWithExtrasForTesting} that has mocked
 * {@link CatalogTracker} etc.
 * @param server
 * @param manager
 * @return An AssignmentManagerWithExtras with mock connections, etc.
 * @throws IOException
 * @throws KeeperException
 */
private AssignmentManagerWithExtrasForTesting setUpMockedAssignmentManager(final Server server,
    final ServerManager manager)
throws IOException, KeeperException {
  // We need a mocked catalog tracker. Its used by our AM instance.
  CatalogTracker ct = Mockito.mock(CatalogTracker.class);
  // Make an RS Interface implementation. Make it so a scanner can go against
  // it and a get to return the single region, REGIONINFO, this test is
  // messing with. Needed when "new master" joins cluster. AM will try and
  // rebuild its list of user regions and it will also get the HRI that goes
  // with an encoded name by doing a Get on .META.
  HRegionInterface ri = Mockito.mock(HRegionInterface.class);
  // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO
  Result[] result = null;
  if (enabling) {
    result = new Result[2];
    result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
    result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
  }
  Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
  Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
    thenReturn(System.currentTimeMillis());
 if (enabling) {
    Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
        (Result[]) null);
    // If a get, return the above result too for REGIONINFO_2
    Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
        getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
  } else {
    // Return good result 'r' first and then return null to indicate end of scan
    Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
    // If a get, return the above result too for REGIONINFO
    Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
  }
  // Get a connection w/ mocked up common methods.
  HConnection connection = HConnectionTestingUtility.
    getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
      REGIONINFO);
  // Make it so we can get the connection from our mocked catalogtracker
  Mockito.when(ct.getConnection()).thenReturn(connection);
  // Create and startup an executor. Used by AM handling zk callbacks.
  ExecutorService executor = startupMasterExecutor("mockedAMExecutor");
  this.balancer = LoadBalancerFactory.getLoadBalancer(server.getConfiguration());
  AssignmentManagerWithExtrasForTesting am = new AssignmentManagerWithExtrasForTesting(
      server, manager, ct, balancer, executor);
  return am;
}

public long getConnectionCount(){
  return HConnectionTestingUtility.getConnectionCount();
}

@Override
public int getVal(Phase phase) {
  return HConnectionTestingUtility.getConnectionCount();
}

@Test
public void testNotPullingDeadRegionServerFromZK()
    throws IOException, KeeperException, InterruptedException {
  final Configuration conf = TESTUTIL.getConfiguration();
  final ServerName newServer = ServerName.valueOf("test.sample", 1, 101);
  final ServerName deadServer = ServerName.valueOf("test.sample", 1, 100);
  final MockRegionServer rs0 = new MockRegionServer(conf, newServer);

  CoordinatedStateManager cp = CoordinatedStateManagerFactory.getCoordinatedStateManager(
    TESTUTIL.getConfiguration());
  HMaster master = new HMaster(conf, cp) {
    @Override
    void assignMeta(MonitoredTask status, Set<ServerName> previouslyFailedMeatRSs) {
    }

    @Override
    void initializeZKBasedSystemTrackers() throws IOException,
    InterruptedException, KeeperException, CoordinatedStateException {
      super.initializeZKBasedSystemTrackers();
      // Record a newer server in server manager at first
      serverManager.recordNewServerWithLock(newServer, ServerLoad.EMPTY_SERVERLOAD);

      List<ServerName> onlineServers = new ArrayList<ServerName>();
      onlineServers.add(deadServer);
      onlineServers.add(newServer);
      // Mock the region server tracker to pull the dead server from zk
      regionServerTracker = Mockito.spy(regionServerTracker);
      Mockito.doReturn(onlineServers).when(
        regionServerTracker).getOnlineServers();
    }

    @Override
    public ClusterConnection getConnection() {
      // Insert a mock for the connection, use TESTUTIL.getConfiguration rather than
      // the conf from the master; the conf will already have an HConnection
      // associate so the below mocking of a connection will fail.
      try {
        return HConnectionTestingUtility.getMockedConnectionAndDecorate(
          TESTUTIL.getConfiguration(), rs0, rs0, rs0.getServerName(),
          HRegionInfo.FIRST_META_REGIONINFO);
      } catch (IOException e) {
        return null;
      }
    }

    @Override
    void initNamespace() {
    }
  };
  master.start();

  try {
    // Wait till master is initialized.
    while (!master.initialized) Threads.sleep(10);
    LOG.info("Master is initialized");

    assertFalse("The dead server should not be pulled in",
      master.serverManager.isServerOnline(deadServer));
  } finally {
    master.stopMaster();
    master.join();
  }
}

@Before
public void before() throws ZooKeeperConnectionException, IOException {
  // TODO: Make generic versions of what we do below and put up in a mocking
  // utility class or move up into HBaseTestingUtility.

  // Mock a Server.  Have it return a legit Configuration and ZooKeeperWatcher.
  // If abort is called, be sure to fail the test (don't just swallow it
  // silently as is mockito default).
  this.server = Mockito.mock(Server.class);
  Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("master,1,1"));
  Mockito.when(server.getConfiguration()).thenReturn(HTU.getConfiguration());
  this.watcher =
    new ZooKeeperWatcher(HTU.getConfiguration(), "mockedServer", this.server, true);
  Mockito.when(server.getZooKeeper()).thenReturn(this.watcher);
  Mockito.doThrow(new RuntimeException("Aborted")).
    when(server).abort(Mockito.anyString(), (Throwable)Mockito.anyObject());

  cp = new ZkCoordinatedStateManager();
  cp.initialize(this.server);
  cp.start();

  mtl = Mockito.mock(MetaTableLocator.class);

  Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp);
  Mockito.when(server.getMetaTableLocator()).thenReturn(mtl);

  // Get a connection w/ mocked up common methods.
  this.connection =
    (ClusterConnection)HConnectionTestingUtility.getMockedConnection(HTU.getConfiguration());

  // Make it so we can get a catalogtracker from servermanager.. .needed
  // down in guts of server shutdown handler.
  Mockito.when(server.getConnection()).thenReturn(connection);

  // Mock a ServerManager.  Say server SERVERNAME_{A,B} are online.  Also
  // make it so if close or open, we return 'success'.
  this.serverManager = Mockito.mock(ServerManager.class);
  Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
  Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true);
  Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer());
  final Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>();
  onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD);
  onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD);
  Mockito.when(this.serverManager.getOnlineServersList()).thenReturn(
      new ArrayList<ServerName>(onlineServers.keySet()));
  Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers);

  List<ServerName> avServers = new ArrayList<ServerName>();
  avServers.addAll(onlineServers.keySet());
  Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers);
  Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers);

  Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)).
    thenReturn(true);
  Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)).
    thenReturn(true);
  // Ditto on open.
  Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)).
    thenReturn(RegionOpeningState.OPENED);
  Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)).
    thenReturn(RegionOpeningState.OPENED);
  this.master = Mockito.mock(HMaster.class);

  Mockito.when(this.master.getServerManager()).thenReturn(serverManager);
}

private void processServerShutdownHandler(AssignmentManager am, boolean splitRegion)
    throws IOException, ServiceException {
  // Make sure our new AM gets callbacks; once registered, can't unregister.
  // Thats ok because we make a new zk watcher for each test.
  this.watcher.registerListenerFirst(am);

  // Need to set up a fake scan of meta for the servershutdown handler
  // Make an RS Interface implementation.  Make it so a scanner can go against it.
  ClientProtos.ClientService.BlockingInterface implementation =
    Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
  // Get a meta row result that has region up on SERVERNAME_A

  Result r;
  if (splitRegion) {
    r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A);
  } else {
    r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
  }

  final ScanResponse.Builder builder = ScanResponse.newBuilder();
  builder.setMoreResults(true);
  builder.addCellsPerResult(r.size());
  final List<CellScannable> cellScannables = new ArrayList<CellScannable>(1);
  cellScannables.add(r);
  Mockito.when(implementation.scan(
    (RpcController)Mockito.any(), (ScanRequest)Mockito.any())).
    thenAnswer(new Answer<ScanResponse>() {
        @Override
        public ScanResponse answer(InvocationOnMock invocation) throws Throwable {
          PayloadCarryingRpcController controller = (PayloadCarryingRpcController) invocation
              .getArguments()[0];
          if (controller != null) {
            controller.setCellScanner(CellUtil.createCellScanner(cellScannables));
          }
          return builder.build();
        }
    });

  // Get a connection w/ mocked up common methods.
  ClusterConnection connection =
    HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(),
      null, implementation, SERVERNAME_B, REGIONINFO);
  // These mocks were done up when all connections were managed.  World is different now we
  // moved to unmanaged connections.  It messes up the intercepts done in these tests.
  // Just mark connections as marked and then down in MetaTableAccessor, it will go the path
  // that picks up the above mocked up 'implementation' so 'scans' of meta return the expected
  // result.  Redo in new realm of unmanaged connections.
  Mockito.when(connection.isManaged()).thenReturn(true);
  try {
    // Make it so we can get a catalogtracker from servermanager.. .needed
    // down in guts of server shutdown handler.
    Mockito.when(this.server.getConnection()).thenReturn(connection);

    // Now make a server shutdown handler instance and invoke process.
    // Have it that SERVERNAME_A died.
    DeadServer deadServers = new DeadServer();
    deadServers.add(SERVERNAME_A);
    // I need a services instance that will return the AM
    MasterFileSystem fs = Mockito.mock(MasterFileSystem.class);
    Mockito.doNothing().when(fs).setLogRecoveryMode();
    Mockito.when(fs.getLogRecoveryMode()).thenReturn(RecoveryMode.LOG_REPLAY);
    MasterServices services = Mockito.mock(MasterServices.class);
    Mockito.when(services.getAssignmentManager()).thenReturn(am);
    Mockito.when(services.getServerManager()).thenReturn(this.serverManager);
    Mockito.when(services.getZooKeeper()).thenReturn(this.watcher);
    Mockito.when(services.getMasterFileSystem()).thenReturn(fs);
    Mockito.when(services.getConnection()).thenReturn(connection);
    ServerShutdownHandler handler = new ServerShutdownHandler(this.server,
        services, deadServers, SERVERNAME_A, false);
    am.failoverCleanupDone.set(true);
    handler.process();
    // The region in r will have been assigned.  It'll be up in zk as unassigned.
  } finally {
    if (connection != null) connection.close();
  }
}