Java 类org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker 实例源码

/**
 * This creates a table and then corrupts an hfile.  Hbck should quarantine the file.
 */
@Test(timeout=180000)
public void testQuarantineCorruptHFile() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  try {
    setupTable(table);
    assertEquals(ROWKEYS.length, countRows());
    admin.flush(table); // flush is async.

    FileSystem fs = FileSystem.get(conf);
    Path hfile = getFlushedHFile(fs, table);

    // Mess it up by leaving a hole in the assignment, meta, and hdfs data
    admin.disableTable(table);

    // create new corrupt file called deadbeef (valid hfile name)
    Path corrupt = new Path(hfile.getParent(), "deadbeef");
    TestHFile.truncateFile(fs, hfile, corrupt);
    LOG.info("Created corrupted file " + corrupt);
    HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));

    // we cannot enable here because enable never finished due to the corrupt region.
    HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
    assertEquals(res.getRetCode(), 0);
    HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
    assertEquals(hfcc.getHFilesChecked(), 5);
    assertEquals(hfcc.getCorrupted().size(), 1);
    assertEquals(hfcc.getFailures().size(), 0);
    assertEquals(hfcc.getQuarantined().size(), 1);
    assertEquals(hfcc.getMissing().size(), 0);

    // Its been fixed, verify that we can enable.
    admin.enableTable(table);
  } finally {
    cleanupTable(table);
  }
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=180000)
public void testQuarantineMissingHFile() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());

  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
        @Override
        protected void checkHFile(Path p) throws IOException {
          if (attemptedFirstHFile.compareAndSet(false, true)) {
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
  hbck.close();
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
// Disabled because fails sporadically.  Is this test right?  Timing-wise, there could be no
// files in a column family on initial creation -- as suggested by Matteo.
@Ignore @Test(timeout=180000)
public void testQuarantineMissingFamdir() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
        @Override
        protected void checkColFamDir(Path p) throws IOException {
          if (attemptedFirstHFile.compareAndSet(false, true)) {
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
  hbck.close();
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingRegionDir() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)
    throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
        @Override
        protected void checkRegionDir(Path p) throws IOException {
          if (attemptedFirstHFile.compareAndSet(false, true)) {
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
  hbck.close();
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=180000)
public void testQuarantineMissingHFile() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingFamdir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstFamDir = false;
        protected void checkColFamDir(Path p) throws IOException {
          if (!attemptedFirstFamDir) {
            attemptedFirstFamDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingRegionDir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=180000)
public void testQuarantineMissingHFile() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());

  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
        @Override
        protected void checkHFile(Path p) throws IOException {
          if (attemptedFirstHFile.compareAndSet(false, true)) {
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
  hbck.close();
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
// Disabled because fails sporadically.  Is this test right?  Timing-wise, there could be no
// files in a column family on initial creation -- as suggested by Matteo.
@Ignore @Test(timeout=180000)
public void testQuarantineMissingFamdir() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
        @Override
        protected void checkColFamDir(Path p) throws IOException {
          if (attemptedFirstHFile.compareAndSet(false, true)) {
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
  hbck.close();
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingRegionDir() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, hbfsckExecutorService) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles)
    throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        AtomicBoolean attemptedFirstHFile = new AtomicBoolean(false);
        @Override
        protected void checkRegionDir(Path p) throws IOException {
          if (attemptedFirstHFile.compareAndSet(false, true)) {
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
  hbck.close();
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=180000)
public void testQuarantineMissingHFile() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        @Override
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingRegionDir() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        @Override
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=180000)
public void testQuarantineMissingHFile() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingFamdir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstFamDir = false;
        protected void checkColFamDir(Path p) throws IOException {
          if (!attemptedFirstFamDir) {
            attemptedFirstFamDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingRegionDir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=180000)
public void testQuarantineMissingHFile() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        @Override
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingRegionDir() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        @Override
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=180000)
public void testQuarantineMissingHFile() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        @Override
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=180000)
public void testQuarantineMissingRegionDir() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    @Override
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        @Override
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=120000)
public void testQuarantineMissingHFile() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingFamdir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstFamDir = false;
        protected void checkColFamDir(Path p) throws IOException {
          if (!attemptedFirstFamDir) {
            attemptedFirstFamDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingRegionDir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=120000)
public void testQuarantineMissingHFile() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingFamdir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstFamDir = false;
        protected void checkColFamDir(Path p) throws IOException {
          if (!attemptedFirstFamDir) {
            attemptedFirstFamDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingRegionDir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=120000)
public void testQuarantineMissingHFile() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingFamdir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstFamDir = false;
        protected void checkColFamDir(Path p) throws IOException {
          if (!attemptedFirstFamDir) {
            attemptedFirstFamDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingRegionDir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=120000)
public void testQuarantineMissingHFile() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingFamdir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstFamDir = false;
        protected void checkColFamDir(Path p) throws IOException {
          if (!attemptedFirstFamDir) {
            attemptedFirstFamDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingRegionDir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an hfile after the corruption checker learned about it.
 */
@Test(timeout=120000)
public void testQuarantineMissingHFile() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstHFile = false;
        protected void checkHFile(Path p) throws IOException {
          if (!attemptedFirstHFile) {
            attemptedFirstHFile = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkHFile(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 4, 0, 0, 0, 1); // 4 attempted, but 1 missing.
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed an colfam dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingFamdir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstFamDir = false;
        protected void checkColFamDir(Path p) throws IOException {
          if (!attemptedFirstFamDir) {
            attemptedFirstFamDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkColFamDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * This creates a table and simulates the race situation where a concurrent compaction or split
 * has removed a region dir before the corruption checker got to it.
 */
@Test(timeout=120000)
public void testQuarantineMissingRegionDir() throws Exception {
  String table = name.getMethodName();
  ExecutorService exec = new ScheduledThreadPoolExecutor(10);
  // inject a fault in the hfcc created.
  final FileSystem fs = FileSystem.get(conf);
  HBaseFsck hbck = new HBaseFsck(conf, exec) {
    public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
      return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
        boolean attemptedFirstRegionDir = false;
        protected void checkRegionDir(Path p) throws IOException {
          if (!attemptedFirstRegionDir) {
            attemptedFirstRegionDir = true;
            assertTrue(fs.delete(p, true)); // make sure delete happened.
          }
          super.checkRegionDir(p);
        }
      };
    }
  };
  doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
}

/**
 * Test that use this should have a timeout, because this method could potentially wait forever.
*/
private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
                              int corrupt, int fail, int quar, int missing) throws Exception {
  try {
    setupTable(table);
    assertEquals(ROWKEYS.length, countRows());
    admin.flush(table); // flush is async.

    // Mess it up by leaving a hole in the assignment, meta, and hdfs data
    admin.disableTable(table);

    String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
        table.getNameAsString()};
    HBaseFsck res = hbck.exec(hbfsckExecutorService, args);

    HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
    assertEquals(hfcc.getHFilesChecked(), check);
    assertEquals(hfcc.getCorrupted().size(), corrupt);
    assertEquals(hfcc.getFailures().size(), fail);
    assertEquals(hfcc.getQuarantined().size(), quar);
    assertEquals(hfcc.getMissing().size(), missing);

    // its been fixed, verify that we can enable
    admin.enableTableAsync(table);
    while (!admin.isTableEnabled(table)) {
      try {
        Thread.sleep(250);
      } catch (InterruptedException e) {
        e.printStackTrace();
        fail("Interrupted when trying to enable table " + table);
      }
    }
  } finally {
    cleanupTable(table);
  }
}

@Test
public void testFsckWithEncryption() throws Exception {
  // Populate the table with some data
  Table table = new HTable(conf, htd.getTableName());
  try {
    byte[] values = { 'A', 'B', 'C', 'D' };
    for (int i = 0; i < values.length; i++) {
      for (int j = 0; j < values.length; j++) {
        Put put = new Put(new byte[] { values[i], values[j] });
        put.add(Bytes.toBytes("cf"), new byte[] {}, new byte[] { values[i],
          values[j] });
        table.put(put);
      }
    }
  } finally {
    table.close();
  }
  // Flush it
  TEST_UTIL.getHBaseAdmin().flush(htd.getTableName());

  // Verify we have encrypted store files on disk
  final List<Path> paths = findStorefilePaths(htd.getTableName());
  assertTrue(paths.size() > 0);
  for (Path path: paths) {
    assertTrue("Store file " + path + " has incorrect key",
      Bytes.equals(cfKey.getEncoded(), extractHFileKey(path)));
  }

  // Insure HBck doesn't consider them corrupt
  HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, htd.getTableName());
  assertEquals(res.getRetCode(), 0);
  HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
  assertEquals(hfcc.getCorrupted().size(), 0);
  assertEquals(hfcc.getFailures().size(), 0);
  assertEquals(hfcc.getQuarantined().size(), 0);
  assertEquals(hfcc.getMissing().size(), 0);
}

/**
 * This creates a table and then corrupts an hfile.  Hbck should quarantine the file.
 */
@Test(timeout=180000)
public void testQuarantineCorruptHFile() throws Exception {
  String table = name.getMethodName();
  try {
    setupTable(table);
    assertEquals(ROWKEYS.length, countRows());
    TEST_UTIL.getHBaseAdmin().flush(table); // flush is async.

    FileSystem fs = FileSystem.get(conf);
    Path hfile = getFlushedHFile(fs, table);

    // Mess it up by leaving a hole in the assignment, meta, and hdfs data
    TEST_UTIL.getHBaseAdmin().disableTable(table);

    // create new corrupt file called deadbeef (valid hfile name)
    Path corrupt = new Path(hfile.getParent(), "deadbeef");
    TestHFile.truncateFile(fs, hfile, corrupt);
    LOG.info("Created corrupted file " + corrupt);
    HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));

    // we cannot enable here because enable never finished due to the corrupt region.
    HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
    assertEquals(res.getRetCode(), 0);
    HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
    assertEquals(hfcc.getHFilesChecked(), 5);
    assertEquals(hfcc.getCorrupted().size(), 1);
    assertEquals(hfcc.getFailures().size(), 0);
    assertEquals(hfcc.getQuarantined().size(), 1);
    assertEquals(hfcc.getMissing().size(), 0);

    // Its been fixed, verify that we can enable.
    TEST_UTIL.getHBaseAdmin().enableTable(table);
  } finally {
    deleteTable(table);
  }
}

/**
* Test that use this should have a timeout, because this method could potentially wait forever.
*/
private void doQuarantineTest(String table, HBaseFsck hbck, int check, int corrupt, int fail,
    int quar, int missing) throws Exception {
  try {
    setupTable(table);
    assertEquals(ROWKEYS.length, countRows());
    TEST_UTIL.getHBaseAdmin().flush(table); // flush is async.

    // Mess it up by leaving a hole in the assignment, meta, and hdfs data
    TEST_UTIL.getHBaseAdmin().disableTable(table);

    String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission", table};
    ExecutorService exec = new ScheduledThreadPoolExecutor(10);
    HBaseFsck res = hbck.exec(exec, args);

    HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
    assertEquals(hfcc.getHFilesChecked(), check);
    assertEquals(hfcc.getCorrupted().size(), corrupt);
    assertEquals(hfcc.getFailures().size(), fail);
    assertEquals(hfcc.getQuarantined().size(), quar);
    assertEquals(hfcc.getMissing().size(), missing);

    // its been fixed, verify that we can enable
    HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
    admin.enableTableAsync(table);
    while (!admin.isTableEnabled(table)) {
      try {
        Thread.sleep(250);
      } catch (InterruptedException e) {
        e.printStackTrace();
        fail("Interrupted when trying to enable table " + table);
      }
    }
  } finally {
    deleteTable(table);
  }
}

/**
 * This creates a table and then corrupts an hfile.  Hbck should quarantine the file.
 */
@Test(timeout=180000)
public void testQuarantineCorruptHFile() throws Exception {
  TableName table = TableName.valueOf(name.getMethodName());
  try {
    setupTable(table);
    assertEquals(ROWKEYS.length, countRows());
    admin.flush(table); // flush is async.

    FileSystem fs = FileSystem.get(conf);
    Path hfile = getFlushedHFile(fs, table);

    // Mess it up by leaving a hole in the assignment, meta, and hdfs data
    admin.disableTable(table);

    // create new corrupt file called deadbeef (valid hfile name)
    Path corrupt = new Path(hfile.getParent(), "deadbeef");
    TestHFile.truncateFile(fs, hfile, corrupt);
    LOG.info("Created corrupted file " + corrupt);
    HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));

    // we cannot enable here because enable never finished due to the corrupt region.
    HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
    assertEquals(res.getRetCode(), 0);
    HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
    assertEquals(hfcc.getHFilesChecked(), 5);
    assertEquals(hfcc.getCorrupted().size(), 1);
    assertEquals(hfcc.getFailures().size(), 0);
    assertEquals(hfcc.getQuarantined().size(), 1);
    assertEquals(hfcc.getMissing().size(), 0);

    // Its been fixed, verify that we can enable.
    admin.enableTable(table);
  } finally {
    cleanupTable(table);
  }
}

/**
 * Test that use this should have a timeout, because this method could potentially wait forever.
*/
private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
                              int corrupt, int fail, int quar, int missing) throws Exception {
  try {
    setupTable(table);
    assertEquals(ROWKEYS.length, countRows());
    admin.flush(table); // flush is async.

    // Mess it up by leaving a hole in the assignment, meta, and hdfs data
    admin.disableTable(table);

    String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
        table.getNameAsString()};
    HBaseFsck res = hbck.exec(hbfsckExecutorService, args);

    HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
    assertEquals(hfcc.getHFilesChecked(), check);
    assertEquals(hfcc.getCorrupted().size(), corrupt);
    assertEquals(hfcc.getFailures().size(), fail);
    assertEquals(hfcc.getQuarantined().size(), quar);
    assertEquals(hfcc.getMissing().size(), missing);

    // its been fixed, verify that we can enable
    admin.enableTableAsync(table);
    while (!admin.isTableEnabled(table)) {
      try {
        Thread.sleep(250);
      } catch (InterruptedException e) {
        e.printStackTrace();
        fail("Interrupted when trying to enable table " + table);
      }
    }
  } finally {
    cleanupTable(table);
  }
}