Java 类org.apache.hadoop.hbase.regionserver.wal.FSHLog 实例源码

项目：ditb 文件：BoundedRegionGroupingProvider.java

/**
 * iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or non-meta,
 * count the number of files (rolled and active). if either of them isn't, count 0
 * for that provider.
 * @param walFactory may not be null.
 */
public static long getNumLogFiles(WALFactory walFactory) {
  long result = 0;
  if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
    BoundedRegionGroupingProvider groupProviders =
        (BoundedRegionGroupingProvider)walFactory.provider;
    for (int i = 0; i < groupProviders.delegates.length; i++) {
      result +=
          ((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getNumLogFiles();
    }
  }
  WALProvider meta = walFactory.metaProvider.get();
  if (meta instanceof BoundedRegionGroupingProvider) {
    for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++) {
      result += ((FSHLog)
          ((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
          .getNumLogFiles();      }
  }
  return result;
}

项目：ditb 文件：BoundedRegionGroupingProvider.java

/**
 * iff the given WALFactory is using the BoundedRegionGroupingProvider for meta and/or non-meta,
 * count the size of files (rolled and active). if either of them isn't, count 0
 * for that provider.
 * @param walFactory may not be null.
 */
public static long getLogFileSize(WALFactory walFactory) {
  long result = 0;
  if (walFactory.provider instanceof BoundedRegionGroupingProvider) {
    BoundedRegionGroupingProvider groupProviders =
        (BoundedRegionGroupingProvider)walFactory.provider;
    for (int i = 0; i < groupProviders.delegates.length; i++) {
      result +=
          ((FSHLog)((DefaultWALProvider)(groupProviders.delegates[i])).log).getLogFileSize();
    }
  }
  WALProvider meta = walFactory.metaProvider.get();
  if (meta instanceof BoundedRegionGroupingProvider) {
    for (int i = 0; i < ((BoundedRegionGroupingProvider)meta).delegates.length; i++) {
      result += ((FSHLog)
          ((DefaultWALProvider)(((BoundedRegionGroupingProvider)meta).delegates[i])).log)
          .getLogFileSize();
    }
  }
  return result;
}

项目：hbase 文件：TestCompactionPolicy.java

/**
 * Setting up a Store
 * @throws IOException with error
 */
protected void initialize() throws IOException {
  Path basedir = new Path(DIR);
  String logName = "logs";
  Path logdir = new Path(DIR, logName);
  HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family"));
  FileSystem fs = FileSystem.get(conf);

  fs.delete(logdir, true);

  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table")));
  htd.addFamily(hcd);
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);

  hlog = new FSHLog(fs, basedir, logName, conf);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  region = HRegion.createHRegion(info, basedir, conf, htd, hlog);
  region.close();
  Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName());
  region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);

  store = new HStore(region, hcd, conf);

  TEST_FILE = region.getRegionFileSystem().createTempName();
  fs.createNewFile(TEST_FILE);
}

项目：ditb 文件：DefaultWALProvider.java

/**
 * iff the given WALFactory is using the DefaultWALProvider for meta and/or non-meta,
 * count the number of files (rolled and active). if either of them aren't, count 0
 * for that provider.
 * @param walFactory may not be null.
 */
public static long getNumLogFiles(WALFactory walFactory) {
  long result = 0;
  if (walFactory.provider instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getNumLogFiles();
  }
  WALProvider meta = walFactory.metaProvider.get();
  if (meta instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)meta).log).getNumLogFiles();
  }
  return result;
}

项目：ditb 文件：DefaultWALProvider.java

/**
 * iff the given WALFactory is using the DefaultWALProvider for meta and/or non-meta,
 * count the size of files (rolled and active). if either of them aren't, count 0
 * for that provider.
 * @param walFactory may not be null.
 */
public static long getLogFileSize(WALFactory walFactory) {
  long result = 0;
  if (walFactory.provider instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getLogFileSize();
  }
  WALProvider meta = walFactory.metaProvider.get();
  if (meta instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)meta).log).getLogFileSize();
  }
  return result;
}

项目：ditb 文件：DefaultWALProvider.java

/**
 * It returns the file create timestamp from the file name.
 * For name format see {@link #validateWALFilename(String)}
 * public until remaining tests move to o.a.h.h.wal
 * @param wal must not be null
 * @return the file number that is part of the WAL file name
 */
@VisibleForTesting
public static long extractFileNumFromWAL(final WAL wal) {
  final Path walName = ((FSHLog)wal).getCurrentFileName();
  if (walName == null) {
    throw new IllegalArgumentException("The WAL path couldn't be null");
  }
  final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER);
  return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]);
}

项目：ditb 文件：TestHRegion.java

/**
 * Test for HBASE-14229: Flushing canceled by coprocessor still leads to memstoreSize set down
 */
@Test
public void testMemstoreSizeWithFlushCanceling() throws IOException {
  FileSystem fs = FileSystem.get(CONF);
  Path rootDir = new Path(dir + "testMemstoreSizeWithFlushCanceling");
  FSHLog hLog = new FSHLog(fs, rootDir, "testMemstoreSizeWithFlushCanceling", CONF);
  HRegion region = initHRegion(tableName, null, null, name.getMethodName(),
      CONF, false, Durability.SYNC_WAL, hLog, COLUMN_FAMILY_BYTES);
  Store store = region.getStore(COLUMN_FAMILY_BYTES);
  assertEquals(0, region.getMemstoreSize());

  // Put some value and make sure flush could be completed normally
  byte [] value = Bytes.toBytes(name.getMethodName());
  Put put = new Put(value);
  put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
  region.put(put);
  long onePutSize = region.getMemstoreSize();
  assertTrue(onePutSize > 0);
  region.flush(true);
  assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
  assertEquals("flushable size should be zero", 0, store.getFlushableSize());

  // save normalCPHost and replaced by mockedCPHost, which will cancel flush requests
  RegionCoprocessorHost normalCPHost = region.getCoprocessorHost();
  RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
  when(mockedCPHost.preFlush(Mockito.isA(HStore.class), Mockito.isA(InternalScanner.class))).
    thenReturn(null);
  region.setCoprocessorHost(mockedCPHost);
  region.put(put);
  region.flush(true);
  assertEquals("memstoreSize should NOT be zero", onePutSize, region.getMemstoreSize());
  assertEquals("flushable size should NOT be zero", onePutSize, store.getFlushableSize());

  // set normalCPHost and flush again, the snapshot will be flushed
  region.setCoprocessorHost(normalCPHost);
  region.flush(true);
  assertEquals("memstoreSize should be zero", 0, region.getMemstoreSize());
  assertEquals("flushable size should be zero", 0, store.getFlushableSize());
  HRegion.closeHRegion(region);
}

项目：ditb 文件：TestRegionIncrement.java

private HRegion getRegion(final Configuration conf, final String tableName) throws IOException {
  WAL wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDataTestDir(),
    TEST_UTIL.getDataTestDir().toString(), conf);
  return (HRegion)TEST_UTIL.createLocalHRegion(Bytes.toBytes(tableName),
    HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, tableName, conf,
    false, Durability.SKIP_WAL, wal, INCREMENT_BYTES);
}

项目：pbase 文件：DefaultWALProvider.java

/**
 * iff the given WALFactory is using the DefaultWALProvider for meta and/or non-meta,
 * count the number of files (rolled and active). if either of them aren't, count 0
 * for that provider.
 * @param walFactory may not be null.
 */
public static long getNumLogFiles(WALFactory walFactory) {
  long result = 0;
  if (walFactory.provider instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getNumLogFiles();
  }
  WALProvider meta = walFactory.metaProvider.get();
  if (meta instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)meta).log).getNumLogFiles();
  }
  return result;
}

项目：pbase 文件：DefaultWALProvider.java

/**
 * iff the given WALFactory is using the DefaultWALProvider for meta and/or non-meta,
 * count the size of files (rolled and active). if either of them aren't, count 0
 * for that provider.
 * @param walFactory may not be null.
 */
public static long getLogFileSize(WALFactory walFactory) {
  long result = 0;
  if (walFactory.provider instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getLogFileSize();
  }
  WALProvider meta = walFactory.metaProvider.get();
  if (meta instanceof DefaultWALProvider) {
    result += ((FSHLog)((DefaultWALProvider)meta).log).getLogFileSize();
  }
  return result;
}

项目：pbase 文件：DefaultWALProvider.java

/**
 * It returns the file create timestamp from the file name.
 * For name format see {@link #validateWALFilename(String)}
 * public until remaining tests move to o.a.h.h.wal
 * @param wal must not be null
 * @return the file number that is part of the WAL file name
 */
@VisibleForTesting
public static long extractFileNumFromWAL(final WAL wal) {
  final Path walName = ((FSHLog)wal).getCurrentFileName();
  if (walName == null) {
    throw new IllegalArgumentException("The WAL path couldn't be null");
  }
  final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER);
  return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]);
}

项目：hbase 文件：FSHLogProvider.java

@Override
protected FSHLog createWAL() throws IOException {
  return new FSHLog(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf),
      getWALDirectoryName(factory.factoryId),
      getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix,
      META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
}

项目：hbase 文件：IOTestProvider.java

private FSHLog createWAL() throws IOException {
  String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId;
  return new IOTestWAL(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf),
      AbstractFSWALProvider.getWALDirectoryName(factory.factoryId),
      HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners, true, logPrefix,
      META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
}

项目：hbase 文件：IOTestProvider.java

@Override
public WAL getWAL(RegionInfo region) throws IOException {
  FSHLog log = this.log;
  if (log != null) {
    return log;
  }
  synchronized (this) {
    log = this.log;
    if (log == null) {
      log = createWAL();
      this.log = log;
    }
  }
  return log;
}

项目：hbase 文件：IOTestProvider.java

@Override
public void close() throws IOException {
  FSHLog log = this.log;
  if (log != null) {
    log.close();
  }
}

项目：hbase 文件：IOTestProvider.java

@Override
public void shutdown() throws IOException {
  FSHLog log = this.log;
  if (log != null) {
    log.shutdown();
  }
}

项目：hbase 文件：TestHRegion.java

@Test
public void testMemstoreSizeAccountingWithFailedPostBatchMutate() throws IOException {
  String testName = "testMemstoreSizeAccountingWithFailedPostBatchMutate";
  FileSystem fs = FileSystem.get(CONF);
  Path rootDir = new Path(dir + testName);
  FSHLog hLog = new FSHLog(fs, rootDir, testName, CONF);
  HRegion region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, hLog,
      COLUMN_FAMILY_BYTES);
  HStore store = region.getStore(COLUMN_FAMILY_BYTES);
  assertEquals(0, region.getMemStoreSize());

  // Put one value
  byte [] value = Bytes.toBytes(method);
  Put put = new Put(value);
  put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value);
  region.put(put);
  long onePutSize = region.getMemStoreSize();
  assertTrue(onePutSize > 0);

  RegionCoprocessorHost mockedCPHost = Mockito.mock(RegionCoprocessorHost.class);
  doThrow(new IOException())
     .when(mockedCPHost).postBatchMutate(Mockito.<MiniBatchOperationInProgress<Mutation>>any());
  region.setCoprocessorHost(mockedCPHost);

  put = new Put(value);
  put.addColumn(COLUMN_FAMILY_BYTES, Bytes.toBytes("dfg"), value);
  try {
    region.put(put);
    fail("Should have failed with IOException");
  } catch (IOException expected) {
  }
  long expectedSize = onePutSize * 2;
  assertEquals("memstoreSize should be incremented", expectedSize, region.getMemStoreSize());
  assertEquals("flushable size should be incremented", expectedSize,
      store.getFlushableSize().getDataSize());

  region.setCoprocessorHost(null);
  HBaseTestingUtility.closeRegionAndWAL(region);
}

项目：hbase 文件：TestRegionIncrement.java

private HRegion getRegion(final Configuration conf, final String tableName) throws IOException {
  WAL wal = new FSHLog(FileSystem.get(conf), TEST_UTIL.getDataTestDir(),
    TEST_UTIL.getDataTestDir().toString(), conf);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  return (HRegion)TEST_UTIL.createLocalHRegion(Bytes.toBytes(tableName),
    HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, tableName, conf,
    false, Durability.SKIP_WAL, wal, INCREMENT_BYTES);
}

项目：ditb 文件：DefaultWALProvider.java

/**
 * returns the number of rolled WAL files.
 */
@VisibleForTesting
public static int getNumRolledLogFiles(WAL wal) {
  return ((FSHLog)wal).getNumRolledLogFiles();
}

项目：ditb 文件：DefaultWALProvider.java

/**
 * return the current filename from the current wal.
 */
@VisibleForTesting
public static Path getCurrentFileName(final WAL wal) {
  return ((FSHLog)wal).getCurrentFileName();
}

项目：ditb 文件：DefaultWALProvider.java

/**
 * request a log roll, but don't actually do it.
 */
@VisibleForTesting
static void requestLogRoll(final WAL wal) {
  ((FSHLog)wal).requestLogRoll();
}

项目：ditb 文件：TestPerColumnFamilyFlush.java

private int getNumRolledLogFiles(Region region) {
  return ((FSHLog)getWAL(region)).getNumRolledLogFiles();
}

项目：pbase 文件：DefaultWALProvider.java

/**
 * returns the number of rolled WAL files.
 */
@VisibleForTesting
public static int getNumRolledLogFiles(WAL wal) {
  return ((FSHLog)wal).getNumRolledLogFiles();
}

项目：pbase 文件：DefaultWALProvider.java

/**
 * return the current filename from the current wal.
 */
@VisibleForTesting
public static Path getCurrentFileName(final WAL wal) {
  return ((FSHLog)wal).getCurrentFileName();
}

项目：pbase 文件：DefaultWALProvider.java

/**
 * request a log roll, but don't actually do it.
 */
@VisibleForTesting
static void requestLogRoll(final WAL wal) {
  ((FSHLog)wal).requestLogRoll();
}

项目：hbase 文件：TestHRegion.java

@Test
public void testDataInMemoryWithoutWAL() throws IOException {
  FileSystem fs = FileSystem.get(CONF);
  Path rootDir = new Path(dir + "testDataInMemoryWithoutWAL");
  FSHLog hLog = new FSHLog(fs, rootDir, "testDataInMemoryWithoutWAL", CONF);
  // This chunk creation is done throughout the code base. Do we want to move it into core?
  // It is missing from this test. W/o it we NPE.
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  HRegion region = initHRegion(tableName, null, null, false, Durability.SYNC_WAL, hLog,
      COLUMN_FAMILY_BYTES);

  Cell originalCell = CellUtil.createCell(row, COLUMN_FAMILY_BYTES, qual1,
    System.currentTimeMillis(), KeyValue.Type.Put.getCode(), value1);
  final long originalSize = KeyValueUtil.length(originalCell);

  Cell addCell = CellUtil.createCell(row, COLUMN_FAMILY_BYTES, qual1,
    System.currentTimeMillis(), KeyValue.Type.Put.getCode(), Bytes.toBytes("xxxxxxxxxx"));
  final long addSize = KeyValueUtil.length(addCell);

  LOG.info("originalSize:" + originalSize
    + ", addSize:" + addSize);
  // start test. We expect that the addPut's durability will be replaced
  // by originalPut's durability.

  // case 1:
  testDataInMemoryWithoutWAL(region,
          new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
          new Put(row).add(addCell).setDurability(Durability.SKIP_WAL),
          originalSize + addSize);

  // case 2:
  testDataInMemoryWithoutWAL(region,
          new Put(row).add(originalCell).setDurability(Durability.SKIP_WAL),
          new Put(row).add(addCell).setDurability(Durability.SYNC_WAL),
          originalSize + addSize);

  // case 3:
  testDataInMemoryWithoutWAL(region,
          new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
          new Put(row).add(addCell).setDurability(Durability.SKIP_WAL),
          0);

  // case 4:
  testDataInMemoryWithoutWAL(region,
          new Put(row).add(originalCell).setDurability(Durability.SYNC_WAL),
          new Put(row).add(addCell).setDurability(Durability.SYNC_WAL),
          0);
}