Java 类org.apache.hadoop.hbase.util.CollectionBackedScanner 实例源码

项目:ditb    文件:DefaultMemStore.java   
/**
 * Creates a snapshot of the current memstore.
 * Snapshot must be cleared by call to {@link #clearSnapshot(long)}
 */
@Override
public MemStoreSnapshot snapshot() {
  // If snapshot currently has entries, then flusher failed or didn't call
  // cleanup.  Log a warning.
  if (!this.snapshot.isEmpty()) {
    LOG.warn("Snapshot called again without clearing previous. " +
        "Doing nothing. Another ongoing flush or did we fail last attempt?");
  } else {
    this.snapshotId = EnvironmentEdgeManager.currentTime();
    this.snapshotSize = keySize();
    if (!this.cellSet.isEmpty()) {
      this.snapshot = this.cellSet;
      this.cellSet = new CellSkipListSet(this.comparator);
      this.snapshotTimeRangeTracker = this.timeRangeTracker;
      this.timeRangeTracker = new TimeRangeTracker();
      // Reset heap to not include any keys
      this.size.set(DEEP_OVERHEAD);
      this.snapshotAllocator = this.allocator;
      // Reset allocator so we get a fresh buffer for the new memstore
      if (allocator != null) {
        String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName());
        this.allocator = ReflectionUtils.instantiateWithCustomCtor(className,
            new Class[] { Configuration.class }, new Object[] { conf });
      }
      timeOfOldestEdit = Long.MAX_VALUE;
    }
  }
  MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize,
      this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator),
      this.tagsPresent);
  this.tagsPresent = false;
  return memStoreSnapshot;
}
项目:ditb    文件:TestPrefixTreeEncoding.java   
private void verifySeeking(EncodedSeeker encodeSeeker,
    ByteBuffer encodedData, int batchId) {
  List<KeyValue> kvList = new ArrayList<KeyValue>();
  for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
    kvList.clear();
    encodeSeeker.setCurrentBuffer(encodedData);
    KeyValue firstOnRow = KeyValueUtil.createFirstOnRow(getRowKey(batchId, i));
    encodeSeeker.seekToKeyInBlock(
        new KeyValue.KeyOnlyKeyValue(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(),
            firstOnRow.getKeyLength()), false);
    boolean hasMoreOfEncodeScanner = encodeSeeker.next();
    CollectionBackedScanner collectionScanner = new CollectionBackedScanner(
        this.kvset);
    boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow);
    if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) {
      dumpInputKVSet();
      fail("Get error result after seeking " + firstOnRow);
    }
    if (hasMoreOfEncodeScanner) {
      if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(),
          collectionScanner.peek()) != 0) {
        dumpInputKVSet();
        fail("Expected " + collectionScanner.peek() + " actual "
            + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow);
      }
    }
  }
}
项目:pbase    文件:DefaultMemStore.java   
/**
 * Creates a snapshot of the current memstore.
 * Snapshot must be cleared by call to {@link #clearSnapshot(long)}
 */
@Override
public MemStoreSnapshot snapshot() {
    // If snapshot currently has entries, then flusher failed or didn't call
    // cleanup.  Log a warning.
    if (!this.snapshot.isEmpty()) {
        LOG.warn("Snapshot called again without clearing previous. " +
                "Doing nothing. Another ongoing flush or did we fail last attempt?");
    } else {
        this.snapshotId = EnvironmentEdgeManager.currentTime();
        this.snapshotSize = keySize();
        if (!this.cellSet.isEmpty()) {
            this.snapshot = this.cellSet;
            this.cellSet = new CellSkipListSet(this.comparator);
            this.snapshotTimeRangeTracker = this.timeRangeTracker;
            this.timeRangeTracker = new TimeRangeTracker();
            // Reset heap to not include any keys
            this.size.set(DEEP_OVERHEAD);
            this.snapshotAllocator = this.allocator;
            // Reset allocator so we get a fresh buffer for the new memstore
            if (allocator != null) {
                String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName());
                this.allocator = ReflectionUtils.instantiateWithCustomCtor(className,
                        new Class[]{Configuration.class}, new Object[]{conf});
            }
            timeOfOldestEdit = Long.MAX_VALUE;
        }
    }
    return new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize,
            this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator));
}
项目:pbase    文件:TestPrefixTreeEncoding.java   
private void verifySeeking(EncodedSeeker encodeSeeker,
    ByteBuffer encodedData, int batchId) {
  List<KeyValue> kvList = new ArrayList<KeyValue>();
  for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
    kvList.clear();
    encodeSeeker.setCurrentBuffer(encodedData);
    KeyValue firstOnRow = KeyValueUtil.createFirstOnRow(getRowKey(batchId, i));
    encodeSeeker.seekToKeyInBlock(
        new KeyValue.KeyOnlyKeyValue(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(),
            firstOnRow.getKeyLength()), false);
    boolean hasMoreOfEncodeScanner = encodeSeeker.next();
    CollectionBackedScanner collectionScanner = new CollectionBackedScanner(
        this.kvset);
    boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow);
    if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) {
      dumpInputKVSet();
      fail("Get error result after seeking " + firstOnRow);
    }
    if (hasMoreOfEncodeScanner) {
      if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(),
          collectionScanner.peek()) != 0) {
        dumpInputKVSet();
        fail("Expected " + collectionScanner.peek() + " actual "
            + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow);
      }
    }
  }
}
项目:HIndex    文件:TestPrefixTreeEncoding.java   
private void verifySeeking(EncodedSeeker encodeSeeker,
    ByteBuffer encodedData, int batchId) {
  List<KeyValue> kvList = new ArrayList<KeyValue>();
  for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
    kvList.clear();
    encodeSeeker.setCurrentBuffer(encodedData);
    KeyValue firstOnRow = KeyValue.createFirstOnRow(getRowKey(batchId, i));
    encodeSeeker.seekToKeyInBlock(firstOnRow.getBuffer(),
        firstOnRow.getKeyOffset(), firstOnRow.getKeyLength(), false);
    boolean hasMoreOfEncodeScanner = encodeSeeker.next();
    CollectionBackedScanner collectionScanner = new CollectionBackedScanner(
        this.kvset);
    boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow);
    if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) {
      dumpInputKVSet();
      fail("Get error result after seeking " + firstOnRow);
    }
    if (hasMoreOfEncodeScanner) {
      if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(),
          collectionScanner.peek()) != 0) {
        dumpInputKVSet();
        fail("Expected " + collectionScanner.peek() + " actual "
            + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow);
      }
    }
  }
}
项目:PyroDB    文件:DefaultMemStore.java   
/**
 * Creates a snapshot of the current memstore.
 * Snapshot must be cleared by call to {@link #clearSnapshot(long)}
 */
@Override
public MemStoreSnapshot snapshot() {
  // If snapshot currently has entries, then flusher failed or didn't call
  // cleanup.  Log a warning.
  if (!this.snapshot.isEmpty()) {
    LOG.warn("Snapshot called again without clearing previous. " +
        "Doing nothing. Another ongoing flush or did we fail last attempt?");
  } else {
    this.snapshotId = EnvironmentEdgeManager.currentTimeMillis();
    this.snapshotSize = keySize();
    if (!this.kvset.isEmpty()) {
      this.snapshot = this.kvset;
      this.kvset = new KeyValueSkipListSet(this.comparator);
      this.snapshotTimeRangeTracker = this.timeRangeTracker;
      this.timeRangeTracker = new TimeRangeTracker();
      // Reset heap to not include any keys
      this.size.set(DEEP_OVERHEAD);
      this.snapshotAllocator = this.allocator;
      // Reset allocator so we get a fresh buffer for the new memstore
      if (allocator != null) {
        String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName());
        this.allocator = ReflectionUtils.instantiateWithCustomCtor(className,
            new Class[] { Configuration.class }, new Object[] { conf });
      }
      timeOfOldestEdit = Long.MAX_VALUE;
    }
  }
  return new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize,
      this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator));
}
项目:PyroDB    文件:TestPrefixTreeEncoding.java   
private void verifySeeking(EncodedSeeker encodeSeeker,
    ByteBuffer encodedData, int batchId) {
  List<KeyValue> kvList = new ArrayList<KeyValue>();
  for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
    kvList.clear();
    encodeSeeker.setCurrentBuffer(encodedData);
    KeyValue firstOnRow = KeyValueUtil.createFirstOnRow(getRowKey(batchId, i));
    encodeSeeker.seekToKeyInBlock(
        new KeyValue.KeyOnlyKeyValue(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(),
            firstOnRow.getKeyLength()), false);
    boolean hasMoreOfEncodeScanner = encodeSeeker.next();
    CollectionBackedScanner collectionScanner = new CollectionBackedScanner(
        this.kvset);
    boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow);
    if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) {
      dumpInputKVSet();
      fail("Get error result after seeking " + firstOnRow);
    }
    if (hasMoreOfEncodeScanner) {
      if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(),
          collectionScanner.peek()) != 0) {
        dumpInputKVSet();
        fail("Expected " + collectionScanner.peek() + " actual "
            + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow);
      }
    }
  }
}
项目:c5    文件:TestPrefixTreeEncoding.java   
private void verifySeeking(EncodedSeeker encodeSeeker,
    ByteBuffer encodedData, int batchId) {
  List<KeyValue> kvList = new ArrayList<KeyValue>();
  for (int i = 0; i < NUM_ROWS_PER_BATCH; ++i) {
    kvList.clear();
    encodeSeeker.setCurrentBuffer(encodedData);
    KeyValue firstOnRow = KeyValue.createFirstOnRow(getRowKey(batchId, i));
    encodeSeeker.seekToKeyInBlock(firstOnRow.getBuffer(),
        firstOnRow.getKeyOffset(), firstOnRow.getKeyLength(), false);
    boolean hasMoreOfEncodeScanner = encodeSeeker.next();
    CollectionBackedScanner collectionScanner = new CollectionBackedScanner(
        this.kvset);
    boolean hasMoreOfCollectionScanner = collectionScanner.seek(firstOnRow);
    if (hasMoreOfEncodeScanner != hasMoreOfCollectionScanner) {
      dumpInputKVSet();
      fail("Get error result after seeking " + firstOnRow);
    }
    if (hasMoreOfEncodeScanner) {
      if (KeyValue.COMPARATOR.compare(encodeSeeker.getKeyValue(),
          collectionScanner.peek()) != 0) {
        dumpInputKVSet();
        fail("Expected " + collectionScanner.peek() + " actual "
            + encodeSeeker.getKeyValue() + ", after seeking " + firstOnRow);
      }
    }
  }
}
项目:c5    文件:DefaultStoreFlusher.java   
@Override
public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
    TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize,
    MonitoredTask status) throws IOException {
  ArrayList<Path> result = new ArrayList<Path>();
  if (snapshot.size() == 0) return result; // don't flush if there are no entries

  // Use a store scanner to find which rows to flush.
  long smallestReadPoint = store.getSmallestReadPoint();
  KeyValueScanner memstoreScanner =
      new CollectionBackedScanner(snapshot, store.getComparator());
  InternalScanner scanner = preCreateCoprocScanner(memstoreScanner);
  if (scanner == null) {
    scanner = createStoreScanner(smallestReadPoint, memstoreScanner);
  }
  scanner = postCreateCoprocScanner(scanner);
  if (scanner == null) {
    return result; // NULL scanner returned from coprocessor hooks means skip normal processing
  }

  StoreFile.Writer writer;
  long flushed = 0;
  try {
    // TODO:  We can fail in the below block before we complete adding this flush to
    //        list of store files.  Add cleanup of anything put on filesystem if we fail.
    synchronized (flushLock) {
      status.setStatus("Flushing " + store + ": creating writer");
      // Write the map out to the disk
      writer = store.createWriterInTmp(
          snapshot.size(), store.getFamily().getCompression(), false, true);
      writer.setTimeRangeTracker(snapshotTimeRangeTracker);
      try {
        flushed = performFlush(scanner, writer, smallestReadPoint);
      } finally {
        finalizeWriter(writer, cacheFlushId, status);
      }
    }
  } finally {
    flushedSize.set(flushed);
    scanner.close();
  }
  LOG.info("Flushed, sequenceid=" + cacheFlushId +", memsize="
      + StringUtils.humanReadableInt(flushed) +
      ", hasBloomFilter=" + writer.hasGeneralBloom() +
      ", into tmp file " + writer.getPath());
  result.add(writer.getPath());
  return result;
}