Python mmap 模块，ALLOCATIONGRANULARITY 实例源码

def compute_etag_from_file_obj(file_obj, offset=0, size=None, chunk_size=1024 * 1024):
    etag = hashlib.sha256()

    size = size or os.fstat(file_obj.fileno()).st_size - offset

    if size != 0 and offset % mmap.ALLOCATIONGRANULARITY == 0:
        target = mmap.mmap(file_obj.fileno(), length=size,
                           offset=offset,
                           access=mmap.ACCESS_READ)
    else:
        target = file_obj
        target.seek(offset)

    while size > 0:
        data = target.read(chunk_size)
        etag.update(data[:min(len(data), size)])
        size -= len(data)

    if target is file_obj:
        file_obj.seek(offset)
    else:
        target.close()
    s = etag.hexdigest()
    return s

def compute_tree_etag_from_file_obj(file_obj, offset=0, size=None,
                                    chunk_size=1024 * 1024):
    generator = TreeHashGenerator()

    size = size or os.fstat(file_obj.fileno()).st_size - offset
    if size != 0 and offset % mmap.ALLOCATIONGRANULARITY == 0:
        target = mmap.mmap(file_obj.fileno(), length=size,
                           offset=offset,
                           access=mmap.ACCESS_READ)
    else:
        target = file_obj
        target.seek(offset)

    while size > 0:
        data = target.read(chunk_size)
        generator.update(data[:min(len(data), size)])
        size -= len(data)

    if target is file_obj:
        file_obj.seek(offset)
    else:
        target.close()
    return generator.generate().digest()

def compute_hash_from_file_obj(file_obj, offset=0, size=None, chunk_size=1024 * 1024):
    etag = hashlib.sha256()
    generator = TreeHashGenerator()

    size = size or os.fstat(file_obj.fileno()).st_size - offset

    if size != 0 and offset % mmap.ALLOCATIONGRANULARITY == 0:
        target = mmap.mmap(file_obj.fileno(), length=size,
                           offset=offset,
                           access=mmap.ACCESS_READ)
    else:
        target = file_obj
        target.seek(offset)

    while size > 0:
        data = target.read(chunk_size)
        generator.update(data[:min(len(data), size)])
        etag.update(data[:min(len(data), size)])
        size -= len(data)

    if target is file_obj:
        file_obj.seek(offset)
    else:
        target.close()
    return etag.hexdigest(), generator.generate().digest()

def __init__(self, path, page_count, read_only=False):
        # XXX TODO NOTE remove this line when write functionality is added.
        read_only = True

        # getting 'too many files open' error? increase the constant on the next line
        # (must be an exponent of 2)
        self._page_size = page_count * mmap.PAGESIZE

        # make sure we're sane here - allocation granularity needs to divide into page size!
        assert (self._page_size % mmap.ALLOCATIONGRANULARITY) == 0, 'page size is not a multiple of allocation granularity!'

        self._file = open(path, 'r+b')
        self._pages = dict()

        self.read_only = read_only
        self._path = path

        self.cursor = 0
        super(MappedFile, self).__init__(self, base_offset=0, size=len(self))

def map_file(cls, fileobj, offset = 0, size = None):
        fileobj.seek(offset)
        total_size = cls._Header.unpack(fileobj.read(cls._Header.size))[0]
        map_start = offset - offset % mmap.ALLOCATIONGRANULARITY
        buf = mmap.mmap(fileobj.fileno(), total_size + offset - map_start, 
            access = mmap.ACCESS_READ, offset = map_start)
        rv = cls(buffer(buf, offset - map_start))
        rv._file = fileobj
        rv._mmap = buf
        return rv

def map_file(cls, fileobj, offset = 0, size = None):
        map_start = offset - offset % mmap.ALLOCATIONGRANULARITY
        fileobj.seek(map_start)
        buf = mmap.mmap(fileobj.fileno(), 0, access = mmap.ACCESS_READ, offset = map_start)
        rv = cls(buf, offset - map_start)
        rv._file = fileobj
        return rv

def map_file(cls, fileobj, offset = 0, size = None):
        map_start = offset - offset % mmap.ALLOCATIONGRANULARITY
        fileobj.seek(map_start)
        buf = mmap.mmap(fileobj.fileno(), 0, access = mmap.ACCESS_READ, offset = map_start)
        rv = cls(buf, offset - map_start)
        rv._file = fileobj
        return rv