Python os 模块，sendfile() 实例源码
我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用os.sendfile()。
def test_offset_overflow(self):
# specify an offset > file size
offset = len(self.DATA) + 4096
try:
sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
except OSError as e:
# Solaris can raise EINVAL if offset >= file length, ignore.
if e.errno != errno.EINVAL:
raise
else:
self.assertEqual(sent, 0)
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b'')
def sendfile(self, file, offset=0, count=None):
"""sendfile(file[, offset[, count]]) -> sent
Send a file until EOF is reached by using high-performance
os.sendfile() and return the total number of bytes which
were sent.
*file* must be a regular file object opened in binary mode.
If os.sendfile() is not available (e.g. Windows) or file is
not a regular file socket.send() will be used instead.
*offset* tells from where to start reading the file.
If specified, *count* is the total number of bytes to transmit
as opposed to sending the file until EOF is reached.
File position is updated on return or also in case of error in
which case file.tell() can be used to figure out the number of
bytes which were sent.
The socket must be of SOCK_STREAM type.
Non-blocking sockets are not supported.
.. versionadded:: 1.1rc4
Added in Python 3.5, but available under all Python 3 versions in
gevent.
"""
return self._sendfile_use_send(file, offset, count)
# get/set_inheritable new in 3.4
def sendfile(self, file, offset=0, count=None):
"""sendfile(file[, offset[, count]]) -> sent
Send a file until EOF is reached by using high-performance
os.sendfile() and return the total number of bytes which
were sent.
*file* must be a regular file object opened in binary mode.
If os.sendfile() is not available (e.g. Windows) or file is
not a regular file socket.send() will be used instead.
*offset* tells from where to start reading the file.
If specified, *count* is the total number of bytes to transmit
as opposed to sending the file until EOF is reached.
File position is updated on return or also in case of error in
which case file.tell() can be used to figure out the number of
bytes which were sent.
The socket must be of SOCK_STREAM type.
Non-blocking sockets are not supported.
.. versionadded:: 1.1rc4
Added in Python 3.5, but available under all Python 3 versions in
gevent.
"""
return self._sendfile_use_send(file, offset, count)
# get/set_inheritable new in 3.4
def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
while 1:
try:
if self.SUPPORT_HEADERS_TRAILERS:
return os.sendfile(sock, file, offset, nbytes, headers,
trailers)
else:
return os.sendfile(sock, file, offset, nbytes)
except OSError as err:
if err.errno == errno.ECONNRESET:
# disconnected
raise
elif err.errno in (errno.EAGAIN, errno.EBUSY):
# we have to retry send data
continue
else:
raise
def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
while 1:
try:
if self.SUPPORT_HEADERS_TRAILERS:
return os.sendfile(sock, file, offset, nbytes, headers,
trailers)
else:
return os.sendfile(sock, file, offset, nbytes)
except OSError as err:
if err.errno == errno.ECONNRESET:
# disconnected
raise
elif err.errno in (errno.EAGAIN, errno.EBUSY):
# we have to retry send data
continue
else:
raise
def test_offset_overflow(self):
# specify an offset > file size
offset = len(self.DATA) + 4096
try:
sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
except OSError as e:
# Solaris can raise EINVAL if offset >= file length, ignore.
if e.errno != errno.EINVAL:
raise
else:
self.assertEqual(sent, 0)
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b'')
def test_headers(self):
total_sent = 0
sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
headers=[b"x" * 512])
total_sent += sent
offset = 4096
nbytes = 4096
while 1:
sent = self.sendfile_wrapper(self.sockno, self.fileno,
offset, nbytes)
if sent == 0:
break
total_sent += sent
offset += sent
expected_data = b"x" * 512 + self.DATA
self.assertEqual(total_sent, len(expected_data))
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(hash(data), hash(expected_data))
def _import_sendfile():
# By default attempt to use os.sendfile introduced in Python 3.3:
# http://bugs.python.org/issue10882
# ...otherwise fallback on using third-party pysendfile module:
# https://github.com/giampaolo/pysendfile/
if os.name == 'posix':
try:
return os.sendfile # py >= 3.3
except AttributeError:
try:
import sendfile as sf
# dirty hack to detect whether old 1.2.4 version is installed
if hasattr(sf, 'has_sf_hdtr'):
raise ImportError
return sf.sendfile
except ImportError:
pass
def get_repr_info(self, as_str=False, extra_info={}):
info = OrderedDict()
info['id'] = id(self)
info['addr'] = "%s:%s" % (self.remote_ip, self.remote_port)
if _is_ssl_sock(self.socket):
info['ssl'] = True
if self.username:
info['user'] = self.username
# If threads are involved sometimes "self" may be None (?!?).
dc = getattr(self, 'data_channel', None)
if dc is not None:
if _is_ssl_sock(dc.socket):
info['ssl-data'] = True
if dc.file_obj:
if self.data_channel.receive:
info['sending-file'] = dc.file_obj
if dc.use_sendfile():
info['use-sendfile(2)'] = True
else:
info['receiving-file'] = dc.file_obj
info['bytes-trans'] = dc.get_transmitted_bytes()
info.update(extra_info)
if as_str:
return ', '.join(['%s=%r' % (k, v) for (k, v) in info.items()])
return info
def sendfile(self, path):
with open(path, mode='r') as file:
offset = 0
blocksize = os.path.getsize(path)
logger.debug("RETR DTP Server is waiting for writable.")
while True:
if self.sock_client:
break
await sleep(1)
await self.sock_client.writeable()
logger.debug("RETR DTP Server is writable.")
while True:
try:
sent = sendfile(self.sock_client.fileno(), file.fileno(), offset, blocksize)
except BlockingIOError:
continue
logger.debug("RETR DTP Server send {}".format(offset))
if sent == 0:
await self.sock_client.close()
await self.client.sendall(parse_message(226, 'Transfer complete.'))
await self.close()
break
offset += sent
def _sendfile_cb(self, fut, out_fd, in_fd, offset,
count, loop, registered):
if registered:
loop.remove_writer(out_fd)
if fut.cancelled():
return
try:
n = os.sendfile(out_fd, in_fd, offset, count)
if n == 0: # EOF reached
n = count
except (BlockingIOError, InterruptedError):
n = 0
except Exception as exc:
fut.set_exception(exc)
return
if n < count:
loop.add_writer(out_fd, self._sendfile_cb, fut, out_fd, in_fd,
offset + n, count - n, loop, True)
else:
fut.set_result(None)
def sendfile(self, file, offset=0, count=None):
"""sendfile(file[, offset[, count]]) -> sent
Send a file until EOF is reached by using high-performance
os.sendfile() and return the total number of bytes which
were sent.
*file* must be a regular file object opened in binary mode.
If os.sendfile() is not available (e.g. Windows) or file is
not a regular file socket.send() will be used instead.
*offset* tells from where to start reading the file.
If specified, *count* is the total number of bytes to transmit
as opposed to sending the file until EOF is reached.
File position is updated on return or also in case of error in
which case file.tell() can be used to figure out the number of
bytes which were sent.
The socket must be of SOCK_STREAM type.
Non-blocking sockets are not supported.
.. versionadded:: 1.1rc4
Added in Python 3.5, but available under all Python 3 versions in
gevent.
"""
return self._sendfile_use_send(file, offset, count)
# get/set_inheritable new in 3.4
def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
while 1:
try:
if self.SUPPORT_HEADERS_TRAILERS:
return os.sendfile(sock, file, offset, nbytes, headers,
trailers)
else:
return os.sendfile(sock, file, offset, nbytes)
except OSError as err:
if err.errno == errno.ECONNRESET:
# disconnected
raise
elif err.errno in (errno.EAGAIN, errno.EBUSY):
# we have to retry send data
continue
else:
raise
def test_offset_overflow(self):
# specify an offset > file size
offset = len(self.DATA) + 4096
try:
sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
except OSError as e:
# Solaris can raise EINVAL if offset >= file length, ignore.
if e.errno != errno.EINVAL:
raise
else:
self.assertEqual(sent, 0)
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b'')
def test_headers(self):
total_sent = 0
sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
headers=[b"x" * 512])
total_sent += sent
offset = 4096
nbytes = 4096
while 1:
sent = self.sendfile_wrapper(self.sockno, self.fileno,
offset, nbytes)
if sent == 0:
break
total_sent += sent
offset += sent
expected_data = b"x" * 512 + self.DATA
self.assertEqual(total_sent, len(expected_data))
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(hash(data), hash(expected_data))
def sendfile(self, file, offset=0, count=None):
"""sendfile(file[, offset[, count]]) -> sent
Send a file until EOF is reached by using high-performance
os.sendfile() and return the total number of bytes which
were sent.
*file* must be a regular file object opened in binary mode.
If os.sendfile() is not available (e.g. Windows) or file is
not a regular file socket.send() will be used instead.
*offset* tells from where to start reading the file.
If specified, *count* is the total number of bytes to transmit
as opposed to sending the file until EOF is reached.
File position is updated on return or also in case of error in
which case file.tell() can be used to figure out the number of
bytes which were sent.
The socket must be of SOCK_STREAM type.
Non-blocking sockets are not supported.
.. versionadded:: 1.1rc4
Added in Python 3.5, but available under all Python 3 versions in
gevent.
"""
return self._sendfile_use_send(file, offset, count)
# get/set_inheritable new in 3.4
def sendfile(self, file, offset=0, count=None):
"""Send a file, possibly by using os.sendfile() if this is a
clear-text socket. Return the total number of bytes sent.
"""
if self._sslobj is None:
# os.sendfile() works with plain sockets only
return super().sendfile(file, offset, count)
else:
return self._sendfile_use_send(file, offset, count)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def shutdown(self, how):
if how == 0: # SHUT_RD
self.hub.cancel_wait(self._read_event, cancel_wait_ex)
elif how == 1: # SHUT_WR
self.hub.cancel_wait(self._write_event, cancel_wait_ex)
else:
self.hub.cancel_wait(self._read_event, cancel_wait_ex)
self.hub.cancel_wait(self._write_event, cancel_wait_ex)
self._sock.shutdown(how)
# sendfile: new in 3.5. But there's no real reason to not
# support it everywhere. Note that we can't use os.sendfile()
# because it's not cooperative.
def shutdown(self, how):
if how == 0: # SHUT_RD
self.hub.cancel_wait(self._read_event, cancel_wait_ex)
elif how == 1: # SHUT_WR
self.hub.cancel_wait(self._write_event, cancel_wait_ex)
else:
self.hub.cancel_wait(self._read_event, cancel_wait_ex)
self.hub.cancel_wait(self._write_event, cancel_wait_ex)
self._sock.shutdown(how)
# sendfile: new in 3.5. But there's no real reason to not
# support it everywhere. Note that we can't use os.sendfile()
# because it's not cooperative.
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def test_invalid_offset(self):
with self.assertRaises(OSError) as cm:
os.sendfile(self.sockno, self.fileno, -1, 4096)
self.assertEqual(cm.exception.errno, errno.EINVAL)
# --- headers / trailers tests
def test_headers(self):
total_sent = 0
sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
headers=[b"x" * 512])
total_sent += sent
offset = 4096
nbytes = 4096
while 1:
sent = self.sendfile_wrapper(self.sockno, self.fileno,
offset, nbytes)
if sent == 0:
break
total_sent += sent
offset += sent
expected_data = b"x" * 512 + self.DATA
self.assertEqual(total_sent, len(expected_data))
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(hash(data), hash(expected_data))
def test_trailers(self):
TESTFN2 = support.TESTFN + "2"
file_data = b"abcdef"
with open(TESTFN2, 'wb') as f:
f.write(file_data)
with open(TESTFN2, 'rb')as f:
self.addCleanup(os.remove, TESTFN2)
os.sendfile(self.sockno, f.fileno(), 0, len(file_data),
trailers=[b"1234"])
self.client.close()
self.server.wait()
data = self.server.handler_instance.get_data()
self.assertEqual(data, b"abcdef1234")
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def test_invalid_offset(self):
with self.assertRaises(OSError) as cm:
os.sendfile(self.sockno, self.fileno, -1, 4096)
self.assertEqual(cm.exception.errno, errno.EINVAL)
def test_keywords(self):
# Keyword arguments should be supported
os.sendfile(out=self.sockno, offset=0, count=4096,
**{'in': self.fileno})
if self.SUPPORT_HEADERS_TRAILERS:
os.sendfile(self.sockno, self.fileno, offset=0, count=4096,
headers=(), trailers=(), flags=0)
# --- headers / trailers tests
def test_flags(self):
try:
os.sendfile(self.sockno, self.fileno, 0, 4096,
flags=os.SF_NODISKIO)
except OSError as err:
if err.errno not in (errno.EBUSY, errno.EAGAIN):
raise
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def can_sendfile(self):
return self.cfg.sendfile is not False and sendfile is not None
def write_file(self, respiter):
if not self.sendfile(respiter):
for item in respiter:
self.write(item)
def initiate_sendfile(self):
"""A wrapper around sendfile."""
try:
sent = sendfile(self._fileno, self._filefd, self._offset,
self.ac_out_buffer_size)
except OSError as err:
if err.errno in _ERRNOS_RETRY or err.errno == errno.EBUSY:
return
elif err.errno in _ERRNOS_DISCONNECTED:
self.handle_close()
else:
if self.tot_bytes_sent == 0:
logger.warning(
"sendfile() failed; falling back on using plain send")
raise _GiveUpOnSendfile
else:
raise
else:
if sent == 0:
# this signals the channel that the transfer is completed
self.discard_buffers()
self.handle_close()
else:
self._offset += sent
self.tot_bytes_sent += sent
# --- utility methods
def test_fallback(self):
# Makes sure that if sendfile() fails and no bytes were
# transmitted yet the server falls back on using plain
# send()
data = b'abcde12345' * 100000
self.dummy_sendfile.write(data)
self.dummy_sendfile.seek(0)
self.client.storbinary('stor ' + TESTFN, self.dummy_sendfile)
with mock.patch('pyftpdlib.handlers.sendfile',
side_effect=OSError(errno.EINVAL)) as fun:
try:
self.client.retrbinary(
'retr ' + TESTFN, self.dummy_recvfile.write)
assert fun.called
self.dummy_recvfile.seek(0)
datafile = self.dummy_recvfile.read()
self.assertEqual(len(data), len(datafile))
self.assertEqual(hash(data), hash(datafile))
finally:
# We do not use os.remove() because file could still be
# locked by ftpd thread. If DELE through FTP fails try
# os.remove() as last resort.
if os.path.exists(TESTFN):
try:
self.client.delete(TESTFN)
except (ftplib.Error, EOFError, socket.error):
safe_remove(TESTFN)