Python mechanize 模块，urlopen() 实例源码
我们从Python开源项目中，提取了以下16个代码示例，用于说明如何使用mechanize.urlopen()。
def test_404(self):
expected_response = 'Bad bad bad...'
handler = self._make_request_handler([(404, [], expected_response)])
try:
mechanize.urlopen('http://localhost:%s/weeble' % handler.port)
except mechanize.URLError as f:
pass
else:
self.fail('404 should raise URLError')
data = f.read()
f.close()
self.assertEquals(data, expected_response)
self.assertEquals(handler.requests, ['/weeble'])
def test_trivial(self):
# A couple trivial tests
self.assertRaises(ValueError, mechanize.urlopen, 'bogus url')
fname = os.path.join(self.make_temp_dir(), "test.txt")
write_file(fname, "data")
if fname[1:2] == ":":
fname = fname[2:]
# And more hacking to get it to work on MacOS. This assumes
# urllib.pathname2url works, unfortunately...
if os.name == 'mac':
fname = '/' + fname.replace(':', '/')
elif os.name == 'riscos':
import string
fname = os.expand(fname)
fname = fname.translate(string.maketrans("/.", "./"))
file_url = "file://%s" % sanepathname2url(fname)
f = mechanize.urlopen(file_url)
buf = f.read()
f.close()
def extract_cookies(self, response, request):
"""Extract cookies from response, where allowable given the request.
Look for allowable Set-Cookie: and Set-Cookie2: headers in the response
object passed as argument. Any of these headers that are found are
used to update the state of the object (subject to the policy.set_ok
method's approval).
The response object (usually be the result of a call to
mechanize.urlopen, or similar) should support an info method, which
returns a mimetools.Message object (in fact, the 'mimetools.Message
object' may be any object that provides a getheaders method).
The request object (usually a mechanize.Request instance) must support
the methods get_full_url, get_type, get_host, and is_unverifiable, as
documented by mechanize, and the port attribute (the port number). The
request is used to set default values for cookie-attributes as well as
for checking that the cookie is OK to be set.
"""
debug("extract_cookies: %s", response.info())
self._policy._now = self._now = int(time.time())
for cookie in self._make_cookies(response, request):
if cookie.expires is not None and cookie.expires <= self._now:
# Expiry date in past is request to delete cookie. This can't be
# in DefaultCookiePolicy, because can't delete cookies there.
try:
self.clear(cookie.domain, cookie.path, cookie.name)
except KeyError:
pass
debug("Expiring cookie, domain='%s', path='%s', name='%s'",
cookie.domain, cookie.path, cookie.name)
elif self._policy.set_ok(cookie, request):
debug(" setting cookie: %s", cookie)
self.set_cookie(cookie)
def extract_cookies(self, response, request):
"""Extract cookies from response, where allowable given the request.
Look for allowable Set-Cookie: and Set-Cookie2: headers in the response
object passed as argument. Any of these headers that are found are
used to update the state of the object (subject to the policy.set_ok
method's approval).
The response object (usually be the result of a call to
mechanize.urlopen, or similar) should support an info method, which
returns a mimetools.Message object (in fact, the 'mimetools.Message
object' may be any object that provides a getheaders method).
The request object (usually a mechanize.Request instance) must support
the methods get_full_url, get_type, get_host, and is_unverifiable, as
documented by mechanize, and the port attribute (the port number). The
request is used to set default values for cookie-attributes as well as
for checking that the cookie is OK to be set.
"""
debug("extract_cookies: %s", response.info())
self._policy._now = self._now = int(time.time())
for cookie in self._make_cookies(response, request):
if cookie.expires is not None and cookie.expires <= self._now:
# Expiry date in past is request to delete cookie. This can't be
# in DefaultCookiePolicy, because can't delete cookies there.
try:
self.clear(cookie.domain, cookie.path, cookie.name)
except KeyError:
pass
debug("Expiring cookie, domain='%s', path='%s', name='%s'",
cookie.domain, cookie.path, cookie.name)
elif self._policy.set_ok(cookie, request):
debug(" setting cookie: %s", cookie)
self.set_cookie(cookie)
def extract_cookies(self, response, request):
"""Extract cookies from response, where allowable given the request.
Look for allowable Set-Cookie: and Set-Cookie2: headers in the response
object passed as argument. Any of these headers that are found are
used to update the state of the object (subject to the policy.set_ok
method's approval).
The response object (usually be the result of a call to
mechanize.urlopen, or similar) should support an info method, which
returns a mimetools.Message object (in fact, the 'mimetools.Message
object' may be any object that provides a getheaders method).
The request object (usually a mechanize.Request instance) must support
the methods get_full_url, get_type, get_host, and is_unverifiable, as
documented by mechanize, and the port attribute (the port number). The
request is used to set default values for cookie-attributes as well as
for checking that the cookie is OK to be set.
"""
debug("extract_cookies: %s", response.info())
self._policy._now = self._now = int(time.time())
for cookie in self._make_cookies(response, request):
if cookie.expires is not None and cookie.expires <= self._now:
# Expiry date in past is request to delete cookie. This can't be
# in DefaultCookiePolicy, because can't delete cookies there.
try:
self.clear(cookie.domain, cookie.path, cookie.name)
except KeyError:
pass
debug("Expiring cookie, domain='%s', path='%s', name='%s'",
cookie.domain, cookie.path, cookie.name)
elif self._policy.set_ok(cookie, request):
debug(" setting cookie: %s", cookie)
self.set_cookie(cookie)
def test_urlopen_with_default_timeout(self):
timeout_log = self._monkey_patch_socket()
response = mechanize.urlopen(self.test_uri)
self.assert_contains(response.read(), "Python bits")
timeout_log.verify_default()
def test_urlopen_with_timeout(self):
timeout_log = self._monkey_patch_socket()
timeout = 10.
response = mechanize.urlopen(self.test_uri, timeout=timeout)
self.assert_contains(response.read(), "Python bits")
timeout_log.verify(timeout)
def _check_retrieve(self, url, filename, headers):
from urllib import urlopen
self.assertEqual(headers.get('Content-Type'), 'text/html')
if self.no_proxies:
proxies = {}
else:
proxies = None
self.assertEqual(read_file(filename),
urlopen(url, proxies=proxies).read())
def test_redirection(self):
expected_response = 'We got here...'
responses = [
(302, [('Location', 'http://localhost:%s/somewhere_else')], ''),
(200, [], expected_response)
]
handler = self._make_request_handler(responses)
f = mechanize.urlopen('http://localhost:%s/' % handler.port)
data = f.read()
f.close()
self.assertEquals(data, expected_response)
self.assertEquals(handler.requests, ['/', '/somewhere_else'])
def test_200_with_parameters(self):
expected_response = 'pycon 2008...'
handler = self._make_request_handler([(200, [], expected_response)])
f = mechanize.urlopen('http://localhost:%s/bizarre' % handler.port,
'get=with_feeling')
data = f.read()
f.close()
self.assertEquals(data, expected_response)
self.assertEquals(handler.requests, ['/bizarre', 'get=with_feeling'])
def test_sending_headers(self):
handler = self._make_request_handler([(200, [], "we don't care")])
req = mechanize.Request("http://localhost:%s/" % handler.port,
headers={'Range': 'bytes=20-39'})
mechanize.urlopen(req)
self.assertEqual(handler.received_headers['Range'], 'bytes=20-39')
def test_basic(self):
handler = self._make_request_handler([(200, [], "we don't care")])
open_url = mechanize.urlopen("http://localhost:%s" % handler.port)
for attr in ("read", "close", "info", "geturl"):
self.assertTrue(hasattr(open_url, attr), "object returned from "
"urlopen lacks the %s attribute" % attr)
try:
self.assertTrue(open_url.read(), "calling 'read' failed")
finally:
open_url.close()
def test_info(self):
handler = self._make_request_handler([(200, [], "we don't care")])
open_url = mechanize.urlopen("http://localhost:%s" % handler.port)
info_obj = open_url.info()
self.assertTrue(isinstance(info_obj, mimetools.Message),
"object returned by 'info' is not an instance of "
"mimetools.Message")
self.assertEqual(info_obj.getsubtype(), "plain")
def test_geturl(self):
# Make sure same URL as opened is returned by geturl.
handler = self._make_request_handler([(200, [], "we don't care")])
open_url = mechanize.urlopen("http://localhost:%s" % handler.port)
url = open_url.geturl()
self.assertEqual(url, "http://localhost:%s" % handler.port)
def extract_cookies(self, response, request):
"""Extract cookies from response, where allowable given the request.
Look for allowable Set-Cookie: and Set-Cookie2: headers in the response
object passed as argument. Any of these headers that are found are
used to update the state of the object (subject to the policy.set_ok
method's approval).
The response object (usually be the result of a call to
mechanize.urlopen, or similar) should support an info method, which
returns a mimetools.Message object (in fact, the 'mimetools.Message
object' may be any object that provides a getheaders method).
The request object (usually a mechanize.Request instance) must support
the methods get_full_url, get_type, get_host, and is_unverifiable, as
documented by mechanize, and the port attribute (the port number). The
request is used to set default values for cookie-attributes as well as
for checking that the cookie is OK to be set.
"""
debug("extract_cookies: %s", response.info())
self._policy._now = self._now = int(time.time())
for cookie in self._make_cookies(response, request):
if cookie.expires is not None and cookie.expires <= self._now:
# Expiry date in past is request to delete cookie. This can't be
# in DefaultCookiePolicy, because can't delete cookies there.
try:
self.clear(cookie.domain, cookie.path, cookie.name)
except KeyError:
pass
debug("Expiring cookie, domain='%s', path='%s', name='%s'",
cookie.domain, cookie.path, cookie.name)
elif self._policy.set_ok(cookie, request):
debug(" setting cookie: %s", cookie)
self.set_cookie(cookie)
def extract_cookies(self, response, request):
"""Extract cookies from response, where allowable given the request.
Look for allowable Set-Cookie: and Set-Cookie2: headers in the response
object passed as argument. Any of these headers that are found are
used to update the state of the object (subject to the policy.set_ok
method's approval).
The response object (usually be the result of a call to
mechanize.urlopen, or similar) should support an info method, which
returns a mimetools.Message object (in fact, the 'mimetools.Message
object' may be any object that provides a getheaders method).
The request object (usually a mechanize.Request instance) must support
the methods get_full_url, get_type, get_host, and is_unverifiable, as
documented by mechanize, and the port attribute (the port number). The
request is used to set default values for cookie-attributes as well as
for checking that the cookie is OK to be set.
"""
debug("extract_cookies: %s", response.info())
self._policy._now = self._now = int(time.time())
for cookie in self._make_cookies(response, request):
if cookie.expires is not None and cookie.expires <= self._now:
# Expiry date in past is request to delete cookie. This can't be
# in DefaultCookiePolicy, because can't delete cookies there.
try:
self.clear(cookie.domain, cookie.path, cookie.name)
except KeyError:
pass
debug("Expiring cookie, domain='%s', path='%s', name='%s'",
cookie.domain, cookie.path, cookie.name)
elif self._policy.set_ok(cookie, request):
debug(" setting cookie: %s", cookie)
self.set_cookie(cookie)