diff options
Diffstat (limited to 'Lib/urllib2.py')
| -rw-r--r-- | Lib/urllib2.py | 53 |
1 files changed, 37 insertions, 16 deletions
diff --git a/Lib/urllib2.py b/Lib/urllib2.py index 2c1c0c5afc..3be7c8b228 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -109,7 +109,7 @@ except ImportError: from StringIO import StringIO from urllib import (unwrap, unquote, splittype, splithost, quote, - addinfourl, splitport, + addinfourl, splitport, splittag, splitattr, ftpwrapper, splituser, splitpasswd, splitvalue) # support for FileHandler, proxies via environment variables @@ -190,6 +190,7 @@ class Request: origin_req_host=None, unverifiable=False): # unwrap('<URL:type://host/path>') --> 'type://host/path' self.__original = unwrap(url) + self.__original, fragment = splittag(self.__original) self.type = None # self.__r_type is what's left after doing the splittype self.host = None @@ -298,8 +299,9 @@ class OpenerDirector: def __init__(self): client_version = "Python-urllib/%s" % __version__ self.addheaders = [('User-agent', client_version)] - # manage the individual handlers + # self.handlers is retained only for backward compatibility self.handlers = [] + # manage the individual handlers self.handle_open = {} self.handle_error = {} self.process_response = {} @@ -349,8 +351,6 @@ class OpenerDirector: added = True if added: - # the handlers must work in an specific order, the order - # is specified in a Handler attribute bisect.insort(self.handlers, handler) handler.add_parent(self) @@ -449,7 +449,7 @@ def build_opener(*handlers): """ import types def isclass(obj): - return isinstance(obj, types.ClassType) or hasattr(obj, "__bases__") + return isinstance(obj, (types.ClassType, type)) opener = OpenerDirector() default_classes = [ProxyHandler, UnknownHandler, HTTPHandler, @@ -832,6 +832,9 @@ class AbstractBasicAuthHandler: self.add_password = self.passwd.add_password self.retried = 0 + def reset_retry_count(self): + self.retried = 0 + def http_error_auth_reqed(self, authreq, host, req, headers): # host may be an authority (without userinfo) or a URL with an # authority @@ -850,7 +853,10 @@ class AbstractBasicAuthHandler: if mo: scheme, quote, realm = mo.groups() if scheme.lower() == 'basic': - return self.retry_http_basic_auth(host, req, realm) + response = self.retry_http_basic_auth(host, req, realm) + if response and response.code != 401: + self.retried = 0 + return response def retry_http_basic_auth(self, host, req, realm): user, pw = self.passwd.find_user_password(realm, host) @@ -871,8 +877,10 @@ class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler): def http_error_401(self, req, fp, code, msg, headers): url = req.get_full_url() - return self.http_error_auth_reqed('www-authenticate', - url, req, headers) + response = self.http_error_auth_reqed('www-authenticate', + url, req, headers) + self.reset_retry_count() + return response class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler): @@ -885,8 +893,10 @@ class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler): # should not, RFC 3986 s. 3.2.1) support requests for URLs containing # userinfo. authority = req.get_host() - return self.http_error_auth_reqed('proxy-authenticate', + response = self.http_error_auth_reqed('proxy-authenticate', authority, req, headers) + self.reset_retry_count() + return response def randombytes(n): @@ -1127,8 +1137,10 @@ class AbstractHTTPHandler(BaseHandler): h = http_class(host, timeout=req.timeout) # will parse host:port h.set_debuglevel(self._debuglevel) - headers = dict(req.headers) - headers.update(req.unredirected_hdrs) + headers = dict(req.unredirected_hdrs) + headers.update(dict((k, v) for k, v in req.headers.items() + if k not in headers)) + # We want to make an HTTP/1.1 request, but the addinfourl # class isn't prepared to deal with a persistent connection. # It will try to read all remaining data from the socket, @@ -1147,11 +1159,14 @@ class AbstractHTTPHandler(BaseHandler): # Proxy-Authorization should not be sent to origin # server. del headers[proxy_auth_hdr] - h._set_tunnel(req._tunnel_host, headers=tunnel_headers) + h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: h.request(req.get_method(), req.get_selector(), req.data, headers) - r = h.getresponse() + try: + r = h.getresponse(buffering=True) + except TypeError: #buffering kw not supported + r = h.getresponse() except socket.error, err: # XXX what error? raise URLError(err) @@ -1266,6 +1281,12 @@ def parse_http_list(s): return [part.strip() for part in res] +def _safe_gethostbyname(host): + try: + return socket.gethostbyname(host) + except socket.gaierror: + return None + class FileHandler(BaseHandler): # Use local file or FTP depending on form of URL def file_open(self, req): @@ -1307,7 +1328,7 @@ class FileHandler(BaseHandler): if host: host, port = splitport(host) if not host or \ - (not port and socket.gethostbyname(host) in self.get_names()): + (not port and _safe_gethostbyname(host) in self.get_names()): if host: origurl = 'file://' + host + filename else: @@ -1338,8 +1359,8 @@ class FTPHandler(BaseHandler): else: passwd = None host = unquote(host) - user = unquote(user or '') - passwd = unquote(passwd or '') + user = user or '' + passwd = passwd or '' try: host = socket.gethostbyname(host) |