From 6f3378c098a481588f5ec5813060376351e5a576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Fri, 11 Sep 2009 23:49:34 +0200 Subject: Shortcut User-agent computation, as 2to3 won't update urllib2.__version__ correctly. --HG-- branch : distribute extra : rebase_source : 9035aa6fb13225181f7ce22429efa91c771247a6 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 220cdec7..48494aff 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -143,7 +143,7 @@ def find_external_links(url, page): yield urlparse.urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s distribute/%s" % ( - urllib2.__version__, require('distribute')[0].version + sys.version[:3], require('distribute')[0].version ) -- cgit v1.2.1 From bd900046b3457bfd98ac36dfbdaac63d7acf0a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 13 Sep 2009 15:33:43 +0200 Subject: Convert HTML pages to text. --HG-- branch : distribute extra : rebase_source : 9ab9cc8b84da2be50c520cd6f23efb15b8744bd9 --- setuptools/package_index.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 48494aff..b6283dbc 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -197,6 +197,8 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() + charset = f.headers.get_param('charset') or 'latin-1' + page = page.decode(charset, "ignore") f.close() if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) -- cgit v1.2.1 From 55413ad562eddc5ffc235bde5471fdf010421f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 13 Sep 2009 21:30:44 +0200 Subject: Only decode HTML pages in 3.x. --HG-- branch : distribute extra : rebase_source : 58a286fe297947272398fc9ddd8f50594f54d4ae --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b6283dbc..084370d5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -197,8 +197,9 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - charset = f.headers.get_param('charset') or 'latin-1' - page = page.decode(charset, "ignore") + if sys.version_info >= (3,): + charset = f.headers.get_param('charset') or 'latin-1' + page = page.decode(charset, "ignore") f.close() if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) -- cgit v1.2.1