diff options
| -rwxr-xr-x | EasyInstall.txt | 73 | ||||
| -rwxr-xr-x | setuptools.txt | 3 | ||||
| -rwxr-xr-x | setuptools/command/easy_install.py | 6 | ||||
| -rwxr-xr-x | setuptools/package_index.py | 98 |
4 files changed, 101 insertions, 79 deletions
diff --git a/EasyInstall.txt b/EasyInstall.txt index 9886e8fb..fa9dc10a 100755 --- a/EasyInstall.txt +++ b/EasyInstall.txt @@ -545,14 +545,14 @@ Command-Line Options file(s)) you must also use ``require()`` to enable packages at runtime. ``--upgrade, -U`` (New in 0.5a4) - By default, EasyInstall only searches the Python Package Index if a - project/version requirement can't be met by distributions already installed + By default, EasyInstall only searches online if a project/version + requirement can't be met by distributions already installed on sys.path or the installation directory. However, if you supply the ``--upgrade`` or ``-U`` flag, EasyInstall will always check the package - index before selecting a version to install. In this way, you can force - EasyInstall to use the latest available version of any package it installs - (subject to any version requirements that might exclude such later - versions). + index and ``--find-links`` URLs before selecting a version to install. In + this way, you can force EasyInstall to use the latest available version of + any package it installs (subject to any version requirements that might + exclude such later versions). ``--install-dir=DIR, -d DIR`` Set the installation directory. It is up to you to ensure that this @@ -597,29 +597,41 @@ Command-Line Options messages for any eggs that EasyInstall skips, before it falls back to an older version or attempts to download a fresh copy. -``--find-links=URL, -f URL`` (Option renamed in 0.4a2) - Scan the specified "download pages" for direct links to downloadable eggs - or source distributions. Any usable packages will be downloaded if they - are required by a command line argument. For example, this:: +``--find-links=URLS_OR_FILENAMES, -f URLS_OR_FILENAMES`` + Scan the specified "download pages" or directories for direct links to eggs + or other distributions. Any existing file or directory names or direct + download URLs are immediately added to EasyInstall's search cache, and any + indirect URLs (ones that don't point to eggs or other recognized archive + formats) are added to a list of additional places to search for download + links. As soon as EasyInstall has to go online to find a package (either + because it doesn't exist locally, or because ``--upgrade`` or ``-U`` was + used), the specified URLs will be downloaded and scanned for additional + direct links. + + Eggs and archives found by way of ``--find-links`` are only downloaded if + they are needed to meet a requirement specified on the command line; links + to unneeded packages are ignored. - easy_install -f http://peak.telecommunity.com/dist PyProtocols - - will download and install the latest version of PyProtocols linked from - the PEAK downloads page, but ignore the other download links on that page. If all requested packages can be found using links on the specified - download pages, the Python Package Index will *not* be consulted. You can - use a ``file:`` URL to reference a local HTML file containing links, or you - can just use the name of a directory containing "distribution files" - (source archives, eggs, Windows installers, etc.), and EasyInstall will - then be aware of the files available there. - - You may specify multiple URLs or directories with this option, separated by - whitespace. Note that on the command line, you will probably have to - surround the URL list with quotes, so that it is recognized as a single - option value. You can also specify URLs in a configuration file; see - `Configuration Files`_, above; but note that this means the specified pages - will be downloaded every time you use EasyInstall (unless overridden on the - command line) and thus may make startup slower. + download pages, the Python Package Index will not be consulted unless you + also specified the ``--upgrade`` or ``-U`` option. + + (Note: if you want to refer to a local HTML file containing links, you must + use a ``file:`` URL, as filenames that do not refer to a directory, egg, or + archive are ignored.) + + You may specify multiple URLs or file/directory names with this option, + separated by whitespace. Note that on the command line, you will probably + have to surround the URL list with quotes, so that it is recognized as a + single option value. You can also specify URLs in a configuration file; + see `Configuration Files`_, above. + + Changed in 0.6a10: previously all URLs and directories passed to this + option were scanned as early as possible, but from 0.6a10 on, only + directories and direct archive links are scanned immediately; URLs are not + retrieved unless a package search was already going to go online due to a + package not being available locally, or due to the use of the ``--update`` + or ``-U`` option. ``--delete-conflicting, -D`` (New in 0.5a9) If you are replacing a package that was previously installed *without* @@ -995,6 +1007,13 @@ Known Issues choose an older version of a package than what you expected, or it may cause downloading and installation of a fresh version of what's already installed. + * The ``--find-links`` option previously scanned all supplied URLs and + directories as early as possible, but now only directories and direct + archive links are scanned immediately. URLs are not retrieved unless a + package search was already going to go online due to a package not being + available locally, or due to the use of the ``--update`` or ``-U`` option. + + 0.6a9 * Fixed ``.pth`` file processing picking up nested eggs (i.e. ones inside "baskets") when they weren't explicitly listed in the ``.pth`` file. diff --git a/setuptools.txt b/setuptools.txt index a6485abb..b1bc873f 100755 --- a/setuptools.txt +++ b/setuptools.txt @@ -2342,6 +2342,9 @@ XXX Release Notes/Change History ---------------------------- +0.6a10 + * Fixed the ``develop`` command ignoring ``--find-links``. + 0.6a9 * The ``sdist`` command no longer uses the traditional ``MANIFEST`` file to create source distributions. ``MANIFEST.in`` is still read and processed, diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 31975f63..abf2ea14 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -194,7 +194,7 @@ class easy_install(Command): self.find_links = self.find_links.split() else: self.find_links = [] - + self.package_index.add_find_links(self.find_links) self.set_undefined_options('install_lib', ('optimize','optimize')) if not isinstance(self.optimize,int): try: @@ -224,8 +224,6 @@ class easy_install(Command): if self.verbose<>self.distribution.verbose: log.set_verbosity(self.verbose) try: - for link in self.find_links: - self.package_index.scan_url(link) for spec in self.args: self.easy_install(spec, not self.no_deps) if self.record: @@ -244,6 +242,8 @@ class easy_install(Command): log.set_verbosity(self.distribution.verbose) + + def install_egg_scripts(self, dist): """Write all the scripts for `dist`, unless scripts are excluded""" diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 669692b6..c02f3b4e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -131,6 +131,7 @@ class PackageIndex(Environment): self.fetched_urls = {} self.package_pages = {} self.allows = re.compile('|'.join(map(translate,hosts))).match + self.to_scan = [] def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -139,18 +140,8 @@ class PackageIndex(Environment): return self.scanned_urls[url] = True if not URL_SCHEME(url): - # process filenames or directories - if os.path.isfile(url): - map(self.add, distros_for_filename(url)) - return # no need to retrieve anything - elif os.path.isdir(url): - url = os.path.realpath(url) - for item in os.listdir(url): - self.process_url(os.path.join(url,item)) - return - else: - self.warn("Not found: %s", url) - return + self.process_filename(url) + return else: dists = list(distros_for_url(url)) if dists: @@ -170,6 +161,7 @@ class PackageIndex(Environment): f = self.open_url(url) self.fetched_urls[url] = self.fetched_urls[f.url] = True + if 'html' not in f.headers['content-type'].lower(): f.close() # not html, we can't process it return @@ -184,6 +176,21 @@ class PackageIndex(Environment): link = urlparse.urljoin(base, match.group(1)) self.process_url(link) + def process_filename(self, fn, nested=False): + # process filenames or directories + if not os.path.exists(fn): + self.warn("Not found: %s", url) + return + + if os.path.isdir(fn): + path = os.path.realpath(fn) + for item in os.listdir(path): + self.process_filename(os.path.join(path,item), True) + + dists = distros_for_filename(fn) + if dists: + self.debug("Found: %s", fn) + map(self.add, dists) def url_ok(self, url, fatal=False): if self.allows(urlparse.urlparse(url)[1]): @@ -196,13 +203,6 @@ class PackageIndex(Environment): - - - - - - - def process_index(self,url,page): """Process the contents of a PyPI page""" def scan(link): @@ -260,9 +260,11 @@ class PackageIndex(Environment): def find_packages(self, requirement): self.scan_url(self.index_url + requirement.unsafe_name+'/') + if not self.package_pages.get(requirement.key): # Fall back to safe version of the name self.scan_url(self.index_url + requirement.project_name+'/') + if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too self.warn( @@ -276,15 +278,13 @@ class PackageIndex(Environment): self.scan_url(url) def obtain(self, requirement, installer=None): - self.find_packages(requirement) + self.prescan(); self.find_packages(requirement) for dist in self[requirement.key]: if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) return super(PackageIndex, self).obtain(requirement,installer) - - def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) @@ -296,26 +296,26 @@ class PackageIndex(Environment): "; possible download problem?" ) + def add_find_links(self, urls): + """Add `urls` to the list that will be prescanned for searches""" + for url in urls: + if ( + self.to_scan is None # if we have already "gone online" + or not URL_SCHEME(url) # or it's a local file/directory + or url.startswith('file:') + or list(distros_for_url(url)) # or a direct package link + ): + # then go ahead and process it now + self.scan_url(url) + else: + # otherwise, defer retrieval till later + self.to_scan.append(url) - - - - - - - - - - - - - - - - - - - + def prescan(self): + """Scan urls scheduled for prescanning (e.g. --find-links)""" + if self.to_scan: + map(self.scan_url, self.to_scan) + self.to_scan = None # from now on, go ahead and process immediately @@ -409,13 +409,17 @@ class PackageIndex(Environment): ) if force_scan: + self.prescan() self.find_packages(requirement) + + dist = find(requirement) + if dist is None and self.to_scan is not None: + self.prescan() dist = find(requirement) - else: + + if dist is None and not force_scan: + self.find_packages(requirement) dist = find(requirement) - if dist is None: - self.find_packages(requirement) - dist = find(requirement) if dist is None: self.warn( @@ -445,10 +449,6 @@ class PackageIndex(Environment): - - - - def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment); #import pdb; pdb.set_trace() dists = match and [d for d in |