diff options
| -rw-r--r-- | distutils2/install.py | 136 | ||||
| -rw-r--r-- | distutils2/tests/test_install.py | 173 | ||||
| -rw-r--r-- | docs/source/library/distutils2.install.rst | 25 |
3 files changed, 302 insertions, 32 deletions
diff --git a/distutils2/install.py b/distutils2/install.py index b598c10..21510e8 100644 --- a/distutils2/install.py +++ b/distutils2/install.py @@ -1,26 +1,30 @@ +from tempfile import mkdtemp import logging +import shutil +import os +import errno + +from distutils2._backport.pkgutil import get_distributions +from distutils2.depgraph import generate_graph from distutils2.index import wrapper from distutils2.index.errors import ProjectNotFound, ReleaseNotFound -from distutils2.depgraph import generate_graph -from distutils2._backport.pkgutil import get_distributions - -"""Provides the installation script. +"""Provides installations scripts. The goal of this script is to install a release from the indexes (eg. PyPI), including the dependencies of the releases if needed. It uses the work made in pkgutil and by the index crawlers to browse the -installed distributions, and rely on the instalation command to install. - -Please note that this installation *script* iis different of the installation -*command*. While the command only install one distribution, the script installs -all the dependencies from a distribution, in a secure way. +installed distributions, and rely on the instalation commands to install. """ class InstallationException(Exception): - pass + """Base exception for installation scripts""" + + +class InstallationConflict(InstallationException): + """Raised when a conflict is detected""" def _update_infos(infos, new_infos): @@ -32,8 +36,113 @@ def _update_infos(infos, new_infos): infos[key].extend(new_infos[key]) -def get_infos(requirements, index=None, installed=None, - prefer_final=True): +def move_files(files, destination=None): + """Move the list of files in the destination folder, keeping the same + structure. + + Return a list of tuple (old, new) emplacement of files + + :param files: a list of files to move. + :param destination: the destination directory to put on the files. + if not defined, create a new one, using mkdtemp + """ + if not destination: + destination = mkdtemp() + + for old in files: + new = '%s%s' % (destination, old) + + # try to make the paths. + try: + os.makedirs(os.path.dirname(new)) + except OSError, e: + if e.errno == errno.EEXIST: + pass + else: + raise e + os.rename(old, new) + yield(old, new) + + +def install_dists(dists, path=None): + """Install all distributions provided in dists, with the given prefix. + + If an error occurs while installing one of the distributions, uninstall all + the installed distribution (in the context if this function). + + Return a list of installed files. + + :param dists: distributions to install + :param path: base path to install distribution on + """ + if not path: + path = mkdtemp() + + installed_dists, installed_files = [], [] + for d in dists: + try: + installed_files.extend(d.install(path)) + installed_dists.append(d) + except Exception, e : + for d in installed_dists: + d.uninstall() + raise e + return installed_files + + +def install_from_infos(install=[], remove=[], conflicts=[], install_path=None): + """Install and remove the given distributions. + + The function signature is made to be compatible with the one of get_infos. + The aim of this script is to povide a way to install/remove what's asked, + and to rollback if needed. + + So, it's not possible to be in an inconsistant state, it could be either + installed, either uninstalled, not half-installed. + + The process follow those steps: + + 1. Move all distributions that will be removed in a temporary location + 2. Install all the distributions that will be installed in a temp. loc. + 3. If the installation fails, rollback (eg. move back) those + distributions, or remove what have been installed. + 4. Else, move the distributions to the right locations, and remove for + real the distributions thats need to be removed. + + :param install: list of distributions that will be installed. + :param remove: list of distributions that will be removed. + :param conflicts: list of conflicting distributions, eg. that will be in + conflict once the install and remove distribution will be + processed. + :param install_path: the installation path where we want to install the + distributions. + """ + # first of all, if we have conflicts, stop here. + if conflicts: + raise InstallationConflict(conflicts) + + # before removing the files, we will start by moving them away + # then, if any error occurs, we could replace them in the good place. + temp_files = {} # contains lists of {dist: (old, new)} paths + if remove: + for dist in remove: + files = dist.get_installed_files() + temp_files[dist] = move_files(files) + try: + if install: + installed_files = install_dists(install, install_path) # install to tmp first + for files in temp_files.values(): + for old, new in files: + os.remove(new) + + except Exception,e: + # if an error occurs, put back the files in the good place. + for files in temp_files.values(): + for old, new in files: + shutil.move(new, old) + + +def get_infos(requirements, index=None, installed=None, prefer_final=True): """Return the informations on what's going to be installed and upgraded. :param requirements: is a *string* containing the requirements for this @@ -63,14 +172,13 @@ def get_infos(requirements, index=None, installed=None, # Get all the releases that match the requirements try: releases = index.get_releases(requirements) - except (ReleaseNotFound, ProjectNotFound): + except (ReleaseNotFound, ProjectNotFound), e: raise InstallationException('Release not found: "%s"' % requirements) # Pick up a release, and try to get the dependency tree release = releases.get_last(requirements, prefer_final=prefer_final) # Iter since we found something without conflicts - # XXX the metadata object is not used, remove the call or the binding metadata = release.fetch_metadata() # Get the distributions already_installed on the system diff --git a/distutils2/tests/test_install.py b/distutils2/tests/test_install.py index 84ad1f9..fde33e1 100644 --- a/distutils2/tests/test_install.py +++ b/distutils2/tests/test_install.py @@ -1,14 +1,20 @@ -"""Tests for the distutils2.index.xmlrpc module.""" +"""Tests for the distutils2.install module.""" -from distutils2.tests.pypi_server import use_xmlrpc_server -from distutils2.tests import unittest, run_unittest +import os +from tempfile import mkstemp + +from distutils2 import install from distutils2.index.xmlrpc import Client -from distutils2.install import (get_infos, InstallationException) from distutils2.metadata import DistributionMetadata +from distutils2.tests import run_unittest +from distutils2.tests.support import TempdirManager +from distutils2.tests.pypi_server import use_xmlrpc_server +from distutils2.tests.support import unittest -class FakeDist(object): - """A fake distribution object, for tests""" +class InstalledDist(object): + """Distribution object, represent distributions currently installed on the + system""" def __init__(self, name, version, deps): self.name = name self.version = version @@ -17,17 +23,49 @@ class FakeDist(object): self.metadata['Provides-Dist'] = ['%s (%s)' % (name, version)] def __repr__(self): - return '<FakeDist %s>' % self.name + return '<InstalledDist %s>' % self.name + + +class ToInstallDist(object): + """Distribution that will be installed""" + + def __init__(self, raise_error=False, files=False): + self._raise_error = raise_error + self._files = files + self.install_called = False + self.install_called_with = {} + self.uninstall_called = False + self._real_files = [] + if files: + for f in range(0,3): + self._real_files.append(mkstemp()) + def install(self, *args): + self.install_called = True + self.install_called_with = args + if self._raise_error: + raise Exception('Oops !') + return ['/path/to/foo', '/path/to/bar'] -def get_fake_dists(dists): + def uninstall(self, **args): + self.uninstall_called = True + + def get_installed_files(self, **args): + if self._files: + return [f[1] for f in self._real_files] + + def get_install(self, **args): + return self.get_installed_files() + + +def get_installed_dists(dists): objects = [] for (name, version, deps) in dists: - objects.append(FakeDist(name, version, deps)) + objects.append(InstalledDist(name, version, deps)) return objects -class TestInstallWithDeps(unittest.TestCase): +class TestInstall(TempdirManager, unittest.TestCase): def _get_client(self, server, *args, **kwargs): return Client(server.full_address, *args, **kwargs) @@ -62,8 +100,8 @@ class TestInstallWithDeps(unittest.TestCase): 'requires_dist': [], 'url': archive_path}, ]) - installed = get_fake_dists([('bacon', '0.1', []),]) - output = get_infos("choxie", index=client, + installed = get_installed_dists([('bacon', '0.1', []),]) + output = install.get_infos("choxie", index=client, installed=installed) # we dont have installed bacon as it's already installed on the system. @@ -94,8 +132,8 @@ class TestInstallWithDeps(unittest.TestCase): 'url': archive_path}, ]) - output = get_infos("choxie", index=client, installed= - get_fake_dists([('bacon', '0.1', []),])) + output = install.get_infos("choxie", index=client, installed= + get_installed_dists([('bacon', '0.1', []),])) installed = [(o.name, '%s' % o.version) for o in output['install']] # we need bacon 0.2, but 0.1 is installed. @@ -128,8 +166,8 @@ class TestInstallWithDeps(unittest.TestCase): ]) already_installed = [('bacon', '0.1', []), ('chicken', '1.1', ['bacon (0.1)'])] - output = get_infos("choxie", index=client, installed= - get_fake_dists(already_installed)) + output = install.get_infos("choxie", index=client, installed= + get_installed_dists(already_installed)) # we need bacon 0.2, but 0.1 is installed. # So we expect to remove 0.1 and to install 0.2 instead. @@ -145,13 +183,112 @@ class TestInstallWithDeps(unittest.TestCase): # Test that the isntalled raises an exception if the project does not # exists. client = self._get_client(server) - self.assertRaises(InstallationException, get_infos, + self.assertRaises(install.InstallationException, + install.get_infos, 'unexistant project', index=client) + def test_move_files(self): + # test that the files are really moved, and that the new path is + # returned. + path = self.mkdtemp() + newpath = self.mkdtemp() + files = [os.path.join(path, '%s' % x) for x in range(1, 20)] + for f in files: + file(f, 'a+') + output = [o for o in install.move_files(files, newpath)] + + # check that output return the list of old/new places + for f in files: + self.assertIn((f, '%s%s' % (newpath, f)), output) + + # remove the files + for f in [o[1] for o in output]: # o[1] is the new place + os.remove(f) + + def test_update_infos(self): + tests = [[{'foo': ['foobar', 'foo', 'baz'], 'baz': ['foo', 'foo']}, + {'foo': ['additional_content', 'yeah'], + 'baz': ['test', 'foo']}, + {'foo': ['foobar', 'foo', 'baz', 'additional_content', 'yeah'], + 'baz': ['foo', 'foo', 'test', 'foo']}],] + + for dict1, dict2, expect in tests: + install._update_infos(dict1, dict2) + for key in expect.keys(): + self.assertEqual(expect[key], dict1[key]) + + def test_install_dists_rollback(self): + # if one of the distribution installation fails, call uninstall on all + # installed distributions. + + d1 = ToInstallDist() + d2 = ToInstallDist(raise_error=True) + self.assertRaises(Exception, install.install_dists, [d1, d2]) + for dist in (d1, d2): + self.assertTrue(dist.install_called) + self.assertTrue(d1.uninstall_called) + self.assertFalse(d2.uninstall_called) + + def test_install_dists_success(self): + # test that the install method is called on each of the distributions. + d1 = ToInstallDist() + d2 = ToInstallDist() + install.install_dists([d1, d2]) + for dist in (d1, d2): + self.assertTrue(dist.install_called) + self.assertFalse(d1.uninstall_called) + self.assertFalse(d2.uninstall_called) + + def test_install_from_infos_conflict(self): + # assert conflicts raise an exception + self.assertRaises(install.InstallationConflict, + install.install_from_infos, + conflicts=[ToInstallDist()]) + + def test_install_from_infos_remove_success(self): + old_install_dists = install.install_dists + install.install_dists = lambda x,y=None: None + try: + dists = [] + for i in range(0,2): + dists.append(ToInstallDist(files=True)) + install.install_from_infos(remove=dists) + + # assert that the files have been removed + for dist in dists: + for f in dist.get_installed_files(): + self.assertFalse(os.path.exists(f)) + finally: + install.install_dists = old_install_dists + + def test_install_from_infos_remove_rollback(self): + # assert that if an error occurs, the removed files are restored. + remove = [] + for i in range(0,2): + remove.append(ToInstallDist(files=True, raise_error=True)) + to_install = [ToInstallDist(raise_error=True), + ToInstallDist()] + + install.install_from_infos(remove=remove, install=to_install) + # assert that the files are in the same place + # assert that the files have been removed + for dist in remove: + for f in dist.get_installed_files(): + self.assertTrue(os.path.exists(f)) + + def test_install_from_infos_install_succes(self): + # assert that the distribution can be installed + install_path = "my_install_path" + to_install = [ToInstallDist(), ToInstallDist()] + + install.install_from_infos(install=to_install, + install_path=install_path) + for dist in to_install: + self.assertEquals(dist.install_called_with, (install_path,)) def test_suite(): suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestInstallWithDeps)) + suite.addTest(unittest.makeSuite(TestInstall)) return suite if __name__ == '__main__': diff --git a/docs/source/library/distutils2.install.rst b/docs/source/library/distutils2.install.rst new file mode 100644 index 0000000..a557778 --- /dev/null +++ b/docs/source/library/distutils2.install.rst @@ -0,0 +1,25 @@ +================== +Installation tools +================== + +In addition to the install commands, distutils2 provides a set of tools to deal +with installation of distributions. + +Basically, they're intended to download the distribution from indexes, to +resolve the dependencies, and to provide a safe way to install all the +distributions. + +You can find those tools in :module distutils2.install_tools:. + + +API +--- + +.. automodule:: distutils2.install + :members: + +Example usage +-------------- + +Get the scheme of what's gonna be installed if we install "foobar": + |