merged upstream

11 files changed, 165 insertions, 140 deletions

diff --git a/src/distutils2/command/bdist.py b/src/distutils2/command/bdist.py
index 27f9ec3..147a31e 100644
--- a/src/distutils2/command/bdist.py
+++ b/src/distutils2/command/bdist.py
@@ -62,7 +62,7 @@ class bdist(Command):
                       'os2': 'zip'}
 
     # Establish the preferred order (for the --help-formats option).
-    format_commands = ['rpm', 'gztar', 'bztar', 'ztar', 'tar',
+    format_commands = ['gztar', 'bztar', 'ztar', 'tar',
                        'wininst', 'zip', 'msi']
 
     # And the real information.
@@ -96,7 +96,7 @@ class bdist(Command):
 
         # 'bdist_base' -- parent of per-built-distribution-format
         # temporary directories (eg. we'll probably have
-        # "build/bdist.<plat>/dumb", "build/bdist.<plat>/rpm", etc.)
+        # "build/bdist.<plat>/dumb", etc.)
         if self.bdist_base is None:
             build_base = self.get_finalized_command('build').build_base
             self.bdist_base = os.path.join(build_base,
@@ -126,7 +126,7 @@ class bdist(Command):
         # Reinitialize and run each command.
         for i in range(len(self.formats)):
             cmd_name = commands[i]
-            sub_cmd = self.reinitialize_command(cmd_name)
+            sub_cmd = self.get_reinitialized_command(cmd_name)
 
             # passing the owner and group names for tar archiving
             if cmd_name == 'bdist_dumb':
diff --git a/src/distutils2/command/bdist_dumb.py b/src/distutils2/command/bdist_dumb.py
index 429725c..604bcb7 100644
--- a/src/distutils2/command/bdist_dumb.py
+++ b/src/distutils2/command/bdist_dumb.py
@@ -85,7 +85,7 @@ class bdist_dumb (Command):
         if not self.skip_build:
             self.run_command('build')
 
-        install = self.reinitialize_command('install', reinit_subcommands=1)
+        install = self.get_reinitialized_command('install', reinit_subcommands=1)
         install.root = self.bdist_dir
         install.skip_build = self.skip_build
         install.warn_dir = 0
@@ -115,8 +115,9 @@ class bdist_dumb (Command):
                        % (repr(install.install_base),
                           repr(install.install_platbase)))
             else:
-                archive_root = os.path.join(self.bdist_dir,
-                                   ensure_relative(install.install_base))
+                archive_root = os.path.join(
+                    self.bdist_dir,
+                    self._ensure_relative(install.install_base))
 
         # Make the archive
         filename = self.make_archive(pseudoinstall_root,
@@ -135,3 +136,9 @@ class bdist_dumb (Command):
             else:
                 rmtree(self.bdist_dir)
 
+    def _ensure_relative(self, path):
+        # copied from dir_util, deleted
+        drive, path = os.path.splitdrive(path)
+        if path[0:1] == os.sep:
+            path = drive + path[1:]
+        return path
diff --git a/src/distutils2/command/bdist_msi.py b/src/distutils2/command/bdist_msi.py
index 158065b..bbbcbcc 100644
--- a/src/distutils2/command/bdist_msi.py
+++ b/src/distutils2/command/bdist_msi.py
@@ -177,12 +177,12 @@ class bdist_msi (Command):
         if not self.skip_build:
             self.run_command('build')
 
-        install = self.reinitialize_command('install', reinit_subcommands=1)
+        install = self.get_reinitialized_command('install', reinit_subcommands=1)
         install.prefix = self.bdist_dir
         install.skip_build = self.skip_build
         install.warn_dir = 0
 
-        install_lib = self.reinitialize_command('install_lib')
+        install_lib = self.get_reinitialized_command('install_lib')
         # we do not want to include pyc or pyo files
         install_lib.compile = 0
         install_lib.optimize = 0
diff --git a/src/distutils2/command/bdist_wininst.py b/src/distutils2/command/bdist_wininst.py
index 10f8bec..7b55de1 100644
--- a/src/distutils2/command/bdist_wininst.py
+++ b/src/distutils2/command/bdist_wininst.py
@@ -8,6 +8,7 @@ __revision__ = "$Id: bdist_wininst.py 77761 2010-01-26 22:46:15Z tarek.ziade $"
 import sys
 import os
 import string
+from shutil import rmtree
 try:
     from sysconfig import get_python_version
 except ImportError:
@@ -126,13 +127,13 @@ class bdist_wininst (Command):
         if not self.skip_build:
             self.run_command('build')
 
-        install = self.reinitialize_command('install', reinit_subcommands=1)
+        install = self.get_reinitialized_command('install', reinit_subcommands=1)
         install.root = self.bdist_dir
         install.skip_build = self.skip_build
         install.warn_dir = 0
         install.plat_name = self.plat_name
 
-        install_lib = self.reinitialize_command('install_lib')
+        install_lib = self.get_reinitialized_command('install_lib')
         # we do not want to include pyc or pyo files
         install_lib.compile = 0
         install_lib.optimize = 0
diff --git a/src/distutils2/command/build_ext.py b/src/distutils2/command/build_ext.py
index ae11330..7e160ab 100644
--- a/src/distutils2/command/build_ext.py
+++ b/src/distutils2/command/build_ext.py
@@ -188,7 +188,24 @@ class build_ext(Command):
         if self.package is None:
             self.package = self.distribution.ext_package
 
+        # Ensure that the list of extensions is valid, i.e. it is a list of
+        # Extension objects.
         self.extensions = self.distribution.ext_modules
+        if self.extensions:
+            if not isinstance(self.extensions, (list, tuple)):
+                type_name = (self.extensions is None and 'None'
+                            or type(self.extensions).__name__)
+                raise DistutilsSetupError(
+                    "'ext_modules' must be a sequence of Extension instances,"
+                    " not %s" % (type_name,))
+            for i, ext in enumerate(self.extensions):
+                if isinstance(ext, Extension):
+                    continue                # OK! (assume type-checking done
+                                            # by Extension constructor)
+                type_name = (ext is None and 'None' or type(ext).__name__)
+                raise DistutilsSetupError(
+                    "'ext_modules' item %d must be an Extension instance,"
+                    " not %s" % (i, type_name))
 
         # Make sure Python's include directories (for Python.h, pyconfig.h,
         # etc.) are in the include search path.
@@ -396,86 +413,7 @@ class build_ext(Command):
         # Now actually compile and link everything.
         self.build_extensions()
 
-    def check_extensions_list(self, extensions):
-        """Ensure that the list of extensions (presumably provided as a
-        command option 'extensions') is valid, i.e. it is a list of
-        Extension objects.  We also support the old-style list of 2-tuples,
-        where the tuples are (ext_name, build_info), which are converted to
-        Extension instances here.
-
-        Raise DistutilsSetupError if the structure is invalid anywhere;
-        just returns otherwise.
-        """
-        if not isinstance(extensions, list):
-            raise DistutilsSetupError, \
-                  "'ext_modules' option must be a list of Extension instances"
-
-        for i, ext in enumerate(extensions):
-            if isinstance(ext, Extension):
-                continue                # OK! (assume type-checking done
-                                        # by Extension constructor)
-
-            if not isinstance(ext, tuple) or len(ext) != 2:
-                raise DistutilsSetupError, \
-                      ("each element of 'ext_modules' option must be an "
-                       "Extension instance or 2-tuple")
-
-            ext_name, build_info = ext
-
-            log.warn(("old-style (ext_name, build_info) tuple found in "
-                      "ext_modules for extension '%s'"
-                      "-- please convert to Extension instance" % ext_name))
-
-            if not (isinstance(ext_name, str) and
-                    extension_name_re.match(ext_name)):
-                raise DistutilsSetupError, \
-                      ("first element of each tuple in 'ext_modules' "
-                       "must be the extension name (a string)")
-
-            if not isinstance(build_info, dict):
-                raise DistutilsSetupError, \
-                      ("second element of each tuple in 'ext_modules' "
-                       "must be a dictionary (build info)")
-
-            # OK, the (ext_name, build_info) dict is type-safe: convert it
-            # to an Extension instance.
-            ext = Extension(ext_name, build_info['sources'])
-
-            # Easy stuff: one-to-one mapping from dict elements to
-            # instance attributes.
-            for key in ('include_dirs', 'library_dirs', 'libraries',
-                        'extra_objects', 'extra_compile_args',
-                        'extra_link_args'):
-                val = build_info.get(key)
-                if val is not None:
-                    setattr(ext, key, val)
-
-            # Medium-easy stuff: same syntax/semantics, different names.
-            ext.runtime_library_dirs = build_info.get('rpath')
-            if 'def_file' in build_info:
-                log.warn("'def_file' element of build info dict "
-                         "no longer supported")
-
-            # Non-trivial stuff: 'macros' split into 'define_macros'
-            # and 'undef_macros'.
-            macros = build_info.get('macros')
-            if macros:
-                ext.define_macros = []
-                ext.undef_macros = []
-                for macro in macros:
-                    if not (isinstance(macro, tuple) and len(macro) in (1, 2)):
-                        raise DistutilsSetupError, \
-                              ("'macros' element of build info dict "
-                               "must be 1- or 2-tuple")
-                    if len(macro) == 1:
-                        ext.undef_macros.append(macro[0])
-                    elif len(macro) == 2:
-                        ext.define_macros.append(macro)
-
-            extensions[i] = ext
-
     def get_source_files(self):
-        self.check_extensions_list(self.extensions)
         filenames = []
 
         # Wouldn't it be neat if we knew the names of header files too...
@@ -485,11 +423,6 @@ class build_ext(Command):
         return filenames
 
     def get_outputs(self):
-        # Sanity check the 'extensions' list -- can't assume this is being
-        # done in the same run as a 'build_extensions()' call (in fact, we
-        # can probably assume that it *isn't*!).
-        self.check_extensions_list(self.extensions)
-
         # And build the list of output (built) filenames.  Note that this
         # ignores the 'inplace' flag, and assumes everything goes in the
         # "build" tree.
@@ -499,9 +432,6 @@ class build_ext(Command):
         return outputs
 
     def build_extensions(self):
-        # First, sanity-check the 'extensions' list
-        self.check_extensions_list(self.extensions)
-
         for ext in self.extensions:
             try:
                 self.build_extension(ext)
diff --git a/src/distutils2/command/build_py.py b/src/distutils2/command/build_py.py
index 821128f..c4c6d68 100644
--- a/src/distutils2/command/build_py.py
+++ b/src/distutils2/command/build_py.py
@@ -6,14 +6,66 @@ __revision__ = "$Id: build_py.py 76956 2009-12-21 01:22:46Z tarek.ziade $"
 
 import os
 import sys
+import logging
 from glob import glob
 
+import distutils2
 from distutils2.core import Command
 from distutils2.errors import DistutilsOptionError, DistutilsFileError
 from distutils2.util import convert_path
-from distutils2 import log
-
-class build_py(Command):
+from distutils2.converter.refactor import DistutilsRefactoringTool
+
+# marking public APIs
+__all__ = ['Mixin2to3', 'build_py']
+
+try:
+    from distutils2.util import Mixin2to3 as _Mixin2to3
+    from lib2to3.refactor import get_fixers_from_package
+    _CONVERT = True
+    _KLASS = _Mixin2to3
+except ImportError:
+    _CONVERT = False
+    _KLASS = object
+
+class Mixin2to3(_KLASS):
+    """ The base class which can be used for refactoring. When run under
+    Python 3.0, the run_2to3 method provided by Mixin2to3 is overridden.
+    When run on Python 2.x, it merely creates a class which overrides run_2to3,
+    yet does nothing in particular with it.
+    """
+    if _CONVERT:
+        def _run_2to3(self, files, doctests=[]):
+            """ Takes a list of files and doctests, and performs conversion
+            on those.
+              - First, the files which contain the code(`files`) are converted.
+              - Second, the doctests in `files` are converted.
+              - Thirdly, the doctests in `doctests` are converted.
+            """
+
+            # Convert the ".py" files.
+            logging.info("Converting Python code")
+            _KLASS.run_2to3(self, files)
+
+            # Convert the doctests in the ".py" files.
+            logging.info("Converting doctests with '.py' files")
+            _KLASS.run_2to3(self, files, doctests_only=True)
+
+            # If the following conditions are met, then convert:-
+            # 1. User has specified the 'convert_2to3_doctests' option. So, we
+            #    can expect that the list 'doctests' is not empty.
+            # 2. The default is allow distutils2 to allow conversion of text files
+            #    containing doctests. It is set as
+            #    distutils2.run_2to3_on_doctests
+
+            if doctests != [] and distutils2.run_2to3_on_doctests:
+                logging.info("Converting text files which contain doctests")
+                _KLASS.run_2to3(self, doctests, doctests_only=True)
+    else:
+        # If run on Python 2.x, there is nothing to do.
+        def _run_2to3(self, files, doctests=[]):
+            pass
+
+class build_py(Command, Mixin2to3):
 
     description = "\"build\" pure Python modules (copy to build directory)"
 
@@ -39,6 +91,8 @@ class build_py(Command):
         self.compile = 0
         self.optimize = 0
         self.force = None
+        self._updated_files = []
+        self._doctests_2to3 = []
 
     def finalize_options(self):
         self.set_undefined_options('build',
@@ -93,10 +147,18 @@ class build_py(Command):
             self.build_packages()
             self.build_package_data()
 
+        if self.distribution.use_2to3 and self_updated_files:
+            self.run_2to3(self._updated_files, self._doctests_2to3)
+
         self.byte_compile(self.get_outputs(include_bytecode=0))
 
+    # -- Top-level worker functions ------------------------------------
+
     def get_data_files(self):
-        """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
+        """Generate list of '(package,src_dir,build_dir,filenames)' tuples.
+
+        Helper function for `finalize_options()`.
+        """
         data = []
         if not self.packages:
             return data
@@ -120,7 +182,10 @@ class build_py(Command):
         return data
 
     def find_data_files(self, package, src_dir):
-        """Return filenames for package's data files in 'src_dir'"""
+        """Return filenames for package's data files in 'src_dir'.
+
+        Helper function for `get_data_files()`.
+        """
         globs = (self.package_data.get('', [])
                  + self.package_data.get(package, []))
         files = []
@@ -132,14 +197,21 @@ class build_py(Command):
         return files
 
     def build_package_data(self):
-        """Copy data files into build directory"""
+        """Copy data files into build directory.
+
+        Helper function for `run()`.
+        """
         for package, src_dir, build_dir, filenames in self.data_files:
             for filename in filenames:
                 target = os.path.join(build_dir, filename)
                 self.mkpath(os.path.dirname(target))
-                self.copy_file(os.path.join(src_dir, filename), target,
-                               preserve_mode=False)
+                outf, copied = self.copy_file(os.path.join(src_dir, filename),
+                               target, preserve_mode=False)
+                if copied and srcfile in self.distribution.convert_2to3.doctests:
+                    self._doctests_2to3.append(outf)
 
+    # XXX - this should be moved to the Distribution class as it is not
+    # only needed for build_py. It also has no dependencies on this class.
     def get_package_dir(self, package):
         """Return the directory, relative to the top of the source
            distribution, where package 'package' should be found
@@ -181,6 +253,8 @@ class build_py(Command):
                     return ''
 
     def check_package(self, package, package_dir):
+        """Helper function for `find_package_modules()` and `find_modules()'.
+        """
         # Empty dir name means current directory, which we can probably
         # assume exists.  Also, os.path.exists and isdir don't know about
         # my "empty string means current dir" convention, so we have to
@@ -200,8 +274,8 @@ class build_py(Command):
             if os.path.isfile(init_py):
                 return init_py
             else:
-                log.warn(("package init file '%s' not found " +
-                          "(or not a regular file)"), init_py)
+                logging.warning(("package init file '%s' not found " +
+                                 "(or not a regular file)"), init_py)
 
         # Either not in a package at all (__init__.py not expected), or
         # __init__.py doesn't exist -- so don't return the filename.
@@ -209,7 +283,8 @@ class build_py(Command):
 
     def check_module(self, module, module_file):
         if not os.path.isfile(module_file):
-            log.warn("file %s (for module %s) not found", module_file, module)
+            logging.warning("file %s (for module %s) not found",
+                            module_file, module)
             return False
         else:
             return True
diff --git a/src/distutils2/command/cmd.py b/src/distutils2/command/cmd.py
index ba88b89..fa6f281 100644
--- a/src/distutils2/command/cmd.py
+++ b/src/distutils2/command/cmd.py
@@ -6,7 +6,7 @@ in the distutils.command package.
 
 __revision__ = "$Id: cmd.py 75192 2009-10-02 23:49:48Z tarek.ziade $"
 
-import sys, os, re
+import os, re
 from distutils2.errors import DistutilsOptionError
 from distutils2 import util
 from distutils2 import log
@@ -19,7 +19,7 @@ try:
 except ImportError:
     from distutils2._backport.shutil import make_archive
 
-class Command:
+class Command(object):
     """Abstract base class for defining command classes, the "worker bees"
     of the Distutils.  A useful analogy for command classes is to think of
     them as subroutines with local variables called "options".  The options
@@ -188,6 +188,31 @@ class Command:
         """
         log.log(level, msg)
 
+    # -- External interface --------------------------------------------
+    # (called by outsiders)
+
+    def get_source_files(self):
+        """Return the list of files that are used as inputs to this command,
+        i.e. the files used to generate the output files.  The result is used
+        by the `sdist` command in determining the set of default files.
+
+        Command classes should implement this method if they operate on files
+        from the source tree.
+        """
+        return []
+
+    def get_outputs(self):
+        """Return the list of files that would be produced if this command
+        were actually run.  Not affected by the "dry-run" flag or whether
+        any other commands have been run.
+
+        Command classes should implement this method if they produce any
+        output files that get consumed by another command.  e.g., `build_ext`
+        returns the list of built extension modules, but not any temporary
+        files used in the compilation process.
+        """
+        return []
+
     # -- Option validation methods -------------------------------------
     # (these are very handy in writing the 'finalize_options()' method)
     #
@@ -307,10 +332,8 @@ class Command:
         cmd_obj.ensure_finalized()
         return cmd_obj
 
-    # XXX rename to 'get_reinitialized_command()'? (should do the
-    # same in dist.py, if so)
-    def reinitialize_command(self, command, reinit_subcommands=0):
-        return self.distribution.reinitialize_command(
+    def get_reinitialized_command(self, command, reinit_subcommands=0):
+        return self.distribution.get_reinitialized_command(
             command, reinit_subcommands)
 
     def run_command(self, command):
@@ -350,8 +373,10 @@ class Command:
         if os.path.isdir(name) or name == '':
             return
         if dry_run:
+            head = ''
             for part in name.split(os.sep):
-                self.log(part)
+                log.info("created directory %s%s", head, part)
+                head += part + os.sep
             return
         os.makedirs(name, mode)
 
@@ -386,7 +411,7 @@ class Command:
 
     def spawn(self, cmd, search_path=1, level=1):
         """Spawn an external command respecting dry-run flag."""
-        from distutils2.spawn import spawn
+        from distutils2.util import spawn
         spawn(cmd, search_path, dry_run= self.dry_run)
 
     def make_archive(self, base_name, format, root_dir=None, base_dir=None,
@@ -422,7 +447,7 @@ class Command:
         # If 'outfile' must be regenerated (either because it doesn't
         # exist, is out-of-date, or the 'force' flag is true) then
         # perform the action that presumably regenerates it
-        if self.force or dep_util.newer_group(infiles, outfile):
+        if self.force or util.newer_group(infiles, outfile):
             self.execute(func, args, exec_msg, level)
 
         # Otherwise, print the "skip" message
diff --git a/src/distutils2/command/register.py b/src/distutils2/command/register.py
index 8abb3c8..29b87e4 100644
--- a/src/distutils2/command/register.py
+++ b/src/distutils2/command/register.py
@@ -28,8 +28,6 @@ class register(PyPIRCCommand):
     boolean_options = PyPIRCCommand.boolean_options + [
         'verify', 'list-classifiers', 'strict']
 
-    sub_commands = [('check', lambda self: True)]
-
     def initialize_options(self):
         PyPIRCCommand.initialize_options(self)
         self.list_classifiers = 0
@@ -46,9 +44,8 @@ class register(PyPIRCCommand):
         self.finalize_options()
         self._set_config()
 
-        # Run sub commands
-        for cmd_name in self.get_sub_commands():
-            self.run_command(cmd_name)
+        # Check the package metadata
+        self.run_command('check')
 
         if self.dry_run:
             self.verify_metadata()
diff --git a/src/distutils2/command/sdist.py b/src/distutils2/command/sdist.py
index bb4cd1f..37f74c5 100644
--- a/src/distutils2/command/sdist.py
+++ b/src/distutils2/command/sdist.py
@@ -18,12 +18,11 @@ except ImportError:
     from distutils2._backport.shutil import get_archive_formats
 
 from distutils2.core import Command
-from distutils2 import util
 from distutils2.errors import (DistutilsPlatformError, DistutilsOptionError,
-                              DistutilsTemplateError)
+                               DistutilsTemplateError)
 from distutils2.manifest import Manifest
 from distutils2 import log
-from distutils2.util import convert_path, newer
+from distutils2.util import convert_path
 
 def show_formats():
     """Print all possible values for the 'formats' option (used by
@@ -45,12 +44,6 @@ class sdist(Command):
 
     description = "create a source distribution (tarball, zip file, etc.)"
 
-    def checking_metadata(self):
-        """Callable used for the check sub-command.
-
-        Placed here so user_options can view it"""
-        return self.metadata_check
-
     user_options = [
         ('template=', 't',
          "name of manifest template file [default: MANIFEST.in]"),
@@ -100,8 +93,6 @@ class sdist(Command):
     default_format = {'posix': 'gztar',
                       'nt': 'zip' }
 
-    sub_commands = [('check', checking_metadata)]
-
     def initialize_options(self):
         # 'template' and 'manifest' are, respectively, the names of
         # the manifest template and manifest file.
@@ -162,9 +153,9 @@ class sdist(Command):
         # manifest
         self.filelist.clear()
 
-        # Run sub commands
-        for cmd_name in self.get_sub_commands():
-            self.run_command(cmd_name)
+        # Check the package metadata
+        if self.metadata_check:
+            self.run_command('check')
 
         # Do whatever it takes to get the list of files to process
         # (process the manifest template, read an existing manifest,
diff --git a/src/distutils2/command/upload.py b/src/distutils2/command/upload.py
index 14fe119..796ca73 100644
--- a/src/distutils2/command/upload.py
+++ b/src/distutils2/command/upload.py
@@ -15,7 +15,7 @@ except ImportError:
 
 from distutils2.errors import DistutilsOptionError
 from distutils2.core import PyPIRCCommand
-from distutils2.spawn import spawn
+from distutils2.util import spawn
 from distutils2 import log
 
 class upload(PyPIRCCommand):
diff --git a/src/distutils2/command/upload_docs.py b/src/distutils2/command/upload_docs.py
index 860f07f..120837b 100644
--- a/src/distutils2/command/upload_docs.py
+++ b/src/distutils2/command/upload_docs.py
@@ -1,4 +1,4 @@
-import base64, httplib, os.path, socket, tempfile, urlparse, zipfile
+import base64, httplib, os.path, socket, urlparse, zipfile
 from cStringIO import StringIO
 from distutils2 import log
 from distutils2.command.upload import upload
@@ -81,7 +81,6 @@ class upload_docs(PyPIRCCommand):
             raise DistutilsFileError(mesg % upload_dir)
 
     def run(self):
-        tmp_dir = tempfile.mkdtemp()
         name = self.distribution.metadata['Name']
         version = self.distribution.metadata['Version']
         zip_file = zip_dir(self.upload_dir)
@@ -125,7 +124,7 @@ class upload_docs(PyPIRCCommand):
         elif r.status == 301:
             location = r.getheader('Location')
             if location is None:
-                location = 'http://packages.python.org/%s/' % meta.get_name()
+                location = 'http://packages.python.org/%s/' % name
             self.announce('Upload successful. Visit %s' % location,
                           log.INFO)
         else: