apidoc: Move apidoc to ext/apidoc

The 'sphinx-apidoc' tool is no longer the only kid on the block when it
comes to automatic documentation of Python projects, with the likes of
'sphinx-autoapi' in development [1], and is not really maintained. Given
the fact that the tool is tangential to Sphinx's main goal, there isn't
really anything that warrants its continue existence as a core module.

Signify this by moving the feature to 'ext'. This allows the feature to
continue to exist in the package, but indicates that stability might be
worse than one would expect from the core library.

To avoid breaking packages that are using this feature directly, such as
pbr [3], aliases for the old 'main' method are included. This is based
on what Django does and, like Django, will allow us to safely remove the
old modules in Sphinx 2.0.

[1] https://github.com/rtfd/sphinx-autoapi
[2] https://github.com/sphinx-doc/sphinx/pull/3485#issuecomment-288081223
[3] https://github.com/django/django/blob/1.11/django/test/runner.py#L688-L695

Signed-off-by: Stephen Finucane <stephen@that.guru>

1 files changed, 16 insertions, 421 deletions

diff --git a/sphinx/apidoc.py b/sphinx/apidoc.py
index da908da04..0a924b31d 100644
--- a/sphinx/apidoc.py
+++ b/sphinx/apidoc.py
@@ -3,437 +3,32 @@
     sphinx.apidoc
     ~~~~~~~~~~~~~
 
-    Parses a directory tree looking for Python modules and packages and creates
-    ReST files appropriately to create code documentation with Sphinx.  It also
-    creates a modules index (named modules.<suffix>).
-
-    This is derived from the "sphinx-autopackage" script, which is:
-    Copyright 2008 Société des arts technologiques (SAT),
-    http://www.sat.qc.ca/
+    This file has moved to :py:mod:`sphinx.ext.apidoc`.
 
     :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-from __future__ import print_function
-
-import os
-import sys
-import optparse
-from os import path
-from six import binary_type
-from fnmatch import fnmatch
-
-from sphinx import __display_version__
-from sphinx.quickstart import EXTENSIONS
-from sphinx.util import rst
-from sphinx.util.osutil import FileAvoidWrite, walk
-
-if False:
-    # For type annotation
-    from typing import Any, List, Tuple  # NOQA
-
-# automodule options
-if 'SPHINX_APIDOC_OPTIONS' in os.environ:
-    OPTIONS = os.environ['SPHINX_APIDOC_OPTIONS'].split(',')
-else:
-    OPTIONS = [
-        'members',
-        'undoc-members',
-        # 'inherited-members', # disabled because there's a bug in sphinx
-        'show-inheritance',
-    ]
-
-INITPY = '__init__.py'
-PY_SUFFIXES = set(['.py', '.pyx'])
-
-
-def makename(package, module):
-    # type: (unicode, unicode) -> unicode
-    """Join package and module with a dot."""
-    # Both package and module can be None/empty.
-    if package:
-        name = package
-        if module:
-            name += '.' + module
-    else:
-        name = module
-    return name
-
-
-def write_file(name, text, opts):
-    # type: (unicode, unicode, Any) -> None
-    """Write the output file for module/package <name>."""
-    fname = path.join(opts.destdir, '%s.%s' % (name, opts.suffix))
-    if opts.dryrun:
-        print('Would create file %s.' % fname)
-        return
-    if not opts.force and path.isfile(fname):
-        print('File %s already exists, skipping.' % fname)
-    else:
-        print('Creating file %s.' % fname)
-        with FileAvoidWrite(fname) as f:
-            f.write(text)
-
-
-def format_heading(level, text, escape=True):
-    # type: (int, unicode, bool) -> unicode
-    """Create a heading of <level> [1, 2 or 3 supported]."""
-    if escape:
-        text = rst.escape(text)
-    underlining = ['=', '-', '~', ][level - 1] * len(text)
-    return '%s\n%s\n\n' % (text, underlining)
-
-
-def format_directive(module, package=None):
-    # type: (unicode, unicode) -> unicode
-    """Create the automodule directive and add the options."""
-    directive = '.. automodule:: %s\n' % makename(package, module)
-    for option in OPTIONS:
-        directive += '    :%s:\n' % option
-    return directive
-
-
-def create_module_file(package, module, opts):
-    # type: (unicode, unicode, Any) -> None
-    """Build the text of the file and write the file."""
-    if not opts.noheadings:
-        text = format_heading(1, '%s module' % module)
-    else:
-        text = ''
-    # text += format_heading(2, ':mod:`%s` Module' % module)
-    text += format_directive(module, package)
-    write_file(makename(package, module), text, opts)
-
-
-def create_package_file(root, master_package, subroot, py_files, opts, subs, is_namespace):
-    # type: (unicode, unicode, unicode, List[unicode], Any, List[unicode], bool) -> None
-    """Build the text of the file and write the file."""
-    text = format_heading(1, ('%s package' if not is_namespace else "%s namespace")
-                          % makename(master_package, subroot))
-
-    if opts.modulefirst and not is_namespace:
-        text += format_directive(subroot, master_package)
-        text += '\n'
-
-    # build a list of directories that are szvpackages (contain an INITPY file)
-    subs = [sub for sub in subs if path.isfile(path.join(root, sub, INITPY))]
-    # if there are some package directories, add a TOC for theses subpackages
-    if subs:
-        text += format_heading(2, 'Subpackages')
-        text += '.. toctree::\n\n'
-        for sub in subs:
-            text += '    %s.%s\n' % (makename(master_package, subroot), sub)
-        text += '\n'
-
-    submods = [path.splitext(sub)[0] for sub in py_files
-               if not shall_skip(path.join(root, sub), opts) and
-               sub != INITPY]
-    if submods:
-        text += format_heading(2, 'Submodules')
-        if opts.separatemodules:
-            text += '.. toctree::\n\n'
-            for submod in submods:
-                modfile = makename(master_package, makename(subroot, submod))
-                text += '   %s\n' % modfile
-
-                # generate separate file for this module
-                if not opts.noheadings:
-                    filetext = format_heading(1, '%s module' % modfile)
-                else:
-                    filetext = ''
-                filetext += format_directive(makename(subroot, submod),
-                                             master_package)
-                write_file(modfile, filetext, opts)
-        else:
-            for submod in submods:
-                modfile = makename(master_package, makename(subroot, submod))
-                if not opts.noheadings:
-                    text += format_heading(2, '%s module' % modfile)
-                text += format_directive(makename(subroot, submod),
-                                         master_package)
-                text += '\n'
-        text += '\n'
-
-    if not opts.modulefirst and not is_namespace:
-        text += format_heading(2, 'Module contents')
-        text += format_directive(subroot, master_package)
-
-    write_file(makename(master_package, subroot), text, opts)
-
-
-def create_modules_toc_file(modules, opts, name='modules'):
-    # type: (List[unicode], Any, unicode) -> None
-    """Create the module's index."""
-    text = format_heading(1, '%s' % opts.header, escape=False)
-    text += '.. toctree::\n'
-    text += '   :maxdepth: %s\n\n' % opts.maxdepth
-
-    modules.sort()
-    prev_module = ''  # type: unicode
-    for module in modules:
-        # look if the module is a subpackage and, if yes, ignore it
-        if module.startswith(prev_module + '.'):
-            continue
-        prev_module = module
-        text += '   %s\n' % module
-
-    write_file(name, text, opts)
-
-
-def shall_skip(module, opts):
-    # type: (unicode, Any) -> bool
-    """Check if we want to skip this module."""
-    # skip if the file doesn't exist and not using implicit namespaces
-    if not opts.implicit_namespaces and not path.exists(module):
-        return True
-
-    # skip it if there is nothing (or just \n or \r\n) in the file
-    if path.exists(module) and path.getsize(module) <= 2:
-        return True
-
-    # skip if it has a "private" name and this is selected
-    filename = path.basename(module)
-    if filename != '__init__.py' and filename.startswith('_') and \
-       not opts.includeprivate:
-        return True
-    return False
-
-
-def recurse_tree(rootpath, excludes, opts):
-    # type: (unicode, List[unicode], Any) -> List[unicode]
-    """
-    Look for every file in the directory tree and create the corresponding
-    ReST files.
-    """
-    followlinks = getattr(opts, 'followlinks', False)
-    includeprivate = getattr(opts, 'includeprivate', False)
-    implicit_namespaces = getattr(opts, 'implicit_namespaces', False)
-
-    # check if the base directory is a package and get its name
-    if INITPY in os.listdir(rootpath) or implicit_namespaces:
-        root_package = rootpath.split(path.sep)[-1]
-    else:
-        # otherwise, the base is a directory with packages
-        root_package = None
-
-    toplevels = []
-    for root, subs, files in walk(rootpath, followlinks=followlinks):
-        # document only Python module files (that aren't excluded)
-        py_files = sorted(f for f in files
-                          if path.splitext(f)[1] in PY_SUFFIXES and
-                          not is_excluded(path.join(root, f), excludes))
-        is_pkg = INITPY in py_files
-        is_namespace = INITPY not in py_files and implicit_namespaces
-        if is_pkg:
-            py_files.remove(INITPY)
-            py_files.insert(0, INITPY)
-        elif root != rootpath:
-            # only accept non-package at toplevel unless using implicit namespaces
-            if not implicit_namespaces:
-                del subs[:]
-                continue
-        # remove hidden ('.') and private ('_') directories, as well as
-        # excluded dirs
-        if includeprivate:
-            exclude_prefixes = ('.',)  # type: Tuple[unicode, ...]
-        else:
-            exclude_prefixes = ('.', '_')
-        subs[:] = sorted(sub for sub in subs if not sub.startswith(exclude_prefixes) and
-                         not is_excluded(path.join(root, sub), excludes))
-
-        if is_pkg or is_namespace:
-            # we are in a package with something to document
-            if subs or len(py_files) > 1 or not shall_skip(path.join(root, INITPY), opts):
-                subpackage = root[len(rootpath):].lstrip(path.sep).\
-                    replace(path.sep, '.')
-                # if this is not a namespace or
-                # a namespace and there is something there to document
-                if not is_namespace or len(py_files) > 0:
-                    create_package_file(root, root_package, subpackage,
-                                        py_files, opts, subs, is_namespace)
-                    toplevels.append(makename(root_package, subpackage))
-        else:
-            # if we are at the root level, we don't require it to be a package
-            assert root == rootpath and root_package is None
-            for py_file in py_files:
-                if not shall_skip(path.join(rootpath, py_file), opts):
-                    module = path.splitext(py_file)[0]
-                    create_module_file(root_package, module, opts)
-                    toplevels.append(module)
-
-    return toplevels
-
-
-def normalize_excludes(rootpath, excludes):
-    # type: (unicode, List[unicode]) -> List[unicode]
-    """Normalize the excluded directory list."""
-    return [path.abspath(exclude) for exclude in excludes]
-
-
-def is_excluded(root, excludes):
-    # type: (unicode, List[unicode]) -> bool
-    """Check if the directory is in the exclude list.
-
-    Note: by having trailing slashes, we avoid common prefix issues, like
-          e.g. an exlude "foo" also accidentally excluding "foobar".
-    """
-    for exclude in excludes:
-        if fnmatch(root, exclude):
-            return True
-    return False
-
-
-def main(argv=sys.argv[1:]):
-    # type: (List[str]) -> int
-    """Parse and check the command line arguments."""
-    parser = optparse.OptionParser(
-        usage="""\
-usage: %prog [options] -o <output_path> <module_path> [exclude_pattern, ...]
-
-Look recursively in <module_path> for Python modules and packages and create
-one reST file with automodule directives per package in the <output_path>.
-
-The <exclude_pattern>s can be file and/or directory patterns that will be
-excluded from generation.
-
-Note: By default this script will not overwrite already created files.""")
-
-    parser.add_option('-o', '--output-dir', action='store', dest='destdir',
-                      help='Directory to place all output', default='')
-    parser.add_option('-d', '--maxdepth', action='store', dest='maxdepth',
-                      help='Maximum depth of submodules to show in the TOC '
-                      '(default: 4)', type='int', default=4)
-    parser.add_option('-f', '--force', action='store_true', dest='force',
-                      help='Overwrite existing files')
-    parser.add_option('-l', '--follow-links', action='store_true',
-                      dest='followlinks', default=False,
-                      help='Follow symbolic links. Powerful when combined '
-                      'with collective.recipe.omelette.')
-    parser.add_option('-n', '--dry-run', action='store_true', dest='dryrun',
-                      help='Run the script without creating files')
-    parser.add_option('-e', '--separate', action='store_true',
-                      dest='separatemodules',
-                      help='Put documentation for each module on its own page')
-    parser.add_option('-P', '--private', action='store_true',
-                      dest='includeprivate',
-                      help='Include "_private" modules')
-    parser.add_option('-T', '--no-toc', action='store_true', dest='notoc',
-                      help='Don\'t create a table of contents file')
-    parser.add_option('-E', '--no-headings', action='store_true',
-                      dest='noheadings',
-                      help='Don\'t create headings for the module/package '
-                           'packages (e.g. when the docstrings already contain '
-                           'them)')
-    parser.add_option('-M', '--module-first', action='store_true',
-                      dest='modulefirst',
-                      help='Put module documentation before submodule '
-                      'documentation')
-    parser.add_option('--implicit-namespaces', action='store_true',
-                      dest='implicit_namespaces',
-                      help='Interpret module paths according to PEP-0420 '
-                           'implicit namespaces specification')
-    parser.add_option('-s', '--suffix', action='store', dest='suffix',
-                      help='file suffix (default: rst)', default='rst')
-    parser.add_option('-F', '--full', action='store_true', dest='full',
-                      help='Generate a full project with sphinx-quickstart')
-    parser.add_option('-a', '--append-syspath', action='store_true',
-                      dest='append_syspath',
-                      help='Append module_path to sys.path, used when --full is given')
-    parser.add_option('-H', '--doc-project', action='store', dest='header',
-                      help='Project name (default: root module name)')
-    parser.add_option('-A', '--doc-author', action='store', dest='author',
-                      type='str',
-                      help='Project author(s), used when --full is given')
-    parser.add_option('-V', '--doc-version', action='store', dest='version',
-                      help='Project version, used when --full is given')
-    parser.add_option('-R', '--doc-release', action='store', dest='release',
-                      help='Project release, used when --full is given, '
-                      'defaults to --doc-version')
-    parser.add_option('--version', action='store_true', dest='show_version',
-                      help='Show version information and exit')
-    group = parser.add_option_group('Extension options')
-    for ext in EXTENSIONS:
-        group.add_option('--ext-' + ext, action='store_true',
-                         dest='ext_' + ext, default=False,
-                         help='enable %s extension' % ext)
-
-    (opts, args) = parser.parse_args(argv)
-
-    if opts.show_version:
-        print('Sphinx (sphinx-apidoc) %s' % __display_version__)
-        return 0
 
-    if not args:
-        parser.error('A package path is required.')
+import warnings
 
-    rootpath, excludes = args[0], args[1:]
-    if not opts.destdir:
-        parser.error('An output directory is required.')
-    if opts.header is None:
-        opts.header = path.abspath(rootpath).split(path.sep)[-1]
-    if opts.suffix.startswith('.'):
-        opts.suffix = opts.suffix[1:]
-    if not path.isdir(rootpath):
-        print('%s is not a directory.' % rootpath, file=sys.stderr)
-        sys.exit(1)
-    if not path.isdir(opts.destdir):
-        if not opts.dryrun:
-            os.makedirs(opts.destdir)
-    rootpath = path.abspath(rootpath)
-    excludes = normalize_excludes(rootpath, excludes)
-    modules = recurse_tree(rootpath, excludes, opts)
-    if opts.full:
-        from sphinx import quickstart as qs
-        modules.sort()
-        prev_module = ''  # type: unicode
-        text = ''
-        for module in modules:
-            if module.startswith(prev_module + '.'):
-                continue
-            prev_module = module
-            text += '   %s\n' % module
-        d = dict(
-            path = opts.destdir,
-            sep = False,
-            dot = '_',
-            project = opts.header,
-            author = opts.author or 'Author',
-            version = opts.version or '',
-            release = opts.release or opts.version or '',
-            suffix = '.' + opts.suffix,
-            master = 'index',
-            epub = True,
-            ext_autodoc = True,
-            ext_viewcode = True,
-            ext_todo = True,
-            makefile = True,
-            batchfile = True,
-            mastertocmaxdepth = opts.maxdepth,
-            mastertoctree = text,
-            language = 'en',
-            module_path = rootpath,
-            append_syspath = opts.append_syspath,
-        )
-        enabled_exts = {'ext_' + ext: getattr(opts, 'ext_' + ext)
-                        for ext in EXTENSIONS if getattr(opts, 'ext_' + ext)}
-        d.update(enabled_exts)
+from sphinx.deprecation import RemovedInSphinx20Warning
+from sphinx.ext.apidoc import main as _main
 
-        if isinstance(opts.header, binary_type):
-            d['project'] = d['project'].decode('utf-8')
-        if isinstance(opts.author, binary_type):
-            d['author'] = d['author'].decode('utf-8')
-        if isinstance(opts.version, binary_type):
-            d['version'] = d['version'].decode('utf-8')
-        if isinstance(opts.release, binary_type):
-            d['release'] = d['release'].decode('utf-8')
 
-        if not opts.dryrun:
-            qs.generate(d, silent=True, overwrite=opts.force)
-    elif not opts.notoc:
-        create_modules_toc_file(modules, opts)
-    return 0
+def main(*args, **kwargs):
+    warnings.warn(
+        '`sphinx.apidoc.main()` has moved to `sphinx.ext.apidoc.main()`.',
+        RemovedInSphinx20Warning,
+        stacklevel=2,
+    )
+    _main(*args, **kwargs)
 
 
 # So program can be started with "python -m sphinx.apidoc ..."
 if __name__ == "__main__":
+    warnings.warn(
+        '`sphinx.apidoc` has moved to `sphinx.ext.apidoc`.',
+        RemovedInSphinx20Warning,
+        stacklevel=2,
+    )
     main()