Merge branch '2.0'

17 files changed, 200 insertions, 62 deletions

diff --git a/CHANGES b/CHANGES
index d7074ec81..feb1f1060 100644
--- a/CHANGES
+++ b/CHANGES
@@ -62,7 +62,7 @@ Bugs fixed
 Testing
 --------
 
-Release 2.4.0 (in development)
+Release 2.4.1 (in development)
 ==============================
 
 Dependencies
@@ -74,11 +74,28 @@ Incompatible changes
 Deprecated
 ----------
 
+Features added
+--------------
+
+Bugs fixed
+----------
+
+Testing
+--------
+
+Release 2.4.0 (released Feb 09, 2020)
+=====================================
+
+Deprecated
+----------
+
 * The ``decode`` argument of ``sphinx.pycode.ModuleAnalyzer()``
 * ``sphinx.directives.other.Index``
 * ``sphinx.environment.temp_data['gloss_entries']``
 * ``sphinx.environment.BuildEnvironment.indexentries``
 * ``sphinx.environment.collectors.indexentries.IndexEntriesCollector``
+* ``sphinx.ext.apidoc.INITPY``
+* ``sphinx.ext.apidoc.shall_skip()``
 * ``sphinx.io.FiletypeNotFoundError``
 * ``sphinx.io.get_filetype()``
 * ``sphinx.pycode.ModuleAnalyzer.encoding``
@@ -106,6 +123,8 @@ Features added
 * #6446: duration: Add ``sphinx.ext.durations`` to inspect which documents slow
   down the build
 * #6837: LaTeX: Support a nested table
+* #7115: LaTeX: Allow to override LATEXOPTS and LATEXMKOPTS via environment
+  variable
 * #6966: graphviz: Support ``:class:`` option
 * #6696: html: ``:scale:`` option of image/figure directive not working for SVG
   images (imagesize-1.2.0 or above is required)
@@ -133,6 +152,7 @@ Bugs fixed
 ----------
 
 * #6925: html: Remove redundant type="text/javascript" from <script> elements
+* #7112: html: SVG image is not layouted as float even if aligned
 * #6906, #6907: autodoc: failed to read the source codes encoeded in cp1251
 * #6961: latex: warning for babel shown twice
 * #7059: latex: LaTeX compilation falls into infinite loop (wrapfig issue)
@@ -140,6 +160,7 @@ Bugs fixed
 * #6559: Wrong node-ids are generated in glossary directive
 * #6986: apidoc: misdetects module name for .so file inside module
 * #6899: apidoc: private members are not shown even if ``--private`` given
+* #6327: apidoc: Support a python package consisted of __init__.so file
 * #6999: napoleon: fails to parse tilde in :exc: role
 * #7019: gettext: Absolute path used in message catalogs
 * #7023: autodoc: nested partial functions are not listed
@@ -153,34 +174,19 @@ Bugs fixed
   modifier keys are ignored, which means the feature can interfere with browser
   features
 * #7090: std domain: Can't assign numfig-numbers for custom container nodes
+* #7106: std domain: enumerated nodes are marked as duplicated when extensions
+  call ``note_explicit_target()``
+* #7095: dirhtml: Cross references are broken via intersphinx and ``:doc:`` role
+* C++:
 
-Testing
---------
-
-Release 2.3.2 (in development)
-==============================
-
-Dependencies
-------------
-
-Incompatible changes
---------------------
-
-Deprecated
-----------
-
-Features added
---------------
-
-Bugs fixed
-----------
-
-* C++, don't crash when using the ``struct`` role in some cases.
-* C++, don't warn when using the ``var``/``member`` role for function
-  parameters.
-
-Testing
---------
+  - Don't crash when using the ``struct`` role in some cases.
+  - Don't warn when using the ``var``/``member`` role for function
+    parameters.
+  - Render call and braced-init expressions correctly.
+* #7097: Filenames of images generated by
+  ``sphinx.transforms.post_transforms.images.ImageConverter``
+  or its subclasses (used for latex build) are now sanitized,
+  to prevent broken paths
 
 Release 2.3.1 (released Dec 22, 2019)
 =====================================
diff --git a/doc/extdev/deprecated.rst b/doc/extdev/deprecated.rst
index a1528b763..c507adbc9 100644
--- a/doc/extdev/deprecated.rst
+++ b/doc/extdev/deprecated.rst
@@ -71,6 +71,16 @@ The following is a list of deprecated interfaces.
      - 4.0
      - ``sphinx.errors.FiletypeNotFoundError``
 
+   * - ``sphinx.ext.apidoc.INITPY``
+     - 2.4
+     - 4.0
+     - N/A
+
+   * - ``sphinx.ext.apidoc.shall_skip()``
+     - 2.4
+     - 4.0
+     - ``sphinx.ext.apidoc.is_skipped_package``
+
    * - ``sphinx.io.get_filetype()``
      - 2.4
      - 4.0
diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py
index 93cf2e646..c89067852 100644
--- a/sphinx/domains/cpp.py
+++ b/sphinx/domains/cpp.py
@@ -3007,7 +3007,7 @@ class ASTParenExprList(ASTBase):
                 signode.append(nodes.Text(', '))
             else:
                 first = False
-                e.describe_signature(signode, mode, env, symbol)
+            e.describe_signature(signode, mode, env, symbol)
         signode.append(nodes.Text(')'))
 
 
@@ -3034,7 +3034,7 @@ class ASTBracedInitList(ASTBase):
                 signode.append(nodes.Text(', '))
             else:
                 first = False
-                e.describe_signature(signode, mode, env, symbol)
+            e.describe_signature(signode, mode, env, symbol)
         if self.trailingComma:
             signode.append(nodes.Text(','))
         signode.append(nodes.Text('}'))
diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py
index 99cf67016..a9196d3a6 100644
--- a/sphinx/ext/apidoc.py
+++ b/sphinx/ext/apidoc.py
@@ -29,7 +29,7 @@ from typing import Any, List, Tuple
 import sphinx.locale
 from sphinx import __display_version__, package_dir
 from sphinx.cmd.quickstart import EXTENSIONS
-from sphinx.deprecation import RemovedInSphinx40Warning
+from sphinx.deprecation import RemovedInSphinx40Warning, deprecated_alias
 from sphinx.locale import __
 from sphinx.util import rst
 from sphinx.util.osutil import FileAvoidWrite, ensuredir
@@ -46,7 +46,6 @@ else:
         'show-inheritance',
     ]
 
-INITPY = '__init__.py'
 PY_SUFFIXES = ('.py', '.pyx') + tuple(EXTENSION_SUFFIXES)
 
 template_dir = path.join(package_dir, 'templates', 'apidoc')
@@ -66,11 +65,31 @@ def makename(package: str, module: str) -> str:
     return name
 
 
+def is_initpy(filename: str) -> bool:
+    """Check *filename* is __init__ file or not."""
+    basename = path.basename(filename)
+    for suffix in sorted(PY_SUFFIXES, key=len, reverse=True):
+        if basename == '__init__' + suffix:
+            return True
+    else:
+        return False
+
+
 def module_join(*modnames: str) -> str:
     """Join module names with dots."""
     return '.'.join(filter(None, modnames))
 
 
+def is_packagedir(dirname: str = None, files: List[str] = None) -> bool:
+    """Check given *files* contains __init__ file."""
+    if files is None and dirname is None:
+        return False
+
+    if files is None:
+        files = os.listdir(dirname)
+    return any(f for f in files if is_initpy(f))
+
+
 def write_file(name: str, text: str, opts: Any) -> None:
     """Write the output file for module/package <name>."""
     quiet = getattr(opts, 'quiet', None)
@@ -132,15 +151,14 @@ def create_package_file(root: str, master_package: str, subroot: str, py_files:
                         opts: Any, subs: List[str], is_namespace: bool,
                         excludes: List[str] = [], user_template_dir: str = None) -> None:
     """Build the text of the file and write the file."""
-    # build a list of sub packages (directories containing an INITPY file)
-    subpackages = [sub for sub in subs if not
-                   shall_skip(path.join(root, sub, INITPY), opts, excludes)]
+    # build a list of sub packages (directories containing an __init__ file)
     subpackages = [module_join(master_package, subroot, pkgname)
-                   for pkgname in subpackages]
+                   for pkgname in subs
+                   if not is_skipped_package(path.join(root, pkgname), opts, excludes)]
     # build a list of sub modules
     submodules = [sub.split('.')[0] for sub in py_files
                   if not is_skipped_module(path.join(root, sub), opts, excludes) and
-                  sub != INITPY]
+                  not is_initpy(sub)]
     submodules = [module_join(master_package, subroot, modname)
                   for modname in submodules]
     options = copy(OPTIONS)
@@ -189,12 +207,14 @@ def create_modules_toc_file(modules: List[str], opts: Any, name: str = 'modules'
 
 def shall_skip(module: str, opts: Any, excludes: List[str] = []) -> bool:
     """Check if we want to skip this module."""
+    warnings.warn('shall_skip() is deprecated.',
+                  RemovedInSphinx40Warning)
     # skip if the file doesn't exist and not using implicit namespaces
     if not opts.implicit_namespaces and not path.exists(module):
         return True
 
     # Are we a package (here defined as __init__.py, not the folder in itself)
-    if os.path.basename(module) == INITPY:
+    if is_initpy(module):
         # Yes, check if we have any non-excluded modules at all here
         all_skipped = True
         basemodule = path.dirname(module)
@@ -207,12 +227,30 @@ def shall_skip(module: str, opts: Any, excludes: List[str] = []) -> bool:
 
     # skip if it has a "private" name and this is selected
     filename = path.basename(module)
-    if filename != '__init__.py' and filename.startswith('_') and \
-       not opts.includeprivate:
+    if is_initpy(filename) and filename.startswith('_') and not opts.includeprivate:
         return True
     return False
 
 
+def is_skipped_package(dirname: str, opts: Any, excludes: List[str] = []) -> bool:
+    """Check if we want to skip this module."""
+    if not path.isdir(dirname):
+        return False
+
+    files = glob.glob(path.join(dirname, '*.py'))
+    regular_package = any(f for f in files if is_initpy(f))
+    if not regular_package and not opts.implicit_namespaces:
+        # *dirname* is not both a regular package and an implicit namespace pacage
+        return True
+
+    # Check there is some showable module inside package
+    if all(is_excluded(path.join(dirname, f), excludes) for f in files):
+        # all submodules are excluded
+        return True
+    else:
+        return False
+
+
 def is_skipped_module(filename: str, opts: Any, excludes: List[str]) -> bool:
     """Check if we want to skip this module."""
     if not path.exists(filename):
@@ -236,7 +274,7 @@ def recurse_tree(rootpath: str, excludes: List[str], opts: Any,
     implicit_namespaces = getattr(opts, 'implicit_namespaces', False)
 
     # check if the base directory is a package and get its name
-    if INITPY in os.listdir(rootpath) or implicit_namespaces:
+    if is_packagedir(rootpath) or implicit_namespaces:
         root_package = rootpath.split(path.sep)[-1]
     else:
         # otherwise, the base is a directory with packages
@@ -248,11 +286,13 @@ def recurse_tree(rootpath: str, excludes: List[str], opts: Any,
         py_files = sorted(f for f in files
                           if f.endswith(PY_SUFFIXES) and
                           not is_excluded(path.join(root, f), excludes))
-        is_pkg = INITPY in py_files
-        is_namespace = INITPY not in py_files and implicit_namespaces
+        is_pkg = is_packagedir(None, py_files)
+        is_namespace = not is_pkg and implicit_namespaces
         if is_pkg:
-            py_files.remove(INITPY)
-            py_files.insert(0, INITPY)
+            for f in py_files[:]:
+                if is_initpy(f):
+                    py_files.remove(f)
+                    py_files.insert(0, f)
         elif root != rootpath:
             # only accept non-package at toplevel unless using implicit namespaces
             if not implicit_namespaces:
@@ -269,7 +309,7 @@ def recurse_tree(rootpath: str, excludes: List[str], opts: Any,
 
         if is_pkg or is_namespace:
             # we are in a package with something to document
-            if subs or len(py_files) > 1 or not shall_skip(path.join(root, INITPY), opts):
+            if subs or len(py_files) > 1 or not is_skipped_package(root, opts):
                 subpackage = root[len(rootpath):].lstrip(path.sep).\
                     replace(path.sep, '.')
                 # if this is not a namespace or
@@ -475,6 +515,13 @@ def main(argv: List[str] = sys.argv[1:]) -> int:
     return 0
 
 
+deprecated_alias('sphinx.ext.apidoc',
+                 {
+                     'INITPY': '__init__.py',
+                 },
+                 RemovedInSphinx40Warning)
+
+
 # So program can be started with "python -m sphinx.apidoc ..."
 if __name__ == "__main__":
     main()
diff --git a/sphinx/texinputs/Makefile_t b/sphinx/texinputs/Makefile_t
index 90cee829b..96bb0fed1 100644
--- a/sphinx/texinputs/Makefile_t
+++ b/sphinx/texinputs/Makefile_t
@@ -14,13 +14,13 @@ ALLPS  = $(addsuffix .ps,$(ALLDOCS))
 # Prefix for archive names
 ARCHIVEPREFIX =
 # Additional LaTeX options (passed via variables in latexmkrc/latexmkjarc file)
-export LATEXOPTS =
+export LATEXOPTS ?=
 # Additional latexmk options
 {% if latex_engine == 'xelatex' -%}
 # with latexmk version 4.52b or higher set LATEXMKOPTS to -xelatex either here
 # or on command line for faster builds.
 {% endif -%}
-LATEXMKOPTS =
+LATEXMKOPTS ?=
 {% if xindy_use -%}
 export XINDYOPTS = {{ xindy_lang_option }} -M sphinx.xdy
 {% if latex_engine == 'pdflatex' -%}
diff --git a/sphinx/transforms/__init__.py b/sphinx/transforms/__init__.py
index 922b22e46..a00f04fdf 100644
--- a/sphinx/transforms/__init__.py
+++ b/sphinx/transforms/__init__.py
@@ -173,7 +173,9 @@ class AutoNumbering(SphinxTransform):
         domain = self.env.get_domain('std')  # type: StandardDomain
 
         for node in self.document.traverse(nodes.Element):
-            if domain.is_enumerable_node(node) and domain.get_numfig_title(node) is not None:
+            if (domain.is_enumerable_node(node) and
+                    domain.get_numfig_title(node) is not None and
+                    node['ids'] == []):
                 self.document.note_implicit_target(node)
 
 
diff --git a/sphinx/transforms/post_transforms/images.py b/sphinx/transforms/post_transforms/images.py
index 758e92f0d..d1b513b27 100644
--- a/sphinx/transforms/post_transforms/images.py
+++ b/sphinx/transforms/post_transforms/images.py
@@ -9,6 +9,7 @@
 """
 
 import os
+import re
 from hashlib import sha1
 from math import ceil
 from typing import Any, Dict, List, Tuple
@@ -27,6 +28,7 @@ from sphinx.util.osutil import ensuredir, movefile
 logger = logging.getLogger(__name__)
 
 MAX_FILENAME_LEN = 32
+CRITICAL_PATH_CHAR_RE = re.compile('[:;<>|*" ]')
 
 
 class BaseImageConverter(SphinxTransform):
@@ -65,6 +67,7 @@ class ImageDownloader(BaseImageConverter):
             if basename == '' or len(basename) > MAX_FILENAME_LEN:
                 filename, ext = os.path.splitext(node['uri'])
                 basename = sha1(filename.encode()).hexdigest() + ext
+            basename = re.sub(CRITICAL_PATH_CHAR_RE, "_", basename)
 
             dirname = node['uri'].replace('://', '/').translate({ord("?"): "/",
                                                                  ord("&"): "/"})
@@ -146,6 +149,7 @@ class DataURIExtractor(BaseImageConverter):
 
 def get_filename_for(filename: str, mimetype: str) -> str:
     basename = os.path.basename(filename)
+    basename = re.sub(CRITICAL_PATH_CHAR_RE, "_", basename)
     return os.path.splitext(basename)[0] + get_image_extension(mimetype)
 
 
diff --git a/sphinx/util/inventory.py b/sphinx/util/inventory.py
index 64f4ef4e3..9b647ccac 100644
--- a/sphinx/util/inventory.py
+++ b/sphinx/util/inventory.py
@@ -122,7 +122,7 @@ class InventoryFile:
 
         for line in stream.read_compressed_lines():
             # be careful to handle names with embedded spaces correctly
-            m = re.match(r'(?x)(.+?)\s+(\S*:\S*)\s+(-?\d+)\s+(\S+)\s+(.*)',
+            m = re.match(r'(?x)(.+?)\s+(\S*:\S*)\s+(-?\d+)\s+?(\S*)\s+(.*)',
                          line.rstrip())
             if not m:
                 continue
diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py
index aa7afd4a1..ba202b549 100644
--- a/sphinx/writers/html.py
+++ b/sphinx/writers/html.py
@@ -608,11 +608,7 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator):
                     atts['height'] = int(atts['height']) * scale
             atts['alt'] = node.get('alt', uri)
             if 'align' in node:
-                self.body.append('<div align="%s" class="align-%s">' %
-                                 (node['align'], node['align']))
-                self.context.append('</div>\n')
-            else:
-                self.context.append('')
+                atts['class'] = 'align-%s' % node['align']
             self.body.append(self.emptytag(node, 'img', '', **atts))
             return
 
@@ -621,7 +617,7 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator):
     # overwritten
     def depart_image(self, node: Element) -> None:
         if node['uri'].lower().endswith(('svg', 'svgz')):
-            self.body.append(self.context.pop())
+            pass
         else:
             super().depart_image(node)
 
diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py
index b8adb7ea3..a9f39cfdc 100644
--- a/sphinx/writers/html5.py
+++ b/sphinx/writers/html5.py
@@ -549,11 +549,7 @@ class HTML5Translator(SphinxTranslator, BaseTranslator):
                     atts['height'] = int(atts['height']) * scale
             atts['alt'] = node.get('alt', uri)
             if 'align' in node:
-                self.body.append('<div align="%s" class="align-%s">' %
-                                 (node['align'], node['align']))
-                self.context.append('</div>\n')
-            else:
-                self.context.append('')
+                atts['class'] = 'align-%s' % node['align']
             self.body.append(self.emptytag(node, 'img', '', **atts))
             return
 
@@ -562,7 +558,7 @@ class HTML5Translator(SphinxTranslator, BaseTranslator):
     # overwritten
     def depart_image(self, node: Element) -> None:
         if node['uri'].lower().endswith(('svg', 'svgz')):
-            self.body.append(self.context.pop())
+            pass
         else:
             super().depart_image(node)
 
diff --git a/tests/roots/test-builder-dirhtml/bar.rst b/tests/roots/test-builder-dirhtml/bar.rst
new file mode 100644
index 000000000..11f287a18
--- /dev/null
+++ b/tests/roots/test-builder-dirhtml/bar.rst
@@ -0,0 +1,4 @@
+.. _bar:
+
+bar
+===
diff --git a/tests/roots/test-builder-dirhtml/conf.py b/tests/roots/test-builder-dirhtml/conf.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/roots/test-builder-dirhtml/conf.py
diff --git a/tests/roots/test-builder-dirhtml/foo/foo_1.rst b/tests/roots/test-builder-dirhtml/foo/foo_1.rst
new file mode 100644
index 000000000..6db0ea57e
--- /dev/null
+++ b/tests/roots/test-builder-dirhtml/foo/foo_1.rst
@@ -0,0 +1,4 @@
+.. _foo_1:
+
+foo/foo_1
+=========
diff --git a/tests/roots/test-builder-dirhtml/foo/foo_2.rst b/tests/roots/test-builder-dirhtml/foo/foo_2.rst
new file mode 100644
index 000000000..fae7f26ef
--- /dev/null
+++ b/tests/roots/test-builder-dirhtml/foo/foo_2.rst
@@ -0,0 +1,4 @@
+.. _foo_2:
+
+foo/foo_2
+=========
diff --git a/tests/roots/test-builder-dirhtml/foo/index.rst b/tests/roots/test-builder-dirhtml/foo/index.rst
new file mode 100644
index 000000000..92d473c6b
--- /dev/null
+++ b/tests/roots/test-builder-dirhtml/foo/index.rst
@@ -0,0 +1,9 @@
+.. _foo:
+
+foo/index
+=========
+
+.. toctree::
+
+   foo_1
+   foo_2
diff --git a/tests/roots/test-builder-dirhtml/index.rst b/tests/roots/test-builder-dirhtml/index.rst
new file mode 100644
index 000000000..274e17793
--- /dev/null
+++ b/tests/roots/test-builder-dirhtml/index.rst
@@ -0,0 +1,9 @@
+.. _index:
+
+index
+=====
+
+.. toctree::
+
+   foo/index
+   bar
diff --git a/tests/test_build_dirhtml.py b/tests/test_build_dirhtml.py
new file mode 100644
index 000000000..715db1146
--- /dev/null
+++ b/tests/test_build_dirhtml.py
@@ -0,0 +1,47 @@
+"""
+    test_build_dirhtml
+    ~~~~~~~~~~~~~~~~~~
+
+    Test dirhtml builder.
+
+    :copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import posixpath
+
+import pytest
+
+from sphinx.util.inventory import InventoryFile
+
+
+@pytest.mark.sphinx(buildername='dirhtml', testroot='builder-dirhtml')
+def test_dirhtml(app, status, warning):
+    app.build()
+
+    assert (app.outdir / 'index.html').exists()
+    assert (app.outdir / 'foo/index.html').exists()
+    assert (app.outdir / 'foo/foo_1/index.html').exists()
+    assert (app.outdir / 'foo/foo_2/index.html').exists()
+    assert (app.outdir / 'bar/index.html').exists()
+
+    content = (app.outdir / 'index.html').text()
+    assert 'href="foo/"' in content
+    assert 'href="foo/foo_1/"' in content
+    assert 'href="foo/foo_2/"' in content
+    assert 'href="bar/"' in content
+
+    # objects.inv (refs: #7095)
+    f = (app.outdir / 'objects.inv').open('rb')
+    invdata = InventoryFile.load(f, 'path/to', posixpath.join)
+    assert 'index' in invdata.get('std:doc')
+    assert ('Python', '', 'path/to/', '-') == invdata['std:doc']['index']
+
+    assert 'foo/index' in invdata.get('std:doc')
+    assert ('Python', '', 'path/to/foo/', '-') == invdata['std:doc']['foo/index']
+
+    assert 'index' in invdata.get('std:label')
+    assert ('Python', '', 'path/to/#index', '-') == invdata['std:label']['index']
+
+    assert 'foo' in invdata.get('std:label')
+    assert ('Python', '', 'path/to/foo/#foo', 'foo/index') == invdata['std:label']['foo']