Merge branch '1.7'

26 files changed, 414 insertions, 156 deletions

diff --git a/.codecov.yml b/.codecov.yml
index 2ce4fda70..f6272f5f1 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -2,9 +2,7 @@ coverage:
   status:
     project:
       default:
-        # allowed to drop X% and still result in a "success" commit status
-        threshold: 0.05
+        enabled: no
     patch:
       default:
-        # allowed to drop X% and still result in a "success" commit status
-        threshold: 0.05
+        enabled: no
diff --git a/CHANGES b/CHANGES
index 78d42cce1..b1886518f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -71,6 +71,8 @@ Dependencies
 
 Incompatible changes
 --------------------
+* #4520: apidoc: folders with an empty __init__.py are no longer excluded from
+  TOC
 
 Deprecated
 ----------
@@ -95,6 +97,11 @@ Bugs fixed
 * #4720: message when an image is mismatched for builder is not clear
 * #4655, #4684: Incomplete localization strings in Polish and Chinese
 * #2286: Sphinx crashes when error is happens in rendering HTML pages
+* #4688: Error to download remote images having long URL
+* #4754: sphinx/pycode/__init__.py raises AttributeError
+* #1435: qthelp builder should htmlescape keywords
+* epub: Fix docTitle elements of toc.ncx is not escaped
+* #4520: apidoc: Subpackage not in toc (introduced in 1.6.6) now fixed
 
 Testing
 --------
diff --git a/doc/_themes/sphinx13/layout.html b/doc/_themes/sphinx13/layout.html
index ce6f08daa..597b6261f 100644
--- a/doc/_themes/sphinx13/layout.html
+++ b/doc/_themes/sphinx13/layout.html
@@ -13,6 +13,11 @@
 {% block sidebar1 %}{{ sidebar() }}{% endblock %}
 {% block sidebar2 %}{% endblock %}
 
+{% block linktags %}
+{{ super() }}
+    <link rel="canonical" href="http://www.sphinx-doc.org/en/master/{{ pagename }}{{ file_suffix }}" />
+{% endblock %}
+
 {% block extrahead %}
     <link href='https://fonts.googleapis.com/css?family=Open+Sans:300,400,700'
           rel='stylesheet' type='text/css' />
diff --git a/doc/man/sphinx-autogen.rst b/doc/man/sphinx-autogen.rst
index 49a8220d0..ef84afcb9 100644
--- a/doc/man/sphinx-autogen.rst
+++ b/doc/man/sphinx-autogen.rst
@@ -73,7 +73,7 @@ If you run the following:
 
 .. code-block:: bash
 
-    $ sphinx-autodoc doc/index.rst
+    $ PYTHONPATH=. sphinx-autodoc doc/index.rst
 
 then the following stub files will be created in ``docs``::
 
diff --git a/sphinx/builders/_epub_base.py b/sphinx/builders/_epub_base.py
index a2fa6d576..839844055 100644
--- a/sphinx/builders/_epub_base.py
+++ b/sphinx/builders/_epub_base.py
@@ -673,7 +673,7 @@ class EpubBuilder(StandaloneHTMLBuilder):
         """
         metadata = {}  # type: Dict[unicode, Any]
         metadata['uid'] = self.config.epub_uid
-        metadata['title'] = self.config.epub_title
+        metadata['title'] = self.esc(self.config.epub_title)
         metadata['level'] = level
         metadata['navpoints'] = navpoints
         return metadata
diff --git a/sphinx/builders/qthelp.py b/sphinx/builders/qthelp.py
index 0539e18de..44b968111 100644
--- a/sphinx/builders/qthelp.py
+++ b/sphinx/builders/qthelp.py
@@ -19,6 +19,7 @@ from docutils import nodes
 from six import text_type
 
 from sphinx import addnodes
+from sphinx import package_dir
 from sphinx.builders.html import StandaloneHTMLBuilder
 from sphinx.config import string_classes
 from sphinx.environment.adapters.indexentries import IndexEntries
@@ -26,6 +27,7 @@ from sphinx.locale import __
 from sphinx.util import force_decode, logging
 from sphinx.util.osutil import make_filename
 from sphinx.util.pycompat import htmlescape
+from sphinx.util.template import SphinxRenderer
 
 if False:
     # For type annotation
@@ -40,67 +42,13 @@ _idpattern = re.compile(
     r'(?P<title>.+) (\((class in )?(?P<id>[\w\.]+)( (?P<descr>\w+))?\))$')
 
 
-# Qt Help Collection Project (.qhcp).
-# Is the input file for the help collection generator.
-# It contains references to compressed help files which should be
-# included in the collection.
-# It may contain various other information for customizing Qt Assistant.
-collection_template = u'''\
-<?xml version="1.0" encoding="utf-8" ?>
-<QHelpCollectionProject version="1.0">
-    <assistant>
-        <title>%(title)s</title>
-        <homePage>%(homepage)s</homePage>
-        <startPage>%(startpage)s</startPage>
-    </assistant>
-    <docFiles>
-        <generate>
-            <file>
-                <input>%(outname)s.qhp</input>
-                <output>%(outname)s.qch</output>
-            </file>
-        </generate>
-        <register>
-            <file>%(outname)s.qch</file>
-        </register>
-    </docFiles>
-</QHelpCollectionProject>
-'''
-
-# Qt Help Project (.qhp)
-# This is the input file for the help generator.
-# It contains the table of contents, indices and references to the
-# actual documentation files (*.html).
-# In addition it defines a unique namespace for the documentation.
-project_template = u'''\
-<?xml version="1.0" encoding="utf-8" ?>
-<QtHelpProject version="1.0">
-    <namespace>%(namespace)s</namespace>
-    <virtualFolder>doc</virtualFolder>
-    <customFilter name="%(project)s %(version)s">
-        <filterAttribute>%(outname)s</filterAttribute>
-        <filterAttribute>%(version)s</filterAttribute>
-    </customFilter>
-    <filterSection>
-        <filterAttribute>%(outname)s</filterAttribute>
-        <filterAttribute>%(version)s</filterAttribute>
-        <toc>
-            <section title="%(title)s" ref="%(masterdoc)s.html">
-%(sections)s
-            </section>
-        </toc>
-        <keywords>
-%(keywords)s
-        </keywords>
-        <files>
-%(files)s
-        </files>
-    </filterSection>
-</QtHelpProject>
-'''
-
 section_template = '<section title="%(title)s" ref="%(ref)s"/>'
-file_template = ' ' * 12 + '<file>%(filename)s</file>'
+
+
+def render_file(filename, **kwargs):
+    # type: (unicode, Any) -> unicode
+    pathname = os.path.join(package_dir, 'templates', 'qthelp', filename)
+    return SphinxRenderer.render_from_file(pathname, kwargs)
 
 
 class QtHelpBuilder(StandaloneHTMLBuilder):
@@ -184,24 +132,6 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
                 keywords.extend(self.build_keywords(title, refs, subitems))
         keywords = u'\n'.join(keywords)  # type: ignore
 
-        # files
-        if not outdir.endswith(os.sep):
-            outdir += os.sep
-        olen = len(outdir)
-        projectfiles = []
-        staticdir = path.join(outdir, '_static')
-        imagesdir = path.join(outdir, self.imagedir)
-        for root, dirs, files in os.walk(outdir):
-            resourcedir = root.startswith(staticdir) or \
-                root.startswith(imagesdir)
-            for fn in sorted(files):
-                if (resourcedir and not fn.endswith('.js')) or \
-                   fn.endswith('.html'):
-                    filename = path.join(root, fn)[olen:]
-                    projectfiles.append(file_template %
-                                        {'filename': htmlescape(filename)})
-        projectfiles = '\n'.join(projectfiles)  # type: ignore
-
         # it seems that the "namespace" may not contain non-alphanumeric
         # characters, and more than one successive dot, or leading/trailing
         # dots, are also forbidden
@@ -216,16 +146,13 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
 
         # write the project file
         with codecs.open(path.join(outdir, outname + '.qhp'), 'w', 'utf-8') as f:  # type: ignore  # NOQA
-            f.write(project_template % {
-                'outname': htmlescape(outname),
-                'title': htmlescape(self.config.html_title),
-                'version': htmlescape(self.config.version),
-                'project': htmlescape(self.config.project),
-                'namespace': htmlescape(nspace),
-                'masterdoc': htmlescape(self.config.master_doc),
-                'sections': sections,
-                'keywords': keywords,
-                'files': projectfiles})
+            body = render_file('project.qhp', outname=outname,
+                               title=self.config.html_title, version=self.config.version,
+                               project=self.config.project, namespace=nspace,
+                               master_doc=self.config.master_doc,
+                               sections=sections, keywords=keywords,
+                               files=self.get_project_files(outdir))
+            f.write(body)
 
         homepage = 'qthelp://' + posixpath.join(
             nspace, 'doc', self.get_target_uri(self.config.master_doc))
@@ -233,11 +160,10 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
 
         logger.info(__('writing collection project file...'))
         with codecs.open(path.join(outdir, outname + '.qhcp'), 'w', 'utf-8') as f:  # type: ignore  # NOQA
-            f.write(collection_template % {
-                'outname': htmlescape(outname),
-                'title': htmlescape(self.config.html_short_title),
-                'homepage': htmlescape(homepage),
-                'startpage': htmlescape(startpage)})
+            body = render_file('project.qhcp', outname=outname,
+                               title=self.config.html_short_title,
+                               homepage=homepage, startpage=startpage)
+            f.write(body)
 
     def isdocnode(self, node):
         # type: (nodes.Node) -> bool
@@ -299,11 +225,12 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
         else:
             id = None
 
+        nameattr = htmlescape(name, quote=True)
+        refattr = htmlescape(ref[1], quote=True)
         if id:
-            item = ' ' * 12 + '<keyword name="%s" id="%s" ref="%s"/>' % (
-                name, id, ref[1])
+            item = ' ' * 12 + '<keyword name="%s" id="%s" ref="%s"/>' % (nameattr, id, refattr)
         else:
-            item = ' ' * 12 + '<keyword name="%s" ref="%s"/>' % (name, ref[1])
+            item = ' ' * 12 + '<keyword name="%s" ref="%s"/>' % (nameattr, refattr)
         item.encode('ascii', 'xmlcharrefreplace')
         return item
 
@@ -311,7 +238,6 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
         # type: (unicode, List[Any], Any) -> List[unicode]
         keywords = []  # type: List[unicode]
 
-        title = htmlescape(title)
         # if len(refs) == 0: # XXX
         #     write_param('See Also', title)
         if len(refs) == 1:
@@ -331,6 +257,23 @@ class QtHelpBuilder(StandaloneHTMLBuilder):
 
         return keywords
 
+    def get_project_files(self, outdir):
+        # type: (unicode) -> List[unicode]
+        if not outdir.endswith(os.sep):
+            outdir += os.sep
+        olen = len(outdir)
+        project_files = []
+        staticdir = path.join(outdir, '_static')
+        imagesdir = path.join(outdir, self.imagedir)
+        for root, dirs, files in os.walk(outdir):
+            resourcedir = root.startswith((staticdir, imagesdir))
+            for fn in sorted(files):
+                if (resourcedir and not fn.endswith('.js')) or fn.endswith('.html'):
+                    filename = path.join(root, fn)[olen:]
+                    project_files.append(filename)
+
+        return project_files
+
 
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]
diff --git a/sphinx/ext/apidoc.py b/sphinx/ext/apidoc.py
index 9363387a4..ad024ea54 100644
--- a/sphinx/ext/apidoc.py
+++ b/sphinx/ext/apidoc.py
@@ -197,16 +197,16 @@ def shall_skip(module, opts, excludes=[]):
     if not opts.implicit_namespaces and not path.exists(module):
         return True
 
-    # skip it if there is nothing (or just \n or \r\n) in the file
-    if path.exists(module) and path.getsize(module) <= 2:
-        if os.path.basename(module) == '__init__.py':
-            # We only want to skip packages if they do not contain any
-            # .py files other than __init__.py.
-            basemodule = path.dirname(module)
-            for module in glob.glob(path.join(basemodule, '*.py')):
-                if not is_excluded(path.join(basemodule, module), excludes):
-                    return True
-        else:
+    # Are we a package (here defined as __init__.py, not the folder in itself)
+    if os.path.basename(module) == INITPY:
+        # Yes, check if we have any non-excluded modules at all here
+        all_skipped = True
+        basemodule = path.dirname(module)
+        for module in glob.glob(path.join(basemodule, '*.py')):
+            if not is_excluded(path.join(basemodule, module), excludes):
+                # There's a non-excluded module here, we won't skip
+                all_skipped = False
+        if all_skipped:
             return True
 
     # skip if it has a "private" name and this is selected
diff --git a/sphinx/pycode/__init__.py b/sphinx/pycode/__init__.py
index 04353e805..c4c055bf5 100644
--- a/sphinx/pycode/__init__.py
+++ b/sphinx/pycode/__init__.py
@@ -120,23 +120,3 @@ class ModuleAnalyzer(object):
             self.parse()
 
         return self.tags
-
-
-if __name__ == '__main__':
-    import time
-    import pprint
-    x0 = time.time()
-    # ma = ModuleAnalyzer.for_file(__file__.rstrip('c'), 'sphinx.builders.html')
-    ma = ModuleAnalyzer.for_file('sphinx/environment.py',
-                                 'sphinx.environment')
-    ma.tokenize()  # type: ignore
-    x1 = time.time()
-    ma.parse()
-    x2 = time.time()
-    # for (ns, name), doc in iteritems(ma.find_attr_docs()):
-    #     print '>>', ns, name
-    #     print '\n'.join(doc)
-    pprint.pprint(ma.find_tags())
-    x3 = time.time()
-    # print nodes.nice_repr(ma.parsetree, number2name)
-    print("tokenizing %.4f, parsing %.4f, finding %.4f" % (x1 - x0, x2 - x1, x3 - x2))
diff --git a/sphinx/templates/qthelp/project.qhcp b/sphinx/templates/qthelp/project.qhcp
new file mode 100644
index 000000000..fe12eaa14
--- /dev/null
+++ b/sphinx/templates/qthelp/project.qhcp
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<QHelpCollectionProject version="1.0">
+    <assistant>
+        <title>{{ title|e }}</title>
+        <homePage>{{ homepage|e }}</homePage>
+        <startPage>{{ startpage|e }}</startPage>
+    </assistant>
+    <docFiles>
+        <generate>
+            <file>
+                <input>{{ outname|e }}.qhp</input>
+                <output>{{ outname|e }}.qch</output>
+            </file>
+        </generate>
+        <register>
+            <file>{{ outname|e }}.qch</file>
+        </register>
+    </docFiles>
+</QHelpCollectionProject>
diff --git a/sphinx/templates/qthelp/project.qhp b/sphinx/templates/qthelp/project.qhp
new file mode 100644
index 000000000..53f999043
--- /dev/null
+++ b/sphinx/templates/qthelp/project.qhp
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<QtHelpProject version="1.0">
+    <namespace>{{ namespace|e }}</namespace>
+    <virtualFolder>doc</virtualFolder>
+    <customFilter name="{{ project|e }} {{ version|e }}">
+        <filterAttribute>{{ outname|e }}</filterAttribute>
+        <filterAttribute>{{ version|e }}</filterAttribute>
+    </customFilter>
+    <filterSection>
+        <filterAttribute>{{ outname|e }}</filterAttribute>
+        <filterAttribute>{{ version|e }}</filterAttribute>
+        <toc>
+            <section title="{{ title|e }}" ref="{{ master_doc|e }}.html">
+{{ sections }}
+            </section>
+        </toc>
+        <keywords>
+{{ keywords }}
+        </keywords>
+        <files>
+            {%- for filename in files %}
+            <file>{{ filename|e }}</file>
+            {%- endfor %}
+        </files>
+    </filterSection>
+</QtHelpProject>
diff --git a/sphinx/transforms/post_transforms/images.py b/sphinx/transforms/post_transforms/images.py
index 7e73332ff..44b099198 100644
--- a/sphinx/transforms/post_transforms/images.py
+++ b/sphinx/transforms/post_transforms/images.py
@@ -31,6 +31,8 @@ if False:
 
 logger = logging.getLogger(__name__)
 
+MAX_FILENAME_LEN = 32
+
 
 class BaseImageConverter(SphinxTransform):
     def apply(self):
@@ -67,16 +69,21 @@ class ImageDownloader(BaseImageConverter):
 
     def handle(self, node):
         # type: (nodes.Node) -> None
-        basename = os.path.basename(node['uri'])
-        if '?' in basename:
-            basename = basename.split('?')[0]
-        if basename == '':
-            basename = sha1(node['uri'].encode("utf-8")).hexdigest()
-        dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/",
-                                                             ord("&"): u"/"})
-        ensuredir(os.path.join(self.imagedir, dirname))
-        path = os.path.join(self.imagedir, dirname, basename)
         try:
+            basename = os.path.basename(node['uri'])
+            if '?' in basename:
+                basename = basename.split('?')[0]
+            if basename == '' or len(basename) > MAX_FILENAME_LEN:
+                filename, ext = os.path.splitext(node['uri'])
+                basename = sha1(filename.encode("utf-8")).hexdigest() + ext
+
+            dirname = node['uri'].replace('://', '/').translate({ord("?"): u"/",
+                                                                 ord("&"): u"/"})
+            if len(dirname) > MAX_FILENAME_LEN:
+                dirname = sha1(dirname.encode('utf-8')).hexdigest()
+            ensuredir(os.path.join(self.imagedir, dirname))
+            path = os.path.join(self.imagedir, dirname, basename)
+
             headers = {}
             if os.path.exists(path):
                 timestamp = ceil(os.stat(path).st_mtime)  # type: float
diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/roots/test-apidoc-subpackage-in-toc/parent/__init__.py
diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/__init__.py
diff --git a/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py
new file mode 100644
index 000000000..810c96eee
--- /dev/null
+++ b/tests/roots/test-apidoc-subpackage-in-toc/parent/child/foo.py
@@ -0,0 +1 @@
+"foo"
diff --git a/tests/roots/test-need-escaped/bar.rst b/tests/roots/test-need-escaped/bar.rst
new file mode 100644
index 000000000..1cccd3cb7
--- /dev/null
+++ b/tests/roots/test-need-escaped/bar.rst
@@ -0,0 +1,2 @@
+bar
+===
diff --git a/tests/roots/test-need-escaped/baz.rst b/tests/roots/test-need-escaped/baz.rst
new file mode 100644
index 000000000..52e2e72ac
--- /dev/null
+++ b/tests/roots/test-need-escaped/baz.rst
@@ -0,0 +1,2 @@
+baz
+===
diff --git a/tests/roots/test-need-escaped/conf.py b/tests/roots/test-need-escaped/conf.py
new file mode 100644
index 000000000..d65a22e07
--- /dev/null
+++ b/tests/roots/test-need-escaped/conf.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+master_doc = 'index'
+project = 'need <b>"escaped"</b> project'
+smartquotes = False
diff --git a/tests/roots/test-need-escaped/foo.rst b/tests/roots/test-need-escaped/foo.rst
new file mode 100644
index 000000000..70859b3fc
--- /dev/null
+++ b/tests/roots/test-need-escaped/foo.rst
@@ -0,0 +1,15 @@
+<foo>
+=====
+
+.. toctree::
+
+   quux
+
+foo "1"
+-------
+
+foo.1-1
+^^^^^^^
+
+foo.2
+-----
diff --git a/tests/roots/test-need-escaped/index.rst b/tests/roots/test-need-escaped/index.rst
new file mode 100644
index 000000000..9ef74e00a
--- /dev/null
+++ b/tests/roots/test-need-escaped/index.rst
@@ -0,0 +1,30 @@
+.. Sphinx Tests documentation master file, created by sphinx-quickstart on Wed Jun  4 23:49:58 2008.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Sphinx Tests's documentation!
+========================================
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+   :numbered:
+   :caption: Table of Contents
+   :name: mastertoc
+
+   foo
+   bar
+   http://sphinx-doc.org/
+   baz
+   qux
+
+.. index::
+   pair: "subsection"; <subsection>
+
+----------
+subsection
+----------
+
+subsubsection
+-------------
diff --git a/tests/roots/test-need-escaped/quux.rst b/tests/roots/test-need-escaped/quux.rst
new file mode 100644
index 000000000..07dd0a0a3
--- /dev/null
+++ b/tests/roots/test-need-escaped/quux.rst
@@ -0,0 +1,2 @@
+quux
+====
diff --git a/tests/roots/test-need-escaped/qux.rst b/tests/roots/test-need-escaped/qux.rst
new file mode 100644
index 000000000..26176b947
--- /dev/null
+++ b/tests/roots/test-need-escaped/qux.rst
@@ -0,0 +1 @@
+qux.rst has no section title
diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py
index 5505891b2..881df1fda 100644
--- a/tests/test_autodoc.py
+++ b/tests/test_autodoc.py
@@ -11,13 +11,15 @@
 """
 
 import sys
+from warnings import catch_warnings
 
 import pytest
 from docutils.statemachine import ViewList
 from six import PY3
 
-from sphinx.ext.autodoc import AutoDirective, add_documenter, \
-    ModuleLevelDocumenter, FunctionDocumenter, cut_lines, between, ALL
+from sphinx.ext.autodoc import (
+    AutoDirective, ModuleLevelDocumenter, FunctionDocumenter, cut_lines, between, ALL
+)
 from sphinx.testing.util import SphinxTestApp, Struct  # NOQA
 from sphinx.util import logging
 
@@ -550,7 +552,7 @@ def test_new_documenter():
         def document_members(self, all_members=False):
             return
 
-    add_documenter(MyDocumenter)
+    app.add_autodocumenter(MyDocumenter)
 
     def assert_result_contains(item, objtype, name, **kw):
         app._warning.truncate(0)
@@ -591,12 +593,13 @@ def test_attrgetter_using():
             assert fullname not in documented_members, \
                 '%r was not hooked by special_attrgetter function' % fullname
 
-    options.members = ALL
-    options.inherited_members = False
-    assert_getter_works('class', 'target.Class', Class, ['meth'])
+    with catch_warnings(record=True):
+        options.members = ALL
+        options.inherited_members = False
+        assert_getter_works('class', 'target.Class', Class, ['meth'])
 
-    options.inherited_members = True
-    assert_getter_works('class', 'target.Class', Class, ['meth', 'inheritedmeth'])
+        options.inherited_members = True
+        assert_getter_works('class', 'target.Class', Class, ['meth', 'inheritedmeth'])
 
 
 @pytest.mark.usefixtures('setup_test')
diff --git a/tests/test_build_epub.py b/tests/test_build_epub.py
index 3256fcb9f..2f09f6d5a 100644
--- a/tests/test_build_epub.py
+++ b/tests/test_build_epub.py
@@ -19,17 +19,17 @@ import pytest
 # check given command is runnable
 def runnable(command):
     try:
-        p = Popen(command, stdout=PIPE)
+        p = Popen(command, stdout=PIPE, stderr=PIPE)
     except OSError:
         # command not found
         return False
     else:
         p.communicate()
-        return p.returncode
+        return p.returncode == 0
 
 
 class EPUBElementTree(object):
-    """Test helper for content.opf and tox.ncx"""
+    """Test helper for content.opf and toc.ncx"""
     namespaces = {
         'idpf': 'http://www.idpf.org/2007/opf',
         'dc': 'http://purl.org/dc/elements/1.1/',
@@ -226,6 +226,62 @@ def test_nested_toc(app):
     assert navinfo(grandchild[0]) == ('foo.xhtml#foo-1-1', 'foo.1-1')
 
 
+@pytest.mark.sphinx('epub', testroot='need-escaped')
+def test_escaped_toc(app):
+    app.build()
+
+    # toc.ncx
+    toc = EPUBElementTree.fromstring((app.outdir / 'toc.ncx').bytes())
+    assert toc.find("./ncx:docTitle/ncx:text").text == ('need <b>"escaped"</b> '
+                                                        'project  documentation')
+
+    # toc.ncx / navPoint
+    def navinfo(elem):
+        label = elem.find("./ncx:navLabel/ncx:text")
+        content = elem.find("./ncx:content")
+        return (elem.get('id'), elem.get('playOrder'),
+                content.get('src'), label.text)
+
+    navpoints = toc.findall("./ncx:navMap/ncx:navPoint")
+    assert len(navpoints) == 4
+    assert navinfo(navpoints[0]) == ('navPoint1', '1', 'index.xhtml',
+                                     u"Welcome to Sphinx Tests's documentation!")
+    assert navpoints[0].findall("./ncx:navPoint") == []
+
+    # toc.ncx / nested navPoints
+    assert navinfo(navpoints[1]) == ('navPoint2', '2', 'foo.xhtml', '<foo>')
+    navchildren = navpoints[1].findall("./ncx:navPoint")
+    assert len(navchildren) == 4
+    assert navinfo(navchildren[0]) == ('navPoint3', '2', 'foo.xhtml', '<foo>')
+    assert navinfo(navchildren[1]) == ('navPoint4', '3', 'quux.xhtml', 'quux')
+    assert navinfo(navchildren[2]) == ('navPoint5', '4', 'foo.xhtml#foo-1', u'foo “1”')
+    assert navinfo(navchildren[3]) == ('navPoint8', '6', 'foo.xhtml#foo-2', 'foo.2')
+
+    # nav.xhtml / nav
+    def navinfo(elem):
+        anchor = elem.find("./xhtml:a")
+        return (anchor.get('href'), anchor.text)
+
+    nav = EPUBElementTree.fromstring((app.outdir / 'nav.xhtml').bytes())
+    toc = nav.findall("./xhtml:body/xhtml:nav/xhtml:ol/xhtml:li")
+    assert len(toc) == 4
+    assert navinfo(toc[0]) == ('index.xhtml',
+                               "Welcome to Sphinx Tests's documentation!")
+    assert toc[0].findall("./xhtml:ol") == []
+
+    # nav.xhtml / nested toc
+    assert navinfo(toc[1]) == ('foo.xhtml', '<foo>')
+    tocchildren = toc[1].findall("./xhtml:ol/xhtml:li")
+    assert len(tocchildren) == 3
+    assert navinfo(tocchildren[0]) == ('quux.xhtml', 'quux')
+    assert navinfo(tocchildren[1]) == ('foo.xhtml#foo-1', u'foo “1”')
+    assert navinfo(tocchildren[2]) == ('foo.xhtml#foo-2', 'foo.2')
+
+    grandchild = tocchildren[1].findall("./xhtml:ol/xhtml:li")
+    assert len(grandchild) == 1
+    assert navinfo(grandchild[0]) == ('foo.xhtml#foo-1-1', 'foo.1-1')
+
+
 @pytest.mark.sphinx('epub', testroot='basic')
 def test_epub_writing_mode(app):
     # horizontal (default)
@@ -266,7 +322,7 @@ def test_run_epubcheck(app):
     app.build()
 
     epubcheck = os.environ.get('EPUBCHECK_PATH', '/usr/share/java/epubcheck.jar')
-    if runnable('java') and os.path.exists(epubcheck):
+    if runnable(['java', '-version']) and os.path.exists(epubcheck):
         p = Popen(['java', '-jar', epubcheck, app.outdir / 'SphinxTests.epub'],
                   stdout=PIPE, stderr=PIPE)
         stdout, stderr = p.communicate()
diff --git a/tests/test_build_qthelp.py b/tests/test_build_qthelp.py
index de676e6e0..a0d4fcf2a 100644
--- a/tests/test_build_qthelp.py
+++ b/tests/test_build_qthelp.py
@@ -13,16 +13,107 @@
 
 import pytest
 
+from sphinx.testing.util import etree_parse
+
+
+@pytest.mark.sphinx('qthelp', testroot='basic')
+def test_qthelp_basic(app, status, warning):
+    app.builder.build_all()
+
+    qhp = (app.outdir / 'Python.qhp').text()
+    assert '<customFilter name="Python ">' in qhp
+    assert '<filterAttribute>Python</filterAttribute>' in qhp
+    assert '<filterAttribute></filterAttribute>' in qhp
+    assert '<section title="Python  documentation" ref="index.html">' in qhp
+    assert '<file>genindex.html</file>' in qhp
+    assert '<file>index.html</file>' in qhp
+    assert '<file>_static/basic.css</file>' in qhp
+    assert '<file>_static/down.png</file>' in qhp
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<title>Python  documentation</title>' in qhcp
+    assert '<homePage>qthelp://org.sphinx.python/doc/index.html</homePage>' in qhcp
+    assert '<startPage>qthelp://org.sphinx.python/doc/index.html</startPage>' in qhcp
+    assert '<input>Python.qhp</input>' in qhcp
+    assert '<output>Python.qch</output>' in qhcp
+    assert '<file>Python.qch</file>' in qhcp
+
+
+@pytest.mark.sphinx('qthelp', testroot='need-escaped')
+def test_qthelp_escaped(app, status, warning):
+    app.builder.build_all()
+
+    et = etree_parse(app.outdir / 'needbescapedbproject.qhp')
+    customFilter = et.find('.//customFilter')
+    assert len(customFilter) == 2
+    assert customFilter.attrib == {'name': 'need <b>"escaped"</b> project '}
+    assert customFilter[0].text == 'needbescapedbproject'
+    assert customFilter[1].text is None
+
+    toc = et.find('.//toc')
+    assert len(toc) == 1
+    assert toc[0].attrib == {'title': 'need <b>"escaped"</b> project  documentation',
+                             'ref': 'index.html'}
+    assert len(toc[0]) == 4
+    assert toc[0][0].attrib == {'title': '<foo>', 'ref': 'foo.html'}
+    assert toc[0][0][0].attrib == {'title': 'quux', 'ref': 'quux.html'}
+    assert toc[0][0][1].attrib == {'title': 'foo "1"', 'ref': 'foo.html#foo-1'}
+    assert toc[0][0][1][0].attrib == {'title': 'foo.1-1', 'ref': 'foo.html#foo-1-1'}
+    assert toc[0][0][2].attrib == {'title': 'foo.2', 'ref': 'foo.html#foo-2'}
+    assert toc[0][1].attrib == {'title': 'bar', 'ref': 'bar.html'}
+    assert toc[0][2].attrib == {'title': 'http://sphinx-doc.org/',
+                                'ref': 'http://sphinx-doc.org/'}
+    assert toc[0][3].attrib == {'title': 'baz', 'ref': 'baz.html'}
+
+    keywords = et.find('.//keywords')
+    assert len(keywords) == 2
+    assert keywords[0].attrib == {'name': '<subsection>', 'ref': 'index.html#index-0'}
+    assert keywords[1].attrib == {'name': '"subsection"', 'ref': 'index.html#index-0'}
+
 
 @pytest.mark.sphinx('qthelp', testroot='basic')
 def test_qthelp_namespace(app, status, warning):
     # default namespace
     app.builder.build_all()
+
     qhp = (app.outdir / 'Python.qhp').text()
     assert '<namespace>org.sphinx.python</namespace>' in qhp
 
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<homePage>qthelp://org.sphinx.python/doc/index.html</homePage>' in qhcp
+    assert '<startPage>qthelp://org.sphinx.python/doc/index.html</startPage>' in qhcp
+
     # give a namespace
     app.config.qthelp_namespace = 'org.sphinx-doc.sphinx'
     app.builder.build_all()
+
     qhp = (app.outdir / 'Python.qhp').text()
     assert '<namespace>org.sphinxdoc.sphinx</namespace>' in qhp
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<homePage>qthelp://org.sphinxdoc.sphinx/doc/index.html</homePage>' in qhcp
+    assert '<startPage>qthelp://org.sphinxdoc.sphinx/doc/index.html</startPage>' in qhcp
+
+
+@pytest.mark.sphinx('qthelp', testroot='basic')
+def test_qthelp_title(app, status, warning):
+    # default title
+    app.builder.build_all()
+
+    qhp = (app.outdir / 'Python.qhp').text()
+    assert '<section title="Python  documentation" ref="index.html">' in qhp
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<title>Python  documentation</title>' in qhcp
+
+    # give a title
+    app.config.html_title = 'Sphinx <b>"full"</b> title'
+    app.config.html_short_title = 'Sphinx <b>"short"</b> title'
+    app.builder.build_all()
+
+    qhp = (app.outdir / 'Python.qhp').text()
+    assert ('<section title="Sphinx &lt;b&gt;&#34;full&#34;&lt;/b&gt; title" ref="index.html">'
+            in qhp)
+
+    qhcp = (app.outdir / 'Python.qhcp').text()
+    assert '<title>Sphinx &lt;b&gt;&#34;short&#34;&lt;/b&gt; title</title>' in qhcp
diff --git a/tests/test_ext_apidoc.py b/tests/test_ext_apidoc.py
index 836c20b04..d3d61d1e0 100644
--- a/tests/test_ext_apidoc.py
+++ b/tests/test_ext_apidoc.py
@@ -211,7 +211,7 @@ def test_trailing_underscore(make_app, apidoc):
 
 @pytest.mark.apidoc(
     coderoot='test-apidoc-pep420/a',
-    excludes=["b/c/d.py", "b/e/f.py"],
+    excludes=["b/c/d.py", "b/e/f.py", "b/e/__init__.py"],
     options=["--implicit-namespaces", "--separate"],
 )
 def test_excludes(apidoc):
@@ -224,6 +224,45 @@ def test_excludes(apidoc):
 
 
 @pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_subpackage_should_be_skipped(apidoc):
+    """Subpackage exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because 'b/e' subpackage is skipped
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=["b/e/f.py"],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_be_skipped(apidoc):
+    """Module exclusion should work."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert not (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-pep420/a',
+    excludes=[],
+    options=["--implicit-namespaces", "--separate"],
+)
+def test_excludes_module_should_not_be_skipped(apidoc):
+    """Module should be included if no excludes are used."""
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+    assert (outdir / 'a.b.c.rst').isfile()  # generated because not empty
+    assert (outdir / 'a.b.e.f.rst').isfile()  # skipped because of empty after excludes
+
+
+@pytest.mark.apidoc(
     coderoot='test-root',
     options=[
         '--doc-project', u'プロジェクト名',
@@ -339,3 +378,29 @@ def extract_toc(path):
     toctree = rst[start_idx + len(toctree_start):end_idx]
 
     return toctree
+
+
+@pytest.mark.apidoc(
+    coderoot='test-apidoc-subpackage-in-toc',
+    options=['--separate']
+)
+
+
+def test_subpackage_in_toc(make_app, apidoc):
+    """Make sure that empty subpackages with non-empty subpackages in them
+       are not skipped (issue #4520)
+    """
+    outdir = apidoc.outdir
+    assert (outdir / 'conf.py').isfile()
+
+    assert (outdir / 'parent.rst').isfile()
+    with open(outdir / 'parent.rst') as f:
+        parent = f.read()
+    assert 'parent.child' in parent
+
+    assert (outdir / 'parent.child.rst').isfile()
+    with open(outdir / 'parent.child.rst') as f:
+        parent_child = f.read()
+    assert 'parent.child.foo' in parent_child
+
+    assert (outdir / 'parent.child.foo.rst').isfile()
diff --git a/tox.ini b/tox.ini
index b8d6a0e32..16ea8f9ba 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@ envlist = docs,flake8,mypy,coverage,py{27,34,35,36,py},du{11,12,13,14}
 [testenv]
 usedevelop = True
 passenv =
-    https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS
+    https_proxy http_proxy no_proxy PERL PERL5LIB PYTEST_ADDOPTS EPUBCHECK_PATH
 description =
     py{27,34,35,36,py}: Run unit tests against {envname}.
     du{11,12,13,14}: Run unit tests with the given version of docutils.