[1.8] htmlhelp: convert hex escaping to decimal escaping in .hhc/.hhk files (#5853)

* htmlhelp: convert hex escaping to decimal escaping in .hhc/.hhk files

.hhc/.hhk files don't recognize hex escaping, we need convert hex escaping to decimal escaping. for example: `'` -> `'`.

6 files changed, 155 insertions, 3 deletions

diff --git a/CHANGES b/CHANGES
index 3115ad944..78bc97b23 100644
--- a/CHANGES
+++ b/CHANGES
@@ -37,6 +37,7 @@ Bugs fixed
   possibility to use original meaning in place of Sphinx custom one
 * #5834: apidoc: wrong help for ``--tocfile``
 * #5800: todo: crashed if todo is defined in TextElement
+* #5846: htmlhelp: convert hex escaping to decimal escaping in .hhc/.hhk files
 
 Testing
 --------
diff --git a/sphinx/builders/htmlhelp.py b/sphinx/builders/htmlhelp.py
index 49d48361e..42ee05c4a 100644
--- a/sphinx/builders/htmlhelp.py
+++ b/sphinx/builders/htmlhelp.py
@@ -13,6 +13,7 @@ from __future__ import print_function
 
 import codecs
 import os
+import re
 from os import path
 
 from docutils import nodes
@@ -27,7 +28,7 @@ from sphinx.util.pycompat import htmlescape
 
 if False:
     # For type annotation
-    from typing import Any, Dict, IO, List, Tuple  # NOQA
+    from typing import Any, Dict, IO, List, Match, Tuple  # NOQA
     from sphinx.application import Sphinx  # NOQA
 
 
@@ -169,6 +170,24 @@ chm_locales = {
 }
 
 
+def chm_htmlescape(*args, **kwargs):
+    # type: (*Any, **Any) -> unicode
+    """
+    chm_htmlescape() is a wrapper of htmlescape().
+    .hhc/.hhk files don't recognize hex escaping, we need convert
+    hex escaping to decimal escaping. for example: `'` -> `'`
+    htmlescape() may generates a hex escaping `'` for single
+    quote `'`, this wrapper fixes this.
+    """
+    def convert(matchobj):
+        # type: (Match[unicode]) -> unicode
+        codepoint = int(matchobj.group(1), 16)
+        return '&#%d;' % codepoint
+    return re.sub(r'&#[xX]([0-9a-fA-F]+);',
+                  convert,
+                  htmlescape(*args, **kwargs))
+
+
 class HTMLHelpBuilder(StandaloneHTMLBuilder):
     """
     Builder that also outputs Windows HTML help project, contents and
@@ -278,7 +297,7 @@ class HTMLHelpBuilder(StandaloneHTMLBuilder):
                         write_toc(subnode, ullevel)
                 elif isinstance(node, nodes.reference):
                     link = node['refuri']
-                    title = htmlescape(node.astext()).replace('"', '"')
+                    title = chm_htmlescape(node.astext()).replace('"', '"')
                     f.write(object_sitemap % (title, link))
                 elif isinstance(node, nodes.bullet_list):
                     if ullevel != 0:
@@ -311,7 +330,7 @@ class HTMLHelpBuilder(StandaloneHTMLBuilder):
                     item = '    <param name="%s" value="%s">\n' % \
                         (name, value)
                     f.write(item)
-                title = htmlescape(title)
+                title = chm_htmlescape(title)
                 f.write('<LI> <OBJECT type="text/sitemap">\n')
                 write_param('Keyword', title)
                 if len(refs) == 0:
diff --git a/tests/roots/test-build-htmlhelp/conf.py b/tests/roots/test-build-htmlhelp/conf.py
new file mode 100644
index 000000000..95293e956
--- /dev/null
+++ b/tests/roots/test-build-htmlhelp/conf.py
@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+project = 'test'
+master_doc = 'index'
diff --git a/tests/roots/test-build-htmlhelp/index.rst b/tests/roots/test-build-htmlhelp/index.rst
new file mode 100644
index 000000000..641c2467d
--- /dev/null
+++ b/tests/roots/test-build-htmlhelp/index.rst
@@ -0,0 +1,19 @@
+Index markup
+------------
+
+.. index::
+   single: entry
+   pair: entry; pair
+   double: entry; double
+   triple: index; entry; triple
+   keyword: with
+   see: from; to
+   seealso: fromalso; toalso
+
+.. index::
+   !Main, !Other
+   !single: entry; pair
+
+.. index:: triple-quoted string, Unicode Consortium, raw string
+   single: """; string literal
+   single: '''; string literal
\ No newline at end of file
diff --git a/tests/roots/test-build-htmlhelp/make.bat b/tests/roots/test-build-htmlhelp/make.bat
new file mode 100644
index 000000000..333fd1439
--- /dev/null
+++ b/tests/roots/test-build-htmlhelp/make.bat
@@ -0,0 +1,64 @@
+@echo off
+setlocal
+
+pushd %~dp0
+
+set this=%~n0
+
+if not defined PYTHON set PYTHON=py
+
+if not defined SPHINXBUILD (
+    %PYTHON% -c "import sphinx" > nul 2> nul
+    if errorlevel 1 (
+        echo Installing sphinx with %PYTHON%
+        %PYTHON% -m pip install sphinx
+        if errorlevel 1 exit /B
+    )
+    set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())"
+)
+
+rem Search for HHC in likely places
+set HTMLHELP=
+where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch
+where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
+if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
+if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc"
+if not exist "%HTMLHELP%" (
+    echo.
+    echo.The HTML Help Workshop was not found.  Set the HTMLHELP variable
+    echo.to the path to hhc.exe or download and install it from
+    echo.http://msdn.microsoft.com/en-us/library/ms669985
+    exit /B 1
+)
+echo hhc.exe path: %HTMLHELP%
+
+if "%BUILDDIR%" EQU "" set BUILDDIR=build
+
+%SPHINXBUILD% >nul 2> nul
+if errorlevel 9009 (
+    echo.
+    echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+    echo.installed, then set the SPHINXBUILD environment variable to point
+    echo.to the full path of the 'sphinx-build' executable. Alternatively you
+    echo.may add the Sphinx directory to PATH.
+    popd
+    exit /B 1
+)
+
+set SPHINXOPTS=-D html_theme_options.body_max_width=none %SPHINXOPTS%
+
+cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -bhtmlhelp -dbuild\doctrees . "%BUILDDIR%\htmlhelp"
+
+"%HTMLHELP%" "%BUILDDIR%\htmlhelp\test.hhp"
+rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2
+if not errorlevel 2 cmd /C exit /b 0
+
+echo.
+if errorlevel 1 (
+    echo.Build failed (exit code %ERRORLEVEL%^), check for error messages
+    echo.above.  Any output will be found in %BUILDDIR%\%1
+) else (
+    echo.Build succeeded. All output should be in %BUILDDIR%\%1
+)
+
+popd
diff --git a/tests/test_build_htmlhelp.py b/tests/test_build_htmlhelp.py
new file mode 100644
index 000000000..5a47ca580
--- /dev/null
+++ b/tests/test_build_htmlhelp.py
@@ -0,0 +1,45 @@
+"""
+    test_build_htmlhelp
+    ~~~~~~~~~~~~~~~~~~~
+    Test the HTML Help builder and check output against XPath.
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os.path
+import re
+import sys
+from subprocess import Popen, PIPE
+
+import pytest
+
+from sphinx.util.osutil import cd
+
+
+@pytest.mark.skipif(sys.platform != "win32",
+                    reason="hhc.exe only available on Windows.")
+@pytest.mark.sphinx('htmlhelp', testroot='build-htmlhelp')
+def test_chm():
+    # run make.bat
+    with cd(r".\roots\test-build-htmlhelp"):
+        try:
+            p = Popen(['make.bat'],
+                      stdout=PIPE, stderr=PIPE)
+        except:
+            raise
+        else:
+            p.communicate()
+
+    # check .hhk file
+    this_path = os.path.dirname(os.path.abspath(__file__))
+    hhk_file = os.path.join(this_path, 'roots', 'test-build-htmlhelp',
+                            'build', 'htmlhelp', 'test.hhk')
+    if not os.path.isfile(hhk_file):
+        print(".chm build failed, please install HTML Help Workshop.")
+        return
+
+    with open(hhk_file, 'rb') as f:
+        data = f.read()
+    m = re.search(br'&#[xX][0-9a-fA-F]+;', data)
+    assert m == None, 'Hex escaping exists in .hhk file: ' + str(m.group(0))
+