Enable math node rendering by default (without HTML builders)

Nowadays, math elements (inline and block level equations) are
integrated into reST spec by default.  But, in Sphinx, they are
not enabled by default.  For this reason, users have to enable
one of math extensions even if target builder supports math
elements directly.

This change starts to enable them by default.  As a first step,
this replaces math node and its structure by docutils based one.

16 files changed, 203 insertions, 63 deletions

diff --git a/CHANGES b/CHANGES
index faabb32ad..e2fa3d60b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -72,6 +72,8 @@ Deprecated
 * ``BuildEnvironment.dump()`` is deprecated
 * ``BuildEnvironment.dumps()`` is deprecated
 * ``BuildEnvironment.topickle()`` is deprecated
+* ``sphinx.ext.mathbase.math`` node is deprecated
+* ``sphinx.ext.mathbase.is_in_section_title()`` is deprecated
 
 For more details, see `deprecation APIs list
 <http://www.sphinx-doc.org/en/master/extdev/index.html#deprecated-apis>`_
diff --git a/doc/extdev/index.rst b/doc/extdev/index.rst
index b089bdcea..08de4b095 100644
--- a/doc/extdev/index.rst
+++ b/doc/extdev/index.rst
@@ -126,6 +126,16 @@ The following is a list of deprecated interface.
      - 4.0
      - :meth:`~sphinx.application.Sphinx.add_css_file()`
 
+   * - ``sphinx.ext.mathbase.is_in_section_title()``
+     - 1.8
+     - 3.0
+     - N/A
+
+   * - ``sphinx.ext.mathbase.math`` (node)
+     - 1.8
+     - 3.0
+     - ``docutils.nodes.math``
+
    * - ``viewcode_import`` (config value)
      - 1.8
      - 3.0
diff --git a/sphinx/addnodes.py b/sphinx/addnodes.py
index e6999bd16..56f23bba3 100644
--- a/sphinx/addnodes.py
+++ b/sphinx/addnodes.py
@@ -9,8 +9,12 @@
     :license: BSD, see LICENSE for details.
 """
 
+import warnings
+
 from docutils import nodes
 
+from sphinx.deprecation import RemovedInSphinx30Warning
+
 if False:
     # For type annotation
     from typing import List, Sequence  # NOQA
@@ -183,6 +187,27 @@ class production(nodes.Part, nodes.Inline, nodes.FixedTextElement):
     """Node for a single grammar production rule."""
 
 
+# math node
+
+
+class math(nodes.math):
+    """Node for inline equations.
+
+    .. deprecated:: 1.8
+       Use ``docutils.nodes.math`` instead.
+    """
+
+    def __getitem__(self, key):
+        """Special accessor for supporting ``node['latex']``."""
+        if key == 'latex' and 'latex' not in self.attributes:
+            warnings.warn("math node for Sphinx was replaced by docutils'. "
+                          "Therefore please use ``node.astext()`` to get an equation instead.",
+                          RemovedInSphinx30Warning)
+            return self.astext()
+        else:
+            return nodes.math.__getitem__(self, key)
+
+
 # other directive-level nodes
 
 class index(nodes.Invisible, nodes.Inline, nodes.TextElement):
diff --git a/sphinx/application.py b/sphinx/application.py
index 9d3d5de9f..f36135cf8 100644
--- a/sphinx/application.py
+++ b/sphinx/application.py
@@ -97,6 +97,7 @@ builtin_extensions = (
     'sphinx.roles',
     'sphinx.transforms.post_transforms',
     'sphinx.transforms.post_transforms.images',
+    'sphinx.transforms.post_transforms.compat',
     'sphinx.util.compat',
     # collectors should be loaded by specific order
     'sphinx.environment.collectors.dependencies',
diff --git a/sphinx/ext/imgmath.py b/sphinx/ext/imgmath.py
index a1faf2c1b..0e43abc74 100644
--- a/sphinx/ext/imgmath.py
+++ b/sphinx/ext/imgmath.py
@@ -37,7 +37,7 @@ if False:
     from sphinx.application import Sphinx  # NOQA
     from sphinx.builders import Builder  # NOQA
     from sphinx.config import Config  # NOQA
-    from sphinx.ext.mathbase import math as math_node, displaymath  # NOQA
+    from sphinx.ext.mathbase import displaymath  # NOQA
 
 logger = logging.getLogger(__name__)
 
@@ -285,27 +285,30 @@ def cleanup_tempdir(app, exc):
 
 
 def get_tooltip(self, node):
-    # type: (nodes.NodeVisitor, math_node) -> unicode
+    # type: (nodes.NodeVisitor, nodes.math) -> unicode
     if self.builder.config.imgmath_add_tooltips:
-        return ' alt="%s"' % self.encode(node['latex']).strip()
+        if len(node) == 1:
+            return ' alt="%s"' % self.encode(node.astext()).strip()
+        else:
+            return ' alt="%s"' % self.encode(node['latex']).strip()
     return ''
 
 
 def html_visit_math(self, node):
-    # type: (nodes.NodeVisitor, math_node) -> None
+    # type: (nodes.NodeVisitor, nodes.math) -> None
     try:
-        fname, depth = render_math(self, '$' + node['latex'] + '$')
+        fname, depth = render_math(self, '$' + node.astext() + '$')
     except MathExtError as exc:
         msg = text_type(exc)
         sm = nodes.system_message(msg, type='WARNING', level=2,
-                                  backrefs=[], source=node['latex'])
+                                  backrefs=[], source=node.astext())
         sm.walkabout(self)
-        logger.warning(__('display latex %r: %s'), node['latex'], msg)
+        logger.warning(__('display latex %r: %s'), node.astext(), msg)
         raise nodes.SkipNode
     if fname is None:
         # something failed -- use text-only as a bad substitute
         self.body.append('<span class="math">%s</span>' %
-                         self.encode(node['latex']).strip())
+                         self.encode(node.astext()).strip())
     else:
         c = ('<img class="math" src="%s"' % fname) + get_tooltip(self, node)
         if depth is not None:
diff --git a/sphinx/ext/jsmath.py b/sphinx/ext/jsmath.py
index 97ea400a3..fa9eb7913 100644
--- a/sphinx/ext/jsmath.py
+++ b/sphinx/ext/jsmath.py
@@ -27,7 +27,7 @@ if False:
 def html_visit_math(self, node):
     # type: (nodes.NodeVisitor, nodes.Node) -> None
     self.body.append(self.starttag(node, 'span', '', CLASS='math notranslate nohighlight'))
-    self.body.append(self.encode(node['latex']) + '</span>')
+    self.body.append(self.encode(node.astext()) + '</span>')
     raise nodes.SkipNode
 
 
diff --git a/sphinx/ext/mathbase.py b/sphinx/ext/mathbase.py
index e6a6929e6..d40d461c2 100644
--- a/sphinx/ext/mathbase.py
+++ b/sphinx/ext/mathbase.py
@@ -9,11 +9,15 @@
     :license: BSD, see LICENSE for details.
 """
 
-from docutils import nodes, utils
+import warnings
+
+from docutils import nodes
 from docutils.nodes import make_id
 from docutils.parsers.rst import directives
 
+from sphinx.addnodes import math
 from sphinx.config import string_classes
+from sphinx.deprecation import RemovedInSphinx30Warning
 from sphinx.domains import Domain
 from sphinx.locale import __
 from sphinx.roles import XRefRole
@@ -33,10 +37,6 @@ if False:
 logger = logging.getLogger(__name__)
 
 
-class math(nodes.Inline, nodes.TextElement):
-    pass
-
-
 class displaymath(nodes.Part, nodes.Element):
     pass
 
@@ -195,17 +195,14 @@ def wrap_displaymath(math, label, numbering):
     return '%s\n%s%s' % (begin, ''.join(equations), end)
 
 
-def math_role(role, rawtext, text, lineno, inliner, options={}, content=[]):
-    # type: (unicode, unicode, unicode, int, Inliner, Dict, List[unicode]) -> Tuple[List[nodes.Node], List[nodes.Node]]  # NOQA
-    latex = utils.unescape(text, restore_backslashes=True)
-    return [math(latex=latex)], []
-
-
 def is_in_section_title(node):
     # type: (nodes.Node) -> bool
     """Determine whether the node is in a section title"""
     from sphinx.util.nodes import traverse_parent
 
+    warnings.warn('is_in_section_title() is deprecated.',
+                  RemovedInSphinx30Warning)
+
     for ancestor in traverse_parent(node):
         if isinstance(ancestor, nodes.title) and \
            isinstance(ancestor.parent, nodes.section):
@@ -275,17 +272,6 @@ class MathDirective(SphinxDirective):
             self.state_machine.reporter.warning(exc.args[0], line=self.lineno)
 
 
-def latex_visit_math(self, node):
-    # type: (nodes.NodeVisitor, math) -> None
-    if is_in_section_title(node):
-        protect = r'\protect'
-    else:
-        protect = ''
-    equation = protect + r'\(' + node['latex'] + protect + r'\)'
-    self.body.append(equation)
-    raise nodes.SkipNode
-
-
 def latex_visit_displaymath(self, node):
     # type: (nodes.NodeVisitor, displaymath) -> None
     if not node['label']:
@@ -320,12 +306,6 @@ def latex_visit_eqref(self, node):
     raise nodes.SkipNode
 
 
-def text_visit_math(self, node):
-    # type: (nodes.NodeVisitor, math) -> None
-    self.add_text(node['latex'])
-    raise nodes.SkipNode
-
-
 def text_visit_displaymath(self, node):
     # type: (nodes.NodeVisitor, displaymath) -> None
     self.new_state()
@@ -334,12 +314,6 @@ def text_visit_displaymath(self, node):
     raise nodes.SkipNode
 
 
-def man_visit_math(self, node):
-    # type: (nodes.NodeVisitor, math) -> None
-    self.body.append(node['latex'])
-    raise nodes.SkipNode
-
-
 def man_visit_displaymath(self, node):
     # type: (nodes.NodeVisitor, displaymath) -> None
     self.visit_centered(node)
@@ -350,12 +324,6 @@ def man_depart_displaymath(self, node):
     self.depart_centered(node)
 
 
-def texinfo_visit_math(self, node):
-    # type: (nodes.NodeVisitor, math) -> None
-    self.body.append('@math{' + self.escape_arg(node['latex']) + '}')
-    raise nodes.SkipNode
-
-
 def texinfo_visit_displaymath(self, node):
     # type: (nodes.NodeVisitor, displaymath) -> None
     if node.get('label'):
@@ -376,10 +344,6 @@ def setup_math(app, htmlinlinevisitors, htmldisplayvisitors):
     app.add_config_value('math_numfig', True, 'env')
     app.add_domain(MathDomain)
     app.add_node(math, override=True,
-                 latex=(latex_visit_math, None),
-                 text=(text_visit_math, None),
-                 man=(man_visit_math, None),
-                 texinfo=(texinfo_visit_math, None),
                  html=htmlinlinevisitors)
     app.add_node(displaymath,
                  latex=(latex_visit_displaymath, None),
@@ -388,6 +352,5 @@ def setup_math(app, htmlinlinevisitors, htmldisplayvisitors):
                  texinfo=(texinfo_visit_displaymath, texinfo_depart_displaymath),
                  html=htmldisplayvisitors)
     app.add_node(eqref, latex=(latex_visit_eqref, None))
-    app.add_role('math', math_role)
     app.add_role('eq', EqXRefRole(warn_dangling=True))
     app.add_directive('math', MathDirective)
diff --git a/sphinx/ext/mathjax.py b/sphinx/ext/mathjax.py
index 2bb7eec09..f6566a583 100644
--- a/sphinx/ext/mathjax.py
+++ b/sphinx/ext/mathjax.py
@@ -29,7 +29,7 @@ def html_visit_math(self, node):
     # type: (nodes.NodeVisitor, nodes.Node) -> None
     self.body.append(self.starttag(node, 'span', '', CLASS='math notranslate nohighlight'))
     self.body.append(self.builder.config.mathjax_inline[0] +
-                     self.encode(node['latex']) +
+                     self.encode(node.astext()) +
                      self.builder.config.mathjax_inline[1] + '</span>')
     raise nodes.SkipNode
 
diff --git a/sphinx/transforms/post_transforms/compat.py b/sphinx/transforms/post_transforms/compat.py
new file mode 100644
index 000000000..0f0b2367d
--- /dev/null
+++ b/sphinx/transforms/post_transforms/compat.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.transforms.post_transforms.compat
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Post transforms for compatibility
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import warnings
+from typing import TYPE_CHECKING
+
+from docutils import nodes
+
+from sphinx.deprecation import RemovedInSphinx30Warning
+from sphinx.transforms import SphinxTransform
+from sphinx.util import logging
+
+if TYPE_CHECKING:
+    from typing import Any, Callable, Dict, Iterable, List, Tuple  # NOQA
+    from docutils.parsers.rst.states import Inliner  # NOQA
+    from docutils.writers.html4css1 import Writer  # NOQA
+    from sphinx.application import Sphinx  # NOQA
+    from sphinx.builders import Builder  # NOQA
+    from sphinx.environment import BuildEnvironment  # NOQA
+
+logger = logging.getLogger(__name__)
+
+
+class MathNodeMigrator(SphinxTransform):
+    """Migrate a math node to docutils'.
+
+    For a long time, Sphinx uses an original node for math. Since 1.8,
+    Sphinx starts to use a math node of docutils'.  This transform converts
+    old and new nodes to keep compatibility.
+    """
+    default_priority = 999
+
+    def apply(self):
+        # type: () -> None
+        for node in self.document.traverse(nodes.math):
+            if len(node) == 0:
+                # convert an old styled node to new one
+                warnings.warn("math node for Sphinx was replaced by docutils'. "
+                              "Please use ``docutils.nodes.math`` instead.",
+                              RemovedInSphinx30Warning)
+                equation = node['latex']
+                node += nodes.Text(equation, equation)
+
+
+def setup(app):
+    app.add_post_transform(MathNodeMigrator)
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index 6c4b66881..15f72566b 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -2541,14 +2541,20 @@ class LaTeXTranslator(nodes.NodeVisitor):
 
     def visit_math(self, node):
         # type: (nodes.Node) -> None
+        if self.in_title:
+            self.body.append(r'\protect\(%s\protect\)' % node.astext())
+        else:
+            self.body.append(r'\(%s\)' % node.astext())
+        raise nodes.SkipNode
+
+    def visit_math_block(self, node):
+        # type: (nodes.Node) -> None
         logger.warning(__('using "math" markup without a Sphinx math extension '
                           'active, please use one of the math extensions '
                           'described at http://sphinx-doc.org/en/master/ext/math.html'),
                        location=(self.curfilestack[-1], node.line))
         raise nodes.SkipNode
 
-    visit_math_block = visit_math
-
     def unknown_visit(self, node):
         # type: (nodes.Node) -> None
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
diff --git a/sphinx/writers/manpage.py b/sphinx/writers/manpage.py
index c6c8723dd..1f49ced9e 100644
--- a/sphinx/writers/manpage.py
+++ b/sphinx/writers/manpage.py
@@ -513,13 +513,19 @@ class ManualPageTranslator(BaseTranslator):
 
     def visit_math(self, node):
         # type: (nodes.Node) -> None
+        pass
+
+    def depart_math(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_math_block(self, node):
+        # type: (nodes.Node) -> None
         logger.warning(__('using "math" markup without a Sphinx math extension '
                           'active, please use one of the math extensions '
                           'described at http://sphinx-doc.org/en/master/ext/math.html'))
         raise nodes.SkipNode
 
-    visit_math_block = visit_math
-
     def unknown_visit(self, node):
         # type: (nodes.Node) -> None
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
diff --git a/sphinx/writers/texinfo.py b/sphinx/writers/texinfo.py
index e58d659ab..3b2dd8915 100644
--- a/sphinx/writers/texinfo.py
+++ b/sphinx/writers/texinfo.py
@@ -1731,9 +1731,12 @@ class TexinfoTranslator(nodes.NodeVisitor):
 
     def visit_math(self, node):
         # type: (nodes.Node) -> None
+        self.body.append('@math{' + self.escape_arg(node.astext()) + '}')
+        raise nodes.SkipNode
+
+    def visit_math_block(self, node):
+        # type: (nodes.Node) -> None
         logger.warning(__('using "math" markup without a Sphinx math extension '
                           'active, please use one of the math extensions '
                           'described at http://sphinx-doc.org/en/master/ext/math.html'))
         raise nodes.SkipNode
-
-    visit_math_block = visit_math
diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py
index 379f06b46..677a10df3 100644
--- a/sphinx/writers/text.py
+++ b/sphinx/writers/text.py
@@ -1179,14 +1179,20 @@ class TextTranslator(nodes.NodeVisitor):
 
     def visit_math(self, node):
         # type: (nodes.Node) -> None
+        pass
+
+    def depart_math(self, node):
+        # type: (nodes.Node) -> None
+        pass
+
+    def visit_math_block(self, node):
+        # type: (nodes.Node) -> None
         logger.warning(__('using "math" markup without a Sphinx math extension '
                           'active, please use one of the math extensions '
                           'described at http://sphinx-doc.org/en/master/ext/math.html'),
                        location=(self.builder.current_docname, node.line))
         raise nodes.SkipNode
 
-    visit_math_block = visit_math
-
     def unknown_visit(self, node):
         # type: (nodes.Node) -> None
         raise NotImplementedError('Unknown node: ' + node.__class__.__name__)
diff --git a/tests/roots/test-ext-math-compat/conf.py b/tests/roots/test-ext-math-compat/conf.py
new file mode 100644
index 000000000..fc3b3d0b6
--- /dev/null
+++ b/tests/roots/test-ext-math-compat/conf.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+from sphinx.ext.mathbase import math
+
+master_doc = 'index'
+extensions = ['sphinx.ext.mathjax']
+
+
+def my_math_role(role, rawtext, text, lineno, inliner, options={}, content=[]):
+    return [math(latex='E = mc^2')], []
+
+
+def setup(app):
+    app.add_role('my_math', my_math_role)
diff --git a/tests/roots/test-ext-math-compat/index.rst b/tests/roots/test-ext-math-compat/index.rst
new file mode 100644
index 000000000..d5fc071f1
--- /dev/null
+++ b/tests/roots/test-ext-math-compat/index.rst
@@ -0,0 +1,25 @@
+Test Math
+=========
+
+inline
+------
+
+Inline: :math:`E=mc^2`
+Inline my math: :my_math:`:-)`
+
+block
+-----
+
+.. math:: a^2+b^2=c^2
+
+Second math
+
+.. math:: e^{i\pi}+1=0
+
+Multi math equations
+
+.. math::
+
+   S &= \pi r^2
+
+   V &= \frac{4}{3} \pi r^3
diff --git a/tests/test_ext_math.py b/tests/test_ext_math.py
index 28ce094a8..d3580e986 100644
--- a/tests/test_ext_math.py
+++ b/tests/test_ext_math.py
@@ -12,8 +12,12 @@
 import errno
 import re
 import subprocess
+import warnings
 
 import pytest
+from docutils import nodes
+
+from sphinx.testing.util import assert_node
 
 
 def has_binary(binary):
@@ -208,3 +212,21 @@ def test_imgmath_numfig_html(app, status, warning):
             'href="math.html#equation-foo">(1)</a> and '
             '<a class="reference internal" href="#equation-bar">(3)</a>.</p>')
     assert html in content
+
+
+@pytest.mark.sphinx('dummy', testroot='ext-math-compat')
+def test_math_compat(app, status, warning):
+    with warnings.catch_warnings(record=True):
+        app.builder.build_all()
+        doctree = app.env.get_and_resolve_doctree('index', app.builder)
+
+        assert_node(doctree,
+                    [nodes.document, nodes.section, (nodes.title,
+                                                     [nodes.section, (nodes.title,
+                                                                      nodes.paragraph)],
+                                                     nodes.section)])
+        assert_node(doctree[0][1][1],
+                    ('Inline: ',
+                     [nodes.math, "E=mc^2"],
+                     '\nInline my math: ',
+                     [nodes.math, "E = mc^2"]))