Apply :confval:`trim_doctest_flags` to all builders (cf. text, manpages)

8 files changed, 64 insertions, 11 deletions

diff --git a/CHANGES b/CHANGES
index e5d143104..1efe974fb 100644
--- a/CHANGES
+++ b/CHANGES
@@ -113,6 +113,8 @@ Deprecated
 * ``sphinx.ext.mathbase.is_in_section_title()`` is deprecated
 * ``sphinx.ext.mathbase.MathDomain`` is deprecated
 * ``sphinx.highlighting.PygmentsBridge.unhighlight()`` is deprecated
+* The ``trim_doctest_flags`` argument of ``sphinx.highlighting.PygmentsBridge``
+  is deprecated
 
 For more details, see `deprecation APIs list
 <http://www.sphinx-doc.org/en/master/extdev/index.html#deprecated-apis>`_
@@ -173,6 +175,7 @@ Features added
   if :confval:`latex_engine` is ``'xelatex'`` or ``'lualatex'``.
 * #4976: ``SphinxLoggerAdapter.info()`` now supports ``location`` parameter
 * #5122: setuptools: support nitpicky option
+* Apply :confval:`trim_doctest_flags` to all builders (cf. text, manpages)
 
 Bugs fixed
 ----------
diff --git a/doc/extdev/index.rst b/doc/extdev/index.rst
index fe0d1b97e..a18534714 100644
--- a/doc/extdev/index.rst
+++ b/doc/extdev/index.rst
@@ -136,6 +136,12 @@ The following is a list of deprecated interface.
      - 3.0
      - N/A
 
+   * - ``trim_doctest_flags`` arguments of
+       ``sphinx.highlighting.PygmentsBridge``
+     - 1.8
+     - 3.0
+     - N/A
+
    * - ``sphinx.ext.mathbase.MathDomain``
      - 1.8
      - 3.0
diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py
index 16587fd18..67107ba0a 100644
--- a/sphinx/builders/html.py
+++ b/sphinx/builders/html.py
@@ -386,8 +386,7 @@ class StandaloneHTMLBuilder(Builder):
             style = self.theme.get_config('theme', 'pygments_style', 'none')
         else:
             style = 'sphinx'
-        self.highlighter = PygmentsBridge('html', style,
-                                          self.config.trim_doctest_flags)
+        self.highlighter = PygmentsBridge('html', style)
 
     def init_css_files(self):
         # type: () -> None
diff --git a/sphinx/builders/latex/__init__.py b/sphinx/builders/latex/__init__.py
index c17ead5c2..c4e92ae5b 100644
--- a/sphinx/builders/latex/__init__.py
+++ b/sphinx/builders/latex/__init__.py
@@ -169,8 +169,7 @@ class LaTeXBuilder(Builder):
 
     def write_stylesheet(self):
         # type: () -> None
-        highlighter = highlighting.PygmentsBridge(
-            'latex', self.config.pygments_style, self.config.trim_doctest_flags)
+        highlighter = highlighting.PygmentsBridge('latex', self.config.pygments_style)
         stylesheet = path.join(self.outdir, 'sphinxhighlight.sty')
         with open(stylesheet, 'w') as f:
             f.write('\\NeedsTeXFormat{LaTeX2e}[1995/12/01]\n')
diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py
index f31ff88cc..b344dc16b 100644
--- a/sphinx/highlighting.py
+++ b/sphinx/highlighting.py
@@ -68,7 +68,7 @@ class PygmentsBridge(object):
     html_formatter = HtmlFormatter
     latex_formatter = LatexFormatter
 
-    def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=False):
+    def __init__(self, dest='html', stylename='sphinx', trim_doctest_flags=None):
         # type: (unicode, unicode, bool) -> None
         self.dest = dest
         if stylename is None or stylename == 'sphinx':
@@ -81,7 +81,6 @@ class PygmentsBridge(object):
                             stylename)
         else:
             style = get_style_by_name(stylename)
-        self.trim_doctest_flags = trim_doctest_flags
         self.formatter_args = {'style': style}  # type: Dict[unicode, Any]
         if dest == 'html':
             self.formatter = self.html_formatter
@@ -89,6 +88,11 @@ class PygmentsBridge(object):
             self.formatter = self.latex_formatter
             self.formatter_args['commandprefix'] = 'PYG'
 
+        self.trim_doctest_flags = trim_doctest_flags
+        if trim_doctest_flags is not None:
+            warnings.warn('trim_doctest_flags option for PygmentsBridge is now deprecated.',
+                          RemovedInSphinx30Warning)
+
     def get_formatter(self, **kwargs):
         # type: (Any) -> Formatter
         kwargs.update(self.formatter_args)  # type: ignore
diff --git a/sphinx/transforms/post_transforms/code.py b/sphinx/transforms/post_transforms/code.py
index f0c1d0097..86143ffb4 100644
--- a/sphinx/transforms/post_transforms/code.py
+++ b/sphinx/transforms/post_transforms/code.py
@@ -13,9 +13,11 @@ import sys
 from typing import NamedTuple
 
 from docutils import nodes
+from pygments.lexers import PythonConsoleLexer, guess_lexer
 from six import text_type
 
 from sphinx import addnodes
+from sphinx.ext import doctest
 from sphinx.transforms import SphinxTransform
 
 if False:
@@ -95,9 +97,51 @@ class HighlightLanguageVisitor(nodes.NodeVisitor):
             node['linenos'] = (lines >= setting.lineno_threshold - 1)
 
 
+class TrimDoctestFlagsTransform(SphinxTransform):
+    """
+    Trim doctest flags like ``# doctest: +FLAG`` from python code-blocks.
+
+    see :confval:`trim_doctest_flags` for more information.
+    """
+    default_priority = HighlightLanguageTransform.default_priority + 1
+
+    def apply(self):
+        if not self.config.trim_doctest_flags:
+            return
+
+        for node in self.document.traverse(nodes.literal_block):
+            if self.is_pyconsole(node):
+                source = node.rawsource
+                source = doctest.blankline_re.sub('', source)
+                source = doctest.doctestopt_re.sub('', source)
+                node.rawsource = source
+                node[:] = [nodes.Text(source)]
+
+    @staticmethod
+    def is_pyconsole(node):
+        # type: (nodes.literal_block) -> bool
+        if node.rawsource != node.astext():
+            return False  # skip parsed-literal node
+
+        language = node.get('language')
+        if language in ('pycon', 'pycon3'):
+            return True
+        elif language in ('py', 'py3', 'python', 'python3', 'default'):
+            return node.rawsource.startswith('>>>')
+        elif language == 'guess':
+            try:
+                lexer = guess_lexer(node.rawsource)
+                return isinstance(lexer, PythonConsoleLexer)
+            except Exception:
+                pass
+
+        return False
+
+
 def setup(app):
     # type: (Sphinx) -> Dict[unicode, Any]
     app.add_post_transform(HighlightLanguageTransform)
+    app.add_post_transform(TrimDoctestFlagsTransform)
 
     return {
         'version': 'builtin',
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index c657664af..b7d80860b 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -697,9 +697,7 @@ class LaTeXTranslator(nodes.NodeVisitor):
             self.babel_defmacro('\\pageautorefname', self.encode(_('page')))
         self.elements['numfig_format'] = self.generate_numfig_format(builder)
 
-        self.highlighter = highlighting.PygmentsBridge(
-            'latex',
-            builder.config.pygments_style, builder.config.trim_doctest_flags)
+        self.highlighter = highlighting.PygmentsBridge('latex', builder.config.pygments_style)
         self.context = []               # type: List[Any]
         self.descstack = []             # type: List[unicode]
         self.table = None               # type: Table
diff --git a/tests/test_config_trim_doctest_flags.py b/tests/test_transforms_post_transforms_code.py
index 3be1b27a2..e1c45ce65 100644
--- a/tests/test_config_trim_doctest_flags.py
+++ b/tests/test_transforms_post_transforms_code.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    test_config_trim_doctest_flags
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    test_transforms_post_transforms_code
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
     :license: BSD, see LICENSE for details.