Merge commit '54d5fcfaebc3364044761d30c0fed6bd4d3052c3'

8 files changed, 75 insertions, 50 deletions

diff --git a/sphinx/application.py b/sphinx/application.py
index 411fa42d8..577ec9ac8 100644
--- a/sphinx/application.py
+++ b/sphinx/application.py
@@ -19,6 +19,7 @@ from io import StringIO
 from os import path
 
 from docutils.parsers.rst import Directive, roles
+from pygments.lexer import Lexer
 
 import sphinx
 from sphinx import package_dir, locale
@@ -27,6 +28,7 @@ from sphinx.deprecation import RemovedInSphinx40Warning
 from sphinx.environment import BuildEnvironment
 from sphinx.errors import ApplicationError, ConfigError, VersionRequirementError
 from sphinx.events import EventManager
+from sphinx.highlighting import lexer_classes, lexers
 from sphinx.locale import __
 from sphinx.project import Project
 from sphinx.registry import SphinxComponentRegistry
@@ -969,19 +971,24 @@ class Sphinx:
         self.registry.add_latex_package(packagename, options)
 
     def add_lexer(self, alias, lexer):
-        # type: (str, Any) -> None
+        # type: (str, Union[Lexer, Type[Lexer]]) -> None
         """Register a new lexer for source code.
 
-        Use *lexer*, which must be an instance of a Pygments lexer class, to
-        highlight code blocks with the given language *alias*.
+        Use *lexer* to highlight code blocks with the given language *alias*.
 
         .. versionadded:: 0.6
+        .. versionchanged:: 2.1
+           Take a lexer class as an argument.  An instance of lexers are
+           still supported until Sphinx-3.x.
         """
         logger.debug('[app] adding lexer: %r', (alias, lexer))
-        from sphinx.highlighting import lexers
-        if lexers is None:
-            return
-        lexers[alias] = lexer
+        if isinstance(lexer, Lexer):
+            warnings.warn('app.add_lexer() API changed; '
+                          'Please give lexer class instead instance',
+                          RemovedInSphinx40Warning)
+            lexers[alias] = lexer
+        else:
+            lexer_classes[alias] = lexer
 
     def add_autodocumenter(self, cls):
         # type: (Any) -> None
diff --git a/sphinx/directives/code.py b/sphinx/directives/code.py
index 75ab61796..ff21bcdab 100644
--- a/sphinx/directives/code.py
+++ b/sphinx/directives/code.py
@@ -41,6 +41,7 @@ class Highlight(SphinxDirective):
     optional_arguments = 0
     final_argument_whitespace = False
     option_spec = {
+        'force': directives.flag,
         'linenothreshold': directives.positive_int,
     }
 
@@ -48,9 +49,12 @@ class Highlight(SphinxDirective):
         # type: () -> List[nodes.Node]
         language = self.arguments[0].strip()
         linenothreshold = self.options.get('linenothreshold', sys.maxsize)
+        force = 'force' in self.options
 
         self.env.temp_data['highlight_language'] = language
-        return [addnodes.highlightlang(lang=language, linenothreshold=linenothreshold)]
+        return [addnodes.highlightlang(lang=language,
+                                       force=force,
+                                       linenothreshold=linenothreshold)]
 
 
 class HighlightLang(Highlight):
@@ -115,6 +119,7 @@ class CodeBlock(SphinxDirective):
     optional_arguments = 1
     final_argument_whitespace = False
     option_spec = {
+        'force': directives.flag,
         'linenos': directives.flag,
         'dedent': int,
         'lineno-start': int,
@@ -156,17 +161,16 @@ class CodeBlock(SphinxDirective):
         if 'linenos' in self.options or 'lineno-start' in self.options:
             literal['linenos'] = True
         literal['classes'] += self.options.get('class', [])
+        literal['force'] = 'force' in self.options
         if self.arguments:
             # highlight language specified
             literal['language'] = self.arguments[0]
-            literal['force_highlighting'] = True
         else:
             # no highlight language specified.  Then this directive refers the current
             # highlight setting via ``highlight`` directive or ``highlight_language``
             # configuration.
             literal['language'] = self.env.temp_data.get('highlight_language',
                                                          self.config.highlight_language)
-            literal['force_highlighting'] = False
         extra_args = literal['highlight_args'] = {}
         if hl_lines is not None:
             extra_args['hl_lines'] = hl_lines
@@ -409,6 +413,7 @@ class LiteralInclude(SphinxDirective):
         'lineno-match': directives.flag,
         'tab-width': int,
         'language': directives.unchanged_required,
+        'force': directives.flag,
         'encoding': directives.encoding,
         'pyobject': directives.unchanged_required,
         'lines': directives.unchanged_required,
@@ -445,6 +450,7 @@ class LiteralInclude(SphinxDirective):
             text, lines = reader.read(location=location)
 
             retnode = nodes.literal_block(text, text, source=filename)  # type: nodes.Element
+            retnode['force'] = 'force' in self.options
             self.set_source_info(retnode)
             if self.options.get('diff'):  # if diff is set, set udiff
                 retnode['language'] = 'udiff'
diff --git a/sphinx/directives/patches.py b/sphinx/directives/patches.py
index c102f3461..189e1f7e3 100644
--- a/sphinx/directives/patches.py
+++ b/sphinx/directives/patches.py
@@ -119,6 +119,7 @@ class Code(SphinxDirective):
     optional_arguments = 1
     option_spec = {
         'class': directives.class_option,
+        'force': directives.flag,
         'name': directives.unchanged,
         'number-lines': optional_int,
     }
@@ -131,6 +132,7 @@ class Code(SphinxDirective):
         code = '\n'.join(self.content)
         node = nodes.literal_block(code, code,
                                    classes=self.options.get('classes', []),
+                                   force='force' in self.options,
                                    highlight_args={})
         self.add_name(node)
         set_source_info(self, node)
@@ -138,14 +140,12 @@ class Code(SphinxDirective):
         if self.arguments:
             # highlight language specified
             node['language'] = self.arguments[0]
-            node['force_highlighting'] = True
         else:
             # no highlight language specified.  Then this directive refers the current
             # highlight setting via ``highlight`` directive or ``highlight_language``
             # configuration.
             node['language'] = self.env.temp_data.get('highlight_language',
                                                       self.config.highlight_language)
-            node['force_highlighting'] = False
 
         if 'number-lines' in self.options:
             node['linenos'] = True
diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py
index 995ca65d3..45a1298b9 100644
--- a/sphinx/highlighting.py
+++ b/sphinx/highlighting.py
@@ -8,6 +8,8 @@
     :license: BSD, see LICENSE for details.
 """
 
+from functools import partial
+
 from pygments import highlight
 from pygments.filters import ErrorToken
 from pygments.formatters import HtmlFormatter, LatexFormatter
@@ -32,17 +34,16 @@ if False:
 
 logger = logging.getLogger(__name__)
 
-lexers = {
-    'none': TextLexer(stripnl=False),
-    'python': PythonLexer(stripnl=False),
-    'python3': Python3Lexer(stripnl=False),
-    'pycon': PythonConsoleLexer(stripnl=False),
-    'pycon3': PythonConsoleLexer(python3=True, stripnl=False),
-    'rest': RstLexer(stripnl=False),
-    'c': CLexer(stripnl=False),
+lexers = {}  # type: Dict[str, Lexer]
+lexer_classes = {
+    'none': partial(TextLexer, stripnl=False),
+    'python': partial(PythonLexer, stripnl=False),
+    'python3': partial(Python3Lexer, stripnl=False),
+    'pycon': partial(PythonConsoleLexer, stripnl=False),
+    'pycon3': partial(PythonConsoleLexer, python3=True, stripnl=False),
+    'rest': partial(RstLexer, stripnl=False),
+    'c': partial(CLexer, stripnl=False),
 }  # type: Dict[str, Lexer]
-for _lexer in lexers.values():
-    _lexer.add_filter('raiseonerror')
 
 
 escape_hl_chars = {ord('\\'): '\\PYGZbs{}',
@@ -91,46 +92,55 @@ class PygmentsBridge:
         kwargs.update(self.formatter_args)
         return self.formatter(**kwargs)
 
-    def get_lexer(self, source, lang, opts=None, location=None):
-        # type: (str, str, Any, Any) -> Lexer
+    def get_lexer(self, source, lang, opts=None, force=False, location=None):
+        # type: (str, str, Dict, bool, Any) -> Lexer
+        if not opts:
+            opts = {}
+
         # find out which lexer to use
         if lang in ('py', 'python'):
             if source.startswith('>>>'):
                 # interactive session
-                lexer = lexers['pycon']
+                lang = 'pycon'
             else:
-                lexer = lexers['python']
+                lang = 'python'
         elif lang in ('py3', 'python3', 'default'):
             if source.startswith('>>>'):
-                lexer = lexers['pycon3']
+                lang = 'pycon3'
             else:
-                lexer = lexers['python3']
+                lang = 'python3'
         elif lang == 'guess':
             try:
                 lexer = guess_lexer(source)
             except Exception:
                 lexer = lexers['none']
+
+        if lang in lexers:
+            lexer = lexers[lang]
+        elif lang in lexer_classes:
+            lexer = lexer_classes[lang](**opts)
         else:
-            if lang in lexers:
-                lexer = lexers[lang]
-            else:
-                try:
-                    lexer = lexers[lang] = get_lexer_by_name(lang, **(opts or {}))
-                except ClassNotFound:
-                    logger.warning(__('Pygments lexer name %r is not known'), lang,
-                                   location=location)
-                    lexer = lexers['none']
+            try:
+                if lang == 'guess':
+                    lexer = guess_lexer(lang, **opts)
                 else:
-                    lexer.add_filter('raiseonerror')
+                    lexer = get_lexer_by_name(lang, **opts)
+            except ClassNotFound:
+                logger.warning(__('Pygments lexer name %r is not known'), lang,
+                               location=location)
+                lexer = lexer_classes['none'](**opts)
+
+        if not force:
+            lexer.add_filter('raiseonerror')
 
         return lexer
 
-    def highlight_block(self, source, lang, opts=None, location=None, force=False, **kwargs):
-        # type: (str, str, Any, Any, bool, Any) -> str
+    def highlight_block(self, source, lang, opts=None, force=False, location=None, **kwargs):
+        # type: (str, str, Dict, bool, Any, Any) -> str
         if not isinstance(source, str):
             source = source.decode()
 
-        lexer = self.get_lexer(source, lang, opts, location)
+        lexer = self.get_lexer(source, lang, opts, force, location)
 
         # highlight via Pygments
         formatter = self.get_formatter(**kwargs)
@@ -146,7 +156,8 @@ class PygmentsBridge:
                                   'Highlighting skipped.'), lang,
                                type='misc', subtype='highlighting_failure',
                                location=location)
-            hlsource = highlight(source, lexers['none'], formatter)
+            lexer = self.get_lexer(source, 'none', opts, force, location)
+            hlsource = highlight(source, lexer, formatter)
 
         if self.dest == 'html':
             return hlsource
diff --git a/sphinx/transforms/post_transforms/code.py b/sphinx/transforms/post_transforms/code.py
index fb8533e9c..a42de4bf1 100644
--- a/sphinx/transforms/post_transforms/code.py
+++ b/sphinx/transforms/post_transforms/code.py
@@ -25,6 +25,7 @@ if False:
 
 
 HighlightSetting = NamedTuple('HighlightSetting', [('language', str),
+                                                   ('force', bool),
                                                    ('lineno_threshold', int)])
 
 
@@ -51,7 +52,7 @@ class HighlightLanguageTransform(SphinxTransform):
 class HighlightLanguageVisitor(nodes.NodeVisitor):
     def __init__(self, document, default_language):
         # type: (nodes.document, str) -> None
-        self.default_setting = HighlightSetting(default_language, sys.maxsize)
+        self.default_setting = HighlightSetting(default_language, False, sys.maxsize)
         self.settings = []  # type: List[HighlightSetting]
         super().__init__(document)
 
@@ -81,16 +82,16 @@ class HighlightLanguageVisitor(nodes.NodeVisitor):
 
     def visit_highlightlang(self, node):
         # type: (addnodes.highlightlang) -> None
-        self.settings[-1] = HighlightSetting(node['lang'], node['linenothreshold'])
+        self.settings[-1] = HighlightSetting(node['lang'],
+                                             node['force'],
+                                             node['linenothreshold'])
 
     def visit_literal_block(self, node):
         # type: (nodes.literal_block) -> None
         setting = self.settings[-1]
         if 'language' not in node:
             node['language'] = setting.language
-            node['force_highlighting'] = False
-        elif 'force_highlighting' not in node:
-            node['force_highlighting'] = True
+            node['force'] = setting.force
         if 'linenos' not in node:
             lines = node.astext().count('\n')
             node['linenos'] = (lines >= setting.lineno_threshold - 1)
diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py
index 60cc7d999..9ac76901b 100644
--- a/sphinx/writers/html.py
+++ b/sphinx/writers/html.py
@@ -475,7 +475,7 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator):
         lang = node.get('language', 'default')
         linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        highlight_args['force'] = node.get('force_highlighting', False)
+        highlight_args['force'] = node.get('force', False)
         if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
             opts = self.builder.config.highlight_options
diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py
index 38e2f3551..fe57f42bf 100644
--- a/sphinx/writers/html5.py
+++ b/sphinx/writers/html5.py
@@ -422,7 +422,7 @@ class HTML5Translator(SphinxTranslator, BaseTranslator):
         lang = node.get('language', 'default')
         linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        highlight_args['force'] = node.get('force_highlighting', False)
+        highlight_args['force'] = node.get('force', False)
         if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
             opts = self.builder.config.highlight_options
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index ceaf48458..7681f479b 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -2097,7 +2097,7 @@ class LaTeXTranslator(SphinxTranslator):
             lang = node.get('language', 'default')
             linenos = node.get('linenos', False)
             highlight_args = node.get('highlight_args', {})
-            highlight_args['force'] = node.get('force_highlighting', False)
+            highlight_args['force'] = node.get('force', False)
             if lang is self.builder.config.highlight_language:
                 # only pass highlighter options for original language
                 opts = self.builder.config.highlight_options