Add :force: option to code directives

11 files changed, 75 insertions, 27 deletions

diff --git a/doc/usage/restructuredtext/directives.rst b/doc/usage/restructuredtext/directives.rst
index d00979255..1dd9078e8 100644
--- a/doc/usage/restructuredtext/directives.rst
+++ b/doc/usage/restructuredtext/directives.rst
@@ -463,6 +463,12 @@ __ http://pygments.org/docs/lexers/
 
    This will produce line numbers for all code blocks longer than five lines.
 
+   To ignore minor errors on highlighting, you can specifiy ``:force:`` option.
+
+   .. versionchanged:: 2.1
+
+      ``:force:`` option.
+
 .. rst:directive:: .. code-block:: [language]
 
    Example::
@@ -525,6 +531,8 @@ __ http://pygments.org/docs/lexers/
 
              some ruby code
 
+   A ``force`` option can ignore minor errors on highlighting.
+
    .. versionchanged:: 1.1
       The ``emphasize-lines`` option has been added.
 
@@ -538,6 +546,10 @@ __ http://pygments.org/docs/lexers/
    .. versionchanged:: 2.0
       The ``language`` argument becomes optional.
 
+   .. versionchanged:: 2.1
+
+      ``:force:`` option has been added.
+
 .. rst:directive:: .. literalinclude:: filename
 
    Longer displays of verbatim text may be included by storing the example text
@@ -630,6 +642,8 @@ __ http://pygments.org/docs/lexers/
    This shows the diff between ``example.py`` and ``example.py.orig`` with
    unified diff format.
 
+   A ``force`` option can ignore minor errors on highlighting.
+
    .. versionchanged:: 0.4.3
       Added the ``encoding`` option.
 
@@ -651,6 +665,9 @@ __ http://pygments.org/docs/lexers/
       With both ``start-after`` and ``lines`` in use, the first line as per
       ``start-after`` is considered to be with line number ``1`` for ``lines``.
 
+   .. versionchanged:: 2.1
+      Added the ``force`` option.
+
 .. _glossary-directive:
 
 Glossary
diff --git a/sphinx/directives/code.py b/sphinx/directives/code.py
index 75ab61796..ff21bcdab 100644
--- a/sphinx/directives/code.py
+++ b/sphinx/directives/code.py
@@ -41,6 +41,7 @@ class Highlight(SphinxDirective):
     optional_arguments = 0
     final_argument_whitespace = False
     option_spec = {
+        'force': directives.flag,
         'linenothreshold': directives.positive_int,
     }
 
@@ -48,9 +49,12 @@ class Highlight(SphinxDirective):
         # type: () -> List[nodes.Node]
         language = self.arguments[0].strip()
         linenothreshold = self.options.get('linenothreshold', sys.maxsize)
+        force = 'force' in self.options
 
         self.env.temp_data['highlight_language'] = language
-        return [addnodes.highlightlang(lang=language, linenothreshold=linenothreshold)]
+        return [addnodes.highlightlang(lang=language,
+                                       force=force,
+                                       linenothreshold=linenothreshold)]
 
 
 class HighlightLang(Highlight):
@@ -115,6 +119,7 @@ class CodeBlock(SphinxDirective):
     optional_arguments = 1
     final_argument_whitespace = False
     option_spec = {
+        'force': directives.flag,
         'linenos': directives.flag,
         'dedent': int,
         'lineno-start': int,
@@ -156,17 +161,16 @@ class CodeBlock(SphinxDirective):
         if 'linenos' in self.options or 'lineno-start' in self.options:
             literal['linenos'] = True
         literal['classes'] += self.options.get('class', [])
+        literal['force'] = 'force' in self.options
         if self.arguments:
             # highlight language specified
             literal['language'] = self.arguments[0]
-            literal['force_highlighting'] = True
         else:
             # no highlight language specified.  Then this directive refers the current
             # highlight setting via ``highlight`` directive or ``highlight_language``
             # configuration.
             literal['language'] = self.env.temp_data.get('highlight_language',
                                                          self.config.highlight_language)
-            literal['force_highlighting'] = False
         extra_args = literal['highlight_args'] = {}
         if hl_lines is not None:
             extra_args['hl_lines'] = hl_lines
@@ -409,6 +413,7 @@ class LiteralInclude(SphinxDirective):
         'lineno-match': directives.flag,
         'tab-width': int,
         'language': directives.unchanged_required,
+        'force': directives.flag,
         'encoding': directives.encoding,
         'pyobject': directives.unchanged_required,
         'lines': directives.unchanged_required,
@@ -445,6 +450,7 @@ class LiteralInclude(SphinxDirective):
             text, lines = reader.read(location=location)
 
             retnode = nodes.literal_block(text, text, source=filename)  # type: nodes.Element
+            retnode['force'] = 'force' in self.options
             self.set_source_info(retnode)
             if self.options.get('diff'):  # if diff is set, set udiff
                 retnode['language'] = 'udiff'
diff --git a/sphinx/directives/patches.py b/sphinx/directives/patches.py
index c102f3461..189e1f7e3 100644
--- a/sphinx/directives/patches.py
+++ b/sphinx/directives/patches.py
@@ -119,6 +119,7 @@ class Code(SphinxDirective):
     optional_arguments = 1
     option_spec = {
         'class': directives.class_option,
+        'force': directives.flag,
         'name': directives.unchanged,
         'number-lines': optional_int,
     }
@@ -131,6 +132,7 @@ class Code(SphinxDirective):
         code = '\n'.join(self.content)
         node = nodes.literal_block(code, code,
                                    classes=self.options.get('classes', []),
+                                   force='force' in self.options,
                                    highlight_args={})
         self.add_name(node)
         set_source_info(self, node)
@@ -138,14 +140,12 @@ class Code(SphinxDirective):
         if self.arguments:
             # highlight language specified
             node['language'] = self.arguments[0]
-            node['force_highlighting'] = True
         else:
             # no highlight language specified.  Then this directive refers the current
             # highlight setting via ``highlight`` directive or ``highlight_language``
             # configuration.
             node['language'] = self.env.temp_data.get('highlight_language',
                                                       self.config.highlight_language)
-            node['force_highlighting'] = False
 
         if 'number-lines' in self.options:
             node['linenos'] = True
diff --git a/sphinx/highlighting.py b/sphinx/highlighting.py
index fba7226c2..2d825e9f1 100644
--- a/sphinx/highlighting.py
+++ b/sphinx/highlighting.py
@@ -40,13 +40,13 @@ logger = logging.getLogger(__name__)
 
 lexers = {}  # type: Dict[str, Lexer]
 lexer_classes = {
-    'none': TextLexer,
-    'python': PythonLexer,
-    'python3': Python3Lexer,
-    'pycon': PythonConsoleLexer,
-    'pycon3': partial(PythonConsoleLexer, python3=True),
-    'rest': RstLexer,
-    'c': CLexer,
+    'none': partial(TextLexer, stripnl=False),
+    'python': partial(PythonLexer, stripnl=False),
+    'python3': partial(Python3Lexer, stripnl=False),
+    'pycon': partial(PythonConsoleLexer, stripnl=False),
+    'pycon3': partial(PythonConsoleLexer, python3=True, stripnl=False),
+    'rest': partial(RstLexer, stripnl=False),
+    'c': partial(CLexer, stripnl=False),
 }  # type: Dict[str, Lexer]
 
 
@@ -117,8 +117,8 @@ class PygmentsBridge:
 
     def get_lexer(self, source, lang, opts=None, force=False, location=None):
         # type: (str, str, Dict, bool, Any) -> Lexer
-        lexer_options = {'strip': True}
-        lexer_options.update(opts)
+        if not opts:
+            opts = {}
 
         # find out which lexer to use
         if lang in ('py', 'python'):
@@ -141,17 +141,17 @@ class PygmentsBridge:
         if lang in lexers:
             lexer = lexers[lang]
         elif lang in lexer_classes:
-            lexer = lexer_classes[lang](**lexer_options)
+            lexer = lexer_classes[lang](**opts)
         else:
             try:
                 if lang == 'guess':
-                    lexer = guess_lexer(lang, **lexer_options)
+                    lexer = guess_lexer(lang, **opts)
                 else:
-                    lexer = get_lexer_by_name(lang, **lexer_options)
+                    lexer = get_lexer_by_name(lang, **opts)
             except ClassNotFound:
                 logger.warning(__('Pygments lexer name %r is not known'), lang,
                                location=location)
-                lexer = lexer_classes['none'](**lexer_options)
+                lexer = lexer_classes['none'](**opts)
 
         if not force:
             lexer.add_filter('raiseonerror')
@@ -184,7 +184,8 @@ class PygmentsBridge:
                                   'Highlighting skipped.'), lang,
                                type='misc', subtype='highlighting_failure',
                                location=location)
-            hlsource = highlight(source, lexers['none'], formatter)
+            lexer = self.get_lexer(source, 'none', opts, force, location)
+            hlsource = highlight(source, lexer, formatter)
 
         if self.dest == 'html':
             return hlsource
diff --git a/sphinx/transforms/post_transforms/code.py b/sphinx/transforms/post_transforms/code.py
index fb8533e9c..a42de4bf1 100644
--- a/sphinx/transforms/post_transforms/code.py
+++ b/sphinx/transforms/post_transforms/code.py
@@ -25,6 +25,7 @@ if False:
 
 
 HighlightSetting = NamedTuple('HighlightSetting', [('language', str),
+                                                   ('force', bool),
                                                    ('lineno_threshold', int)])
 
 
@@ -51,7 +52,7 @@ class HighlightLanguageTransform(SphinxTransform):
 class HighlightLanguageVisitor(nodes.NodeVisitor):
     def __init__(self, document, default_language):
         # type: (nodes.document, str) -> None
-        self.default_setting = HighlightSetting(default_language, sys.maxsize)
+        self.default_setting = HighlightSetting(default_language, False, sys.maxsize)
         self.settings = []  # type: List[HighlightSetting]
         super().__init__(document)
 
@@ -81,16 +82,16 @@ class HighlightLanguageVisitor(nodes.NodeVisitor):
 
     def visit_highlightlang(self, node):
         # type: (addnodes.highlightlang) -> None
-        self.settings[-1] = HighlightSetting(node['lang'], node['linenothreshold'])
+        self.settings[-1] = HighlightSetting(node['lang'],
+                                             node['force'],
+                                             node['linenothreshold'])
 
     def visit_literal_block(self, node):
         # type: (nodes.literal_block) -> None
         setting = self.settings[-1]
         if 'language' not in node:
             node['language'] = setting.language
-            node['force_highlighting'] = False
-        elif 'force_highlighting' not in node:
-            node['force_highlighting'] = True
+            node['force'] = setting.force
         if 'linenos' not in node:
             lines = node.astext().count('\n')
             node['linenos'] = (lines >= setting.lineno_threshold - 1)
diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py
index 3eb9a0911..381457926 100644
--- a/sphinx/writers/html.py
+++ b/sphinx/writers/html.py
@@ -476,7 +476,7 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator):
         lang = node.get('language', 'default')
         linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        highlight_args['force'] = node.get('force_highlighting', False)
+        highlight_args['force'] = node.get('force', False)
         if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
             opts = self.builder.config.highlight_options
diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py
index dc0895f8a..e412ea167 100644
--- a/sphinx/writers/html5.py
+++ b/sphinx/writers/html5.py
@@ -423,7 +423,7 @@ class HTML5Translator(SphinxTranslator, BaseTranslator):
         lang = node.get('language', 'default')
         linenos = node.get('linenos', False)
         highlight_args = node.get('highlight_args', {})
-        highlight_args['force'] = node.get('force_highlighting', False)
+        highlight_args['force'] = node.get('force', False)
         if lang is self.builder.config.highlight_language:
             # only pass highlighter options for original language
             opts = self.builder.config.highlight_options
diff --git a/sphinx/writers/latex.py b/sphinx/writers/latex.py
index 899df04c9..9afb1f0c7 100644
--- a/sphinx/writers/latex.py
+++ b/sphinx/writers/latex.py
@@ -2139,7 +2139,7 @@ class LaTeXTranslator(SphinxTranslator):
             lang = node.get('language', 'default')
             linenos = node.get('linenos', False)
             highlight_args = node.get('highlight_args', {})
-            highlight_args['force'] = node.get('force_highlighting', False)
+            highlight_args['force'] = node.get('force', False)
             if lang is self.builder.config.highlight_language:
                 # only pass highlighter options for original language
                 opts = self.builder.config.highlight_options
diff --git a/tests/roots/test-directive-code/error.inc b/tests/roots/test-directive-code/error.inc
new file mode 100644
index 000000000..472828080
--- /dev/null
+++ b/tests/roots/test-directive-code/error.inc
@@ -0,0 +1 @@
+not a python script!
diff --git a/tests/roots/test-directive-code/force.rst b/tests/roots/test-directive-code/force.rst
new file mode 100644
index 000000000..1834b3ac6
--- /dev/null
+++ b/tests/roots/test-directive-code/force.rst
@@ -0,0 +1,16 @@
+force option
+============
+
+.. code:: python
+   :force:
+
+   not a python script!
+
+.. code-block:: python
+   :force:
+
+   not a python script!
+
+.. literalinclude:: error.inc
+   :language: python
+   :force:
diff --git a/tests/test_directive_code.py b/tests/test_directive_code.py
index 2e1a1fde2..d1f7c03dc 100644
--- a/tests/test_directive_code.py
+++ b/tests/test_directive_code.py
@@ -308,6 +308,12 @@ def test_code_block(app, status, warning):
 
 
 @pytest.mark.sphinx('html', testroot='directive-code')
+def test_force_option(app, status, warning):
+    app.builder.build(['force'])
+    assert 'force.rst' not in warning.getvalue()
+
+
+@pytest.mark.sphinx('html', testroot='directive-code')
 def test_code_block_caption_html(app, status, warning):
     app.builder.build(['caption'])
     html = (app.outdir / 'caption.html').text()