diff options
| -rw-r--r-- | sphinx/io.py | 55 | ||||
| -rw-r--r-- | sphinx/parsers.py | 22 | ||||
| -rw-r--r-- | sphinx/util/__init__.py | 10 | ||||
| -rw-r--r-- | sphinx/util/docutils.py | 19 | ||||
| -rw-r--r-- | tests/test_io.py | 107 | ||||
| -rw-r--r-- | tests/test_util.py | 25 |
6 files changed, 195 insertions, 43 deletions
diff --git a/sphinx/io.py b/sphinx/io.py index 5868bd7b5..056c763b1 100644 --- a/sphinx/io.py +++ b/sphinx/io.py @@ -8,13 +8,15 @@ :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import re import codecs from docutils.io import FileInput, NullOutput from docutils.core import Publisher from docutils.readers import standalone +from docutils.statemachine import StringList from docutils.writers import UnfilteredWriter -from six import string_types, text_type, iteritems +from six import text_type from typing import Any, Union # NOQA from sphinx.transforms import ( @@ -28,7 +30,6 @@ from sphinx.transforms.i18n import ( PreserveTranslatableMessages, Locale, RemoveTranslatableInline, ) from sphinx.util import logging -from sphinx.util import import_object, split_docinfo from sphinx.util.docutils import LoggingReporter if False: @@ -42,6 +43,8 @@ if False: from sphinx.builders import Builder # NOQA from sphinx.environment import BuildEnvironment # NOQA +docinfo_re = re.compile(':\\w+:.*?') + logger = logging.getLogger(__name__) @@ -59,6 +62,7 @@ class SphinxBaseReader(standalone.Reader): document = standalone.Reader.new_document(self) reporter = document.reporter document.reporter = LoggingReporter.from_reporter(reporter) + document.reporter.set_source(self.source) return document @@ -168,15 +172,50 @@ class SphinxFileInput(SphinxBaseFileInput): class SphinxRSTFileInput(SphinxBaseFileInput): + def prepend_prolog(self, text, prolog): + # type: (StringList, unicode) -> None + docinfo = self.count_docinfo_lines(text) + if docinfo: + # insert a blank line after docinfo + text.insert(docinfo, '', '<generated>', 0) + docinfo += 1 + + # insert prolog (after docinfo if exists) + for lineno, line in enumerate(prolog.splitlines()): + text.insert(docinfo + lineno, line, '<rst_prolog>', lineno) + + text.insert(docinfo + lineno + 1, '', '<generated>', 0) + + def append_epilog(self, text, epilog): + # type: (StringList, unicode) -> None + # append a blank line and rst_epilog + text.append('', '<generated>', 0) + for lineno, line in enumerate(epilog.splitlines()): + text.append(line, '<rst_epilog>', lineno) + def read(self): - # type: () -> unicode + # type: () -> StringList data = SphinxBaseFileInput.read(self) - docinfo, data = split_docinfo(data) - if self.env.config.rst_epilog: - data = data + '\n' + self.env.config.rst_epilog + '\n' + content = StringList() + for lineno, line in enumerate(data.splitlines()): + content.append(line, self.source_path, lineno) + if self.env.config.rst_prolog: - data = self.env.config.rst_prolog + '\n' + data - return docinfo + data + self.prepend_prolog(content, self.env.config.rst_prolog) + if self.env.config.rst_epilog: + self.append_epilog(content, self.env.config.rst_epilog) + + return content + + def count_docinfo_lines(self, content): + # type: (StringList) -> int + if len(content) == 0: + return 0 + else: + for lineno, line in enumerate(content.data): + if not docinfo_re.match(line): + break + return lineno def read_doc(app, env, filename): diff --git a/sphinx/parsers.py b/sphinx/parsers.py index 33556e487..085e45070 100644 --- a/sphinx/parsers.py +++ b/sphinx/parsers.py @@ -11,6 +11,8 @@ import docutils.parsers import docutils.parsers.rst +from docutils.parsers.rst import states +from docutils.statemachine import StringList from docutils.transforms.universal import SmartQuotes from sphinx.transforms import SphinxSmartQuotes @@ -66,6 +68,26 @@ class RSTParser(docutils.parsers.rst.Parser): transforms.append(SphinxSmartQuotes) return transforms + def parse(self, inputstring, document): + # type: (Any, nodes.document) -> None + """Parse text and generate a document tree. + + This derived method accepts StringList as a inputstring parameter. + It enables to handle mixed contents (cf. rst_prolog) correctly. + """ + if isinstance(inputstring, StringList): + self.setup_parse(inputstring, document) + self.statemachine = states.RSTStateMachine( + state_classes=self.state_classes, + initial_state=self.initial_state, + debug=document.reporter.debug_flag) + # Give inputstring directly to statemachine. + self.statemachine.run(inputstring, document, inliner=self.inliner) + self.finish_parse() + else: + # otherwise, inputstring might be a string. It will be handled by superclass. + docutils.parsers.rst.Parser.parse(self, inputstring, document) + def setup(app): # type: (Sphinx) -> Dict[unicode, Any] diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index 03f8ce6a3..da0f6ac3a 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -564,16 +564,6 @@ def encode_uri(uri): return urlunsplit(split) -def split_docinfo(text): - # type: (unicode) -> Sequence[unicode] - docinfo_re = re.compile('\\A((?:\\s*:\\w+:.*?\n(?:[ \\t]+.*?\n)*)+)', re.M) - result = docinfo_re.split(text, 1) # type: ignore - if len(result) == 1: - return '', result[0] - else: - return result[1:] - - def display_chunk(chunk): # type: (Any) -> unicode if isinstance(chunk, (list, tuple)): diff --git a/sphinx/util/docutils.py b/sphinx/util/docutils.py index 8da89713d..d1b7ac431 100644 --- a/sphinx/util/docutils.py +++ b/sphinx/util/docutils.py @@ -18,8 +18,9 @@ from contextlib import contextmanager import docutils from docutils.languages import get_language -from docutils.utils import Reporter +from docutils.statemachine import ViewList from docutils.parsers.rst import directives, roles, convert_directive_function +from docutils.utils import Reporter from sphinx.errors import ExtensionError from sphinx.locale import __ @@ -33,6 +34,7 @@ if False: from typing import Any, Callable, Iterator, List, Tuple # NOQA from docutils import nodes # NOQA from sphinx.environment import BuildEnvironment # NOQA + from sphinx.io import SphinxFileInput # NOQA __version_info__ = tuple(LooseVersion(docutils.__version__).version) @@ -180,6 +182,21 @@ class LoggingReporter(Reporter): stream = WarningStream() Reporter.__init__(self, source, report_level, halt_level, stream, debug, error_handler=error_handler) + self.source_and_line = None + + def set_source(self, source): + # type: (SphinxFileInput) -> None + self.source_and_line = source + + def system_message(self, *args, **kwargs): + # type: (Any, Any) -> Any + if kwargs.get('line') and isinstance(self.source_and_line, ViewList): + # replace source parameter if source is set + source, lineno = self.source_and_line.info(kwargs.get('line')) + kwargs['source'] = source + kwargs['line'] = lineno + + return Reporter.system_message(self, *args, **kwargs) def is_html5_writer_available(): diff --git a/tests/test_io.py b/tests/test_io.py new file mode 100644 index 000000000..a017a2cc0 --- /dev/null +++ b/tests/test_io.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +""" + test_sphinx_io + ~~~~~~~~~~~~~~ + + Tests io modules. + + :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import pytest +from six import StringIO + +from sphinx.io import SphinxRSTFileInput + + +@pytest.mark.sphinx(testroot='basic') +def test_SphinxRSTFileInput(app): + app.env.temp_data['docname'] = 'index' + + # normal case + text = ('hello Sphinx world\n' + 'Sphinx is a document generator') + source = SphinxRSTFileInput(app, app.env, source=StringIO(text), + source_path='dummy.rst', encoding='utf-8') + result = source.read() + assert result.data == ['hello Sphinx world', + 'Sphinx is a document generator'] + assert result.info(0) == ('dummy.rst', 0) + assert result.info(1) == ('dummy.rst', 1) + assert result.info(2) == ('dummy.rst', None) # out of range + + # having rst_prolog ends without CR + app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!' + source = SphinxRSTFileInput(app, app.env, source=StringIO(text), + source_path='dummy.rst', encoding='utf-8') + result = source.read() + assert result.data == ['this is rst_prolog', + 'hello reST!', + '', + 'hello Sphinx world', + 'Sphinx is a document generator'] + assert result.info(0) == ('<rst_prolog>', 0) + assert result.info(1) == ('<rst_prolog>', 1) + assert result.info(2) == ('<generated>', 0) + assert result.info(3) == ('dummy.rst', 0) + assert result.info(4) == ('dummy.rst', 1) + + # having rst_prolog ends with CR + app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!\n' + source = SphinxRSTFileInput(app, app.env, source=StringIO(text), + source_path='dummy.rst', encoding='utf-8') + result = source.read() + assert result.data == ['this is rst_prolog', + 'hello reST!', + '', + 'hello Sphinx world', + 'Sphinx is a document generator'] + + # having docinfo and rst_prolog + docinfo_text = (':title: test of SphinxFileInput\n' + ':author: Sphinx team\n' + '\n' + 'hello Sphinx world\n' + 'Sphinx is a document generator\n') + app.env.config.rst_prolog = 'this is rst_prolog\nhello reST!' + source = SphinxRSTFileInput(app, app.env, source=StringIO(docinfo_text), + source_path='dummy.rst', encoding='utf-8') + result = source.read() + assert result.data == [':title: test of SphinxFileInput', + ':author: Sphinx team', + '', + 'this is rst_prolog', + 'hello reST!', + '', + '', + 'hello Sphinx world', + 'Sphinx is a document generator'] + assert result.info(0) == ('dummy.rst', 0) + assert result.info(1) == ('dummy.rst', 1) + assert result.info(2) == ('<generated>', 0) + assert result.info(3) == ('<rst_prolog>', 0) + assert result.info(4) == ('<rst_prolog>', 1) + assert result.info(5) == ('<generated>', 0) + assert result.info(6) == ('dummy.rst', 2) + assert result.info(7) == ('dummy.rst', 3) + assert result.info(8) == ('dummy.rst', 4) + assert result.info(9) == ('dummy.rst', None) # out of range + + # having rst_epilog + app.env.config.rst_prolog = None + app.env.config.rst_epilog = 'this is rst_epilog\ngood-bye reST!' + source = SphinxRSTFileInput(app, app.env, source=StringIO(text), + source_path='dummy.rst', encoding='utf-8') + result = source.read() + assert result.data == ['hello Sphinx world', + 'Sphinx is a document generator', + '', + 'this is rst_epilog', + 'good-bye reST!'] + assert result.info(0) == ('dummy.rst', 0) + assert result.info(1) == ('dummy.rst', 1) + assert result.info(2) == ('<generated>', 0) + assert result.info(3) == ('<rst_epilog>', 0) + assert result.info(4) == ('<rst_epilog>', 1) + assert result.info(5) == ('<rst_epilog>', None) # out of range diff --git a/tests/test_util.py b/tests/test_util.py index 84ce44007..aae54eaf0 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -14,8 +14,7 @@ from mock import patch from sphinx.util import logging from sphinx.util import ( - display_chunk, encode_uri, parselinenos, split_docinfo, status_iterator, - xmlname_checker + display_chunk, encode_uri, parselinenos, status_iterator, xmlname_checker ) from sphinx.testing.util import strip_escseq @@ -36,28 +35,6 @@ def test_encode_uri(): assert expected, encode_uri(uri) -def test_splitdocinfo(): - source = "Hello world.\n" - docinfo, content = split_docinfo(source) - assert docinfo == '' - assert content == 'Hello world.\n' - - source = ":orphan:\n\nHello world.\n" - docinfo, content = split_docinfo(source) - assert docinfo == ':orphan:\n' - assert content == '\nHello world.\n' - - source = ":author: Georg Brandl\n:title: Manual of Sphinx\n\nHello world.\n" - docinfo, content = split_docinfo(source) - assert docinfo == ':author: Georg Brandl\n:title: Manual of Sphinx\n' - assert content == '\nHello world.\n' - - source = ":multiline: one\n\ttwo\n\tthree\n\nHello world.\n" - docinfo, content = split_docinfo(source) - assert docinfo == ":multiline: one\n\ttwo\n\tthree\n" - assert content == '\nHello world.\n' - - def test_display_chunk(): assert display_chunk('hello') == 'hello' assert display_chunk(['hello']) == 'hello' |