diff options
| -rw-r--r-- | sphinx/application.py | 30 | ||||
| -rw-r--r-- | sphinx/builders/__init__.py | 12 | ||||
| -rw-r--r-- | sphinx/environment/__init__.py | 18 | ||||
| -rw-r--r-- | sphinx/extension.py | 2 | ||||
| -rw-r--r-- | tests/roots/test-extensions/conf.py | 4 | ||||
| -rw-r--r-- | tests/roots/test-extensions/read_parallel.py | 4 | ||||
| -rw-r--r-- | tests/roots/test-extensions/read_serial.py | 4 | ||||
| -rw-r--r-- | tests/roots/test-extensions/write_parallel.py | 4 | ||||
| -rw-r--r-- | tests/roots/test-extensions/write_serial.py | 4 | ||||
| -rw-r--r-- | tests/test_application.py | 36 |
10 files changed, 92 insertions, 26 deletions
diff --git a/sphinx/application.py b/sphinx/application.py index b6fd7feef..2a084ff0c 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -19,7 +19,7 @@ import posixpath from os import path from collections import deque -from six import iteritems +from six import iteritems, itervalues from six.moves import cStringIO from docutils import nodes @@ -673,6 +673,34 @@ class Sphinx(object): logger.debug('[app] adding HTML theme: %r, %r', name, theme_path) self.html_themes[name] = theme_path + # ---- other methods ------------------------------------------------- + def is_parallel_allowed(self, typ): + # type: (unicode) -> bool + """Check parallel processing is allowed or not. + + ``typ`` is a type of processing; ``'read'`` or ``'write'``. + """ + if typ == 'read': + attrname = 'parallel_read_safe' + elif typ == 'write': + attrname = 'parallel_write_safe' + else: + raise ValueError('parallel type %s is not supported' % typ) + + for ext in itervalues(self.extensions): + allowed = getattr(ext, attrname, None) + if allowed is None: + logger.warning(__("the %s extension does not declare if it is safe " + "for parallel %sing, assuming it isn't - please " + "ask the extension author to check and make it " + "explicit"), ext.name, typ) + logger.warning('doing serial %s', typ) + return False + elif not allowed: + return False + + return True + class TemplateBridge(object): """ diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 8acd91729..6bba5c788 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -17,7 +17,6 @@ try: except ImportError: multiprocessing = None -from six import itervalues from docutils import nodes from sphinx.deprecation import RemovedInSphinx20Warning @@ -371,15 +370,10 @@ class Builder(object): docnames = set(docnames) & self.env.found_docs # determine if we can write in parallel - self.parallel_ok = False if parallel_available and self.app.parallel > 1 and self.allow_parallel: - self.parallel_ok = True - for extension in itervalues(self.app.extensions): - if not extension.parallel_write_safe: - logger.warning('the %s extension is not safe for parallel ' - 'writing, doing serial write', extension.name) - self.parallel_ok = False - break + self.parallel_ok = self.app.is_parallel_allowed('write') + else: + self.parallel_ok = False # create a task executor to use for misc. "finish-up" tasks # if self.parallel_ok: diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py index b4c40b608..2cbee2523 100644 --- a/sphinx/environment/__init__.py +++ b/sphinx/environment/__init__.py @@ -40,7 +40,6 @@ from sphinx.util.matching import compile_matchers from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks from sphinx.util.websupport import is_commentable from sphinx.errors import SphinxError, ExtensionError -from sphinx.locale import __ from sphinx.transforms import SphinxTransformer from sphinx.deprecation import RemovedInSphinx20Warning from sphinx.environment.adapters.indexentries import IndexEntries @@ -558,21 +557,10 @@ class BuildEnvironment(object): self.app.emit('env-before-read-docs', self, docnames) # check if we should do parallel or serial read - par_ok = False if parallel_available and len(docnames) > 5 and self.app.parallel > 1: - for ext in itervalues(self.app.extensions): - if ext.parallel_read_safe is None: - logger.warning(__('the %s extension does not declare if it is safe ' - 'for parallel reading, assuming it isn\'t - please ' - 'ask the extension author to check and make it ' - 'explicit'), ext.name) - logger.warning('doing serial read') - break - elif ext.parallel_read_safe is False: - break - else: - # all extensions support parallel-read - par_ok = True + par_ok = self.app.is_parallel_allowed('read') + else: + par_ok = False if par_ok: self._read_parallel(docnames, self.app, nproc=self.app.parallel) diff --git a/sphinx/extension.py b/sphinx/extension.py index 0520bf564..acbe0d865 100644 --- a/sphinx/extension.py +++ b/sphinx/extension.py @@ -38,7 +38,7 @@ class Extension(object): # The extension supports parallel write or not. The default value # is ``True``. Sphinx writes parallelly documents even if # the extension does not tell its status. - self.parallel_write_safe = kwargs.pop('parallel_read_safe', True) + self.parallel_write_safe = kwargs.pop('parallel_write_safe', True) def verify_required_extensions(app, requirements): diff --git a/tests/roots/test-extensions/conf.py b/tests/roots/test-extensions/conf.py new file mode 100644 index 000000000..9a3cbc844 --- /dev/null +++ b/tests/roots/test-extensions/conf.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath('.')) diff --git a/tests/roots/test-extensions/read_parallel.py b/tests/roots/test-extensions/read_parallel.py new file mode 100644 index 000000000..a3e052f95 --- /dev/null +++ b/tests/roots/test-extensions/read_parallel.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_read_safe': True + } diff --git a/tests/roots/test-extensions/read_serial.py b/tests/roots/test-extensions/read_serial.py new file mode 100644 index 000000000..c55570a5c --- /dev/null +++ b/tests/roots/test-extensions/read_serial.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_read_safe': False + } diff --git a/tests/roots/test-extensions/write_parallel.py b/tests/roots/test-extensions/write_parallel.py new file mode 100644 index 000000000..ebc48ef9b --- /dev/null +++ b/tests/roots/test-extensions/write_parallel.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_write_safe': True, + } diff --git a/tests/roots/test-extensions/write_serial.py b/tests/roots/test-extensions/write_serial.py new file mode 100644 index 000000000..75494ce77 --- /dev/null +++ b/tests/roots/test-extensions/write_serial.py @@ -0,0 +1,4 @@ +def setup(app): + return { + 'parallel_write_safe': False + } diff --git a/tests/test_application.py b/tests/test_application.py index 785a78878..1a4b22e3e 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -12,6 +12,7 @@ from docutils import nodes from sphinx.application import ExtensionError from sphinx.domains import Domain +from sphinx.util import logging from sphinx.testing.util import strip_escseq import pytest @@ -86,3 +87,38 @@ def test_add_source_parser(app, status, warning): assert set(app.registry.get_source_parsers().keys()) == set(['*', '.md', '.test']) assert app.registry.get_source_parsers()['.md'].__name__ == 'DummyMarkdownParser' assert app.registry.get_source_parsers()['.test'].__name__ == 'TestSourceParser' + + +@pytest.mark.sphinx(testroot='extensions') +def test_add_is_parallel_allowed(app, status, warning): + logging.setup(app, status, warning) + + assert app.is_parallel_allowed('read') is True + assert app.is_parallel_allowed('write') is True + assert warning.getvalue() == '' + + app.setup_extension('read_parallel') + assert app.is_parallel_allowed('read') is True + assert app.is_parallel_allowed('write') is True + assert warning.getvalue() == '' + app.extensions.pop('read_parallel') + + app.setup_extension('write_parallel') + assert app.is_parallel_allowed('read') is False + assert app.is_parallel_allowed('write') is True + assert 'the write_parallel extension does not declare' in warning.getvalue() + app.extensions.pop('write_parallel') + warning.truncate(0) # reset warnings + + app.setup_extension('read_serial') + assert app.is_parallel_allowed('read') is False + assert app.is_parallel_allowed('write') is True + assert warning.getvalue() == '' + app.extensions.pop('read_serial') + + app.setup_extension('write_serial') + assert app.is_parallel_allowed('read') is False + assert app.is_parallel_allowed('write') is False + assert 'the write_serial extension does not declare' in warning.getvalue() + app.extensions.pop('write_serial') + warning.truncate(0) # reset warnings |