diff options
Diffstat (limited to 'sphinx/environment')
| -rw-r--r-- | sphinx/environment/__init__.py | 1202 | ||||
| -rw-r--r-- | sphinx/environment/managers/__init__.py | 37 | ||||
| -rw-r--r-- | sphinx/environment/managers/indexentries.py | 172 | ||||
| -rw-r--r-- | sphinx/environment/managers/toctree.py | 561 |
4 files changed, 1972 insertions, 0 deletions
diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py new file mode 100644 index 000000000..d750b0284 --- /dev/null +++ b/sphinx/environment/__init__.py @@ -0,0 +1,1202 @@ +# -*- coding: utf-8 -*- +""" + sphinx.environment + ~~~~~~~~~~~~~~~~~~ + + Global creation environment. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re +import os +import sys +import time +import types +import codecs +import fnmatch +from os import path +from glob import glob + +from six import iteritems, itervalues, class_types, next +from six.moves import cPickle as pickle +from docutils import nodes +from docutils.io import NullOutput +from docutils.core import Publisher +from docutils.utils import Reporter, relative_path, get_source_line +from docutils.parsers.rst import roles +from docutils.parsers.rst.languages import en as english +from docutils.frontend import OptionParser + +from sphinx import addnodes +from sphinx.io import SphinxStandaloneReader, SphinxDummyWriter, SphinxFileInput +from sphinx.util import get_matching_docs, docname_join, FilenameUniqDict +from sphinx.util.nodes import clean_astext, WarningStream, is_translatable, \ + process_only_nodes +from sphinx.util.osutil import SEP, getcwd, fs_encoding, ensuredir +from sphinx.util.images import guess_mimetype +from sphinx.util.i18n import find_catalog_files, get_image_filename_for_language, \ + search_image_for_language +from sphinx.util.console import bold, purple +from sphinx.util.docutils import sphinx_domains +from sphinx.util.matching import compile_matchers +from sphinx.util.parallel import ParallelTasks, parallel_available, make_chunks +from sphinx.util.websupport import is_commentable +from sphinx.errors import SphinxError, ExtensionError +from sphinx.versioning import add_uids, merge_doctrees +from sphinx.transforms import SphinxContentsFilter +from sphinx.environment.managers.indexentries import IndexEntries +from sphinx.environment.managers.toctree import Toctree + + +default_settings = { + 'embed_stylesheet': False, + 'cloak_email_addresses': True, + 'pep_base_url': 'https://www.python.org/dev/peps/', + 'rfc_base_url': 'https://tools.ietf.org/html/', + 'input_encoding': 'utf-8-sig', + 'doctitle_xform': False, + 'sectsubtitle_xform': False, + 'halt_level': 5, + 'file_insertion_enabled': True, +} + +# This is increased every time an environment attribute is added +# or changed to properly invalidate pickle files. +# +# NOTE: increase base version by 2 to have distinct numbers for Py2 and 3 +ENV_VERSION = 50 + (sys.version_info[0] - 2) + + +dummy_reporter = Reporter('', 4, 4) + +versioning_conditions = { + 'none': False, + 'text': is_translatable, + 'commentable': is_commentable, +} + + +class NoUri(Exception): + """Raised by get_relative_uri if there is no URI available.""" + pass + + +class BuildEnvironment(object): + """ + The environment in which the ReST files are translated. + Stores an inventory of cross-file targets and provides doctree + transformations to resolve links to them. + """ + + # --------- ENVIRONMENT PERSISTENCE ---------------------------------------- + + @staticmethod + def frompickle(srcdir, config, filename): + with open(filename, 'rb') as picklefile: + env = pickle.load(picklefile) + if env.version != ENV_VERSION: + raise IOError('build environment version not current') + if env.srcdir != srcdir: + raise IOError('source directory has changed') + env.config.values = config.values + return env + + def topickle(self, filename): + # remove unpicklable attributes + warnfunc = self._warnfunc + self.set_warnfunc(None) + values = self.config.values + del self.config.values + domains = self.domains + del self.domains + managers = self.detach_managers() + # remove potentially pickling-problematic values from config + for key, val in list(vars(self.config).items()): + if key.startswith('_') or \ + isinstance(val, types.ModuleType) or \ + isinstance(val, types.FunctionType) or \ + isinstance(val, class_types): + del self.config[key] + with open(filename, 'wb') as picklefile: + pickle.dump(self, picklefile, pickle.HIGHEST_PROTOCOL) + # reset attributes + self.attach_managers(managers) + self.domains = domains + self.config.values = values + self.set_warnfunc(warnfunc) + + # --------- ENVIRONMENT INITIALIZATION ------------------------------------- + + def __init__(self, srcdir, doctreedir, config): + self.doctreedir = doctreedir + self.srcdir = srcdir + self.config = config + + # the method of doctree versioning; see set_versioning_method + self.versioning_condition = None + self.versioning_compare = None + + # the application object; only set while update() runs + self.app = None + + # all the registered domains, set by the application + self.domains = {} + + # the docutils settings for building + self.settings = default_settings.copy() + self.settings['env'] = self + + # the function to write warning messages with + self._warnfunc = None + + # this is to invalidate old pickles + self.version = ENV_VERSION + + # All "docnames" here are /-separated and relative and exclude + # the source suffix. + + self.found_docs = set() # contains all existing docnames + self.all_docs = {} # docname -> mtime at the time of reading + # contains all read docnames + self.dependencies = {} # docname -> set of dependent file + # names, relative to documentation root + self.included = set() # docnames included from other documents + self.reread_always = set() # docnames to re-read unconditionally on + # next build + + # File metadata + self.metadata = {} # docname -> dict of metadata items + + # TOC inventory + self.titles = {} # docname -> title node + self.longtitles = {} # docname -> title node; only different if + # set differently with title directive + self.tocs = {} # docname -> table of contents nodetree + self.toc_num_entries = {} # docname -> number of real entries + # used to determine when to show the TOC + # in a sidebar (don't show if it's only one item) + self.toc_secnumbers = {} # docname -> dict of sectionid -> number + self.toc_fignumbers = {} # docname -> dict of figtype -> + # dict of figureid -> number + + self.toctree_includes = {} # docname -> list of toctree includefiles + self.files_to_rebuild = {} # docname -> set of files + # (containing its TOCs) to rebuild too + self.glob_toctrees = set() # docnames that have :glob: toctrees + self.numbered_toctrees = set() # docnames that have :numbered: toctrees + + # domain-specific inventories, here to be pickled + self.domaindata = {} # domainname -> domain-specific dict + + # Other inventories + self.indexentries = {} # docname -> list of + # (type, string, target, aliasname) + self.versionchanges = {} # version -> list of (type, docname, + # lineno, module, descname, content) + + # these map absolute path -> (docnames, unique filename) + self.images = FilenameUniqDict() + self.dlfiles = FilenameUniqDict() + + # temporary data storage while reading a document + self.temp_data = {} + # context for cross-references (e.g. current module or class) + # this is similar to temp_data, but will for example be copied to + # attributes of "any" cross references + self.ref_context = {} + + self.managers = {} + self.init_managers() + + def init_managers(self): + managers = {} + for manager_class in [IndexEntries, Toctree]: + managers[manager_class.name] = manager_class(self) + self.attach_managers(managers) + + def attach_managers(self, managers): + for name, manager in iteritems(managers): + self.managers[name] = manager + manager.attach(self) + + def detach_managers(self): + managers = self.managers + self.managers = {} + for _, manager in iteritems(managers): + manager.detach(self) + return managers + + def set_warnfunc(self, func): + self._warnfunc = func + self.settings['warning_stream'] = WarningStream(func) + + def set_versioning_method(self, method, compare): + """This sets the doctree versioning method for this environment. + + Versioning methods are a builder property; only builders with the same + versioning method can share the same doctree directory. Therefore, we + raise an exception if the user tries to use an environment with an + incompatible versioning method. + """ + if method not in versioning_conditions: + raise ValueError('invalid versioning method: %r' % method) + condition = versioning_conditions[method] + if self.versioning_condition not in (None, condition): + raise SphinxError('This environment is incompatible with the ' + 'selected builder, please choose another ' + 'doctree directory.') + self.versioning_condition = condition + self.versioning_compare = compare + + def warn(self, docname, msg, lineno=None, **kwargs): + """Emit a warning. + + This differs from using ``app.warn()`` in that the warning may not + be emitted instantly, but collected for emitting all warnings after + the update of the environment. + """ + # strange argument order is due to backwards compatibility + self._warnfunc(msg, (docname, lineno), **kwargs) + + def warn_node(self, msg, node, **kwargs): + """Like :meth:`warn`, but with source information taken from *node*.""" + self._warnfunc(msg, '%s:%s' % get_source_line(node), **kwargs) + + def clear_doc(self, docname): + """Remove all traces of a source file in the inventory.""" + if docname in self.all_docs: + self.all_docs.pop(docname, None) + self.reread_always.discard(docname) + self.metadata.pop(docname, None) + self.dependencies.pop(docname, None) + self.titles.pop(docname, None) + self.longtitles.pop(docname, None) + self.images.purge_doc(docname) + self.dlfiles.purge_doc(docname) + + for version, changes in self.versionchanges.items(): + new = [change for change in changes if change[1] != docname] + changes[:] = new + + for manager in itervalues(self.managers): + manager.clear_doc(docname) + + for domain in self.domains.values(): + domain.clear_doc(docname) + + def merge_info_from(self, docnames, other, app): + """Merge global information gathered about *docnames* while reading them + from the *other* environment. + + This possibly comes from a parallel build process. + """ + docnames = set(docnames) + for docname in docnames: + self.all_docs[docname] = other.all_docs[docname] + if docname in other.reread_always: + self.reread_always.add(docname) + self.metadata[docname] = other.metadata[docname] + if docname in other.dependencies: + self.dependencies[docname] = other.dependencies[docname] + self.titles[docname] = other.titles[docname] + self.longtitles[docname] = other.longtitles[docname] + + self.images.merge_other(docnames, other.images) + self.dlfiles.merge_other(docnames, other.dlfiles) + + for version, changes in other.versionchanges.items(): + self.versionchanges.setdefault(version, []).extend( + change for change in changes if change[1] in docnames) + + for manager in itervalues(self.managers): + manager.merge_other(docnames, other) + for domainname, domain in self.domains.items(): + domain.merge_domaindata(docnames, other.domaindata[domainname]) + app.emit('env-merge-info', self, docnames, other) + + def path2doc(self, filename): + """Return the docname for the filename if the file is document. + + *filename* should be absolute or relative to the source directory. + """ + if filename.startswith(self.srcdir): + filename = filename[len(self.srcdir) + 1:] + for suffix in self.config.source_suffix: + if fnmatch.fnmatch(filename, '*' + suffix): + return filename[:-len(suffix)] + else: + # the file does not have docname + return None + + def doc2path(self, docname, base=True, suffix=None): + """Return the filename for the document name. + + If *base* is True, return absolute path under self.srcdir. + If *base* is None, return relative path to self.srcdir. + If *base* is a path string, return absolute path under that. + If *suffix* is not None, add it instead of config.source_suffix. + """ + docname = docname.replace(SEP, path.sep) + if suffix is None: + for candidate_suffix in self.config.source_suffix: + if path.isfile(path.join(self.srcdir, docname) + + candidate_suffix): + suffix = candidate_suffix + break + else: + # document does not exist + suffix = self.config.source_suffix[0] + if base is True: + return path.join(self.srcdir, docname) + suffix + elif base is None: + return docname + suffix + else: + return path.join(base, docname) + suffix + + def relfn2path(self, filename, docname=None): + """Return paths to a file referenced from a document, relative to + documentation root and absolute. + + In the input "filename", absolute filenames are taken as relative to the + source dir, while relative filenames are relative to the dir of the + containing document. + """ + if filename.startswith('/') or filename.startswith(os.sep): + rel_fn = filename[1:] + else: + docdir = path.dirname(self.doc2path(docname or self.docname, + base=None)) + rel_fn = path.join(docdir, filename) + try: + # the path.abspath() might seem redundant, but otherwise artifacts + # such as ".." will remain in the path + return rel_fn, path.abspath(path.join(self.srcdir, rel_fn)) + except UnicodeDecodeError: + # the source directory is a bytestring with non-ASCII characters; + # let's try to encode the rel_fn in the file system encoding + enc_rel_fn = rel_fn.encode(sys.getfilesystemencoding()) + return rel_fn, path.abspath(path.join(self.srcdir, enc_rel_fn)) + + def find_files(self, config): + """Find all source files in the source dir and put them in + self.found_docs. + """ + matchers = compile_matchers( + config.exclude_patterns[:] + + config.templates_path + + config.html_extra_path + + ['**/_sources', '.#*', '**/.#*', '*.lproj/**'] + ) + self.found_docs = set() + for docname in get_matching_docs(self.srcdir, config.source_suffix, + exclude_matchers=matchers): + if os.access(self.doc2path(docname), os.R_OK): + self.found_docs.add(docname) + else: + self.warn(docname, "document not readable. Ignored.") + + # add catalog mo file dependency + for docname in self.found_docs: + catalog_files = find_catalog_files( + docname, + self.srcdir, + self.config.locale_dirs, + self.config.language, + self.config.gettext_compact) + for filename in catalog_files: + self.dependencies.setdefault(docname, set()).add(filename) + + def get_outdated_files(self, config_changed): + """Return (added, changed, removed) sets.""" + # clear all files no longer present + removed = set(self.all_docs) - self.found_docs + + added = set() + changed = set() + + if config_changed: + # config values affect e.g. substitutions + added = self.found_docs + else: + for docname in self.found_docs: + if docname not in self.all_docs: + added.add(docname) + continue + # if the doctree file is not there, rebuild + if not path.isfile(self.doc2path(docname, self.doctreedir, + '.doctree')): + changed.add(docname) + continue + # check the "reread always" list + if docname in self.reread_always: + changed.add(docname) + continue + # check the mtime of the document + mtime = self.all_docs[docname] + newmtime = path.getmtime(self.doc2path(docname)) + if newmtime > mtime: + changed.add(docname) + continue + # finally, check the mtime of dependencies + for dep in self.dependencies.get(docname, ()): + try: + # this will do the right thing when dep is absolute too + deppath = path.join(self.srcdir, dep) + if not path.isfile(deppath): + changed.add(docname) + break + depmtime = path.getmtime(deppath) + if depmtime > mtime: + changed.add(docname) + break + except EnvironmentError: + # give it another chance + changed.add(docname) + break + + return added, changed, removed + + def update(self, config, srcdir, doctreedir, app): + """(Re-)read all files new or changed since last update. + + Store all environment docnames in the canonical format (ie using SEP as + a separator in place of os.path.sep). + """ + config_changed = False + if self.config is None: + msg = '[new config] ' + config_changed = True + else: + # check if a config value was changed that affects how + # doctrees are read + for key, descr in iteritems(config.values): + if descr[1] != 'env': + continue + if self.config[key] != config[key]: + msg = '[config changed] ' + config_changed = True + break + else: + msg = '' + # this value is not covered by the above loop because it is handled + # specially by the config class + if self.config.extensions != config.extensions: + msg = '[extensions changed] ' + config_changed = True + # the source and doctree directories may have been relocated + self.srcdir = srcdir + self.doctreedir = doctreedir + self.find_files(config) + self.config = config + + # this cache also needs to be updated every time + self._nitpick_ignore = set(self.config.nitpick_ignore) + + app.info(bold('updating environment: '), nonl=1) + + added, changed, removed = self.get_outdated_files(config_changed) + + # allow user intervention as well + for docs in app.emit('env-get-outdated', self, added, changed, removed): + changed.update(set(docs) & self.found_docs) + + # if files were added or removed, all documents with globbed toctrees + # must be reread + if added or removed: + # ... but not those that already were removed + changed.update(self.glob_toctrees & self.found_docs) + + msg += '%s added, %s changed, %s removed' % (len(added), len(changed), + len(removed)) + app.info(msg) + + self.app = app + + # clear all files no longer present + for docname in removed: + app.emit('env-purge-doc', self, docname) + self.clear_doc(docname) + + # read all new and changed files + docnames = sorted(added | changed) + # allow changing and reordering the list of docs to read + app.emit('env-before-read-docs', self, docnames) + + # check if we should do parallel or serial read + par_ok = False + if parallel_available and len(docnames) > 5 and app.parallel > 1: + par_ok = True + for extname, md in app._extension_metadata.items(): + ext_ok = md.get('parallel_read_safe') + if ext_ok: + continue + if ext_ok is None: + app.warn('the %s extension does not declare if it ' + 'is safe for parallel reading, assuming it ' + 'isn\'t - please ask the extension author to ' + 'check and make it explicit' % extname) + app.warn('doing serial read') + else: + app.warn('the %s extension is not safe for parallel ' + 'reading, doing serial read' % extname) + par_ok = False + break + if par_ok: + self._read_parallel(docnames, app, nproc=app.parallel) + else: + self._read_serial(docnames, app) + + if config.master_doc not in self.all_docs: + raise SphinxError('master file %s not found' % + self.doc2path(config.master_doc)) + + self.app = None + + for retval in app.emit('env-updated', self): + if retval is not None: + docnames.extend(retval) + + return sorted(docnames) + + def _read_serial(self, docnames, app): + for docname in app.status_iterator(docnames, 'reading sources... ', + purple, len(docnames)): + # remove all inventory entries for that file + app.emit('env-purge-doc', self, docname) + self.clear_doc(docname) + self.read_doc(docname, app) + + def _read_parallel(self, docnames, app, nproc): + # clear all outdated docs at once + for docname in docnames: + app.emit('env-purge-doc', self, docname) + self.clear_doc(docname) + + def read_process(docs): + self.app = app + self.warnings = [] + self.set_warnfunc(lambda *args, **kwargs: self.warnings.append((args, kwargs))) + for docname in docs: + self.read_doc(docname, app) + # allow pickling self to send it back + self.set_warnfunc(None) + del self.app + del self.domains + del self.config.values + del self.config + return self + + def merge(docs, otherenv): + warnings.extend(otherenv.warnings) + self.merge_info_from(docs, otherenv, app) + + tasks = ParallelTasks(nproc) + chunks = make_chunks(docnames, nproc) + + warnings = [] + for chunk in app.status_iterator( + chunks, 'reading sources... ', purple, len(chunks)): + tasks.add_task(read_process, chunk, merge) + + # make sure all threads have finished + app.info(bold('waiting for workers...')) + tasks.join() + + for warning, kwargs in warnings: + self._warnfunc(*warning, **kwargs) + + def check_dependents(self, already): + to_rewrite = (self.toctree.assign_section_numbers() + + self.toctree.assign_figure_numbers()) + for docname in set(to_rewrite): + if docname not in already: + yield docname + + # --------- SINGLE FILE READING -------------------------------------------- + + def warn_and_replace(self, error): + """Custom decoding error handler that warns and replaces.""" + linestart = error.object.rfind(b'\n', 0, error.start) + lineend = error.object.find(b'\n', error.start) + if lineend == -1: + lineend = len(error.object) + lineno = error.object.count(b'\n', 0, error.start) + 1 + self.warn(self.docname, 'undecodable source characters, ' + 'replacing with "?": %r' % + (error.object[linestart+1:error.start] + b'>>>' + + error.object[error.start:error.end] + b'<<<' + + error.object[error.end:lineend]), lineno) + return (u'?', error.end) + + def read_doc(self, docname, app=None): + """Parse a file and add/update inventory entries for the doctree.""" + + self.temp_data['docname'] = docname + # defaults to the global default, but can be re-set in a document + self.temp_data['default_domain'] = \ + self.domains.get(self.config.primary_domain) + + self.settings['input_encoding'] = self.config.source_encoding + self.settings['trim_footnote_reference_space'] = \ + self.config.trim_footnote_reference_space + self.settings['gettext_compact'] = self.config.gettext_compact + + docutilsconf = path.join(self.srcdir, 'docutils.conf') + # read docutils.conf from source dir, not from current dir + OptionParser.standard_config_files[1] = docutilsconf + if path.isfile(docutilsconf): + self.note_dependency(docutilsconf) + + with sphinx_domains(self): + if self.config.default_role: + role_fn, messages = roles.role(self.config.default_role, english, + 0, dummy_reporter) + if role_fn: + roles._roles[''] = role_fn + else: + self.warn(docname, 'default role %s not found' % + self.config.default_role) + + codecs.register_error('sphinx', self.warn_and_replace) + + # publish manually + reader = SphinxStandaloneReader(self.app, parsers=self.config.source_parsers) + pub = Publisher(reader=reader, + writer=SphinxDummyWriter(), + destination_class=NullOutput) + pub.set_components(None, 'restructuredtext', None) + pub.process_programmatic_settings(None, self.settings, None) + src_path = self.doc2path(docname) + source = SphinxFileInput(app, self, source=None, source_path=src_path, + encoding=self.config.source_encoding) + pub.source = source + pub.settings._source = src_path + pub.set_destination(None, None) + pub.publish() + doctree = pub.document + + # post-processing + self.process_dependencies(docname, doctree) + self.process_images(docname, doctree) + self.process_downloads(docname, doctree) + self.process_metadata(docname, doctree) + self.create_title_from(docname, doctree) + for manager in itervalues(self.managers): + manager.process_doc(docname, doctree) + for domain in itervalues(self.domains): + domain.process_doc(self, docname, doctree) + + # allow extension-specific post-processing + if app: + app.emit('doctree-read', doctree) + + # store time of reading, for outdated files detection + # (Some filesystems have coarse timestamp resolution; + # therefore time.time() can be older than filesystem's timestamp. + # For example, FAT32 has 2sec timestamp resolution.) + self.all_docs[docname] = max( + time.time(), path.getmtime(self.doc2path(docname))) + + if self.versioning_condition: + old_doctree = None + if self.versioning_compare: + # get old doctree + try: + with open(self.doc2path(docname, + self.doctreedir, '.doctree'), 'rb') as f: + old_doctree = pickle.load(f) + except EnvironmentError: + pass + + # add uids for versioning + if not self.versioning_compare or old_doctree is None: + list(add_uids(doctree, self.versioning_condition)) + else: + list(merge_doctrees( + old_doctree, doctree, self.versioning_condition)) + + # make it picklable + doctree.reporter = None + doctree.transformer = None + doctree.settings.warning_stream = None + doctree.settings.env = None + doctree.settings.record_dependencies = None + + # cleanup + self.temp_data.clear() + self.ref_context.clear() + roles._roles.pop('', None) # if a document has set a local default role + + # save the parsed doctree + doctree_filename = self.doc2path(docname, self.doctreedir, + '.doctree') + ensuredir(path.dirname(doctree_filename)) + with open(doctree_filename, 'wb') as f: + pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL) + + # utilities to use while reading a document + + @property + def docname(self): + """Returns the docname of the document currently being parsed.""" + return self.temp_data['docname'] + + @property + def currmodule(self): + """Backwards compatible alias. Will be removed.""" + self.warn(self.docname, 'env.currmodule is being referenced by an ' + 'extension; this API will be removed in the future') + return self.ref_context.get('py:module') + + @property + def currclass(self): + """Backwards compatible alias. Will be removed.""" + self.warn(self.docname, 'env.currclass is being referenced by an ' + 'extension; this API will be removed in the future') + return self.ref_context.get('py:class') + + def new_serialno(self, category=''): + """Return a serial number, e.g. for index entry targets. + + The number is guaranteed to be unique in the current document. + """ + key = category + 'serialno' + cur = self.temp_data.get(key, 0) + self.temp_data[key] = cur + 1 + return cur + + def note_dependency(self, filename): + """Add *filename* as a dependency of the current document. + + This means that the document will be rebuilt if this file changes. + + *filename* should be absolute or relative to the source directory. + """ + self.dependencies.setdefault(self.docname, set()).add(filename) + + def note_included(self, filename): + """Add *filename* as a included from other document. + + This means the document is not orphaned. + + *filename* should be absolute or relative to the source directory. + """ + self.included.add(self.path2doc(filename)) + + def note_reread(self): + """Add the current document to the list of documents that will + automatically be re-read at the next build. + """ + self.reread_always.add(self.docname) + + def note_versionchange(self, type, version, node, lineno): + self.versionchanges.setdefault(version, []).append( + (type, self.temp_data['docname'], lineno, + self.ref_context.get('py:module'), + self.temp_data.get('object'), node.astext())) + + # post-processing of read doctrees + + def process_dependencies(self, docname, doctree): + """Process docutils-generated dependency info.""" + cwd = getcwd() + frompath = path.join(path.normpath(self.srcdir), 'dummy') + deps = doctree.settings.record_dependencies + if not deps: + return + for dep in deps.list: + # the dependency path is relative to the working dir, so get + # one relative to the srcdir + if isinstance(dep, bytes): + dep = dep.decode(fs_encoding) + relpath = relative_path(frompath, + path.normpath(path.join(cwd, dep))) + self.dependencies.setdefault(docname, set()).add(relpath) + + def process_downloads(self, docname, doctree): + """Process downloadable file paths. """ + for node in doctree.traverse(addnodes.download_reference): + targetname = node['reftarget'] + rel_filename, filename = self.relfn2path(targetname, docname) + self.dependencies.setdefault(docname, set()).add(rel_filename) + if not os.access(filename, os.R_OK): + self.warn_node('download file not readable: %s' % filename, + node) + continue + uniquename = self.dlfiles.add_file(docname, filename) + node['filename'] = uniquename + + def process_images(self, docname, doctree): + """Process and rewrite image URIs.""" + def collect_candidates(imgpath, candidates): + globbed = {} + for filename in glob(imgpath): + new_imgpath = relative_path(path.join(self.srcdir, 'dummy'), + filename) + try: + mimetype = guess_mimetype(filename) + if mimetype not in candidates: + globbed.setdefault(mimetype, []).append(new_imgpath) + except (OSError, IOError) as err: + self.warn_node('image file %s not readable: %s' % + (filename, err), node) + for key, files in iteritems(globbed): + candidates[key] = sorted(files, key=len)[0] # select by similarity + + for node in doctree.traverse(nodes.image): + # Map the mimetype to the corresponding image. The writer may + # choose the best image from these candidates. The special key * is + # set if there is only single candidate to be used by a writer. + # The special key ? is set for nonlocal URIs. + node['candidates'] = candidates = {} + imguri = node['uri'] + if imguri.startswith('data:'): + self.warn_node('image data URI found. some builders might not support', node, + type='image', subtype='data_uri') + candidates['?'] = imguri + continue + elif imguri.find('://') != -1: + self.warn_node('nonlocal image URI found: %s' % imguri, node, + type='image', subtype='nonlocal_uri') + candidates['?'] = imguri + continue + rel_imgpath, full_imgpath = self.relfn2path(imguri, docname) + if self.config.language: + # substitute figures (ex. foo.png -> foo.en.png) + i18n_full_imgpath = search_image_for_language(full_imgpath, self) + if i18n_full_imgpath != full_imgpath: + full_imgpath = i18n_full_imgpath + rel_imgpath = relative_path(path.join(self.srcdir, 'dummy'), + i18n_full_imgpath) + # set imgpath as default URI + node['uri'] = rel_imgpath + if rel_imgpath.endswith(os.extsep + '*'): + if self.config.language: + # Search language-specific figures at first + i18n_imguri = get_image_filename_for_language(imguri, self) + _, full_i18n_imgpath = self.relfn2path(i18n_imguri, docname) + collect_candidates(full_i18n_imgpath, candidates) + + collect_candidates(full_imgpath, candidates) + else: + candidates['*'] = rel_imgpath + + # map image paths to unique image names (so that they can be put + # into a single directory) + for imgpath in itervalues(candidates): + self.dependencies.setdefault(docname, set()).add(imgpath) + if not os.access(path.join(self.srcdir, imgpath), os.R_OK): + self.warn_node('image file not readable: %s' % imgpath, + node) + continue + self.images.add_file(docname, imgpath) + + def process_metadata(self, docname, doctree): + """Process the docinfo part of the doctree as metadata. + + Keep processing minimal -- just return what docutils says. + """ + self.metadata[docname] = md = {} + try: + docinfo = doctree[0] + except IndexError: + # probably an empty document + return + if docinfo.__class__ is not nodes.docinfo: + # nothing to see here + return + for node in docinfo: + # nodes are multiply inherited... + if isinstance(node, nodes.authors): + md['authors'] = [author.astext() for author in node] + elif isinstance(node, nodes.TextElement): # e.g. author + md[node.__class__.__name__] = node.astext() + else: + name, body = node + md[name.astext()] = body.astext() + for name, value in md.items(): + if name in ('tocdepth',): + try: + value = int(value) + except ValueError: + value = 0 + md[name] = value + + del doctree[0] + + def create_title_from(self, docname, document): + """Add a title node to the document (just copy the first section title), + and store that title in the environment. + """ + titlenode = nodes.title() + longtitlenode = titlenode + # explicit title set with title directive; use this only for + # the <title> tag in HTML output + if 'title' in document: + longtitlenode = nodes.title() + longtitlenode += nodes.Text(document['title']) + # look for first section title and use that as the title + for node in document.traverse(nodes.section): + visitor = SphinxContentsFilter(document) + node[0].walkabout(visitor) + titlenode += visitor.get_entry_text() + break + else: + # document has no title + titlenode += nodes.Text('<no title>') + self.titles[docname] = titlenode + self.longtitles[docname] = longtitlenode + + def note_toctree(self, docname, toctreenode): + """Note a TOC tree directive in a document and gather information about + file relations from it. + """ + self.toctree.note_toctree(docname, toctreenode) + + def get_toc_for(self, docname, builder): + """Return a TOC nodetree -- for use on the same page only!""" + return self.toctree.get_toc_for(docname, builder) + + def get_toctree_for(self, docname, builder, collapse, **kwds): + """Return the global TOC nodetree.""" + return self.toctree.get_toctree_for(docname, builder, collapse, **kwds) + + def get_domain(self, domainname): + """Return the domain instance with the specified name. + + Raises an ExtensionError if the domain is not registered. + """ + try: + return self.domains[domainname] + except KeyError: + raise ExtensionError('Domain %r is not registered' % domainname) + + # --------- RESOLVING REFERENCES AND TOCTREES ------------------------------ + + def get_doctree(self, docname): + """Read the doctree for a file from the pickle and return it.""" + doctree_filename = self.doc2path(docname, self.doctreedir, '.doctree') + with open(doctree_filename, 'rb') as f: + doctree = pickle.load(f) + doctree.settings.env = self + doctree.reporter = Reporter(self.doc2path(docname), 2, 5, + stream=WarningStream(self._warnfunc)) + return doctree + + def get_and_resolve_doctree(self, docname, builder, doctree=None, + prune_toctrees=True, includehidden=False): + """Read the doctree from the pickle, resolve cross-references and + toctrees and return it. + """ + if doctree is None: + doctree = self.get_doctree(docname) + + # resolve all pending cross-references + self.resolve_references(doctree, docname, builder) + + # now, resolve all toctree nodes + for toctreenode in doctree.traverse(addnodes.toctree): + result = self.resolve_toctree(docname, builder, toctreenode, + prune=prune_toctrees, + includehidden=includehidden) + if result is None: + toctreenode.replace_self([]) + else: + toctreenode.replace_self(result) + + return doctree + + def resolve_toctree(self, docname, builder, toctree, prune=True, maxdepth=0, + titles_only=False, collapse=False, includehidden=False): + """Resolve a *toctree* node into individual bullet lists with titles + as items, returning None (if no containing titles are found) or + a new node. + + If *prune* is True, the tree is pruned to *maxdepth*, or if that is 0, + to the value of the *maxdepth* option on the *toctree* node. + If *titles_only* is True, only toplevel document titles will be in the + resulting tree. + If *collapse* is True, all branches not containing docname will + be collapsed. + """ + return self.toctree.resolve_toctree(docname, builder, toctree, prune, + maxdepth, titles_only, collapse, + includehidden) + + def resolve_references(self, doctree, fromdocname, builder): + for node in doctree.traverse(addnodes.pending_xref): + contnode = node[0].deepcopy() + newnode = None + + typ = node['reftype'] + target = node['reftarget'] + refdoc = node.get('refdoc', fromdocname) + domain = None + + try: + if 'refdomain' in node and node['refdomain']: + # let the domain try to resolve the reference + try: + domain = self.domains[node['refdomain']] + except KeyError: + raise NoUri + newnode = domain.resolve_xref(self, refdoc, builder, + typ, target, node, contnode) + # really hardwired reference types + elif typ == 'any': + newnode = self._resolve_any_reference(builder, refdoc, node, contnode) + elif typ == 'doc': + newnode = self._resolve_doc_reference(builder, refdoc, node, contnode) + # no new node found? try the missing-reference event + if newnode is None: + newnode = builder.app.emit_firstresult( + 'missing-reference', self, node, contnode) + # still not found? warn if node wishes to be warned about or + # we are in nit-picky mode + if newnode is None: + self._warn_missing_reference(refdoc, typ, target, node, domain) + except NoUri: + newnode = contnode + node.replace_self(newnode or contnode) + + # remove only-nodes that do not belong to our builder + process_only_nodes(doctree, builder.tags, warn_node=self.warn_node) + + # allow custom references to be resolved + builder.app.emit('doctree-resolved', doctree, fromdocname) + + def _warn_missing_reference(self, refdoc, typ, target, node, domain): + warn = node.get('refwarn') + if self.config.nitpicky: + warn = True + if self._nitpick_ignore: + dtype = domain and '%s:%s' % (domain.name, typ) or typ + if (dtype, target) in self._nitpick_ignore: + warn = False + # for "std" types also try without domain name + if (not domain or domain.name == 'std') and \ + (typ, target) in self._nitpick_ignore: + warn = False + if not warn: + return + if domain and typ in domain.dangling_warnings: + msg = domain.dangling_warnings[typ] + elif typ == 'doc': + msg = 'unknown document: %(target)s' + elif node.get('refdomain', 'std') not in ('', 'std'): + msg = '%s:%s reference target not found: %%(target)s' % \ + (node['refdomain'], typ) + else: + msg = '%r reference target not found: %%(target)s' % typ + self.warn_node(msg % {'target': target}, node, type='ref', subtype=typ) + + def _resolve_doc_reference(self, builder, refdoc, node, contnode): + # directly reference to document by source name; + # can be absolute or relative + docname = docname_join(refdoc, node['reftarget']) + if docname in self.all_docs: + if node['refexplicit']: + # reference with explicit title + caption = node.astext() + else: + caption = clean_astext(self.titles[docname]) + innernode = nodes.inline(caption, caption) + innernode['classes'].append('doc') + newnode = nodes.reference('', '', internal=True) + newnode['refuri'] = builder.get_relative_uri(refdoc, docname) + newnode.append(innernode) + return newnode + + def _resolve_any_reference(self, builder, refdoc, node, contnode): + """Resolve reference generated by the "any" role.""" + target = node['reftarget'] + results = [] + # first, try resolving as :doc: + doc_ref = self._resolve_doc_reference(builder, refdoc, node, contnode) + if doc_ref: + results.append(('doc', doc_ref)) + # next, do the standard domain (makes this a priority) + results.extend(self.domains['std'].resolve_any_xref( + self, refdoc, builder, target, node, contnode)) + for domain in self.domains.values(): + if domain.name == 'std': + continue # we did this one already + try: + results.extend(domain.resolve_any_xref(self, refdoc, builder, + target, node, contnode)) + except NotImplementedError: + # the domain doesn't yet support the new interface + # we have to manually collect possible references (SLOW) + for role in domain.roles: + res = domain.resolve_xref(self, refdoc, builder, role, target, + node, contnode) + if res and isinstance(res[0], nodes.Element): + results.append(('%s:%s' % (domain.name, role), res)) + # now, see how many matches we got... + if not results: + return None + if len(results) > 1: + nice_results = ' or '.join(':%s:' % r[0] for r in results) + self.warn_node('more than one target found for \'any\' cross-' + 'reference %r: could be %s' % (target, nice_results), + node) + res_role, newnode = results[0] + # Override "any" class with the actual role type to get the styling + # approximately correct. + res_domain = res_role.split(':')[0] + if newnode and newnode[0].get('classes'): + newnode[0]['classes'].append(res_domain) + newnode[0]['classes'].append(res_role.replace(':', '-')) + return newnode + + def create_index(self, builder, group_entries=True, + _fixre=re.compile(r'(.*) ([(][^()]*[)])')): + return self.indices.create_index(builder, group_entries=group_entries, _fixre=_fixre) + + def collect_relations(self): + traversed = set() + + def traverse_toctree(parent, docname): + if parent == docname: + self.warn(docname, 'self referenced toctree found. Ignored.') + return + + # traverse toctree by pre-order + yield parent, docname + traversed.add(docname) + + for child in (self.toctree_includes.get(docname) or []): + for subparent, subdocname in traverse_toctree(docname, child): + if subdocname not in traversed: + yield subparent, subdocname + traversed.add(subdocname) + + relations = {} + docnames = traverse_toctree(None, self.config.master_doc) + prevdoc = None + parent, docname = next(docnames) + for nextparent, nextdoc in docnames: + relations[docname] = [parent, prevdoc, nextdoc] + prevdoc = docname + docname = nextdoc + parent = nextparent + + relations[docname] = [parent, prevdoc, None] + + return relations + + def check_consistency(self): + """Do consistency checks.""" + for docname in sorted(self.all_docs): + if docname not in self.files_to_rebuild: + if docname == self.config.master_doc: + # the master file is not included anywhere ;) + continue + if docname in self.included: + # the document is included from other documents + continue + if 'orphan' in self.metadata[docname]: + continue + self.warn(docname, 'document isn\'t included in any toctree') diff --git a/sphinx/environment/managers/__init__.py b/sphinx/environment/managers/__init__.py new file mode 100644 index 000000000..963ec54b8 --- /dev/null +++ b/sphinx/environment/managers/__init__.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" + sphinx.environment.managers + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Manager components for sphinx.environment. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + + +class EnvironmentManager(object): + """Base class for sphinx.environment managers.""" + name = None + + def __init__(self, env): + self.env = env + + def attach(self, env): + self.env = env + if self.name: + setattr(env, self.name, self) + + def detach(self, env): + self.env = None + if self.name: + delattr(env, self.name) + + def clear_doc(self, docname): + raise NotImplementedError + + def merge_other(self, docnames, other): + raise NotImplementedError + + def process_doc(self, docname, doctree): + raise NotImplementedError diff --git a/sphinx/environment/managers/indexentries.py b/sphinx/environment/managers/indexentries.py new file mode 100644 index 000000000..c35a161b4 --- /dev/null +++ b/sphinx/environment/managers/indexentries.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" + sphinx.environment.managers.indexentries + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Index entries manager for sphinx.environment. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import re +import bisect +import unicodedata +import string +from itertools import groupby + +from six import text_type + +from sphinx import addnodes +from sphinx.util import iteritems, split_index_msg, split_into +from sphinx.locale import _ +from sphinx.environment.managers import EnvironmentManager + + +class IndexEntries(EnvironmentManager): + name = 'indices' + + def __init__(self, env): + super(IndexEntries, self).__init__(env) + self.data = env.indexentries + + def clear_doc(self, docname): + self.data.pop(docname, None) + + def merge_other(self, docnames, other): + for docname in docnames: + self.data[docname] = other.indexentries[docname] + + def process_doc(self, docname, doctree): + entries = self.data[docname] = [] + for node in doctree.traverse(addnodes.index): + try: + for entry in node['entries']: + split_index_msg(entry[0], entry[1]) + except ValueError as exc: + self.env.warn_node(exc, node) + node.parent.remove(node) + else: + for entry in node['entries']: + if len(entry) == 5: + # Since 1.4: new index structure including index_key (5th column) + entries.append(entry) + else: + entries.append(entry + (None,)) + + def create_index(self, builder, group_entries=True, + _fixre=re.compile(r'(.*) ([(][^()]*[)])')): + """Create the real index from the collected index entries.""" + from sphinx.environment import NoUri + + new = {} + + def add_entry(word, subword, main, link=True, dic=new, key=None): + # Force the word to be unicode if it's a ASCII bytestring. + # This will solve problems with unicode normalization later. + # For instance the RFC role will add bytestrings at the moment + word = text_type(word) + entry = dic.get(word) + if not entry: + dic[word] = entry = [[], {}, key] + if subword: + add_entry(subword, '', main, link=link, dic=entry[1], key=key) + elif link: + try: + uri = builder.get_relative_uri('genindex', fn) + '#' + tid + except NoUri: + pass + else: + # maintain links in sorted/deterministic order + bisect.insort(entry[0], (main, uri)) + + for fn, entries in iteritems(self.data): + # new entry types must be listed in directives/other.py! + for type, value, tid, main, index_key in entries: + try: + if type == 'single': + try: + entry, subentry = split_into(2, 'single', value) + except ValueError: + entry, = split_into(1, 'single', value) + subentry = '' + add_entry(entry, subentry, main, key=index_key) + elif type == 'pair': + first, second = split_into(2, 'pair', value) + add_entry(first, second, main, key=index_key) + add_entry(second, first, main, key=index_key) + elif type == 'triple': + first, second, third = split_into(3, 'triple', value) + add_entry(first, second + ' ' + third, main, key=index_key) + add_entry(second, third + ', ' + first, main, key=index_key) + add_entry(third, first + ' ' + second, main, key=index_key) + elif type == 'see': + first, second = split_into(2, 'see', value) + add_entry(first, _('see %s') % second, None, + link=False, key=index_key) + elif type == 'seealso': + first, second = split_into(2, 'see', value) + add_entry(first, _('see also %s') % second, None, + link=False, key=index_key) + else: + self.env.warn(fn, 'unknown index entry type %r' % type) + except ValueError as err: + self.env.warn(fn, str(err)) + + # sort the index entries; put all symbols at the front, even those + # following the letters in ASCII, this is where the chr(127) comes from + def keyfunc(entry, lcletters=string.ascii_lowercase + '_'): + lckey = unicodedata.normalize('NFD', entry[0].lower()) + if lckey[0:1] in lcletters: + lckey = chr(127) + lckey + # ensure a determinstic order *within* letters by also sorting on + # the entry itself + return (lckey, entry[0]) + newlist = sorted(new.items(), key=keyfunc) + + if group_entries: + # fixup entries: transform + # func() (in module foo) + # func() (in module bar) + # into + # func() + # (in module foo) + # (in module bar) + oldkey = '' + oldsubitems = None + i = 0 + while i < len(newlist): + key, (targets, subitems, _key) = newlist[i] + # cannot move if it has subitems; structure gets too complex + if not subitems: + m = _fixre.match(key) + if m: + if oldkey == m.group(1): + # prefixes match: add entry as subitem of the + # previous entry + oldsubitems.setdefault(m.group(2), [[], {}, _key])[0].\ + extend(targets) + del newlist[i] + continue + oldkey = m.group(1) + else: + oldkey = key + oldsubitems = subitems + i += 1 + + # group the entries by letter + def keyfunc2(item, letters=string.ascii_uppercase + '_'): + # hack: mutating the subitems dicts to a list in the keyfunc + k, v = item + v[1] = sorted((si, se) for (si, (se, void, void)) in iteritems(v[1])) + if v[2] is None: + # now calculate the key + letter = unicodedata.normalize('NFD', k[0])[0].upper() + if letter in letters: + return letter + else: + # get all other symbols under one heading + return _('Symbols') + else: + return v[2] + return [(key_, list(group)) + for (key_, group) in groupby(newlist, keyfunc2)] diff --git a/sphinx/environment/managers/toctree.py b/sphinx/environment/managers/toctree.py new file mode 100644 index 000000000..d4848a72c --- /dev/null +++ b/sphinx/environment/managers/toctree.py @@ -0,0 +1,561 @@ +# -*- coding: utf-8 -*- +""" + sphinx.environment.managers.toctree + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Toctree manager for sphinx.environment. + + :copyright: Copyright 2007-2016 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from six import iteritems +from docutils import nodes + +from sphinx import addnodes +from sphinx.util import url_re +from sphinx.util.nodes import clean_astext, process_only_nodes +from sphinx.transforms import SphinxContentsFilter +from sphinx.environment.managers import EnvironmentManager + + +class Toctree(EnvironmentManager): + name = 'toctree' + + def __init__(self, env): + super(Toctree, self).__init__(env) + + self.tocs = env.tocs + self.toc_num_entries = env.toc_num_entries + self.toc_secnumbers = env.toc_secnumbers + self.toc_fignumbers = env.toc_fignumbers + self.toctree_includes = env.toctree_includes + self.files_to_rebuild = env.files_to_rebuild + self.glob_toctrees = env.glob_toctrees + self.numbered_toctrees = env.numbered_toctrees + + def clear_doc(self, docname): + self.tocs.pop(docname, None) + self.toc_secnumbers.pop(docname, None) + self.toc_fignumbers.pop(docname, None) + self.toc_num_entries.pop(docname, None) + self.toctree_includes.pop(docname, None) + self.glob_toctrees.discard(docname) + self.numbered_toctrees.discard(docname) + + for subfn, fnset in list(self.files_to_rebuild.items()): + fnset.discard(docname) + if not fnset: + del self.files_to_rebuild[subfn] + + def merge_other(self, docnames, other): + for docname in docnames: + self.tocs[docname] = other.tocs[docname] + self.toc_num_entries[docname] = other.toc_num_entries[docname] + if docname in other.toctree_includes: + self.toctree_includes[docname] = other.toctree_includes[docname] + if docname in other.glob_toctrees: + self.glob_toctrees.add(docname) + if docname in other.numbered_toctrees: + self.numbered_toctrees.add(docname) + + for subfn, fnset in other.files_to_rebuild.items(): + self.files_to_rebuild.setdefault(subfn, set()).update(fnset & docnames) + + def process_doc(self, docname, doctree): + """Build a TOC from the doctree and store it in the inventory.""" + numentries = [0] # nonlocal again... + + def traverse_in_section(node, cls): + """Like traverse(), but stay within the same section.""" + result = [] + if isinstance(node, cls): + result.append(node) + for child in node.children: + if isinstance(child, nodes.section): + continue + result.extend(traverse_in_section(child, cls)) + return result + + def build_toc(node, depth=1): + entries = [] + for sectionnode in node: + # find all toctree nodes in this section and add them + # to the toc (just copying the toctree node which is then + # resolved in self.get_and_resolve_doctree) + if isinstance(sectionnode, addnodes.only): + onlynode = addnodes.only(expr=sectionnode['expr']) + blist = build_toc(sectionnode, depth) + if blist: + onlynode += blist.children + entries.append(onlynode) + continue + if not isinstance(sectionnode, nodes.section): + for toctreenode in traverse_in_section(sectionnode, + addnodes.toctree): + item = toctreenode.copy() + entries.append(item) + # important: do the inventory stuff + self.note_toctree(docname, toctreenode) + continue + title = sectionnode[0] + # copy the contents of the section title, but without references + # and unnecessary stuff + visitor = SphinxContentsFilter(doctree) + title.walkabout(visitor) + nodetext = visitor.get_entry_text() + if not numentries[0]: + # for the very first toc entry, don't add an anchor + # as it is the file's title anyway + anchorname = '' + else: + anchorname = '#' + sectionnode['ids'][0] + numentries[0] += 1 + # make these nodes: + # list_item -> compact_paragraph -> reference + reference = nodes.reference( + '', '', internal=True, refuri=docname, + anchorname=anchorname, *nodetext) + para = addnodes.compact_paragraph('', '', reference) + item = nodes.list_item('', para) + sub_item = build_toc(sectionnode, depth + 1) + item += sub_item + entries.append(item) + if entries: + return nodes.bullet_list('', *entries) + return [] + toc = build_toc(doctree) + if toc: + self.tocs[docname] = toc + else: + self.tocs[docname] = nodes.bullet_list('') + self.toc_num_entries[docname] = numentries[0] + + def note_toctree(self, docname, toctreenode): + """Note a TOC tree directive in a document and gather information about + file relations from it. + """ + if toctreenode['glob']: + self.glob_toctrees.add(docname) + if toctreenode.get('numbered'): + self.numbered_toctrees.add(docname) + includefiles = toctreenode['includefiles'] + for includefile in includefiles: + # note that if the included file is rebuilt, this one must be + # too (since the TOC of the included file could have changed) + self.files_to_rebuild.setdefault(includefile, set()).add(docname) + self.toctree_includes.setdefault(docname, []).extend(includefiles) + + def get_toc_for(self, docname, builder): + """Return a TOC nodetree -- for use on the same page only!""" + tocdepth = self.env.metadata[docname].get('tocdepth', 0) + try: + toc = self.tocs[docname].deepcopy() + self._toctree_prune(toc, 2, tocdepth) + except KeyError: + # the document does not exist anymore: return a dummy node that + # renders to nothing + return nodes.paragraph() + process_only_nodes(toc, builder.tags, warn_node=self.env.warn_node) + for node in toc.traverse(nodes.reference): + node['refuri'] = node['anchorname'] or '#' + return toc + + def get_toctree_for(self, docname, builder, collapse, **kwds): + """Return the global TOC nodetree.""" + doctree = self.env.get_doctree(self.env.config.master_doc) + toctrees = [] + if 'includehidden' not in kwds: + kwds['includehidden'] = True + if 'maxdepth' not in kwds: + kwds['maxdepth'] = 0 + kwds['collapse'] = collapse + for toctreenode in doctree.traverse(addnodes.toctree): + toctree = self.env.resolve_toctree(docname, builder, toctreenode, + prune=True, **kwds) + if toctree: + toctrees.append(toctree) + if not toctrees: + return None + result = toctrees[0] + for toctree in toctrees[1:]: + result.extend(toctree.children) + return result + + def resolve_toctree(self, docname, builder, toctree, prune=True, maxdepth=0, + titles_only=False, collapse=False, includehidden=False): + """Resolve a *toctree* node into individual bullet lists with titles + as items, returning None (if no containing titles are found) or + a new node. + + If *prune* is True, the tree is pruned to *maxdepth*, or if that is 0, + to the value of the *maxdepth* option on the *toctree* node. + If *titles_only* is True, only toplevel document titles will be in the + resulting tree. + If *collapse* is True, all branches not containing docname will + be collapsed. + """ + if toctree.get('hidden', False) and not includehidden: + return None + + # For reading the following two helper function, it is useful to keep + # in mind the node structure of a toctree (using HTML-like node names + # for brevity): + # + # <ul> + # <li> + # <p><a></p> + # <p><a></p> + # ... + # <ul> + # ... + # </ul> + # </li> + # </ul> + # + # The transformation is made in two passes in order to avoid + # interactions between marking and pruning the tree (see bug #1046). + + toctree_ancestors = self.get_toctree_ancestors(docname) + + def _toctree_add_classes(node, depth): + """Add 'toctree-l%d' and 'current' classes to the toctree.""" + for subnode in node.children: + if isinstance(subnode, (addnodes.compact_paragraph, + nodes.list_item)): + # for <p> and <li>, indicate the depth level and recurse + subnode['classes'].append('toctree-l%d' % (depth-1)) + _toctree_add_classes(subnode, depth) + elif isinstance(subnode, nodes.bullet_list): + # for <ul>, just recurse + _toctree_add_classes(subnode, depth+1) + elif isinstance(subnode, nodes.reference): + # for <a>, identify which entries point to the current + # document and therefore may not be collapsed + if subnode['refuri'] == docname: + if not subnode['anchorname']: + # give the whole branch a 'current' class + # (useful for styling it differently) + branchnode = subnode + while branchnode: + branchnode['classes'].append('current') + branchnode = branchnode.parent + # mark the list_item as "on current page" + if subnode.parent.parent.get('iscurrent'): + # but only if it's not already done + return + while subnode: + subnode['iscurrent'] = True + subnode = subnode.parent + + def _entries_from_toctree(toctreenode, parents, + separate=False, subtree=False): + """Return TOC entries for a toctree node.""" + refs = [(e[0], e[1]) for e in toctreenode['entries']] + entries = [] + for (title, ref) in refs: + try: + refdoc = None + if url_re.match(ref): + if title is None: + title = ref + reference = nodes.reference('', '', internal=False, + refuri=ref, anchorname='', + *[nodes.Text(title)]) + para = addnodes.compact_paragraph('', '', reference) + item = nodes.list_item('', para) + toc = nodes.bullet_list('', item) + elif ref == 'self': + # 'self' refers to the document from which this + # toctree originates + ref = toctreenode['parent'] + if not title: + title = clean_astext(self.titles[ref]) + reference = nodes.reference('', '', internal=True, + refuri=ref, + anchorname='', + *[nodes.Text(title)]) + para = addnodes.compact_paragraph('', '', reference) + item = nodes.list_item('', para) + # don't show subitems + toc = nodes.bullet_list('', item) + else: + if ref in parents: + self.env.warn(ref, 'circular toctree references ' + 'detected, ignoring: %s <- %s' % + (ref, ' <- '.join(parents))) + continue + refdoc = ref + toc = self.tocs[ref].deepcopy() + maxdepth = self.env.metadata[ref].get('tocdepth', 0) + if ref not in toctree_ancestors or (prune and maxdepth > 0): + self._toctree_prune(toc, 2, maxdepth, collapse) + process_only_nodes(toc, builder.tags, warn_node=self.env.warn_node) + if title and toc.children and len(toc.children) == 1: + child = toc.children[0] + for refnode in child.traverse(nodes.reference): + if refnode['refuri'] == ref and \ + not refnode['anchorname']: + refnode.children = [nodes.Text(title)] + if not toc.children: + # empty toc means: no titles will show up in the toctree + self.env.warn_node( + 'toctree contains reference to document %r that ' + 'doesn\'t have a title: no link will be generated' + % ref, toctreenode) + except KeyError: + # this is raised if the included file does not exist + self.env.warn_node( + 'toctree contains reference to nonexisting document %r' + % ref, toctreenode) + else: + # if titles_only is given, only keep the main title and + # sub-toctrees + if titles_only: + # delete everything but the toplevel title(s) + # and toctrees + for toplevel in toc: + # nodes with length 1 don't have any children anyway + if len(toplevel) > 1: + subtrees = toplevel.traverse(addnodes.toctree) + if subtrees: + toplevel[1][:] = subtrees + else: + toplevel.pop(1) + # resolve all sub-toctrees + for subtocnode in toc.traverse(addnodes.toctree): + if not (subtocnode.get('hidden', False) and + not includehidden): + i = subtocnode.parent.index(subtocnode) + 1 + for item in _entries_from_toctree( + subtocnode, [refdoc] + parents, + subtree=True): + subtocnode.parent.insert(i, item) + i += 1 + subtocnode.parent.remove(subtocnode) + if separate: + entries.append(toc) + else: + entries.extend(toc.children) + if not subtree and not separate: + ret = nodes.bullet_list() + ret += entries + return [ret] + return entries + + maxdepth = maxdepth or toctree.get('maxdepth', -1) + if not titles_only and toctree.get('titlesonly', False): + titles_only = True + if not includehidden and toctree.get('includehidden', False): + includehidden = True + + # NOTE: previously, this was separate=True, but that leads to artificial + # separation when two or more toctree entries form a logical unit, so + # separating mode is no longer used -- it's kept here for history's sake + tocentries = _entries_from_toctree(toctree, [], separate=False) + if not tocentries: + return None + + newnode = addnodes.compact_paragraph('', '') + caption = toctree.attributes.get('caption') + if caption: + caption_node = nodes.caption(caption, '', *[nodes.Text(caption)]) + caption_node.line = toctree.line + caption_node.source = toctree.source + caption_node.rawsource = toctree['rawcaption'] + if hasattr(toctree, 'uid'): + # move uid to caption_node to translate it + caption_node.uid = toctree.uid + del toctree.uid + newnode += caption_node + newnode.extend(tocentries) + newnode['toctree'] = True + + # prune the tree to maxdepth, also set toc depth and current classes + _toctree_add_classes(newnode, 1) + self._toctree_prune(newnode, 1, prune and maxdepth or 0, collapse) + + if len(newnode[-1]) == 0: # No titles found + return None + + # set the target paths in the toctrees (they are not known at TOC + # generation time) + for refnode in newnode.traverse(nodes.reference): + if not url_re.match(refnode['refuri']): + refnode['refuri'] = builder.get_relative_uri( + docname, refnode['refuri']) + refnode['anchorname'] + return newnode + + def get_toctree_ancestors(self, docname): + parent = {} + for p, children in iteritems(self.toctree_includes): + for child in children: + parent[child] = p + ancestors = [] + d = docname + while d in parent and d not in ancestors: + ancestors.append(d) + d = parent[d] + return ancestors + + def _toctree_prune(self, node, depth, maxdepth, collapse=False): + """Utility: Cut a TOC at a specified depth.""" + for subnode in node.children[:]: + if isinstance(subnode, (addnodes.compact_paragraph, + nodes.list_item)): + # for <p> and <li>, just recurse + self._toctree_prune(subnode, depth, maxdepth, collapse) + elif isinstance(subnode, nodes.bullet_list): + # for <ul>, determine if the depth is too large or if the + # entry is to be collapsed + if maxdepth > 0 and depth > maxdepth: + subnode.parent.replace(subnode, []) + else: + # cull sub-entries whose parents aren't 'current' + if (collapse and depth > 1 and + 'iscurrent' not in subnode.parent): + subnode.parent.remove(subnode) + else: + # recurse on visible children + self._toctree_prune(subnode, depth+1, maxdepth, collapse) + + def assign_section_numbers(self): + """Assign a section number to each heading under a numbered toctree.""" + # a list of all docnames whose section numbers changed + rewrite_needed = [] + + assigned = set() + old_secnumbers = self.toc_secnumbers + self.toc_secnumbers = self.env.toc_secnumbers = {} + + def _walk_toc(node, secnums, depth, titlenode=None): + # titlenode is the title of the document, it will get assigned a + # secnumber too, so that it shows up in next/prev/parent rellinks + for subnode in node.children: + if isinstance(subnode, nodes.bullet_list): + numstack.append(0) + _walk_toc(subnode, secnums, depth-1, titlenode) + numstack.pop() + titlenode = None + elif isinstance(subnode, nodes.list_item): + _walk_toc(subnode, secnums, depth, titlenode) + titlenode = None + elif isinstance(subnode, addnodes.only): + # at this stage we don't know yet which sections are going + # to be included; just include all of them, even if it leads + # to gaps in the numbering + _walk_toc(subnode, secnums, depth, titlenode) + titlenode = None + elif isinstance(subnode, addnodes.compact_paragraph): + numstack[-1] += 1 + if depth > 0: + number = tuple(numstack) + else: + number = None + secnums[subnode[0]['anchorname']] = \ + subnode[0]['secnumber'] = number + if titlenode: + titlenode['secnumber'] = number + titlenode = None + elif isinstance(subnode, addnodes.toctree): + _walk_toctree(subnode, depth) + + def _walk_toctree(toctreenode, depth): + if depth == 0: + return + for (title, ref) in toctreenode['entries']: + if url_re.match(ref) or ref == 'self' or ref in assigned: + # don't mess with those + continue + if ref in self.tocs: + secnums = self.toc_secnumbers[ref] = {} + assigned.add(ref) + _walk_toc(self.tocs[ref], secnums, depth, + self.env.titles.get(ref)) + if secnums != old_secnumbers.get(ref): + rewrite_needed.append(ref) + + for docname in self.numbered_toctrees: + assigned.add(docname) + doctree = self.env.get_doctree(docname) + for toctreenode in doctree.traverse(addnodes.toctree): + depth = toctreenode.get('numbered', 0) + if depth: + # every numbered toctree gets new numbering + numstack = [0] + _walk_toctree(toctreenode, depth) + + return rewrite_needed + + def assign_figure_numbers(self): + """Assign a figure number to each figure under a numbered toctree.""" + + rewrite_needed = [] + + assigned = set() + old_fignumbers = self.toc_fignumbers + self.toc_fignumbers = self.env.toc_fignumbers = {} + fignum_counter = {} + + def get_section_number(docname, section): + anchorname = '#' + section['ids'][0] + secnumbers = self.toc_secnumbers.get(docname, {}) + if anchorname in secnumbers: + secnum = secnumbers.get(anchorname) + else: + secnum = secnumbers.get('') + + return secnum or tuple() + + def get_next_fignumber(figtype, secnum): + counter = fignum_counter.setdefault(figtype, {}) + + secnum = secnum[:self.env.config.numfig_secnum_depth] + counter[secnum] = counter.get(secnum, 0) + 1 + return secnum + (counter[secnum],) + + def register_fignumber(docname, secnum, figtype, fignode): + self.toc_fignumbers.setdefault(docname, {}) + fignumbers = self.toc_fignumbers[docname].setdefault(figtype, {}) + figure_id = fignode['ids'][0] + + fignumbers[figure_id] = get_next_fignumber(figtype, secnum) + + def _walk_doctree(docname, doctree, secnum): + for subnode in doctree.children: + if isinstance(subnode, nodes.section): + next_secnum = get_section_number(docname, subnode) + if next_secnum: + _walk_doctree(docname, subnode, next_secnum) + else: + _walk_doctree(docname, subnode, secnum) + continue + elif isinstance(subnode, addnodes.toctree): + for title, subdocname in subnode['entries']: + if url_re.match(subdocname) or subdocname == 'self': + # don't mess with those + continue + + _walk_doc(subdocname, secnum) + + continue + + figtype = self.env.domains['std'].get_figtype(subnode) + if figtype and subnode['ids']: + register_fignumber(docname, secnum, figtype, subnode) + + _walk_doctree(docname, subnode, secnum) + + def _walk_doc(docname, secnum): + if docname not in assigned: + assigned.add(docname) + doctree = self.env.get_doctree(docname) + _walk_doctree(docname, doctree, secnum) + + if self.env.config.numfig: + _walk_doc(self.env.config.master_doc, tuple()) + for docname, fignums in iteritems(self.toc_fignumbers): + if fignums != old_fignumbers.get(docname): + rewrite_needed.append(docname) + + return rewrite_needed |