Only use ``_write_doc_doctree_cache`` in serial mode (#11290)

When using multiple processes, the content of
``self.env._write_doc_doctree_cache`` is not synchronised with the
main process, meaning we can only use it in serial mode.

Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com>

1 files changed, 11 insertions, 5 deletions

diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py
index bb02058f7..3a9203626 100644
--- a/sphinx/builders/__init__.py
+++ b/sphinx/builders/__init__.py
@@ -470,7 +470,7 @@ class Builder:
         def read_process(docs: list[str]) -> bytes:
             self.env.app = self.app
             for docname in docs:
-                self.read_doc(docname)
+                self.read_doc(docname, _cache=False)
             # allow pickling self to send it back
             return pickle.dumps(self.env, pickle.HIGHEST_PROTOCOL)
 
@@ -488,7 +488,7 @@ class Builder:
         tasks.join()
         logger.info('')
 
-    def read_doc(self, docname: str) -> None:
+    def read_doc(self, docname: str, *, _cache: bool = True) -> None:
         """Parse a file and add/update inventory entries for the doctree."""
         self.env.prepare_settings(docname)
 
@@ -522,9 +522,11 @@ class Builder:
         self.env.temp_data.clear()
         self.env.ref_context.clear()
 
-        self.write_doctree(docname, doctree)
+        self.write_doctree(docname, doctree, _cache=_cache)
 
-    def write_doctree(self, docname: str, doctree: nodes.document) -> None:
+    def write_doctree(
+        self, docname: str, doctree: nodes.document, *, _cache: bool = True,
+    ) -> None:
         """Write the doctree to a file."""
         # make it picklable
         doctree.reporter = None
@@ -542,7 +544,11 @@ class Builder:
         with open(doctree_filename, 'wb') as f:
             pickle.dump(doctree, f, pickle.HIGHEST_PROTOCOL)
 
-        self.env._write_doc_doctree_cache[docname] = doctree
+        # When Sphinx is running in parallel mode, ``write_doctree()`` is invoked
+        # in the context of a process worker, and thus it does not make sense to
+        # pickle the doctree and send it to the main process
+        if _cache:
+            self.env._write_doc_doctree_cache[docname] = doctree
 
     def write(
         self,