diff options
Diffstat (limited to 'sphinx/pycode')
| -rw-r--r-- | sphinx/pycode/__init__.py | 41 | ||||
| -rw-r--r-- | sphinx/pycode/ast.py | 45 | ||||
| -rw-r--r-- | sphinx/pycode/parser.py | 69 |
3 files changed, 73 insertions, 82 deletions
diff --git a/sphinx/pycode/__init__.py b/sphinx/pycode/__init__.py index ab0dfdbf8..c55a4fe4a 100644 --- a/sphinx/pycode/__init__.py +++ b/sphinx/pycode/__init__.py @@ -19,14 +19,14 @@ from os import path from typing import IO, Any, Dict, List, Optional, Tuple from zipfile import ZipFile -from sphinx.deprecation import RemovedInSphinx40Warning, RemovedInSphinx50Warning +from sphinx.deprecation import RemovedInSphinx50Warning from sphinx.errors import PycodeError from sphinx.pycode.parser import Parser class ModuleAnalyzer: # cache for analyzer objects -- caches both by module and file name - cache = {} # type: Dict[Tuple[str, str], Any] + cache: Dict[Tuple[str, str], Any] = {} @staticmethod def get_module_source(modname: str) -> Tuple[Optional[str], Optional[str]]: @@ -79,7 +79,7 @@ class ModuleAnalyzer: @classmethod def for_string(cls, string: str, modname: str, srcname: str = '<string>' ) -> "ModuleAnalyzer": - return cls(StringIO(string), modname, srcname, decoded=True) + return cls(StringIO(string), modname, srcname) @classmethod def for_file(cls, filename: str, modname: str) -> "ModuleAnalyzer": @@ -87,7 +87,7 @@ class ModuleAnalyzer: return cls.cache['file', filename] try: with tokenize.open(filename) as f: - obj = cls(f, modname, filename, decoded=True) + obj = cls(f, modname, filename) cls.cache['file', filename] = obj except Exception as err: if '.egg' + path.sep in filename: @@ -127,29 +127,20 @@ class ModuleAnalyzer: cls.cache['module', modname] = obj return obj - def __init__(self, source: IO, modname: str, srcname: str, decoded: bool = False) -> None: + def __init__(self, source: IO, modname: str, srcname: str) -> None: self.modname = modname # name of the module self.srcname = srcname # name of the source file # cache the source code as well - pos = source.tell() - if not decoded: - warnings.warn('decode option for ModuleAnalyzer is deprecated.', - RemovedInSphinx40Warning, stacklevel=2) - self._encoding, _ = tokenize.detect_encoding(source.readline) - source.seek(pos) - self.code = source.read().decode(self._encoding) - else: - self._encoding = None - self.code = source.read() + self.code = source.read() # will be filled by analyze() - self.annotations = None # type: Dict[Tuple[str, str], str] - self.attr_docs = None # type: Dict[Tuple[str, str], List[str]] - self.finals = None # type: List[str] - self.overloads = None # type: Dict[str, List[Signature]] - self.tagorder = None # type: Dict[str, int] - self.tags = None # type: Dict[str, Tuple[str, int, int]] + self.annotations: Dict[Tuple[str, str], str] = None + self.attr_docs: Dict[Tuple[str, str], List[str]] = None + self.finals: List[str] = None + self.overloads: Dict[str, List[Signature]] = None + self.tagorder: Dict[str, int] = None + self.tags: Dict[str, Tuple[str, int, int]] = None self._analyzed = False def parse(self) -> None: @@ -164,7 +155,7 @@ class ModuleAnalyzer: return None try: - parser = Parser(self.code, self._encoding) + parser = Parser(self.code) parser.parse() self.attr_docs = OrderedDict() @@ -192,9 +183,3 @@ class ModuleAnalyzer: """Find class, function and method definitions and their location.""" self.analyze() return self.tags - - @property - def encoding(self) -> str: - warnings.warn('ModuleAnalyzer.encoding is deprecated.', - RemovedInSphinx40Warning, stacklevel=2) - return self._encoding diff --git a/sphinx/pycode/ast.py b/sphinx/pycode/ast.py index 65534f958..f541ec0a9 100644 --- a/sphinx/pycode/ast.py +++ b/sphinx/pycode/ast.py @@ -9,7 +9,7 @@ """ import sys -from typing import Dict, List, Optional, Type +from typing import Dict, List, Optional, Type, overload if sys.version_info > (3, 8): import ast @@ -21,7 +21,7 @@ else: import ast # type: ignore -OPERATORS = { +OPERATORS: Dict[Type[ast.AST], str] = { ast.Add: "+", ast.And: "and", ast.BitAnd: "&", @@ -41,7 +41,7 @@ OPERATORS = { ast.Sub: "-", ast.UAdd: "+", ast.USub: "-", -} # type: Dict[Type[ast.AST], str] +} def parse(code: str, mode: str = 'exec') -> "ast.AST": @@ -62,6 +62,16 @@ def parse(code: str, mode: str = 'exec') -> "ast.AST": return ast.parse(code, mode=mode) +@overload +def unparse(node: None, code: str = '') -> None: + ... + + +@overload +def unparse(node: ast.AST, code: str = '') -> str: + ... + + def unparse(node: Optional[ast.AST], code: str = '') -> Optional[str]: """Unparse an AST to string.""" if node is None: @@ -98,7 +108,7 @@ class _UnparseVisitor(ast.NodeVisitor): return name def visit_arguments(self, node: ast.arguments) -> str: - defaults = list(node.defaults) + defaults: List[Optional[ast.expr]] = list(node.defaults) positionals = len(node.args) posonlyargs = 0 if hasattr(node, "posonlyargs"): # for py38+ @@ -107,11 +117,11 @@ class _UnparseVisitor(ast.NodeVisitor): for _ in range(len(defaults), positionals): defaults.insert(0, None) - kw_defaults = list(node.kw_defaults) + kw_defaults: List[Optional[ast.expr]] = list(node.kw_defaults) for _ in range(len(kw_defaults), len(node.kwonlyargs)): kw_defaults.insert(0, None) - args = [] # type: List[str] + args: List[str] = [] if hasattr(node, "posonlyargs"): # for py38+ for i, arg in enumerate(node.posonlyargs): # type: ignore args.append(self._visit_arg_with_default(arg, defaults[i])) @@ -150,6 +160,17 @@ class _UnparseVisitor(ast.NodeVisitor): ["%s=%s" % (k.arg, self.visit(k.value)) for k in node.keywords]) return "%s(%s)" % (self.visit(node.func), ", ".join(args)) + def visit_Constant(self, node: ast.Constant) -> str: # type: ignore + if node.value is Ellipsis: + return "..." + elif isinstance(node.value, (int, float, complex)): + if self.code and sys.version_info > (3, 8): + return ast.get_source_segment(self.code, node) # type: ignore + else: + return repr(node.value) + else: + return repr(node.value) + def visit_Dict(self, node: ast.Dict) -> str: keys = (self.visit(k) for k in node.keys) values = (self.visit(v) for v in node.values) @@ -197,18 +218,6 @@ class _UnparseVisitor(ast.NodeVisitor): else: return "()" - if sys.version_info >= (3, 6): - def visit_Constant(self, node: ast.Constant) -> str: - if node.value is Ellipsis: - return "..." - elif isinstance(node.value, (int, float, complex)): - if self.code and sys.version_info > (3, 8): - return ast.get_source_segment(self.code, node) - else: - return repr(node.value) - else: - return repr(node.value) - if sys.version_info < (3, 8): # these ast nodes were deprecated in python 3.8 def visit_Bytes(self, node: ast.Bytes) -> str: diff --git a/sphinx/pycode/parser.py b/sphinx/pycode/parser.py index dca59acd4..fa249d8c5 100644 --- a/sphinx/pycode/parser.py +++ b/sphinx/pycode/parser.py @@ -10,7 +10,6 @@ import inspect import itertools import re -import sys import tokenize from collections import OrderedDict from inspect import Signature @@ -26,12 +25,6 @@ indent_re = re.compile('^\\s*$') emptyline_re = re.compile('^\\s*(#.*)?$') -if sys.version_info >= (3, 6): - ASSIGN_NODES = (ast.Assign, ast.AnnAssign) -else: - ASSIGN_NODES = (ast.Assign) - - def filter_whitespace(code: str) -> str: return code.replace('\f', ' ') # replace FF (form feed) with whitespace @@ -94,7 +87,10 @@ def dedent_docstring(s: str) -> str: dummy.__doc__ = s docstring = inspect.getdoc(dummy) - return docstring.lstrip("\r\n").rstrip("\r\n") + if docstring: + return docstring.lstrip("\r\n").rstrip("\r\n") + else: + return "" class Token: @@ -133,8 +129,8 @@ class TokenProcessor: lines = iter(buffers) self.buffers = buffers self.tokens = tokenize.generate_tokens(lambda: next(lines)) - self.current = None # type: Token - self.previous = None # type: Token + self.current: Token = None + self.previous: Token = None def get_line(self, lineno: int) -> str: """Returns specified line.""" @@ -182,7 +178,7 @@ class AfterCommentParser(TokenProcessor): def __init__(self, lines: List[str]) -> None: super().__init__(lines) - self.comment = None # type: str + self.comment: str = None def fetch_rvalue(self) -> List[Token]: """Fetch right-hand value of assignment.""" @@ -225,18 +221,18 @@ class VariableCommentPicker(ast.NodeVisitor): self.counter = itertools.count() self.buffers = buffers self.encoding = encoding - self.context = [] # type: List[str] - self.current_classes = [] # type: List[str] - self.current_function = None # type: ast.FunctionDef - self.comments = OrderedDict() # type: Dict[Tuple[str, str], str] - self.annotations = {} # type: Dict[Tuple[str, str], str] - self.previous = None # type: ast.AST - self.deforders = {} # type: Dict[str, int] - self.finals = [] # type: List[str] - self.overloads = {} # type: Dict[str, List[Signature]] - self.typing = None # type: str - self.typing_final = None # type: str - self.typing_overload = None # type: str + self.context: List[str] = [] + self.current_classes: List[str] = [] + self.current_function: ast.FunctionDef = None + self.comments: Dict[Tuple[str, str], str] = OrderedDict() + self.annotations: Dict[Tuple[str, str], str] = {} + self.previous: ast.AST = None + self.deforders: Dict[str, int] = {} + self.finals: List[str] = [] + self.overloads: Dict[str, List[Signature]] = {} + self.typing: str = None + self.typing_final: str = None + self.typing_overload: str = None super().__init__() def get_qualname_for(self, name: str) -> Optional[List[str]]: @@ -354,7 +350,7 @@ class VariableCommentPicker(ast.NodeVisitor): """Handles Assign node and pick up a variable comment.""" try: targets = get_assign_targets(node) - varnames = sum([get_lvar_names(t, self=self.get_self()) for t in targets], []) # type: List[str] # NOQA + varnames: List[str] = sum([get_lvar_names(t, self=self.get_self()) for t in targets], []) # NOQA current_line = self.get_line(node.lineno) except TypeError: return # this assignment is not new definition! @@ -398,13 +394,14 @@ class VariableCommentPicker(ast.NodeVisitor): for varname in varnames: self.add_entry(varname) - def visit_AnnAssign(self, node: ast.AST) -> None: # Note: ast.AnnAssign not found in py35 + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: """Handles AnnAssign node and pick up a variable comment.""" self.visit_Assign(node) # type: ignore def visit_Expr(self, node: ast.Expr) -> None: """Handles Expr node and pick up a comment if string.""" - if (isinstance(self.previous, ASSIGN_NODES) and isinstance(node.value, ast.Str)): + if (isinstance(self.previous, (ast.Assign, ast.AnnAssign)) and + isinstance(node.value, ast.Str)): try: targets = get_assign_targets(self.previous) varnames = get_lvar_names(targets[0], self.get_self()) @@ -469,10 +466,10 @@ class DefinitionFinder(TokenProcessor): def __init__(self, lines: List[str]) -> None: super().__init__(lines) - self.decorator = None # type: Token - self.context = [] # type: List[str] - self.indents = [] # type: List - self.definitions = {} # type: Dict[str, Tuple[str, int, int]] + self.decorator: Token = None + self.context: List[str] = [] + self.indents: List = [] + self.definitions: Dict[str, Tuple[str, int, int]] = {} def add_definition(self, name: str, entry: Tuple[str, int, int]) -> None: """Add a location of definition.""" @@ -546,12 +543,12 @@ class Parser: def __init__(self, code: str, encoding: str = 'utf-8') -> None: self.code = filter_whitespace(code) self.encoding = encoding - self.annotations = {} # type: Dict[Tuple[str, str], str] - self.comments = {} # type: Dict[Tuple[str, str], str] - self.deforders = {} # type: Dict[str, int] - self.definitions = {} # type: Dict[str, Tuple[str, int, int]] - self.finals = [] # type: List[str] - self.overloads = {} # type: Dict[str, List[Signature]] + self.annotations: Dict[Tuple[str, str], str] = {} + self.comments: Dict[Tuple[str, str], str] = {} + self.deforders: Dict[str, int] = {} + self.definitions: Dict[str, Tuple[str, int, int]] = {} + self.finals: List[str] = [] + self.overloads: Dict[str, List[Signature]] = {} def parse(self) -> None: """Parse the source code.""" |