diff options
| -rw-r--r-- | docs/source/plugin-development/plugin-parameters.rst | 22 | ||||
| -rw-r--r-- | src/flake8/processor.py | 21 |
2 files changed, 32 insertions, 11 deletions
diff --git a/docs/source/plugin-development/plugin-parameters.rst b/docs/source/plugin-development/plugin-parameters.rst index fd644bc..1625098 100644 --- a/docs/source/plugin-development/plugin-parameters.rst +++ b/docs/source/plugin-development/plugin-parameters.rst @@ -40,25 +40,25 @@ a file, a plugin can ask for any of the following: - :attr:`~flake8.processor.FileProcessor.previous_logical` - :attr:`~flake8.processor.FileProcessor.tokens` -Some properties are set once per file being processed: +Some properties are set once per file for plugins which iterate itself over +the data instead of being called on each physical or logical line. - :attr:`~flake8.processor.FileProcessor.filename` +- :attr:`~flake8.processor.FileProcessor.file_tokens` - :attr:`~flake8.processor.FileProcessor.lines` - :attr:`~flake8.processor.FileProcessor.max_line_length` - :attr:`~flake8.processor.FileProcessor.total_lines` - :attr:`~flake8.processor.FileProcessor.verbose` These parameters can also be supplied to plugins working on each line -separately. Additionally, plugins called once per file can also accept ``tree`` -which is not supplied as a parameter of -:class:`~flake8.processor.FileProcessor`, which will be a parsed abstract -syntax tree. It is used by plugins like PyFlakes and McCabe. - -When the plugin is run depends on the first parameter, not counting ``self``. -It can be either ``physical_line``, ``logical_line`` or ``tree``. If the -parameter is ``tree``, it is run once per file, otherwise once per physical -line or logical line respectively. If the plugin is using neither of them it -won't be run at all. +separately. + +Plugins that depend on ``physical_line`` or ``logical_line`` are run on each +physical or logical line once. These parameters should be the first in the +list of arguments (with the exception of ``self``). Plugins that need an AST +(e.g., PyFlakes and McCabe) should depend on ``tree``. These plugins will run +once per file. The parameters listed above can be combined with +``physical_line``, ``logical_line``, and ``tree``. Registering Options diff --git a/src/flake8/processor.py b/src/flake8/processor.py index 44024e5..8490092 100644 --- a/src/flake8/processor.py +++ b/src/flake8/processor.py @@ -42,6 +42,7 @@ class FileProcessor(object): - :attr:`previous_indent_level` - :attr:`previous_logical` - :attr:`tokens` + - :attr:`file_tokens` - :attr:`total_lines` - :attr:`verbose` """ @@ -98,6 +99,26 @@ class FileProcessor(object): self.statistics = { 'logical lines': 0, } + self._file_tokens = None + + @property + def file_tokens(self): + """The complete set of tokens for a file. + + Accessing this attribute *may* raise an InvalidSyntax exception. + + :raises: flake8.exceptions.InvalidSyntax + """ + if self._file_tokens is None: + line_iter = iter(self.lines) + try: + self._file_tokens = list(tokenize.generate_tokens( + lambda: next(line_iter) + )) + except tokenize.TokenError as exc: + raise exceptions.InvalidSyntax(exc.message, exception=exc) + + return self._file_tokens @contextlib.contextmanager def inside_multiline(self, line_number): |