Merge branch 'mr/80' into 'master'

Provide complete list of file's tokens to plugins

Supersedes !80

See merge request !127

1 files changed, 21 insertions, 0 deletions

diff --git a/src/flake8/processor.py b/src/flake8/processor.py
index 44024e5..8490092 100644
--- a/src/flake8/processor.py
+++ b/src/flake8/processor.py
@@ -42,6 +42,7 @@ class FileProcessor(object):
     - :attr:`previous_indent_level`
     - :attr:`previous_logical`
     - :attr:`tokens`
+    - :attr:`file_tokens`
     - :attr:`total_lines`
     - :attr:`verbose`
     """
@@ -98,6 +99,26 @@ class FileProcessor(object):
         self.statistics = {
             'logical lines': 0,
         }
+        self._file_tokens = None
+
+    @property
+    def file_tokens(self):
+        """The complete set of tokens for a file.
+
+        Accessing this attribute *may* raise an InvalidSyntax exception.
+
+        :raises: flake8.exceptions.InvalidSyntax
+        """
+        if self._file_tokens is None:
+            line_iter = iter(self.lines)
+            try:
+                self._file_tokens = list(tokenize.generate_tokens(
+                    lambda: next(line_iter)
+                ))
+            except tokenize.TokenError as exc:
+                raise exceptions.InvalidSyntax(exc.message, exception=exc)
+
+        return self._file_tokens
 
     @contextlib.contextmanager
     def inside_multiline(self, line_number):