Merge branch 'bug/179' into 'master'

Serialize Checkers PluginTypeManager to a dict

*Description of changes*

Try to side-step issues with attributes not being set/updated on plugins when used with multiprocessing and Queues.

*Related to:*  #179, #164 

See merge request !98

5 files changed, 56 insertions, 25 deletions

diff --git a/src/flake8/checker.py b/src/flake8/checker.py
index 57fd896..2dc3481 100644
--- a/src/flake8/checker.py
+++ b/src/flake8/checker.py
@@ -38,21 +38,6 @@ SERIAL_RETRY_ERRNOS = set([
 ])
 
 
-def _run_checks_from_queue(process_queue, results_queue, statistics_queue):
-    LOG.info('Running checks in parallel')
-    try:
-        for checker in iter(process_queue.get, 'DONE'):
-            LOG.info('Checking "%s"', checker.filename)
-            checker.run_checks(results_queue, statistics_queue)
-    except exceptions.PluginRequestedUnknownParameters as exc:
-        print(str(exc))
-    except Exception as exc:
-        LOG.error('Unhandled exception occurred')
-        raise
-    finally:
-        results_queue.put('DONE')
-
-
 class Manager(object):
     """Manage the parallelism and checker instances for each plugin and file.
 
@@ -284,8 +269,9 @@ class Manager(object):
             file_exists = os.path.exists(filename)
             return (file_exists and matches_filename_patterns) or is_stdin
 
+        checks = self.checks.to_dictionary()
         self.checkers = [
-            FileChecker(filename, self.checks, self.options)
+            FileChecker(filename, checks, self.options)
             for argument in paths
             for filename in utils.filenames_from(argument,
                                                  self.is_path_excluded)
@@ -405,7 +391,7 @@ class FileChecker(object):
         :param checks:
             The plugins registered to check the file.
         :type checks:
-            flake8.plugins.manager.Checkers
+            dict
         :param options:
             Parsed option values from config and command-line.
         :type options:
@@ -458,14 +444,17 @@ class FileChecker(object):
         """Run the check in a single plugin."""
         LOG.debug('Running %r with %r', plugin, arguments)
         try:
-            self.processor.keyword_arguments_for(plugin.parameters, arguments)
+            self.processor.keyword_arguments_for(
+                plugin['parameters'],
+                arguments,
+            )
         except AttributeError as ae:
             LOG.error('Plugin requested unknown parameters.')
             raise exceptions.PluginRequestedUnknownParameters(
                 plugin=plugin,
                 exception=ae,
             )
-        return plugin.execute(**arguments)
+        return plugin['plugin'](**arguments)
 
     def run_ast_checks(self):
         """Run all checks expecting an abstract syntax tree."""
@@ -484,7 +473,7 @@ class FileChecker(object):
                         (exc_type.__name__, exception.args[0]))
             return
 
-        for plugin in self.checks.ast_plugins:
+        for plugin in self.checks['ast_plugins']:
             checker = self.run_check(plugin, tree=ast)
             # If the plugin uses a class, call the run method of it, otherwise
             # the call should return something iterable itself
@@ -509,7 +498,7 @@ class FileChecker(object):
 
         LOG.debug('Logical line: "%s"', logical_line.rstrip())
 
-        for plugin in self.checks.logical_line_plugins:
+        for plugin in self.checks['logical_line_plugins']:
             self.processor.update_checker_state_for(plugin)
             results = self.run_check(plugin, logical_line=logical_line) or ()
             for offset, text in results:
@@ -526,7 +515,7 @@ class FileChecker(object):
 
     def run_physical_checks(self, physical_line, override_error_line=None):
         """Run all checks for a given physical line."""
-        for plugin in self.checks.physical_line_plugins:
+        for plugin in self.checks['physical_line_plugins']:
             self.processor.update_checker_state_for(plugin)
             result = self.run_check(plugin, physical_line=physical_line)
             if result is not None:
@@ -636,6 +625,21 @@ class FileChecker(object):
                                              override_error_line=token[4])
 
 
+def _run_checks_from_queue(process_queue, results_queue, statistics_queue):
+    LOG.info('Running checks in parallel')
+    try:
+        for checker in iter(process_queue.get, 'DONE'):
+            LOG.info('Checking "%s"', checker.filename)
+            checker.run_checks(results_queue, statistics_queue)
+    except exceptions.PluginRequestedUnknownParameters as exc:
+        print(str(exc))
+    except Exception as exc:
+        LOG.error('Unhandled exception occurred')
+        raise
+    finally:
+        results_queue.put('DONE')
+
+
 def find_offset(offset, mapping):
     """Find the offset tuple for a single offset."""
     if isinstance(offset, tuple):
diff --git a/src/flake8/exceptions.py b/src/flake8/exceptions.py
index da650f9..349d471 100644
--- a/src/flake8/exceptions.py
+++ b/src/flake8/exceptions.py
@@ -65,7 +65,7 @@ class PluginRequestedUnknownParameters(Flake8Exception):
 
     def __str__(self):
         """Format our exception message."""
-        return self.FORMAT % {'name': self.plugin.plugin_name,
+        return self.FORMAT % {'name': self.plugin['plugin_name'],
                               'exc': self.original_exception}
 
 
diff --git a/src/flake8/plugins/manager.py b/src/flake8/plugins/manager.py
index 28f38a6..a5f1c3b 100644
--- a/src/flake8/plugins/manager.py
+++ b/src/flake8/plugins/manager.py
@@ -49,6 +49,16 @@ class Plugin(object):
             self.name, self.entry_point
         )
 
+    def to_dictionary(self):
+        """Convert this plugin to a dictionary."""
+        return {
+            'name': self.name,
+            'parameters': self.parameters,
+            'parameter_names': self.parameter_names,
+            'plugin': self.plugin,
+            'plugin_name': self.plugin_name,
+        }
+
     def is_in_a_group(self):
         """Determine if this plugin is in a group.
 
@@ -433,6 +443,20 @@ class Checkers(PluginTypeManager):
             if argument_name == plugin.parameter_names[0]:
                 yield plugin
 
+    def to_dictionary(self):
+        """Return a dictionary of AST and line-based plugins."""
+        return {
+            'ast_plugins': [
+                plugin.to_dictionary() for plugin in self.ast_plugins
+            ],
+            'logical_line_plugins': [
+                plugin.to_dictionary() for plugin in self.logical_line_plugins
+            ],
+            'physical_line_plugins': [
+                plugin.to_dictionary() for plugin in self.physical_line_plugins
+            ],
+        }
+
     def register_options(self, optmanager):
         """Register all of the checkers' options to the OptionManager.
 
diff --git a/src/flake8/plugins/pyflakes.py b/src/flake8/plugins/pyflakes.py
index 158d5a2..542e5fa 100644
--- a/src/flake8/plugins/pyflakes.py
+++ b/src/flake8/plugins/pyflakes.py
@@ -46,6 +46,9 @@ class FlakesChecker(pyflakes.checker.Checker):
 
     name = 'pyflakes'
     version = pyflakes.__version__
+    with_doctest = False
+    include_in_doctest = []
+    exclude_from_doctest = []
 
     def __init__(self, tree, filename):
         """Initialize the PyFlakes plugin with an AST tree and filename."""
diff --git a/src/flake8/processor.py b/src/flake8/processor.py
index a3c9a6b..e9d1b8a 100644
--- a/src/flake8/processor.py
+++ b/src/flake8/processor.py
@@ -129,9 +129,9 @@ class FileProcessor(object):
 
     def update_checker_state_for(self, plugin):
         """Update the checker_state attribute for the plugin."""
-        if 'checker_state' in plugin.parameters:
+        if 'checker_state' in plugin['parameters']:
             self.checker_state = self._checker_states.setdefault(
-                plugin.name, {}
+                plugin['name'], {}
             )
 
     def next_logical_line(self):