Add support for local (in-repo, non-setuptools) plugins.

Closes #357

5 files changed, 206 insertions, 77 deletions

diff --git a/src/flake8/api/legacy.py b/src/flake8/api/legacy.py
index 2b983c8..b332860 100644
--- a/src/flake8/api/legacy.py
+++ b/src/flake8/api/legacy.py
@@ -6,6 +6,7 @@ In 3.0 we no longer have an "engine" module but we maintain the API from it.
 import logging
 import os.path
 
+import flake8
 from flake8.formatting import base as formatter
 from flake8.main import application as app
 
@@ -26,6 +27,10 @@ def get_style_guide(**kwargs):
         :class:`StyleGuide`
     """
     application = app.Application()
+    application.parse_preliminary_options_and_args([])
+    flake8.configure_logging(
+        application.prelim_opts.verbose, application.prelim_opts.output_file)
+    application.make_config_finder()
     application.find_plugins()
     application.register_plugin_options()
     application.parse_configuration_and_cli([])
diff --git a/src/flake8/main/application.py b/src/flake8/main/application.py
index 293ac92..f1052b9 100644
--- a/src/flake8/main/application.py
+++ b/src/flake8/main/application.py
@@ -2,7 +2,6 @@
 from __future__ import print_function
 
 import logging
-import sys
 import time
 
 import flake8
@@ -12,7 +11,7 @@ from flake8 import exceptions
 from flake8 import style_guide
 from flake8 import utils
 from flake8.main import options
-from flake8.options import aggregator
+from flake8.options import aggregator, config
 from flake8.options import manager
 from flake8.plugins import manager as plugin_manager
 
@@ -45,34 +44,16 @@ class Application(object):
             prog='flake8', version=flake8.__version__
         )
         options.register_default_options(self.option_manager)
-
-        # We haven't found or registered our plugins yet, so let's defer
-        # printing the version until we aggregate options from config files
-        # and the command-line. First, let's clone our arguments on the CLI,
-        # then we'll attempt to remove ``--version`` so that we can avoid
-        # triggering the "version" action in optparse. If it's not there, we
-        # do not need to worry and we can continue. If it is, we successfully
-        # defer printing the version until just a little bit later.
-        # Similarly we have to defer printing the help text until later.
-        args = sys.argv[:]
-        try:
-            args.remove('--version')
-        except ValueError:
-            pass
-        try:
-            args.remove('--help')
-        except ValueError:
-            pass
-        try:
-            args.remove('-h')
-        except ValueError:
-            pass
-
-        preliminary_opts, _ = self.option_manager.parse_known_args(args)
-        # Set the verbosity of the program
-        flake8.configure_logging(preliminary_opts.verbose,
-                                 preliminary_opts.output_file)
-
+        #: The preliminary options parsed from CLI before plugins are loaded,
+        #: into a :class:`optparse.Values` instance
+        self.prelim_opts = None
+        #: The preliminary arguments parsed from CLI before plugins are loaded
+        self.prelim_args = None
+        #: The instance of :class:`flake8.options.config.ConfigFileFinder`
+        self.config_finder = None
+
+        #: The :class:`flake8.options.config.LocalPlugins` found in config
+        self.local_plugins = None
         #: The instance of :class:`flake8.plugins.manager.Checkers`
         self.check_plugins = None
         #: The instance of :class:`flake8.plugins.manager.Listeners`
@@ -111,6 +92,48 @@ class Application(object):
         #: The parsed diff information
         self.parsed_diff = {}
 
+    def parse_preliminary_options_and_args(self, argv):
+        """Get preliminary options and args from CLI, pre-plugin-loading.
+
+        We need to know the values of a few standard options and args now, so
+        that we can find config files and configure logging.
+
+        Since plugins aren't loaded yet, there may be some as-yet-unknown
+        options; we ignore those for now, they'll be parsed later when we do
+        real option parsing.
+
+        Sets self.prelim_opts and self.prelim_args.
+
+        :param list argv:
+            Command-line arguments passed in directly.
+        """
+        # We haven't found or registered our plugins yet, so let's defer
+        # printing the version until we aggregate options from config files
+        # and the command-line. First, let's clone our arguments on the CLI,
+        # then we'll attempt to remove ``--version`` so that we can avoid
+        # triggering the "version" action in optparse. If it's not there, we
+        # do not need to worry and we can continue. If it is, we successfully
+        # defer printing the version until just a little bit later.
+        # Similarly we have to defer printing the help text until later.
+        args = argv[:]
+        try:
+            args.remove('--version')
+        except ValueError:
+            pass
+        try:
+            args.remove('--help')
+        except ValueError:
+            pass
+        try:
+            args.remove('-h')
+        except ValueError:
+            pass
+
+        opts, args = self.option_manager.parse_known_args(args)
+        # parse_known_args includes unknown options as args; get rid of them
+        args = [a for a in args if not a.startswith('-')]
+        self.prelim_opts, self.prelim_args = opts, args
+
     def exit(self):
         # type: () -> NoneType
         """Handle finalization and exiting the program.
@@ -125,6 +148,17 @@ class Application(object):
             raise SystemExit((self.result_count > 0) or
                              self.catastrophic_failure)
 
+    def make_config_finder(self):
+        """Make our ConfigFileFinder based on preliminary opts and args."""
+        if self.config_finder is None:
+            extra_config_files = utils.normalize_paths(
+                self.prelim_opts.append_config)
+            self.config_finder = config.ConfigFileFinder(
+                self.option_manager.program_name,
+                self.prelim_args,
+                extra_config_files,
+            )
+
     def find_plugins(self):
         # type: () -> NoneType
         """Find and load the plugins for this application.
@@ -135,14 +169,23 @@ class Application(object):
         of finding plugins (via :mod:`pkg_resources`) we want this to be
         idempotent and so only update those attributes if they are ``None``.
         """
+        if self.local_plugins is None:
+            self.local_plugins = config.get_local_plugins(
+                self.config_finder,
+                self.prelim_opts.config,
+                self.prelim_opts.isolated,
+            )
+
         if self.check_plugins is None:
-            self.check_plugins = plugin_manager.Checkers()
+            self.check_plugins = plugin_manager.Checkers(
+                self.local_plugins.extension)
 
         if self.listening_plugins is None:
             self.listening_plugins = plugin_manager.Listeners()
 
         if self.formatting_plugins is None:
-            self.formatting_plugins = plugin_manager.ReportFormatters()
+            self.formatting_plugins = plugin_manager.ReportFormatters(
+                self.local_plugins.report)
 
         self.check_plugins.load_plugins()
         self.listening_plugins.load_plugins()
@@ -165,7 +208,7 @@ class Application(object):
         """
         if self.options is None and self.args is None:
             self.options, self.args = aggregator.aggregate_options(
-                self.option_manager, argv
+                self.option_manager, self.config_finder, argv
             )
 
         self.running_against_diff = self.options.diff
@@ -314,6 +357,10 @@ class Application(object):
         """
         # NOTE(sigmavirus24): When updating this, make sure you also update
         # our legacy API calls to these same methods.
+        self.parse_preliminary_options_and_args(argv)
+        flake8.configure_logging(
+            self.prelim_opts.verbose, self.prelim_opts.output_file)
+        self.make_config_finder()
         self.find_plugins()
         self.register_plugin_options()
         self.parse_configuration_and_cli(argv)
diff --git a/src/flake8/options/aggregator.py b/src/flake8/options/aggregator.py
index 4075dc9..5b8ab9c 100644
--- a/src/flake8/options/aggregator.py
+++ b/src/flake8/options/aggregator.py
@@ -5,17 +5,18 @@ applies the user-specified command-line configuration on top of it.
 """
 import logging
 
-from flake8 import utils
 from flake8.options import config
 
 LOG = logging.getLogger(__name__)
 
 
-def aggregate_options(manager, arglist=None, values=None):
+def aggregate_options(manager, config_finder, arglist=None, values=None):
     """Aggregate and merge CLI and config file options.
 
-    :param flake8.option.manager.OptionManager manager:
+    :param flake8.options.manager.OptionManager manager:
         The instance of the OptionManager that we're presently using.
+    :param flake8.options.config.ConfigFileFinder config_finder:
+        The config file finder to use.
     :param list arglist:
         The list of arguments to pass to ``manager.parse_args``. In most cases
         this will be None so ``parse_args`` uses ``sys.argv``. This is mostly
@@ -32,14 +33,12 @@ def aggregate_options(manager, arglist=None, values=None):
     default_values, _ = manager.parse_args([], values=values)
     # Get original CLI values so we can find additional config file paths and
     # see if --config was specified.
-    original_values, original_args = manager.parse_args(arglist)
-    extra_config_files = utils.normalize_paths(original_values.append_config)
+    original_values, _ = manager.parse_args(arglist)
 
     # Make our new configuration file mergerator
     config_parser = config.MergedConfigParser(
         option_manager=manager,
-        extra_config_files=extra_config_files,
-        args=original_args,
+        config_finder=config_finder,
     )
 
     # Get the parsed config
diff --git a/src/flake8/options/config.py b/src/flake8/options/config.py
index a6ac63f..251697a 100644
--- a/src/flake8/options/config.py
+++ b/src/flake8/options/config.py
@@ -1,4 +1,5 @@
 """Config handling logic for Flake8."""
+import collections
 import configparser
 import logging
 import os.path
@@ -49,6 +50,11 @@ class ConfigFileFinder(object):
             args = ['.']
         self.parent = self.tail = os.path.abspath(os.path.commonprefix(args))
 
+        # caches to avoid double-reading config files
+        self._local_configs = None
+        self._user_config = None
+        self._cli_configs = {}
+
     @staticmethod
     def _read_config(files):
         config = configparser.RawConfigParser()
@@ -63,10 +69,12 @@ class ConfigFileFinder(object):
 
     def cli_config(self, files):
         """Read and parse the config file specified on the command-line."""
-        config, found_files = self._read_config(files)
-        if found_files:
-            LOG.debug('Found cli configuration files: %s', found_files)
-        return config
+        if files not in self._cli_configs:
+            config, found_files = self._read_config(files)
+            if found_files:
+                LOG.debug('Found cli configuration files: %s', found_files)
+            self._cli_configs[files] = config
+        return self._cli_configs[files]
 
     def generate_possible_local_files(self):
         """Find and generate all local config files."""
@@ -104,10 +112,12 @@ class ConfigFileFinder(object):
 
     def local_configs(self):
         """Parse all local config files into one config object."""
-        config, found_files = self._read_config(self.local_config_files())
-        if found_files:
-            LOG.debug('Found local configuration files: %s', found_files)
-        return config
+        if self._local_configs is None:
+            config, found_files = self._read_config(self.local_config_files())
+            if found_files:
+                LOG.debug('Found local configuration files: %s', found_files)
+            self._local_configs = config
+        return self._local_configs
 
     def user_config_file(self):
         """Find the user-level config file."""
@@ -117,10 +127,12 @@ class ConfigFileFinder(object):
 
     def user_config(self):
         """Parse the user config file into a config object."""
-        config, found_files = self._read_config(self.user_config_file())
-        if found_files:
-            LOG.debug('Found user configuration files: %s', found_files)
-        return config
+        if self._user_config is None:
+            config, found_files = self._read_config(self.user_config_file())
+            if found_files:
+                LOG.debug('Found user configuration files: %s', found_files)
+            self._user_config = config
+        return self._user_config
 
 
 class MergedConfigParser(object):
@@ -138,30 +150,23 @@ class MergedConfigParser(object):
     #: :meth:`~configparser.RawConfigParser.getbool` method.
     GETBOOL_ACTIONS = {'store_true', 'store_false'}
 
-    def __init__(self, option_manager, extra_config_files=None, args=None):
+    def __init__(self, option_manager, config_finder):
         """Initialize the MergedConfigParser instance.
 
-        :param flake8.option.manager.OptionManager option_manager:
+        :param flake8.options.manager.OptionManager option_manager:
             Initialized OptionManager.
-        :param list extra_config_files:
-            List of extra config files to parse.
-        :params list args:
-            The extra parsed arguments from the command-line.
+        :param flake8.options.config.ConfigFileFinder config_finder:
+            Initialized ConfigFileFinder.
         """
         #: Our instance of flake8.options.manager.OptionManager
         self.option_manager = option_manager
         #: The prog value for the cli parser
         self.program_name = option_manager.program_name
-        #: Parsed extra arguments
-        self.args = args
         #: Mapping of configuration option names to
         #: :class:`~flake8.options.manager.Option` instances
         self.config_options = option_manager.config_options_dict
-        #: List of extra config files
-        self.extra_config_files = extra_config_files or []
         #: Our instance of our :class:`~ConfigFileFinder`
-        self.config_finder = ConfigFileFinder(self.program_name, self.args,
-                                              self.extra_config_files)
+        self.config_finder = config_finder
 
     def _normalize_value(self, option, value):
         final_value = option.normalize(
@@ -280,3 +285,49 @@ class MergedConfigParser(object):
             return self.parse_cli_config(cli_config)
 
         return self.merge_user_and_local_config()
+
+
+def get_local_plugins(config_finder, cli_config=None, isolated=False):
+    """Get local plugins lists from config files.
+
+    :param flake8.options.config.ConfigFileFinder config_finder:
+        The config file finder to use.
+    :param str cli_config:
+        Value of --config when specified at the command-line. Overrides
+        all other config files.
+    :param bool isolated:
+        Determines if we should parse configuration files at all or not.
+        If running in isolated mode, we ignore all configuration files
+    :returns:
+        LocalPlugins namedtuple containing two lists of plugin strings,
+        one for extension (checker) plugins and one for report plugins.
+    :rtype:
+        flake8.options.config.LocalPlugins
+    """
+    local_plugins = LocalPlugins(extension=[], report=[])
+    if isolated:
+        LOG.debug('Refusing to look for local plugins in configuration'
+                  'files due to user-requested isolation')
+        return local_plugins
+
+    if cli_config:
+        LOG.debug('Reading local plugins only from "%s" specified via '
+                  '--config by the user', cli_config)
+        configs = [config_finder.cli_config(cli_config)]
+    else:
+        configs = [
+            config_finder.user_config(),
+            config_finder.local_configs(),
+        ]
+
+    section = '%s:local-plugins' % config_finder.program_name
+    for config in configs:
+        for plugin_type in ['extension', 'report']:
+            if config.has_option(section, plugin_type):
+                getattr(local_plugins, plugin_type).extend(
+                    config.get(section, plugin_type).strip().splitlines()
+                )
+    return local_plugins
+
+
+LocalPlugins = collections.namedtuple('LocalPlugins', 'extension report')
diff --git a/src/flake8/plugins/manager.py b/src/flake8/plugins/manager.py
index 80a9ef2..e7b402f 100644
--- a/src/flake8/plugins/manager.py
+++ b/src/flake8/plugins/manager.py
@@ -236,11 +236,14 @@ class Plugin(object):
 class PluginManager(object):  # pylint: disable=too-few-public-methods
     """Find and manage plugins consistently."""
 
-    def __init__(self, namespace, verify_requirements=False):
+    def __init__(self, namespace,
+                 local_plugins=None, verify_requirements=False):
         """Initialize the manager.
 
         :param str namespace:
             Namespace of the plugins to manage, e.g., 'flake8.extension'.
+        :param list local_plugins:
+            Plugins from config (as "X = path.to:Plugin" strings).
         :param bool verify_requirements:
             Whether or not to make setuptools verify that the requirements for
             the plugin are satisfied.
@@ -249,15 +252,34 @@ class PluginManager(object):  # pylint: disable=too-few-public-methods
         self.verify_requirements = verify_requirements
         self.plugins = {}
         self.names = []
-        self._load_all_plugins()
+        self._load_local_plugins(local_plugins or [])
+        self._load_entrypoint_plugins()
 
-    def _load_all_plugins(self):
+    def _load_local_plugins(self, local_plugins):
+        """Load local plugins from config.
+
+        :param list local_plugins:
+            Plugins from config (as "X = path.to:Plugin" strings).
+        """
+        for plugin_str in local_plugins:
+            entry_point = pkg_resources.EntryPoint.parse(plugin_str)
+            self._load_plugin_from_entrypoint(entry_point)
+
+    def _load_entrypoint_plugins(self):
         LOG.info('Loading entry-points for "%s".', self.namespace)
         for entry_point in pkg_resources.iter_entry_points(self.namespace):
-            name = entry_point.name
-            self.plugins[name] = Plugin(name, entry_point)
-            self.names.append(name)
-            LOG.debug('Loaded %r for plugin "%s".', self.plugins[name], name)
+            self._load_plugin_from_entrypoint(entry_point)
+
+    def _load_plugin_from_entrypoint(self, entry_point):
+        """Load a plugin from a setuptools EntryPoint.
+
+        :param EntryPoint entry_point:
+            EntryPoint to load plugin from.
+        """
+        name = entry_point.name
+        self.plugins[name] = Plugin(name, entry_point)
+        self.names.append(name)
+        LOG.debug('Loaded %r for plugin "%s".', self.plugins[name], name)
 
     def map(self, func, *args, **kwargs):
         r"""Call ``func`` with the plugin and \*args and \**kwargs after.
@@ -329,9 +351,14 @@ class PluginTypeManager(object):
 
     namespace = None
 
-    def __init__(self):
-        """Initialize the plugin type's manager."""
-        self.manager = PluginManager(self.namespace)
+    def __init__(self, local_plugins=None):
+        """Initialize the plugin type's manager.
+
+        :param list local_plugins:
+            Plugins from config file instead of entry-points
+        """
+        self.manager = PluginManager(
+            self.namespace, local_plugins=local_plugins)
         self.plugins_loaded = False
 
     def __contains__(self, name):
@@ -436,7 +463,7 @@ class NotifierBuilderMixin(object):  # pylint: disable=too-few-public-methods
 
 
 class Checkers(PluginTypeManager):
-    """All of the checkers registered through entry-ponits."""
+    """All of the checkers registered through entry-points or config."""
 
     namespace = 'flake8.extension'
 
@@ -515,12 +542,12 @@ class Checkers(PluginTypeManager):
 
 
 class Listeners(PluginTypeManager, NotifierBuilderMixin):
-    """All of the listeners registered through entry-points."""
+    """All of the listeners registered through entry-points or config."""
 
     namespace = 'flake8.listen'
 
 
 class ReportFormatters(PluginTypeManager):
-    """All of the report formatters registered through entry-points."""
+    """All of the report formatters registered through entry-points/config."""
 
     namespace = 'flake8.report'