summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAshley Whetter <ashley@awhetter.co.uk>2018-02-03 15:27:53 -0800
committerAshley Whetter <ashley@awhetter.co.uk>2019-02-09 13:25:18 -0800
commit80104b1d8b257b3f711c77d7214d1fccb8bc5329 (patch)
tree607e903acc2769e3239b7471e8c34e659e87af9f
parent5c7c975875bc1d0ac656a190ce3abd0548eb9fc5 (diff)
downloadpylint-git-80104b1d8b257b3f711c77d7214d1fccb8bc5329.tar.gz
Removed command line parsing from Linter
-rw-r--r--pylint/__init__.py4
-rw-r--r--pylint/checkers/__init__.py22
-rw-r--r--pylint/config.py627
-rw-r--r--pylint/lint.py1371
-rw-r--r--pylint/utils.py32
5 files changed, 699 insertions, 1357 deletions
diff --git a/pylint/__init__.py b/pylint/__init__.py
index a5d43d123..11924c366 100644
--- a/pylint/__init__.py
+++ b/pylint/__init__.py
@@ -14,10 +14,10 @@ from .__pkginfo__ import version as __version__
def run_pylint():
"""run pylint"""
- from pylint.lint import Run
+ from pylint.lint import CLIRunner
try:
- Run(sys.argv[1:])
+ CLIRunner().run(sys.argv[1:])
except KeyboardInterrupt:
sys.exit(1)
diff --git a/pylint/checkers/__init__.py b/pylint/checkers/__init__.py
index 1a4c0009f..ed649707d 100644
--- a/pylint/checkers/__init__.py
+++ b/pylint/checkers/__init__.py
@@ -44,7 +44,6 @@ import tokenize
import warnings
from typing import Any
-from pylint.config import OptionsProviderMixIn
from pylint.reporters import diff_string
from pylint.utils import register_plugins
from pylint.interfaces import UNDEFINED
@@ -71,7 +70,7 @@ def table_lines_from_stats(stats, old_stats, columns):
return lines
-class BaseChecker(OptionsProviderMixIn):
+class BaseChecker(object):
"""base class for checkers"""
# checker name (you may reuse an existing one)
@@ -87,14 +86,9 @@ class BaseChecker(OptionsProviderMixIn):
# mark this checker as enabled or not.
enabled = True
- def __init__(self, linter=None):
- """checker instances should have the linter as argument
+ priority = -1
- linter is an object implementing ILinter
- """
- if self.name is not None:
- self.name = self.name.lower()
- OptionsProviderMixIn.__init__(self)
+ def __init__(self, linter=None):
self.linter = linter
def add_message(
@@ -126,9 +120,13 @@ class BaseTokenChecker(BaseChecker):
raise NotImplementedError()
-def initialize(linter):
- """initialize linter with checkers in this package """
- register_plugins(linter, __path__[0])
+def initialize(registry):
+ """Register the checkers in this package.
+
+ :param registry: The registry to register checkers with.
+ :type registry: CheckerRegistry
+ """
+ register_plugins(registry, __path__[0])
__all__ = ("BaseChecker", "BaseTokenChecker", "initialize")
diff --git a/pylint/config.py b/pylint/config.py
index c20c1474d..d70e1123b 100644
--- a/pylint/config.py
+++ b/pylint/config.py
@@ -37,16 +37,11 @@ from __future__ import print_function
import abc
import argparse
-import contextlib
import collections
-import copy
-import functools
-import io
import os
import pickle
import re
import sys
-import time
import configparser
@@ -132,10 +127,10 @@ def find_nearby_pylintrc(search_dir=""):
path = find_pylintrc_in(search_dir)
if not path:
- for search_dir in utils.walk_up(search_dir):
- if not os.path.isfile(os.path.join(search_dir, "__init__.py")):
+ for cur_dir in utils.walk_up(search_dir):
+ if not os.path.isfile(os.path.join(cur_dir, "__init__.py")):
break
- path = find_pylintrc_in(search_dir)
+ path = find_pylintrc_in(cur_dir)
if path:
break
@@ -205,54 +200,23 @@ class UnsupportedAction(Exception):
"""raised by set_option when it doesn't know what to do for an action"""
-def _multiple_choice_validator(choices, name, value):
- values = utils._check_csv(value)
- for csv_value in values:
- if csv_value not in choices:
- msg = "option %s: invalid value: %r, should be in %s"
- raise argparse.ArgumentError(msg % (name, csv_value, choices))
- return values
-
-
-def _choice_validator(choices, name, value):
- if value not in choices:
- msg = "option %s: invalid value: %r, should be in %s"
- raise argparse.ArgumentError(msg % (name, value, choices))
- return value
-
-
-# pylint: disable=unused-argument
-def _csv_validator(_, name, value):
- return utils._check_csv(value)
-
+def _regexp_csv_validator(value):
+ return [re.compile(val) for val in utils._check_csv(value)]
-# pylint: disable=unused-argument
-def _regexp_validator(_, name, value):
- if hasattr(value, "pattern"):
- return value
- return re.compile(value)
-
-# pylint: disable=unused-argument
-def _regexp_csv_validator(_, name, value):
- return [_regexp_validator(_, name, val) for val in _csv_validator(_, name, value)]
-
-
-def _yn_validator(opt, _, value):
- if isinstance(value, int):
- return bool(value)
+def _yn_validator(value):
if value in ("y", "yes"):
return True
if value in ("n", "no"):
return False
- msg = "option %s: invalid yn value %r, should be in (y, yes, n, no)"
- raise argparse.ArgumentError(msg % (opt, value))
+ msg = "invalid yn value %r, should be in (y, yes, n, no)"
+ raise argparse.ArgumentTypeError(msg % (value,))
-def _non_empty_string_validator(opt, _, value):
+def _non_empty_string_validator(value):
if not value:
msg = "indent string can't be empty."
- raise argparse.ArgumentError(msg)
+ raise argparse.ArgumentTypeError(msg)
return utils._unquote(value)
@@ -261,462 +225,21 @@ VALIDATORS = {
"int": int,
"regexp": re.compile,
"regexp_csv": _regexp_csv_validator,
- "csv": _csv_validator,
+ "csv": utils._check_csv,
"yn": _yn_validator,
- "choice": lambda opt, name, value: _choice_validator(opt["choices"], name, value),
- "multiple_choice": lambda opt, name, value: _multiple_choice_validator(
- opt["choices"], name, value
- ),
"non_empty_string": _non_empty_string_validator,
}
-def _call_validator(opttype, optdict, option, value):
- if opttype not in VALIDATORS:
- raise Exception('Unsupported type "%s"' % opttype)
- try:
- return VALIDATORS[opttype](optdict, option, value)
- except TypeError:
- try:
- return VALIDATORS[opttype](value)
- except Exception:
- raise argparse.ArgumentError(
- "%s value (%r) should be of type %s" % (option, value, opttype)
- )
-
-
-def _validate(value, optdict, name=""):
- """return a validated value for an option according to its type
-
- optional argument name is only used for error message formatting
- """
- try:
- _type = optdict["type"]
- except KeyError:
- # FIXME
- return value
- return _call_validator(_type, optdict, name, value)
-
-
-def _level_options(group, outputlevel):
- return [
- option
- for option in group.option_list
- if (getattr(option, "level", 0) or 0) <= outputlevel
- and option.help is not argparse.SUPPRESS
- ]
-
-
-def _multiple_choices_validating_option(opt, name, value):
- return _multiple_choice_validator(opt.choices, name, value)
-
-
-class OptionsManagerMixIn:
- """Handle configuration from both a configuration file and command line options"""
-
- class CallbackAction(argparse.Action):
- """Doesn't store the value on the config."""
-
- def __init__(self, nargs=None, **kwargs):
- nargs = nargs or int("metavar" in kwargs)
- super(OptionsManagerMixIn.CallbackAction, self).__init__(
- nargs=nargs, **kwargs
- )
-
- def __call__(self, parser, namespace, values, option_string):
- # If no value was passed, argparse didn't call the callback via
- # `type`, so we need to do it ourselves.
- if not self.nargs and callable(self.type):
- self.type(self, option_string, values, parser)
-
- def __init__(self, usage, config_file=None, version=None):
- self.config_file = config_file
- self.reset_parsers(usage, version=version)
- # list of registered options providers
- self.options_providers = []
- # dictionary associating option name to checker
- self._all_options = collections.OrderedDict()
- self._short_options = {}
- self._nocallback_options = {}
- self._mygroups = {}
- # verbosity
- self._maxlevel = 0
-
- def reset_parsers(self, usage="", version=None):
- # configuration file parser
- self.cfgfile_parser = IniFileParser()
- # command line parser
- self.cmdline_parser = CLIParser(usage)
-
- def register_options_provider(self, provider, own_group=True):
- """register an options provider"""
- assert provider.priority <= 0, "provider's priority can't be >= 0"
- for i in range(len(self.options_providers)):
- if provider.priority > self.options_providers[i].priority:
- self.options_providers.insert(i, provider)
- break
- else:
- self.options_providers.append(provider)
- non_group_spec_options = [
- option for option in provider.options if "group" not in option[1]
- ]
- groups = getattr(provider, "option_groups", ())
- if own_group and non_group_spec_options:
- self.add_option_group(
- provider.name.upper(),
- provider.__doc__,
- non_group_spec_options,
- provider,
- )
- else:
- for opt, optdict in non_group_spec_options:
- self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
- for gname, gdoc in groups:
- gname = gname.upper()
- goptions = [
- option
- for option in provider.options
- if option[1].get("group", "").upper() == gname
- ]
- self.add_option_group(gname, gdoc, goptions, provider)
-
- def add_option_group(self, group_name, _, options, provider):
- # add option group to the command line parser
- if group_name in self._mygroups:
- group = self._mygroups[group_name]
- else:
- group = self.cmdline_parser._parser.add_argument_group(
- group_name.capitalize(), level=provider.level
- )
- self._mygroups[group_name] = group
- # add section to the config file
- if group_name != "DEFAULT":
- try:
- self.cfgfile_parser._parser.add_section(group_name)
- except configparser.DuplicateSectionError:
- pass
-
- # add provider's specific options
- for opt, optdict in options:
- self.add_optik_option(provider, group, opt, optdict)
-
- def add_optik_option(self, provider, optikcontainer, opt, optdict):
- args, optdict = self.optik_option(provider, opt, optdict)
- if hasattr(optikcontainer, "_parser"):
- optikcontainer = optikcontainer._parser
- if "group" in optdict:
- optikcontainer = self._mygroups[optdict["group"].upper()]
- del optdict["group"]
-
- # Some sanity checks for things that trip up argparse
- assert not any(" " in arg for arg in args)
- assert all(optdict.values())
- assert not ("metavar" in optdict and "[" in optdict["metavar"])
-
- level = optdict.pop("level", 0)
- option = optikcontainer.add_argument(*args, **optdict)
- option.level = level
- self._all_options[opt] = provider
- self._maxlevel = max(self._maxlevel, optdict.get("level", 0))
-
- def optik_option(self, provider, opt, optdict):
- """get our personal option definition and return a suitable form for
- use with optik/argparse
- """
- # TODO: Changed to work with argparse but this should call
- # self.cmdline_parser.add_argument_definitions and not use callbacks
- optdict = copy.copy(optdict)
- if "action" in optdict:
- self._nocallback_options[provider] = opt
- if optdict["action"] == "callback":
- optdict["type"] = optdict["callback"]
- optdict["action"] = self.CallbackAction
- del optdict["callback"]
- else:
- callback = functools.partial(
- self.cb_set_provider_option, None, "--" + str(opt), parser=None
- )
- optdict["type"] = callback
- optdict.setdefault("action", "store")
- # default is handled here and *must not* be given to optik if you
- # want the whole machinery to work
- if "default" in optdict:
- if (
- "help" in optdict
- and optdict.get("default") is not None
- and optdict["action"] not in ("store_true", "store_false")
- ):
- default = optdict["default"]
- if isinstance(default, (tuple, list)):
- default = ",".join(str(x) for x in default)
- optdict["help"] += " [current: {0}]".format(default)
- del optdict["default"]
- args = ["--" + str(opt)]
- if "short" in optdict:
- self._short_options[optdict["short"]] = opt
- args.append("-" + optdict["short"])
- del optdict["short"]
- if optdict.get("action") == "callback":
- optdict["type"] = optdict["callback"]
- del optdict["action"]
- del optdict["callback"]
- if optdict.get("hide"):
- optdict["help"] = argparse.SUPPRESS
- del optdict["hide"]
- return args, optdict
-
- def cb_set_provider_option(self, option, opt, value, parser):
- """optik callback for option setting"""
- if opt.startswith("--"):
- # remove -- on long option
- opt = opt[2:]
- else:
- # short option, get its long equivalent
- opt = self._short_options[opt[1:]]
- # trick since we can't set action='store_true' on options
- if value is None:
- value = 1
- self.global_set_option(opt, value)
- return value
-
- def global_set_option(self, opt, value):
- """set option on the correct option provider"""
- self._all_options[opt].set_option(opt, value)
-
- def generate_config(self, stream=None, skipsections=(), encoding=None):
- """write a configuration file according to the current configuration
- into the given stream or stdout
- """
- options_by_section = {}
- sections = []
- for provider in self.options_providers:
- for section, options in provider.options_by_section():
- if section is None:
- section = provider.name
- if section in skipsections:
- continue
- options = [
- (n, d, v)
- for (n, d, v) in options
- if d.get("type") is not None and not d.get("deprecated")
- ]
- if not options:
- continue
- if section not in sections:
- sections.append(section)
- alloptions = options_by_section.setdefault(section, [])
- alloptions += options
- stream = stream or sys.stdout
- printed = False
- for section in sections:
- if printed:
- print("\n", file=stream)
- utils.format_section(
- stream, section.upper(), sorted(options_by_section[section])
- )
- printed = True
-
- def generate_manpage(self, pkginfo, section=1, stream=None):
- # TODO
- raise NotImplementedError
-
- def load_provider_defaults(self):
- """initialize configuration using default values"""
- for provider in self.options_providers:
- provider.load_defaults()
-
- def read_config_file(self, config_file=None, verbose=None):
- """read the configuration file but do not load it (i.e. dispatching
- values to each options provider)
- """
- if config_file is None:
- config_file = self.config_file
- if config_file is not None:
- config_file = os.path.expanduser(config_file)
- if not os.path.exists(config_file):
- raise IOError("The config file {:s} doesn't exist!".format(config_file))
-
- use_config_file = config_file and os.path.exists(config_file)
- if use_config_file:
- self.cfgfile_parser.parse(config_file, Configuration())
-
- if not verbose:
- return
-
- if use_config_file:
- msg = "Using config file {}".format(os.path.abspath(config_file))
- else:
- msg = "No config file found, using default configuration"
- print(msg, file=sys.stderr)
-
- def load_config_file(self):
- """dispatch values previously read from a configuration file to each
- options provider)
- """
- for section in self.cfgfile_parser._parser.sections():
- for option, value in self.cfgfile_parser._parser.items(section):
- try:
- self.global_set_option(option, value)
- except (KeyError, argparse.ArgumentError):
- # TODO handle here undeclared options appearing in the config file
- continue
-
- def load_configuration(self, **kwargs):
- """override configuration according to given parameters"""
- return self.load_configuration_from_config(kwargs)
-
- def load_configuration_from_config(self, config):
- for opt, opt_value in config.items():
- opt = opt.replace("_", "-")
- provider = self._all_options[opt]
- provider.set_option(opt, opt_value)
-
- def load_command_line_configuration(self, args=None):
- """Override configuration according to command line parameters
-
- return additional arguments
- """
- if args is None:
- args = sys.argv[1:]
- else:
- args = list(args)
- for provider in self._nocallback_options:
- self.cmdline_parser.parse(args, provider.config)
- config = Configuration()
- self.cmdline_parser.parse(args, config)
- return config.module_or_package
-
- def add_help_section(self, title, description, level=0):
- """add a dummy option section for help purpose """
- group = self.cmdline_parser._parser.add_argument_group(
- title.capitalize(), description, level=level
- )
- self._maxlevel = max(self._maxlevel, level)
-
- def help(self, level=0):
- """return the usage string for available options """
- return self.cmdline_parser._parser.format_help(level)
-
-
-class OptionsProviderMixIn:
- """Mixin to provide options to an OptionsManager"""
-
- # those attributes should be overridden
- priority = -1
- name = "default"
- options = ()
- level = 0
-
- def __init__(self):
- self.config = Configuration()
- self.load_defaults()
-
- def load_defaults(self):
- """initialize the provider using default values"""
- for opt, optdict in self.options:
- action = optdict.get("action")
- if action != "callback":
- # callback action have no default
- if optdict is None:
- optdict = self.get_option_def(opt)
- default = optdict.get("default")
- self.set_option(opt, default, action, optdict)
-
- def option_attrname(self, opt, optdict=None):
- """get the config attribute corresponding to opt"""
- if optdict is None:
- optdict = self.get_option_def(opt)
- return optdict.get("dest", opt.replace("-", "_"))
-
- def option_value(self, opt):
- """get the current value for the given option"""
- return getattr(self.config, self.option_attrname(opt), None)
-
- def set_option(self, optname, value, action=None, optdict=None):
- """method called to set an option (registered in the options list)"""
- if optdict is None:
- optdict = self.get_option_def(optname)
- if value is not None:
- value = _validate(value, optdict, optname)
- if action is None:
- action = optdict.get("action", "store")
- if action == "store":
- setattr(self.config, self.option_attrname(optname, optdict), value)
- elif action in ("store_true", "count"):
- setattr(self.config, self.option_attrname(optname, optdict), 0)
- elif action == "store_false":
- setattr(self.config, self.option_attrname(optname, optdict), 1)
- elif action == "append":
- optname = self.option_attrname(optname, optdict)
- _list = getattr(self.config, optname, None)
- if _list is None:
- if isinstance(value, (list, tuple)):
- _list = value
- elif value is not None:
- _list = []
- _list.append(value)
- setattr(self.config, optname, _list)
- elif isinstance(_list, tuple):
- setattr(self.config, optname, _list + (value,))
- else:
- _list.append(value)
- elif action == "callback":
- optdict["callback"](None, optname, value, None)
- else:
- raise UnsupportedAction(action)
-
- def get_option_def(self, opt):
- """return the dictionary defining an option given its name"""
- assert self.options
- for option in self.options:
- if option[0] == opt:
- return option[1]
- raise argparse.ArgumentError(
- "no such option %s in section %r" % (opt, self.name), opt
- )
-
- def options_by_section(self):
- """return an iterator on options grouped by section
-
- (section, [list of (optname, optdict, optvalue)])
- """
- sections = {}
- for optname, optdict in self.options:
- sections.setdefault(optdict.get("group"), []).append(
- (optname, optdict, self.option_value(optname))
- )
- if None in sections:
- yield None, sections.pop(None)
- for section, options in sorted(sections.items()):
- yield section.upper(), options
-
- def options_and_values(self, options=None):
- if options is None:
- options = self.options
- for optname, optdict in options:
- yield (optname, optdict, self.option_value(optname))
-
-
-class ConfigurationMixIn(OptionsManagerMixIn, OptionsProviderMixIn):
- """basic mixin for simple configurations which don't need the
- manager / providers model
- """
-
- def __init__(self, *args, **kwargs):
- if not args:
- kwargs.setdefault("usage", "")
- OptionsManagerMixIn.__init__(self, *args, **kwargs)
- OptionsProviderMixIn.__init__(self)
- if not getattr(self, "option_groups", None):
- self.option_groups = []
- for _, optdict in self.options:
- try:
- gdef = (optdict["group"].upper(), "")
- except KeyError:
- continue
- if gdef not in self.option_groups:
- self.option_groups.append(gdef)
- self.register_options_provider(self, own_group=False)
+UNVALIDATORS = {
+ "string": str,
+ "int": str,
+ "regexp": lambda value: getattr(value, "pattern", value),
+ "regexp_csv": (lambda value: ",".join(r.pattern for r in value)),
+ "csv": (lambda value: ",".join(value)),
+ "yn": (lambda value: "y" if value else "n"),
+ "non_empty_string": str,
+}
OptionDefinition = collections.namedtuple("OptionDefinition", ["name", "definition"])
@@ -730,6 +253,7 @@ class Configuration(object):
name, definition = option_definition
if name in self._option_definitions:
raise exceptions.ConfigurationError('Option "{0}" already exists.')
+
self._option_definitions[name] = definition
def add_options(self, option_definitions):
@@ -737,6 +261,7 @@ class Configuration(object):
self.add_option(option_definition)
def set_option(self, option, value):
+ option = option.replace("-", "_")
setattr(self, option, value)
def copy(self):
@@ -758,8 +283,9 @@ class Configuration(object):
self._option_definitions.update(other._option_definitions)
for option in other._option_definitions:
+ option = option.replace("-", "_")
value = getattr(other, option)
- setattr(result, option, value)
+ setattr(self, option, value)
return self
@@ -830,7 +356,7 @@ class ConfigurationStore(object):
config = self.global_config.copy()
parent_configs = self._get_parent_configs(path)
- for parent_config in reversed(parent_configs):
+ for parent_config in reversed(list(parent_configs)):
config += parent_config
self._cache["path"] = config
@@ -854,7 +380,7 @@ class ConfigParser(metaclass=abc.ABCMeta):
for _, definition_dict in option_definitions:
try:
- group = optdict["group"].upper()
+ group = definition_dict["group"].upper()
except KeyError:
continue
else:
@@ -887,11 +413,12 @@ class CLIParser(ConfigParser):
self._parser.add_argument("module_or_package", nargs=argparse.REMAINDER)
def add_option_definitions(self, option_definitions):
+ self._option_definitions.update(option_definitions)
option_groups = collections.defaultdict(list)
for option, definition in option_definitions:
group, args, kwargs = self._convert_definition(option, definition)
- option_groups[group].append(args, kwargs)
+ option_groups[group].append((args, kwargs))
for args, kwargs in option_groups["DEFAULT"]:
self._parser.add_argument(*args, **kwargs)
@@ -900,9 +427,9 @@ class CLIParser(ConfigParser):
for group, arguments in option_groups.items():
self._option_groups.add(group)
- self._parser.add_argument_group(group.title())
+ group = self._parser.add_argument_group(group.title())
for args, kwargs in arguments:
- self._parser.add_argument(*args, **kwargs)
+ group.add_argument(*args, **kwargs)
@staticmethod
def _convert_definition(option, definition):
@@ -936,7 +463,7 @@ class CLIParser(ConfigParser):
if "choices" not in definition:
msg = 'No choice list given for option "{0}" of type "choice".'
msg = msg.format(option)
- raise ConfigurationError(msg)
+ raise exceptions.ConfigurationError(msg)
if definition["type"] == "multiple_choice":
kwargs["type"] = VALIDATORS["csv"]
@@ -944,7 +471,7 @@ class CLIParser(ConfigParser):
kwargs["choices"] = definition["choices"]
else:
msg = 'Unsupported type "{0}"'.format(definition["type"])
- raise ConfigurationError(msg)
+ raise exception.ConfigurationError(msg)
if definition.get("hide"):
kwargs["help"] = argparse.SUPPRESS
@@ -952,15 +479,15 @@ class CLIParser(ConfigParser):
group = definition.get("group", "DEFAULT").upper()
return group, args, kwargs
- def parse(self, argv, config):
+ def parse(self, to_parse, config):
"""Parse the command line arguments into the given config object.
- :param argv: The command line arguments to parse.
- :type argv: list(str)
+ :param to_parse: The command line arguments to parse.
+ :type to_parse: list(str)
:param config: The config object to parse the command line into.
:type config: Configuration
"""
- self._parser.parse_args(argv, config)
+ self._parser.parse_args(to_parse, config)
def preprocess(self, argv, *options):
"""Do some guess work to get a value for the specified option.
@@ -973,19 +500,34 @@ class CLIParser(ConfigParser):
:returns: A config with the processed options.
:rtype: Configuration
"""
- config = Config()
- config.add_options(self._option_definitions)
+ config = Configuration()
+ config.add_options(self._option_definitions.items())
args = self._parser.parse_known_args(argv)[0]
for option in options:
+ option = option.replace("-", "_")
config.set_option(option, getattr(args, option, None))
return config
+ def add_help_section(self, title, description, level=0):
+ """Add an extra help section to the help message.
+
+ :param title: The title of the section.
+ This is included as part of the help message.
+ :type title: str
+ :param description: The description of the help section.
+ :type description: str
+ :param level: The minimum level of help needed to include this
+ in the help message.
+ :type level: int
+ """
+ self._parser.add_argument_group(title, description, level=level)
+
class FileParser(ConfigParser, metaclass=abc.ABCMeta):
@abc.abstractmethod
- def parse(self, file_path, config):
+ def parse(self, to_parse, config):
pass
@@ -997,17 +539,20 @@ class IniFileParser(FileParser):
self._parser = configparser.ConfigParser(inline_comment_prefixes=("#", ";"))
def add_option_definitions(self, option_definitions):
+ self._option_definitions.update(option_definitions)
for option, definition in option_definitions:
group, default = self._convert_definition(option, definition)
- try:
- self._parser.add_section(group)
- except configparser.DuplicateSectionError:
- pass
- else:
- self._option_groups.add(group)
+ if group != self._parser.default_section:
+ try:
+ self._parser.add_section(group)
+ except configparser.DuplicateSectionError:
+ pass
+ else:
+ self._option_groups.add(group)
- self._parser["DEFAULT"].update(default)
+ if default is not None:
+ self._parser["DEFAULT"].update(default)
@staticmethod
def _convert_definition(option, definition):
@@ -1021,13 +566,17 @@ class IniFileParser(FileParser):
:returns: The converted definition.
:rtype: tuple(str, dict)
"""
- default = {option: definition.get("default")}
+ default = None
+ if definition.get("default"):
+ unvalidator = UNVALIDATORS.get(definition.get("type"), str)
+ default_value = unvalidator(definition["default"])
+ default = {option: default_value}
group = definition.get("group", "DEFAULT").upper()
return group, default
- def parse(self, file_path, config):
- self._parser.read(file_path)
+ def parse(self, to_parse, config):
+ self._parser.read(to_parse)
for section in self._parser.sections():
# Normalise the section titles
@@ -1039,8 +588,17 @@ class IniFileParser(FileParser):
section = section.upper()
for option, value in self._parser.items(section):
+ if isinstance(value, str):
+ definition = self._option_definitions.get(option, {})
+ type_ = definition.get("type")
+ validator = VALIDATORS.get(type_, lambda x: x)
+ value = validator(value)
config.set_option(option, value)
+ def write(self, stream=sys.stdout):
+ # TODO: Check if option descriptions are written out
+ self._parser.write(stream)
+
class LongHelpFormatter(argparse.HelpFormatter):
output_level = None
@@ -1092,6 +650,25 @@ class LongHelpAction(argparse.Action):
parser.exit()
+class LongHelpArgumentGroup(argparse._ArgumentGroup):
+ def __init__(self, *args, level=0, **kwargs):
+ super(LongHelpArgumentGroup, self).__init__(*args, **kwargs)
+ self.level = level
+
+ def add_argument(self, *args, **kwargs):
+ """See :func:`argparse.ArgumentParser.add_argument`.
+
+ Patches in the level to each created action instance.
+
+ :returns: The created action.
+ :rtype: argparse.Action
+ """
+ level = kwargs.pop("level", 0)
+ action = super(LongHelpArgumentGroup, self).add_argument(*args, **kwargs)
+ action.level = level
+ return action
+
+
class LongHelpArgumentParser(argparse.ArgumentParser):
def __init__(self, formatter_class=LongHelpFormatter, **kwargs):
self._max_level = 0
@@ -1140,9 +717,9 @@ class LongHelpArgumentParser(argparse.ArgumentParser):
action.level = level
return action
- def add_argument_group(self, *args, level=0, **kwargs):
- group = super(LongHelpArgumentParser, self).add_argument_group(*args, **kwargs)
- group.level = level
+ def add_argument_group(self, *args, **kwargs):
+ group = LongHelpArgumentGroup(self, *args, **kwargs)
+ self._action_groups.append(group)
return group
# These methods use yucky way of passing the level to the formatter class
diff --git a/pylint/lint.py b/pylint/lint.py
index 4721e911f..34b935064 100644
--- a/pylint/lint.py
+++ b/pylint/lint.py
@@ -64,11 +64,6 @@ import collections
import contextlib
import operator
import os
-
-try:
- import multiprocessing
-except ImportError:
- multiprocessing = None # type: ignore
import sys
import tokenize
import warnings
@@ -215,85 +210,9 @@ MSGS = {
}
-def _cpu_count() -> int:
- """Use sched_affinity if available for virtualized or containerized environments."""
- sched_getaffinity = getattr(os, "sched_getaffinity", None)
- # pylint: disable=not-callable,using-constant-test
- if sched_getaffinity:
- return len(sched_getaffinity(0))
- if multiprocessing:
- return multiprocessing.cpu_count()
- return 1
-
-
-if multiprocessing is not None:
-
- class ChildRunner(multiprocessing.Process):
- def run(self):
- # pylint: disable=no-member, unbalanced-tuple-unpacking
- tasks_queue, results_queue, self._config = self._args
-
- self._config["jobs"] = 1 # Child does not parallelize any further.
- self._python3_porting_mode = self._config.pop("python3_porting_mode", None)
- self._plugins = self._config.pop("plugins", None)
-
- # Run linter for received files/modules.
- for file_or_module in iter(tasks_queue.get, "STOP"):
- try:
- result = self._run_linter(file_or_module[0])
- results_queue.put(result)
- except Exception as ex:
- print(
- "internal error with sending report for module %s"
- % file_or_module,
- file=sys.stderr,
- )
- print(ex, file=sys.stderr)
- results_queue.put({})
-
- def _run_linter(self, file_or_module):
- linter = PyLinter()
-
- # Register standard checkers.
- linter.load_default_plugins()
- # Load command line plugins.
- if self._plugins:
- linter.load_plugin_modules(self._plugins)
-
- linter.load_configuration_from_config(self._config)
-
- # Load plugin specific configuration
- linter.load_plugin_configuration()
-
- linter.set_reporter(reporters.CollectingReporter())
-
- # Enable the Python 3 checker mode. This option is
- # passed down from the parent linter up to here, since
- # the Python 3 porting flag belongs to the Run class,
- # instead of the Linter class.
- if self._python3_porting_mode:
- linter.python3_porting_mode()
-
- # Run the checks.
- linter.check(file_or_module)
-
- msgs = [_get_new_args(m) for m in linter.reporter.messages]
- return (
- file_or_module,
- linter.file_state.base_name,
- linter.current_name,
- msgs,
- linter.stats,
- linter.msg_status,
- )
-
-
# pylint: disable=too-many-instance-attributes
class PyLinter(
- config.OptionsManagerMixIn,
- utils.MessagesHandlerMixIn,
- utils.ReportsHandlerMixIn,
- checkers.BaseTokenChecker,
+ utils.MessagesHandlerMixIn, utils.ReportsHandlerMixIn, checkers.BaseTokenChecker
):
"""lint Python modules using external checkers.
@@ -315,261 +234,246 @@ class PyLinter(
level = 0
msgs = MSGS
- @staticmethod
- def make_options():
- return (
- (
- "ignore",
- {
- "type": "csv",
- "metavar": "<file>,...",
- "dest": "black_list",
- "default": ("CVS",),
- "help": "Add files or directories to the blacklist. "
- "They should be base names, not paths.",
- },
- ),
- (
- "ignore-patterns",
- {
- "type": "regexp_csv",
- "metavar": "<pattern>,...",
- "dest": "black_list_re",
- "default": (),
- "help": "Add files or directories matching the regex patterns to the"
- " blacklist. The regex matches against base names, not paths.",
- },
- ),
- (
- "persistent",
- {
- "default": True,
- "type": "yn",
- "metavar": "<y_or_n>",
- "level": 1,
- "help": "Pickle collected data for later comparisons.",
- },
- ),
- (
- "load-plugins",
- {
- "type": "csv",
- "metavar": "<modules>",
- "default": (),
- "level": 1,
- "help": "List of plugins (as comma separated values of "
- "python modules names) to load, usually to register "
- "additional checkers.",
- },
- ),
- (
- "output-format",
- {
- "default": "text",
- "type": "string",
- "metavar": "<format>",
- "short": "f",
- "group": "Reports",
- "help": "Set the output format. Available formats are text,"
- " parseable, colorized, json and msvs (visual studio)."
- " You can also give a reporter class, e.g. mypackage.mymodule."
- "MyReporterClass.",
- },
- ),
- (
- "reports",
- {
- "default": False,
- "type": "yn",
- "metavar": "<y_or_n>",
- "short": "r",
- "group": "Reports",
- "help": "Tells whether to display a full report or only the "
- "messages.",
- },
- ),
- (
- "evaluation",
- {
- "type": "string",
- "metavar": "<python_expression>",
- "group": "Reports",
- "level": 1,
- "default": "10.0 - ((float(5 * error + warning + refactor + "
- "convention) / statement) * 10)",
- "help": "Python expression which should return a note less "
- "than 10 (10 is the highest note). You have access "
- "to the variables errors warning, statement which "
- "respectively contain the number of errors / "
- "warnings messages and the total number of "
- "statements analyzed. This is used by the global "
- "evaluation report (RP0004).",
- },
- ),
- (
- "score",
- {
- "default": True,
- "type": "yn",
- "metavar": "<y_or_n>",
- "short": "s",
- "group": "Reports",
- "help": "Activate the evaluation score.",
- },
- ),
- (
- "confidence",
- {
- "type": "multiple_choice",
- "metavar": "<levels>",
- "default": "",
- "choices": [c.name for c in interfaces.CONFIDENCE_LEVELS],
- "group": "Messages control",
- "help": "Only show warnings with the listed confidence levels."
- " Leave empty to show all. Valid levels: %s."
- % (", ".join(c.name for c in interfaces.CONFIDENCE_LEVELS),),
- },
- ),
- (
- "enable",
- {
- "type": "csv",
- "metavar": "<msg ids>",
- "short": "e",
- "group": "Messages control",
- "help": "Enable the message, report, category or checker with the "
- "given id(s). You can either give multiple identifier "
- "separated by comma (,) or put this option multiple time "
- "(only on the command line, not in the configuration file "
- "where it should appear only once). "
- 'See also the "--disable" option for examples.',
- },
- ),
- (
- "disable",
- {
- "type": "csv",
- "metavar": "<msg ids>",
- "short": "d",
- "group": "Messages control",
- "help": "Disable the message, report, category or checker "
- "with the given id(s). You can either give multiple identifiers "
- "separated by comma (,) or put this option multiple times "
- "(only on the command line, not in the configuration file "
- "where it should appear only once). "
- 'You can also use "--disable=all" to disable everything first '
- "and then reenable specific checks. For example, if you want "
- "to run only the similarities checker, you can use "
- '"--disable=all --enable=similarities". '
- "If you want to run only the classes checker, but have no "
- "Warning level messages displayed, use "
- '"--disable=all --enable=classes --disable=W".',
- },
- ),
- (
- "msg-template",
- {
- "type": "string",
- "metavar": "<template>",
- "group": "Reports",
- "help": (
- "Template used to display messages. "
- "This is a python new-style format string "
- "used to format the message information. "
- "See doc for all details."
- ),
- },
- ),
- (
- "jobs",
- {
- "type": "int",
- "metavar": "<n-processes>",
- "short": "j",
- "default": 1,
- "help": "Use multiple processes to speed up Pylint. Specifying 0 will "
- "auto-detect the number of processors available to use.",
- },
- ),
- (
- "unsafe-load-any-extension",
- {
- "type": "yn",
- "metavar": "<yn>",
- "default": False,
- "hide": True,
- "help": (
- "Allow loading of arbitrary C extensions. Extensions"
- " are imported into the active Python interpreter and"
- " may run arbitrary code."
- ),
- },
- ),
- (
- "limit-inference-results",
- {
- "type": "int",
- "metavar": "<number-of-results>",
- "default": 100,
- "help": (
- "Control the amount of potential inferred values when inferring "
- "a single object. This can help the performance when dealing with "
- "large functions or complex, nested conditions. "
- ),
- },
- ),
- (
- "extension-pkg-whitelist",
- {
- "type": "csv",
- "metavar": "<pkg>,...",
- "default": [],
- "help": (
- "A comma-separated list of package or module names"
- " from where C extensions may be loaded. Extensions are"
- " loading into the active Python interpreter and may run"
- " arbitrary code."
- ),
- },
- ),
- (
- "suggestion-mode",
- {
- "type": "yn",
- "metavar": "<yn>",
- "default": True,
- "help": (
- "When enabled, pylint would attempt to guess common "
- "misconfiguration and emit user-friendly hints instead "
- "of false-positive error messages."
- ),
- },
- ),
- (
- "exit-zero",
- {
- "action": "store_true",
- "help": (
- "Always return a 0 (non-error) status code, even if "
- "lint errors are found. This is primarily useful in "
- "continuous integration scripts."
- ),
- },
- ),
- )
+ options = (
+ (
+ "ignore",
+ {
+ "type": "csv",
+ "metavar": "<file>,...",
+ "dest": "black_list",
+ "default": ("CVS",),
+ "help": "Add files or directories to the blacklist. "
+ "They should be base names, not paths.",
+ },
+ ),
+ (
+ "ignore-patterns",
+ {
+ "type": "regexp_csv",
+ "metavar": "<pattern>,...",
+ "dest": "black_list_re",
+ "default": (),
+ "help": "Add files or directories matching the regex patterns to the"
+ " blacklist. The regex matches against base names, not paths.",
+ },
+ ),
+ (
+ "persistent",
+ {
+ "default": True,
+ "type": "yn",
+ "metavar": "<y_or_n>",
+ "level": 1,
+ "help": "Pickle collected data for later comparisons.",
+ },
+ ),
+ (
+ "load-plugins",
+ {
+ "type": "csv",
+ "metavar": "<modules>",
+ "default": (),
+ "level": 1,
+ "help": "List of plugins (as comma separated values of "
+ "python modules names) to load, usually to register "
+ "additional checkers.",
+ },
+ ),
+ (
+ "output-format",
+ {
+ "default": "text",
+ "type": "string",
+ "metavar": "<format>",
+ "short": "f",
+ "group": "Reports",
+ "help": "Set the output format. Available formats are text,"
+ " parseable, colorized, json and msvs (visual studio)."
+ " You can also give a reporter class, e.g. mypackage.mymodule."
+ "MyReporterClass.",
+ },
+ ),
+ (
+ "reports",
+ {
+ "default": False,
+ "type": "yn",
+ "metavar": "<y_or_n>",
+ "short": "r",
+ "group": "Reports",
+ "help": "Tells whether to display a full report or only the "
+ "messages",
+ },
+ ),
+ (
+ "evaluation",
+ {
+ "type": "string",
+ "metavar": "<python_expression>",
+ "group": "Reports",
+ "level": 1,
+ "default": "10.0 - ((float(5 * error + warning + refactor + "
+ "convention) / statement) * 10)",
+ "help": "Python expression which should return a note less "
+ "than 10 (10 is the highest note). You have access "
+ "to the variables errors warning, statement which "
+ "respectively contain the number of errors / "
+ "warnings messages and the total number of "
+ "statements analyzed. This is used by the global "
+ "evaluation report (RP0004).",
+ },
+ ),
+ (
+ "score",
+ {
+ "default": True,
+ "type": "yn",
+ "metavar": "<y_or_n>",
+ "short": "s",
+ "group": "Reports",
+ "help": "Activate the evaluation score.",
+ },
+ ),
+ (
+ "confidence",
+ {
+ "type": "multiple_choice",
+ "metavar": "<levels>",
+ "default": "",
+ "choices": [c.name for c in interfaces.CONFIDENCE_LEVELS],
+ "group": "Messages control",
+ "help": "Only show warnings with the listed confidence levels."
+ " Leave empty to show all. Valid levels: %s"
+ % (", ".join(c.name for c in interfaces.CONFIDENCE_LEVELS),),
+ },
+ ),
+ (
+ "enable",
+ {
+ "type": "csv",
+ "metavar": "<msg ids>",
+ "short": "e",
+ "group": "Messages control",
+ "help": "Enable the message, report, category or checker with the "
+ "given id(s). You can either give multiple identifier "
+ "separated by comma (,) or put this option multiple time "
+ "(only on the command line, not in the configuration file "
+ "where it should appear only once). "
+ 'See also the "--disable" option for examples. ',
+ },
+ ),
+ (
+ "disable",
+ {
+ "type": "csv",
+ "metavar": "<msg ids>",
+ "short": "d",
+ "group": "Messages control",
+ "help": "Disable the message, report, category or checker "
+ "with the given id(s). You can either give multiple identifiers"
+ " separated by comma (,) or put this option multiple times "
+ "(only on the command line, not in the configuration file "
+ "where it should appear only once)."
+ 'You can also use "--disable=all" to disable everything first '
+ "and then reenable specific checks. For example, if you want "
+ "to run only the similarities checker, you can use "
+ '"--disable=all --enable=similarities". '
+ "If you want to run only the classes checker, but have no "
+ "Warning level messages displayed, use"
+ '"--disable=all --enable=classes --disable=W"',
+ },
+ ),
+ (
+ "msg-template",
+ {
+ "type": "string",
+ "metavar": "<template>",
+ "default": "",
+ "group": "Reports",
+ "help": (
+ "Template used to display messages. "
+ "This is a python new-style format string "
+ "used to format the message information. "
+ "See doc for all details"
+ ),
+ },
+ ),
+ (
+ "jobs",
+ {
+ "type": "int",
+ "metavar": "<n-processes>",
+ "short": "j",
+ "default": 1,
+ "help": """Use multiple processes to speed up Pylint.""",
+ },
+ ),
+ (
+ "unsafe-load-any-extension",
+ {
+ "type": "yn",
+ "metavar": "<yn>",
+ "default": False,
+ "hide": True,
+ "help": (
+ "Allow loading of arbitrary C extensions. Extensions"
+ " are imported into the active Python interpreter and"
+ " may run arbitrary code."
+ ),
+ },
+ ),
+ (
+ "extension-pkg-whitelist",
+ {
+ "type": "csv",
+ "metavar": "<pkg>,...",
+ "default": [],
+ "help": (
+ "A comma-separated list of package or module names"
+ " from where C extensions may be loaded. Extensions are"
+ " loading into the active Python interpreter and may run"
+ " arbitrary code"
+ ),
+ },
+ ),
+ (
+ "suggestion-mode",
+ {
+ "type": "yn",
+ "metavar": "<yn>",
+ "default": True,
+ "help": (
+ "When enabled, pylint would attempt to guess common "
+ "misconfiguration and emit user-friendly hints instead "
+ "of false-positive error messages"
+ ),
+ },
+ ),
+ (
+ "exit-zero",
+ {
+ "action": "store_true",
+ "help": (
+ "Always return a 0 (non-error) status code, even if "
+ "lint errors are found. This is primarily useful in "
+ "continuous integration scripts."
+ ),
+ },
+ ),
+ )
option_groups = (
("Messages control", "Options controlling analysis messages"),
("Reports", "Options related to output formatting and reporting"),
)
- def __init__(self, options=(), reporter=None, option_groups=(), pylintrc=None):
+ def __init__(self, config=None):
# some stuff has to be done before ancestors initialization...
#
# messages store / checkers / reporter / astroid manager
+ self.config = config
self.msgs_store = utils.MessagesStore()
self.reporter = None
- self._reporter_name = None
self._reporters = {}
self._checkers = collections.defaultdict(list)
self._pragma_lineno = {}
@@ -579,15 +483,8 @@ class PyLinter(
self.current_name = None
self.current_file = None
self.stats = None
- # init options
- self._external_opts = options
- self.options = options + PyLinter.make_options()
- self.option_groups = option_groups + PyLinter.option_groups
- self._options_methods = {"enable": self.enable, "disable": self.disable}
- self._bw_options_methods = {
- "disable-msg": self.disable,
- "enable-msg": self.enable,
- }
+
+ # TODO: Runner needs to give this to parser?
full_version = "%%prog %s\nastroid %s\nPython %s" % (
version,
astroid_version,
@@ -595,9 +492,6 @@ class PyLinter(
)
utils.MessagesHandlerMixIn.__init__(self)
utils.ReportsHandlerMixIn.__init__(self)
- super(PyLinter, self).__init__(
- usage=__doc__, version=full_version, config_file=pylintrc or config.PYLINTRC
- )
checkers.BaseTokenChecker.__init__(self)
# provided reports
self.reports = (
@@ -609,32 +503,9 @@ class PyLinter(
),
("RP0003", "Messages", report_messages_stats),
)
- self.register_checker(self)
self._dynamic_plugins = set()
self._python3_porting_mode = False
self._error_mode = False
- self.load_provider_defaults()
- if reporter:
- self.set_reporter(reporter)
-
- def load_default_plugins(self):
- checkers.initialize(self)
- reporters.initialize(self)
- # Make sure to load the default reporter, because
- # the option has been set before the plugins had been loaded.
- if not self.reporter:
- self._load_reporter()
-
- def load_plugin_modules(self, modnames):
- """take a list of module names which are pylint plugins and load
- and register them
- """
- for modname in modnames:
- if modname in self._dynamic_plugins:
- continue
- self._dynamic_plugins.add(modname)
- module = modutils.load_module_from_name(modname)
- module.register(self)
def load_plugin_configuration(self):
"""Call the configuration hook for plugins
@@ -648,8 +519,8 @@ class PyLinter(
if hasattr(module, "load_configuration"):
module.load_configuration(self)
- def _load_reporter(self):
- name = self._reporter_name.lower()
+ def load_reporter(self):
+ name = self.config.output_format.lower()
if name in self._reporters:
self.set_reporter(self._reporters[name]())
else:
@@ -661,7 +532,7 @@ class PyLinter(
self.set_reporter(reporter_class())
def _load_reporter_class(self):
- qname = self._reporter_name
+ qname = self.config.output_format
module = modutils.load_module_from_name(modutils.get_module_part(qname))
class_name = qname.split(".")[-1]
reporter_class = getattr(module, class_name)
@@ -672,42 +543,10 @@ class PyLinter(
self.reporter = reporter
reporter.linter = self
- def set_option(self, optname, value, action=None, optdict=None):
- """overridden from config.OptionsProviderMixin to handle some
- special options
- """
- if optname in self._options_methods or optname in self._bw_options_methods:
- if value:
- try:
- meth = self._options_methods[optname]
- except KeyError:
- meth = self._bw_options_methods[optname]
- warnings.warn(
- "%s is deprecated, replace it by %s"
- % (optname, optname.split("-")[0]),
- DeprecationWarning,
- )
- value = utils._check_csv(value)
- if isinstance(value, (list, tuple)):
- for _id in value:
- meth(_id, ignore_unknown=True)
- else:
- meth(value)
- return # no need to call set_option, disable/enable methods do it
- elif optname == "output-format":
- self._reporter_name = value
- # If the reporters are already available, load
- # the reporter class.
- if self._reporters:
- self._load_reporter()
-
- try:
- checkers.BaseTokenChecker.set_option(self, optname, value, action, optdict)
- except config.UnsupportedAction:
- print("option %s can't be read from config file" % optname, file=sys.stderr)
-
def register_reporter(self, reporter_class):
self._reporters[reporter_class.name] = reporter_class
+ if reporter_class.name == self.config.output_format.lower():
+ self.load_reporter()
def report_order(self):
reports = sorted(self._reports, key=lambda x: getattr(x, "name", ""))
@@ -723,25 +562,6 @@ class PyLinter(
# checkers manipulation methods ############################################
- def register_checker(self, checker):
- """register a new checker
-
- checker is an object implementing IRawChecker or / and IAstroidChecker
- """
- assert checker.priority <= 0, "checker priority can't be >= 0"
- self._checkers[checker.name].append(checker)
- for r_id, r_title, r_cb in checker.reports:
- self.register_report(r_id, r_title, r_cb, checker)
- self.register_options_provider(checker)
- if hasattr(checker, "msgs"):
- self.msgs_store.register_messages_from_checker(checker)
- checker.load_defaults()
-
- # Register the checker, but disable all of its messages.
- # TODO(cpopa): we should have a better API for this.
- if not getattr(checker, "enabled", True):
- self.disable(checker.name)
-
def disable_noerror_messages(self):
for msgcat, msgids in self.msgs_store._msgs_by_category.items():
# enable only messages with 'error' severity and above ('fatal')
@@ -805,6 +625,7 @@ class PyLinter(
"""process tokens from the current module to search for module/block
level options
"""
+ options_methods = {"enable": self.enable, "disable": self.disable}
control_pragmas = {"disable", "enable"}
for (tok_type, content, start, _, _) in tokens:
if tok_type != tokenize.COMMENT:
@@ -835,17 +656,8 @@ class PyLinter(
)
continue
opt = opt.strip()
- if opt in self._options_methods or opt in self._bw_options_methods:
- try:
- meth = self._options_methods[opt]
- except KeyError:
- meth = self._bw_options_methods[opt]
- # found a "(dis|en)able-msg" pragma deprecated suppression
- self.add_message(
- "deprecated-pragma",
- line=start[0],
- args=(opt, opt.replace("-msg", "")),
- )
+ if opt in options_methods:
+ meth = options_methods[opt]
for msgid in utils._splitstrip(value):
# Add the line where a control pragma was encountered.
if opt in control_pragmas:
@@ -928,6 +740,7 @@ class PyLinter(
"""main checking entry: check a list of files or modules from their
name.
"""
+ assert self.reporter, "A reporter has not been loaded"
# initialize msgs_state now that all messages have been registered into
# the store
self._init_msg_states()
@@ -1230,7 +1043,6 @@ def preprocess_options(args, search_for):
@contextlib.contextmanager
def fix_import_path(args):
"""Prepare sys.path for running the linter checks.
-
Within this context, each of the given arguments is importable.
Paths are added to sys.path in corresponding order to the arguments.
We avoid adding duplicate directories to sys.path.
@@ -1251,13 +1063,207 @@ def fix_import_path(args):
sys.path[:] = orig
-class Run:
- """helper class to use as main for pylint :
+def guess_lint_path(args):
+ """Attempt to determine the file being linted from a list of arguments.
+
+ :param args: The list of command line arguments to guess from.
+ :type args: list(str)
- run(*sys.argv[1:])
+ :returns: The path to file being linted if it can be guessed.
+ None otherwise.
+ :rtype: str or None
"""
+ value = None
+
+ # We only care if it's a path. If it's a module,
+ # we can't get a config from it
+ if args and os.path.exists(args[-1]):
+ value = args[-1]
+
+ return value
+
+
+class CheckerRegistry(object):
+ """A class to register checkers to."""
+
+ def __init__(self, linter):
+ super(CheckerRegistry, self).__init__()
+ self.register_options = lambda options: None
+ self._checkers = collections.defaultdict(list)
+ # TODO: Remove. This is needed for the MessagesHandlerMixIn for now.
+ linter._checkers = self._checkers
+ self._linter = linter
+ self.register_checker(linter)
+
+ def for_all_checkers(self):
+ """Loop through all registered checkers.
+
+ :returns: Each registered checker.
+ :rtype: iterable(BaseChecker)
+ """
+ for checkers in self._checkers.values():
+ yield from checkers
+
+ def register_checker(self, checker):
+ """Register a checker.
+
+ :param checker: The checker to register.
+ :type checker: BaseChecker
+
+ :raises ValueError: If the priority of the checker is invalid.
+ """
+ if checker.name in self._checkers:
+ # TODO: Raise if classes are the same
+ for duplicate in self._checkers[checker.name]:
+ msg = "A checker called {} has already been registered ({})."
+ msg = msg.format(checker.name, duplicate.__class__)
+ warnings.warn(msg)
+
+ if checker.priority > 0:
+ # TODO: Use a custom exception
+ msg = "{}.priority must be <= 0".format(checker.__class__)
+ raise ValueError(msg)
+
+ self._checkers[checker.name].append(checker)
+
+ # TODO: Move elsewhere
+ for r_id, r_title, r_cb in checker.reports:
+ self._linter.register_report(r_id, r_title, r_cb, checker)
+
+ self.register_options(checker.options)
+
+ # TODO: Move elsewhere
+ if hasattr(checker, "msgs"):
+ self._linter.msgs_store.register_messages(checker)
+
+ # Register the checker, but disable all of its messages.
+ # TODO(cpopa): we should have a better API for this.
+ if not getattr(checker, "enabled", True):
+ self._linter.disable(checker.name)
+
+ # For now simply defer missing attributs to the linter,
+ # until we know what API we want.
+ def __getattr__(self, attribute):
+ return getattr(self._linter, attribute)
+
+
+class Runner(object):
+ """A class to manager how the linter runs."""
+
+ option_definitions = ()
+ """The runner specific configuration options.
+
+ :type: set(OptionDefinition)
+ """
+
+
+class CLIRunner(Runner):
+ option_definitions = (
+ (
+ "rcfile",
+ {
+ "type": "string",
+ "metavar": "<file>",
+ "help": "Specify a configuration file.",
+ },
+ ),
+ (
+ "init-hook",
+ {
+ "type": "string",
+ "metavar": "<code>",
+ "level": 1,
+ "help": "Python code to execute, usually for sys.path "
+ "manipulation such as pygtk.require().",
+ },
+ ),
+ (
+ "help-msg",
+ {
+ "type": "string",
+ "metavar": "<msg-id>",
+ "group": "Commands",
+ "default": None,
+ "help": "Display a help message for the given message id and "
+ "exit. The value may be a comma separated list of message ids.",
+ },
+ ),
+ (
+ "list-msgs",
+ {
+ "metavar": "<msg-id>",
+ "group": "Commands",
+ "level": 1,
+ "default": None,
+ "help": "Generate pylint's messages.",
+ },
+ ),
+ (
+ "list-conf-levels",
+ {
+ "group": "Commands",
+ "level": 1,
+ "action": "store_true",
+ "default": False,
+ "help": "Generate pylint's confidence levels.",
+ },
+ ),
+ (
+ "full-documentation",
+ {
+ "metavar": "<msg-id>",
+ "default": None,
+ "group": "Commands",
+ "level": 1,
+ "help": "Generate pylint's full documentation.",
+ },
+ ),
+ (
+ "generate-rcfile",
+ {
+ "group": "Commands",
+ "action": "store_true",
+ "default": False,
+ "help": "Generate a sample configuration file according to "
+ "the current configuration. You can put other options "
+ "before this one to get them in the generated "
+ "configuration.",
+ },
+ ),
+ (
+ "generate-man",
+ {
+ "group": "Commands",
+ "action": "store_true",
+ "default": False,
+ "help": "Generate pylint's man page.",
+ "hide": True,
+ },
+ ),
+ (
+ "errors-only",
+ {
+ "short": "E",
+ "action": "store_true",
+ "default": False,
+ "help": "In error mode, checkers without error messages are "
+ "disabled and for others, only the ERROR messages are "
+ "displayed, and no reports are done by default"
+ "",
+ },
+ ),
+ (
+ "py3k",
+ {
+ "action": "store_true",
+ "default": False,
+ "help": "In Python 3 porting mode, all checkers will be "
+ "disabled and only messages emitted by the porting "
+ "checker will be displayed",
+ },
+ ),
+ )
- LinterClass = PyLinter
option_groups = (
(
"Commands",
@@ -1266,158 +1272,34 @@ group are mutually exclusive.",
),
)
- def __init__(self, args, reporter=None, do_exit=True):
- self._rcfile = None
- self._plugins = []
- self.verbose = None
- try:
- preprocess_options(
- args,
- {
- # option: (callback, takearg)
- "init-hook": (cb_init_hook, True),
- "rcfile": (self.cb_set_rcfile, True),
- "load-plugins": (self.cb_add_plugins, True),
- "verbose": (self.cb_verbose_mode, False),
- },
- )
- except ArgumentPreprocessingError as ex:
- print(ex, file=sys.stderr)
- sys.exit(32)
+ description = (
+ "pylint [options] module_or_package\n"
+ "\n"
+ " Check that a module satisfies a coding standard (and more !).\n"
+ "\n"
+ " pylint --help\n"
+ "\n"
+ " Display this help message and exit.\n"
+ "\n"
+ " pylint --help-msg <msg-id>[,<msg-id>]\n"
+ "\n"
+ " Display help messages about given message identifiers and exit.\n"
+ )
- self.linter = linter = self.LinterClass(
- (
- (
- "rcfile",
- {
- "action": "callback",
- "callback": lambda *args: 1,
- "type": "string",
- "metavar": "<file>",
- "help": "Specify a configuration file.",
- },
- ),
- (
- "init-hook",
- {
- "action": "callback",
- "callback": lambda *args: 1,
- "type": "string",
- "metavar": "<code>",
- "level": 1,
- "help": "Python code to execute, usually for sys.path "
- "manipulation such as pygtk.require().",
- },
- ),
- (
- "help-msg",
- {
- "action": "callback",
- "type": "string",
- "metavar": "<msg-id>",
- "callback": self.cb_help_message,
- "group": "Commands",
- "help": "Display a help message for the given message id and "
- "exit. The value may be a comma separated list of message ids.",
- },
- ),
- (
- "list-msgs",
- {
- "action": "callback",
- "metavar": "<msg-id>",
- "callback": self.cb_list_messages,
- "group": "Commands",
- "level": 1,
- "help": "Generate pylint's messages.",
- },
- ),
- (
- "list-conf-levels",
- {
- "action": "callback",
- "callback": cb_list_confidence_levels,
- "group": "Commands",
- "level": 1,
- "help": "Generate pylint's confidence levels.",
- },
- ),
- (
- "full-documentation",
- {
- "action": "callback",
- "metavar": "<msg-id>",
- "callback": self.cb_full_documentation,
- "group": "Commands",
- "level": 1,
- "help": "Generate pylint's full documentation.",
- },
- ),
- (
- "generate-rcfile",
- {
- "action": "callback",
- "callback": self.cb_generate_config,
- "group": "Commands",
- "help": "Generate a sample configuration file according to "
- "the current configuration. You can put other options "
- "before this one to get them in the generated "
- "configuration.",
- },
- ),
- (
- "generate-man",
- {
- "action": "callback",
- "callback": self.cb_generate_manpage,
- "group": "Commands",
- "help": "Generate pylint's man page.",
- "hide": True,
- },
- ),
- (
- "errors-only",
- {
- "action": "callback",
- "callback": self.cb_error_mode,
- "short": "E",
- "help": "In error mode, checkers without error messages are "
- "disabled and for others, only the ERROR messages are "
- "displayed, and no reports are done by default.",
- },
- ),
- (
- "py3k",
- {
- "action": "callback",
- "callback": self.cb_python3_porting_mode,
- "help": "In Python 3 porting mode, all checkers will be "
- "disabled and only messages emitted by the porting "
- "checker will be displayed.",
- },
- ),
- (
- "verbose",
- {
- "action": "callback",
- "callback": self.cb_verbose_mode,
- "short": "v",
- "help": "In verbose mode, extra non-checker-related info "
- "will be displayed.",
- },
- ),
- ),
- option_groups=self.option_groups,
- pylintrc=self._rcfile,
- )
- # register standard checkers
- linter.load_default_plugins()
- # load command line plugins
- linter.load_plugin_modules(self._plugins)
- # add some help section
- linter.add_help_section("Environment variables", config.ENV_HELP, level=1)
+ def __init__(self):
+ super(CLIRunner, self).__init__()
+ self._linter = PyLinter()
+ self._checker_registry = CheckerRegistry(self._linter)
+ self._loaded_plugins = set()
+
+ def run(self, args):
+ # Phase 1: Preprocessing
+ option_definitions = self.option_definitions + self._linter.options
+ parser = config.CLIParser(self.description)
+ parser.add_option_definitions(option_definitions)
+ parser.add_help_section("Environment variables", config.ENV_HELP, level=1)
# pylint: disable=bad-continuation
- linter.add_help_section(
+ parser.add_help_section(
"Output",
"Using the default text output, the message format is : \n"
" \n"
@@ -1432,7 +1314,7 @@ group are mutually exclusive.",
"processing.\n",
level=1,
)
- linter.add_help_section(
+ parser.add_help_section(
"Output status code",
"Pylint should leave with following status code: \n"
" * 0 if everything went fine \n"
@@ -1447,244 +1329,127 @@ group are mutually exclusive.",
"been issued by analysing pylint output status code\n",
level=1,
)
- # read configuration
- linter.disable("I")
- linter.enable("c-extension-no-member")
- linter.read_config_file(verbose=self.verbose)
- config_parser = linter.cfgfile_parser
- # run init hook, if present, before loading plugins
- if config_parser._parser.has_option("MASTER", "init-hook"):
- cb_init_hook(
- "init-hook",
- utils._unquote(config_parser._parser.get("MASTER", "init-hook")),
- )
- # is there some additional plugins in the file configuration, in
- if config_parser._parser.has_option("MASTER", "load-plugins"):
- plugins = utils._splitstrip(
- config_parser._parser.get("MASTER", "load-plugins")
- )
- linter.load_plugin_modules(plugins)
- # now we can load file config and command line, plugins (which can
- # provide options) have been registered
- linter.load_config_file()
-
- if reporter:
- # if a custom reporter is provided as argument, it may be overridden
- # by file parameters, so re-set it here, but before command line
- # parsing so it's still overrideable by command line option
- linter.set_reporter(reporter)
- try:
- args = linter.load_command_line_configuration(args)
- except SystemExit as exc:
- if exc.code == 2: # bad options
- exc.code = 32
- raise
- if not args:
- print(linter.help())
- sys.exit(32)
-
- if linter.config.jobs < 0:
- print(
- "Jobs number (%d) should be greater than or equal to 0"
- % linter.config.jobs,
- file=sys.stderr,
- )
- sys.exit(32)
- if linter.config.jobs > 1 or linter.config.jobs == 0:
- if multiprocessing is None:
- print(
- "Multiprocessing library is missing, " "fallback to single process",
- file=sys.stderr,
- )
- linter.set_option("jobs", 1)
- elif linter.config.jobs == 0:
- linter.config.jobs = _cpu_count()
-
- # We have loaded configuration from config file and command line. Now, we can
- # load plugin specific configuration.
- #linter.load_plugin_configuration()
-
- # insert current working directory to the python path to have a correct
- # behaviour
- with fix_import_path(args):
- if linter.config.jobs == 1:
- linter.check(args)
- else:
- self._parallel_run(args)
- linter.generate_reports()
- if do_exit:
- if linter.config.exit_zero:
- sys.exit(0)
- else:
- sys.exit(self.linter.msg_status)
+ global_config = config.Configuration()
+ global_config.add_options(option_definitions)
+ self._linter.config = global_config
- def _parallel_run(self, files_or_modules):
- with _patch_sysmodules():
- self.linter._init_msg_states()
- self._parallel_check(files_or_modules)
+ parsed = parser.preprocess(
+ args, "init_hook", "rcfile", "load_plugins", "ignore", "ignore_patterns"
+ )
- def _parallel_task(self, files_or_modules):
- # Prepare configuration for child linters.
- child_config = self._get_jobs_config()
+ # Call init-hook
+ if parsed.init_hook:
+ exec(parsed.init_hook)
- children = []
- manager = multiprocessing.Manager()
- tasks_queue = manager.Queue()
- results_queue = manager.Queue()
+ # Load rcfile, else system rcfile
+ file_parser = config.IniFileParser()
+ file_parser.add_option_definitions(self._linter.options)
+ rcfile = parsed.rcfile or config.PYLINTRC
+ if rcfile:
+ file_parser.parse(rcfile, global_config)
- for _ in range(self.linter.config.jobs):
- child_linter = ChildRunner(args=(tasks_queue, results_queue, child_config))
- child_linter.start()
- children.append(child_linter)
+ def register_options(options):
+ global_config.add_options(options)
+ parser.add_option_definitions(options)
+ file_parser.add_option_definitions(options)
- # Send files to child linters.
- expanded_files = utils.expand_files(
- files_or_modules,
- self.linter,
- self.linter.config.black_list,
- self.linter.config.black_list_re,
- )
- for module_desc in expanded_files:
- tasks_queue.put([module_desc.path])
+ self._checker_registry.register_options = register_options
- # collect results from child linters
- failed = False
- for _ in expanded_files:
- try:
- result = results_queue.get()
- except Exception as ex:
- print(
- "internal error while receiving results from child linter",
- file=sys.stderr,
- )
- print(ex, file=sys.stderr)
- failed = True
- break
- yield result
-
- # Stop child linters and wait for their completion.
- for _ in range(self.linter.config.jobs):
- tasks_queue.put("STOP")
- for child in children:
- child.join()
-
- if failed:
- print("Error occured, stopping the linter.", file=sys.stderr)
- sys.exit(32)
-
- def _parallel_check(self, files_or_modules):
- # Reset stats.
- self.linter.open()
-
- all_stats = []
- module = None
- for result in self._parallel_task(files_or_modules):
- if not result:
- continue
- (
- _,
- self.linter.file_state.base_name,
- module,
- messages,
- stats,
- msg_status,
- ) = result
-
- for msg in messages:
- msg = utils.Message(*msg)
- self.linter.set_current_module(module)
- self.linter.reporter.handle_message(msg)
-
- all_stats.append(stats)
- self.linter.msg_status |= msg_status
-
- self.linter.stats = _merge_stats(all_stats)
- self.linter.current_name = module
-
- # Insert stats data to local checkers.
- for checker in self.linter.get_checkers():
- if checker is not self.linter:
- checker.stats = self.linter.stats
-
- def _get_jobs_config(self):
- child_config = collections.OrderedDict()
- filter_options = {"long-help"}
- filter_options.update((opt_name for opt_name, _ in self.linter._external_opts))
- for opt_providers in six.itervalues(self.linter._all_options):
- for optname, optdict, val in opt_providers.options_and_values():
- if optdict.get("deprecated"):
- continue
-
- if optname not in filter_options:
- child_config[optname] = utils._format_option_value(optdict, val)
- child_config["python3_porting_mode"] = self.linter._python3_porting_mode
- child_config["plugins"] = self.linter._dynamic_plugins
- return child_config
-
- def cb_set_rcfile(self, name, value):
- """callback for option preprocessing (i.e. before option parsing)"""
- self._rcfile = value
-
- def cb_add_plugins(self, name, value):
- """callback for option preprocessing (i.e. before option parsing)"""
- self._plugins.extend(utils._splitstrip(value))
-
- def cb_error_mode(self, *args, **kwargs):
- """error mode:
- * disable all but error messages
- * disable the 'miscellaneous' checker which can be safely deactivated in
- debug
- * disable reports
- * do not save execution information
- """
- self.linter.error_mode()
+ checkers.initialize(self._checker_registry)
+
+ # Load plugins from CLI
+ plugins = parsed.load_plugins or []
+ for plugin in plugins:
+ self.load_plugin(plugin)
+
+ # TODO: This is for per directory config support (#618)
+ # Phase 2: Discover more plugins found in config files
+ # Walk and discover config files, watching for blacklists as we go
+
+ # Load plugins from config files
- def cb_generate_config(self, *args, **kwargs):
- """optik callback for sample config file generation"""
- self.linter.generate_config(skipsections=("COMMANDS",))
- sys.exit(0)
+ # Phase 3: Full load
+ # Fully load config files
+ if rcfile:
+ file_parser.parse(rcfile, global_config)
+ # Fully load CLI into global config
+ parser.parse(args, global_config)
- def cb_generate_manpage(self, *args, **kwargs):
- """optik callback for sample config file generation"""
- from pylint import __pkginfo__
+ if global_config.generate_rcfile:
+ file_parser.write()
+ sys.exit(0)
- self.linter.generate_manpage(__pkginfo__)
- sys.exit(0)
+ # TODO: if global_config.generate_man
- def cb_help_message(self, option, optname, value, parser):
- """optik callback for printing some help about a particular message"""
- self.linter.msgs_store.help_message(utils._splitstrip(value))
- sys.exit(0)
+ if global_config.errors_only:
+ self._linter.errors_mode()
- def cb_full_documentation(self, option, optname, value, parser):
- """optik callback for printing full documentation"""
- self.linter.print_full_documentation()
- sys.exit(0)
+ if global_config.py3k:
+ self._linter.python3_porting_mode()
- def cb_list_messages(self, option, optname, value, parser): # FIXME
- """optik callback for printing available messages"""
- self.linter.msgs_store.list_messages()
- sys.exit(0)
+ if global_config.full_documentation:
+ self._linter.print_full_documentation()
+ sys.exit(0)
- def cb_python3_porting_mode(self, *args, **kwargs):
- """Activate only the python3 porting checker."""
- self.linter.python3_porting_mode()
+ if global_config.list_conf_levels:
+ for level in interfaces.CONFIDENCE_LEVELS:
+ print("%-18s: %s" % level)
+ sys.exit(0)
- def cb_verbose_mode(self, *args, **kwargs):
- self.verbose = True
+ if global_config.list_msgs:
+ self._linter.msgs_store.list_messages()
+ sys.exit(0)
+ if global_config.help_msg:
+ msg = utils._splitstrip(global_config.help_msg)
+ self._linter.msgs_store.help_message(msg)
+ sys.exit(0)
-def cb_list_confidence_levels(option, optname, value, parser):
- for level in interfaces.CONFIDENCE_LEVELS:
- print("%-18s: %s" % level)
- sys.exit(0)
+ self.load_default_plugins()
+ self._linter.disable("I")
+ self._linter.enable("c-extension-no-member")
-def cb_init_hook(optname, value):
- """exec arbitrary code to set sys.path for instance"""
- exec(value) # pylint: disable=exec-used
+ for checker in self._checker_registry.for_all_checkers():
+ checker.config = global_config
+
+ with fix_import_path(global_config.module_or_package):
+ assert self._linter.config.jobs == 1
+ self._linter.check(global_config.module_or_package)
+
+ self._linter.generate_reports()
+
+ if linter.config.exit_zero:
+ sys.exit(0)
+ else:
+ sys.exit(self.linter.msg_status)
+
+ def load_plugin(self, module_name):
+ if module_name in self._loaded_plugins:
+ msg = "Already loaded plugin {0}. Ignoring".format(module_name)
+ warnings.warn(msg)
+ else:
+ module = astroid.modutils.load_module_from_name(module_name)
+ module.register(self._checker_registry)
+
+ def load_plugins(self, module_names):
+ """Load a plugin.
+
+ Args:
+ module_names (list(str)): The name of plugin modules to load.
+ """
+ for module_name in module_names:
+ self.load_plugin(module_name)
+
+ def load_default_plugins(self):
+ """Load all of the default plugins."""
+ reporters.initialize(self._linter)
+ # Make sure to load the default reporter, because
+ # the option has been set before the plugins had been loaded.
+ if not self._linter.reporter:
+ self._linter.load_reporter()
if __name__ == "__main__":
- Run(sys.argv[1:])
+ CLIRunner().run(sys.argv[1:])
diff --git a/pylint/utils.py b/pylint/utils.py
index 3f71cba67..513ed4666 100644
--- a/pylint/utils.py
+++ b/pylint/utils.py
@@ -1340,24 +1340,26 @@ class PyLintASTWalker:
PY_EXTS = (".py", ".pyc", ".pyo", ".pyw", ".so", ".dll")
-def register_plugins(linter, directory):
- """load all module and package in the given directory, looking for a
- 'register' function in each one, used to register pylint checkers
+def register_plugins(registry, directory):
+ """Load plugins from all modules and packages in the given directory.
+
+ Args:
+ registry (CheckerRegistry): The registry to register the checkers with.
+ directory (str): The directory to search for plugins.
"""
- imported = {}
+ imported = {"__init__", "__pycache__"}
for filename in os.listdir(directory):
- base, extension = splitext(filename)
- if base in imported or base == "__pycache__":
+ base, extension = os.path.splitext(filename)
+ if base in imported:
continue
- if (
- extension in PY_EXTS
- and base != "__init__"
- or (not extension and isdir(join(directory, base)))
- ):
+
+ package_dir = os.path.join(directory, base)
+ if extension in PY_EXTS or (not extension and os.path.isdir(package_dir)):
+ file_path = os.path.join(directory, filename)
try:
- module = modutils.load_module_from_file(join(directory, filename))
+ module = modutils.load_module_from_file(file_path)
except ValueError:
- # empty module name (usually emacs auto-save files)
+ # Empty module name
continue
except ImportError as exc:
print(
@@ -1365,8 +1367,8 @@ def register_plugins(linter, directory):
)
else:
if hasattr(module, "register"):
- module.register(linter)
- imported[base] = 1
+ module.register(registry)
+ imported.add(base)
def get_global_option(checker, option, default=None):