diff options
author | Ashley Whetter <ashley@awhetter.co.uk> | 2018-02-03 15:27:53 -0800 |
---|---|---|
committer | Ashley Whetter <ashley@awhetter.co.uk> | 2019-02-09 13:25:18 -0800 |
commit | 80104b1d8b257b3f711c77d7214d1fccb8bc5329 (patch) | |
tree | 607e903acc2769e3239b7471e8c34e659e87af9f | |
parent | 5c7c975875bc1d0ac656a190ce3abd0548eb9fc5 (diff) | |
download | pylint-git-80104b1d8b257b3f711c77d7214d1fccb8bc5329.tar.gz |
Removed command line parsing from Linter
-rw-r--r-- | pylint/__init__.py | 4 | ||||
-rw-r--r-- | pylint/checkers/__init__.py | 22 | ||||
-rw-r--r-- | pylint/config.py | 627 | ||||
-rw-r--r-- | pylint/lint.py | 1371 | ||||
-rw-r--r-- | pylint/utils.py | 32 |
5 files changed, 699 insertions, 1357 deletions
diff --git a/pylint/__init__.py b/pylint/__init__.py index a5d43d123..11924c366 100644 --- a/pylint/__init__.py +++ b/pylint/__init__.py @@ -14,10 +14,10 @@ from .__pkginfo__ import version as __version__ def run_pylint(): """run pylint""" - from pylint.lint import Run + from pylint.lint import CLIRunner try: - Run(sys.argv[1:]) + CLIRunner().run(sys.argv[1:]) except KeyboardInterrupt: sys.exit(1) diff --git a/pylint/checkers/__init__.py b/pylint/checkers/__init__.py index 1a4c0009f..ed649707d 100644 --- a/pylint/checkers/__init__.py +++ b/pylint/checkers/__init__.py @@ -44,7 +44,6 @@ import tokenize import warnings from typing import Any -from pylint.config import OptionsProviderMixIn from pylint.reporters import diff_string from pylint.utils import register_plugins from pylint.interfaces import UNDEFINED @@ -71,7 +70,7 @@ def table_lines_from_stats(stats, old_stats, columns): return lines -class BaseChecker(OptionsProviderMixIn): +class BaseChecker(object): """base class for checkers""" # checker name (you may reuse an existing one) @@ -87,14 +86,9 @@ class BaseChecker(OptionsProviderMixIn): # mark this checker as enabled or not. enabled = True - def __init__(self, linter=None): - """checker instances should have the linter as argument + priority = -1 - linter is an object implementing ILinter - """ - if self.name is not None: - self.name = self.name.lower() - OptionsProviderMixIn.__init__(self) + def __init__(self, linter=None): self.linter = linter def add_message( @@ -126,9 +120,13 @@ class BaseTokenChecker(BaseChecker): raise NotImplementedError() -def initialize(linter): - """initialize linter with checkers in this package """ - register_plugins(linter, __path__[0]) +def initialize(registry): + """Register the checkers in this package. + + :param registry: The registry to register checkers with. + :type registry: CheckerRegistry + """ + register_plugins(registry, __path__[0]) __all__ = ("BaseChecker", "BaseTokenChecker", "initialize") diff --git a/pylint/config.py b/pylint/config.py index c20c1474d..d70e1123b 100644 --- a/pylint/config.py +++ b/pylint/config.py @@ -37,16 +37,11 @@ from __future__ import print_function import abc import argparse -import contextlib import collections -import copy -import functools -import io import os import pickle import re import sys -import time import configparser @@ -132,10 +127,10 @@ def find_nearby_pylintrc(search_dir=""): path = find_pylintrc_in(search_dir) if not path: - for search_dir in utils.walk_up(search_dir): - if not os.path.isfile(os.path.join(search_dir, "__init__.py")): + for cur_dir in utils.walk_up(search_dir): + if not os.path.isfile(os.path.join(cur_dir, "__init__.py")): break - path = find_pylintrc_in(search_dir) + path = find_pylintrc_in(cur_dir) if path: break @@ -205,54 +200,23 @@ class UnsupportedAction(Exception): """raised by set_option when it doesn't know what to do for an action""" -def _multiple_choice_validator(choices, name, value): - values = utils._check_csv(value) - for csv_value in values: - if csv_value not in choices: - msg = "option %s: invalid value: %r, should be in %s" - raise argparse.ArgumentError(msg % (name, csv_value, choices)) - return values - - -def _choice_validator(choices, name, value): - if value not in choices: - msg = "option %s: invalid value: %r, should be in %s" - raise argparse.ArgumentError(msg % (name, value, choices)) - return value - - -# pylint: disable=unused-argument -def _csv_validator(_, name, value): - return utils._check_csv(value) - +def _regexp_csv_validator(value): + return [re.compile(val) for val in utils._check_csv(value)] -# pylint: disable=unused-argument -def _regexp_validator(_, name, value): - if hasattr(value, "pattern"): - return value - return re.compile(value) - -# pylint: disable=unused-argument -def _regexp_csv_validator(_, name, value): - return [_regexp_validator(_, name, val) for val in _csv_validator(_, name, value)] - - -def _yn_validator(opt, _, value): - if isinstance(value, int): - return bool(value) +def _yn_validator(value): if value in ("y", "yes"): return True if value in ("n", "no"): return False - msg = "option %s: invalid yn value %r, should be in (y, yes, n, no)" - raise argparse.ArgumentError(msg % (opt, value)) + msg = "invalid yn value %r, should be in (y, yes, n, no)" + raise argparse.ArgumentTypeError(msg % (value,)) -def _non_empty_string_validator(opt, _, value): +def _non_empty_string_validator(value): if not value: msg = "indent string can't be empty." - raise argparse.ArgumentError(msg) + raise argparse.ArgumentTypeError(msg) return utils._unquote(value) @@ -261,462 +225,21 @@ VALIDATORS = { "int": int, "regexp": re.compile, "regexp_csv": _regexp_csv_validator, - "csv": _csv_validator, + "csv": utils._check_csv, "yn": _yn_validator, - "choice": lambda opt, name, value: _choice_validator(opt["choices"], name, value), - "multiple_choice": lambda opt, name, value: _multiple_choice_validator( - opt["choices"], name, value - ), "non_empty_string": _non_empty_string_validator, } -def _call_validator(opttype, optdict, option, value): - if opttype not in VALIDATORS: - raise Exception('Unsupported type "%s"' % opttype) - try: - return VALIDATORS[opttype](optdict, option, value) - except TypeError: - try: - return VALIDATORS[opttype](value) - except Exception: - raise argparse.ArgumentError( - "%s value (%r) should be of type %s" % (option, value, opttype) - ) - - -def _validate(value, optdict, name=""): - """return a validated value for an option according to its type - - optional argument name is only used for error message formatting - """ - try: - _type = optdict["type"] - except KeyError: - # FIXME - return value - return _call_validator(_type, optdict, name, value) - - -def _level_options(group, outputlevel): - return [ - option - for option in group.option_list - if (getattr(option, "level", 0) or 0) <= outputlevel - and option.help is not argparse.SUPPRESS - ] - - -def _multiple_choices_validating_option(opt, name, value): - return _multiple_choice_validator(opt.choices, name, value) - - -class OptionsManagerMixIn: - """Handle configuration from both a configuration file and command line options""" - - class CallbackAction(argparse.Action): - """Doesn't store the value on the config.""" - - def __init__(self, nargs=None, **kwargs): - nargs = nargs or int("metavar" in kwargs) - super(OptionsManagerMixIn.CallbackAction, self).__init__( - nargs=nargs, **kwargs - ) - - def __call__(self, parser, namespace, values, option_string): - # If no value was passed, argparse didn't call the callback via - # `type`, so we need to do it ourselves. - if not self.nargs and callable(self.type): - self.type(self, option_string, values, parser) - - def __init__(self, usage, config_file=None, version=None): - self.config_file = config_file - self.reset_parsers(usage, version=version) - # list of registered options providers - self.options_providers = [] - # dictionary associating option name to checker - self._all_options = collections.OrderedDict() - self._short_options = {} - self._nocallback_options = {} - self._mygroups = {} - # verbosity - self._maxlevel = 0 - - def reset_parsers(self, usage="", version=None): - # configuration file parser - self.cfgfile_parser = IniFileParser() - # command line parser - self.cmdline_parser = CLIParser(usage) - - def register_options_provider(self, provider, own_group=True): - """register an options provider""" - assert provider.priority <= 0, "provider's priority can't be >= 0" - for i in range(len(self.options_providers)): - if provider.priority > self.options_providers[i].priority: - self.options_providers.insert(i, provider) - break - else: - self.options_providers.append(provider) - non_group_spec_options = [ - option for option in provider.options if "group" not in option[1] - ] - groups = getattr(provider, "option_groups", ()) - if own_group and non_group_spec_options: - self.add_option_group( - provider.name.upper(), - provider.__doc__, - non_group_spec_options, - provider, - ) - else: - for opt, optdict in non_group_spec_options: - self.add_optik_option(provider, self.cmdline_parser, opt, optdict) - for gname, gdoc in groups: - gname = gname.upper() - goptions = [ - option - for option in provider.options - if option[1].get("group", "").upper() == gname - ] - self.add_option_group(gname, gdoc, goptions, provider) - - def add_option_group(self, group_name, _, options, provider): - # add option group to the command line parser - if group_name in self._mygroups: - group = self._mygroups[group_name] - else: - group = self.cmdline_parser._parser.add_argument_group( - group_name.capitalize(), level=provider.level - ) - self._mygroups[group_name] = group - # add section to the config file - if group_name != "DEFAULT": - try: - self.cfgfile_parser._parser.add_section(group_name) - except configparser.DuplicateSectionError: - pass - - # add provider's specific options - for opt, optdict in options: - self.add_optik_option(provider, group, opt, optdict) - - def add_optik_option(self, provider, optikcontainer, opt, optdict): - args, optdict = self.optik_option(provider, opt, optdict) - if hasattr(optikcontainer, "_parser"): - optikcontainer = optikcontainer._parser - if "group" in optdict: - optikcontainer = self._mygroups[optdict["group"].upper()] - del optdict["group"] - - # Some sanity checks for things that trip up argparse - assert not any(" " in arg for arg in args) - assert all(optdict.values()) - assert not ("metavar" in optdict and "[" in optdict["metavar"]) - - level = optdict.pop("level", 0) - option = optikcontainer.add_argument(*args, **optdict) - option.level = level - self._all_options[opt] = provider - self._maxlevel = max(self._maxlevel, optdict.get("level", 0)) - - def optik_option(self, provider, opt, optdict): - """get our personal option definition and return a suitable form for - use with optik/argparse - """ - # TODO: Changed to work with argparse but this should call - # self.cmdline_parser.add_argument_definitions and not use callbacks - optdict = copy.copy(optdict) - if "action" in optdict: - self._nocallback_options[provider] = opt - if optdict["action"] == "callback": - optdict["type"] = optdict["callback"] - optdict["action"] = self.CallbackAction - del optdict["callback"] - else: - callback = functools.partial( - self.cb_set_provider_option, None, "--" + str(opt), parser=None - ) - optdict["type"] = callback - optdict.setdefault("action", "store") - # default is handled here and *must not* be given to optik if you - # want the whole machinery to work - if "default" in optdict: - if ( - "help" in optdict - and optdict.get("default") is not None - and optdict["action"] not in ("store_true", "store_false") - ): - default = optdict["default"] - if isinstance(default, (tuple, list)): - default = ",".join(str(x) for x in default) - optdict["help"] += " [current: {0}]".format(default) - del optdict["default"] - args = ["--" + str(opt)] - if "short" in optdict: - self._short_options[optdict["short"]] = opt - args.append("-" + optdict["short"]) - del optdict["short"] - if optdict.get("action") == "callback": - optdict["type"] = optdict["callback"] - del optdict["action"] - del optdict["callback"] - if optdict.get("hide"): - optdict["help"] = argparse.SUPPRESS - del optdict["hide"] - return args, optdict - - def cb_set_provider_option(self, option, opt, value, parser): - """optik callback for option setting""" - if opt.startswith("--"): - # remove -- on long option - opt = opt[2:] - else: - # short option, get its long equivalent - opt = self._short_options[opt[1:]] - # trick since we can't set action='store_true' on options - if value is None: - value = 1 - self.global_set_option(opt, value) - return value - - def global_set_option(self, opt, value): - """set option on the correct option provider""" - self._all_options[opt].set_option(opt, value) - - def generate_config(self, stream=None, skipsections=(), encoding=None): - """write a configuration file according to the current configuration - into the given stream or stdout - """ - options_by_section = {} - sections = [] - for provider in self.options_providers: - for section, options in provider.options_by_section(): - if section is None: - section = provider.name - if section in skipsections: - continue - options = [ - (n, d, v) - for (n, d, v) in options - if d.get("type") is not None and not d.get("deprecated") - ] - if not options: - continue - if section not in sections: - sections.append(section) - alloptions = options_by_section.setdefault(section, []) - alloptions += options - stream = stream or sys.stdout - printed = False - for section in sections: - if printed: - print("\n", file=stream) - utils.format_section( - stream, section.upper(), sorted(options_by_section[section]) - ) - printed = True - - def generate_manpage(self, pkginfo, section=1, stream=None): - # TODO - raise NotImplementedError - - def load_provider_defaults(self): - """initialize configuration using default values""" - for provider in self.options_providers: - provider.load_defaults() - - def read_config_file(self, config_file=None, verbose=None): - """read the configuration file but do not load it (i.e. dispatching - values to each options provider) - """ - if config_file is None: - config_file = self.config_file - if config_file is not None: - config_file = os.path.expanduser(config_file) - if not os.path.exists(config_file): - raise IOError("The config file {:s} doesn't exist!".format(config_file)) - - use_config_file = config_file and os.path.exists(config_file) - if use_config_file: - self.cfgfile_parser.parse(config_file, Configuration()) - - if not verbose: - return - - if use_config_file: - msg = "Using config file {}".format(os.path.abspath(config_file)) - else: - msg = "No config file found, using default configuration" - print(msg, file=sys.stderr) - - def load_config_file(self): - """dispatch values previously read from a configuration file to each - options provider) - """ - for section in self.cfgfile_parser._parser.sections(): - for option, value in self.cfgfile_parser._parser.items(section): - try: - self.global_set_option(option, value) - except (KeyError, argparse.ArgumentError): - # TODO handle here undeclared options appearing in the config file - continue - - def load_configuration(self, **kwargs): - """override configuration according to given parameters""" - return self.load_configuration_from_config(kwargs) - - def load_configuration_from_config(self, config): - for opt, opt_value in config.items(): - opt = opt.replace("_", "-") - provider = self._all_options[opt] - provider.set_option(opt, opt_value) - - def load_command_line_configuration(self, args=None): - """Override configuration according to command line parameters - - return additional arguments - """ - if args is None: - args = sys.argv[1:] - else: - args = list(args) - for provider in self._nocallback_options: - self.cmdline_parser.parse(args, provider.config) - config = Configuration() - self.cmdline_parser.parse(args, config) - return config.module_or_package - - def add_help_section(self, title, description, level=0): - """add a dummy option section for help purpose """ - group = self.cmdline_parser._parser.add_argument_group( - title.capitalize(), description, level=level - ) - self._maxlevel = max(self._maxlevel, level) - - def help(self, level=0): - """return the usage string for available options """ - return self.cmdline_parser._parser.format_help(level) - - -class OptionsProviderMixIn: - """Mixin to provide options to an OptionsManager""" - - # those attributes should be overridden - priority = -1 - name = "default" - options = () - level = 0 - - def __init__(self): - self.config = Configuration() - self.load_defaults() - - def load_defaults(self): - """initialize the provider using default values""" - for opt, optdict in self.options: - action = optdict.get("action") - if action != "callback": - # callback action have no default - if optdict is None: - optdict = self.get_option_def(opt) - default = optdict.get("default") - self.set_option(opt, default, action, optdict) - - def option_attrname(self, opt, optdict=None): - """get the config attribute corresponding to opt""" - if optdict is None: - optdict = self.get_option_def(opt) - return optdict.get("dest", opt.replace("-", "_")) - - def option_value(self, opt): - """get the current value for the given option""" - return getattr(self.config, self.option_attrname(opt), None) - - def set_option(self, optname, value, action=None, optdict=None): - """method called to set an option (registered in the options list)""" - if optdict is None: - optdict = self.get_option_def(optname) - if value is not None: - value = _validate(value, optdict, optname) - if action is None: - action = optdict.get("action", "store") - if action == "store": - setattr(self.config, self.option_attrname(optname, optdict), value) - elif action in ("store_true", "count"): - setattr(self.config, self.option_attrname(optname, optdict), 0) - elif action == "store_false": - setattr(self.config, self.option_attrname(optname, optdict), 1) - elif action == "append": - optname = self.option_attrname(optname, optdict) - _list = getattr(self.config, optname, None) - if _list is None: - if isinstance(value, (list, tuple)): - _list = value - elif value is not None: - _list = [] - _list.append(value) - setattr(self.config, optname, _list) - elif isinstance(_list, tuple): - setattr(self.config, optname, _list + (value,)) - else: - _list.append(value) - elif action == "callback": - optdict["callback"](None, optname, value, None) - else: - raise UnsupportedAction(action) - - def get_option_def(self, opt): - """return the dictionary defining an option given its name""" - assert self.options - for option in self.options: - if option[0] == opt: - return option[1] - raise argparse.ArgumentError( - "no such option %s in section %r" % (opt, self.name), opt - ) - - def options_by_section(self): - """return an iterator on options grouped by section - - (section, [list of (optname, optdict, optvalue)]) - """ - sections = {} - for optname, optdict in self.options: - sections.setdefault(optdict.get("group"), []).append( - (optname, optdict, self.option_value(optname)) - ) - if None in sections: - yield None, sections.pop(None) - for section, options in sorted(sections.items()): - yield section.upper(), options - - def options_and_values(self, options=None): - if options is None: - options = self.options - for optname, optdict in options: - yield (optname, optdict, self.option_value(optname)) - - -class ConfigurationMixIn(OptionsManagerMixIn, OptionsProviderMixIn): - """basic mixin for simple configurations which don't need the - manager / providers model - """ - - def __init__(self, *args, **kwargs): - if not args: - kwargs.setdefault("usage", "") - OptionsManagerMixIn.__init__(self, *args, **kwargs) - OptionsProviderMixIn.__init__(self) - if not getattr(self, "option_groups", None): - self.option_groups = [] - for _, optdict in self.options: - try: - gdef = (optdict["group"].upper(), "") - except KeyError: - continue - if gdef not in self.option_groups: - self.option_groups.append(gdef) - self.register_options_provider(self, own_group=False) +UNVALIDATORS = { + "string": str, + "int": str, + "regexp": lambda value: getattr(value, "pattern", value), + "regexp_csv": (lambda value: ",".join(r.pattern for r in value)), + "csv": (lambda value: ",".join(value)), + "yn": (lambda value: "y" if value else "n"), + "non_empty_string": str, +} OptionDefinition = collections.namedtuple("OptionDefinition", ["name", "definition"]) @@ -730,6 +253,7 @@ class Configuration(object): name, definition = option_definition if name in self._option_definitions: raise exceptions.ConfigurationError('Option "{0}" already exists.') + self._option_definitions[name] = definition def add_options(self, option_definitions): @@ -737,6 +261,7 @@ class Configuration(object): self.add_option(option_definition) def set_option(self, option, value): + option = option.replace("-", "_") setattr(self, option, value) def copy(self): @@ -758,8 +283,9 @@ class Configuration(object): self._option_definitions.update(other._option_definitions) for option in other._option_definitions: + option = option.replace("-", "_") value = getattr(other, option) - setattr(result, option, value) + setattr(self, option, value) return self @@ -830,7 +356,7 @@ class ConfigurationStore(object): config = self.global_config.copy() parent_configs = self._get_parent_configs(path) - for parent_config in reversed(parent_configs): + for parent_config in reversed(list(parent_configs)): config += parent_config self._cache["path"] = config @@ -854,7 +380,7 @@ class ConfigParser(metaclass=abc.ABCMeta): for _, definition_dict in option_definitions: try: - group = optdict["group"].upper() + group = definition_dict["group"].upper() except KeyError: continue else: @@ -887,11 +413,12 @@ class CLIParser(ConfigParser): self._parser.add_argument("module_or_package", nargs=argparse.REMAINDER) def add_option_definitions(self, option_definitions): + self._option_definitions.update(option_definitions) option_groups = collections.defaultdict(list) for option, definition in option_definitions: group, args, kwargs = self._convert_definition(option, definition) - option_groups[group].append(args, kwargs) + option_groups[group].append((args, kwargs)) for args, kwargs in option_groups["DEFAULT"]: self._parser.add_argument(*args, **kwargs) @@ -900,9 +427,9 @@ class CLIParser(ConfigParser): for group, arguments in option_groups.items(): self._option_groups.add(group) - self._parser.add_argument_group(group.title()) + group = self._parser.add_argument_group(group.title()) for args, kwargs in arguments: - self._parser.add_argument(*args, **kwargs) + group.add_argument(*args, **kwargs) @staticmethod def _convert_definition(option, definition): @@ -936,7 +463,7 @@ class CLIParser(ConfigParser): if "choices" not in definition: msg = 'No choice list given for option "{0}" of type "choice".' msg = msg.format(option) - raise ConfigurationError(msg) + raise exceptions.ConfigurationError(msg) if definition["type"] == "multiple_choice": kwargs["type"] = VALIDATORS["csv"] @@ -944,7 +471,7 @@ class CLIParser(ConfigParser): kwargs["choices"] = definition["choices"] else: msg = 'Unsupported type "{0}"'.format(definition["type"]) - raise ConfigurationError(msg) + raise exception.ConfigurationError(msg) if definition.get("hide"): kwargs["help"] = argparse.SUPPRESS @@ -952,15 +479,15 @@ class CLIParser(ConfigParser): group = definition.get("group", "DEFAULT").upper() return group, args, kwargs - def parse(self, argv, config): + def parse(self, to_parse, config): """Parse the command line arguments into the given config object. - :param argv: The command line arguments to parse. - :type argv: list(str) + :param to_parse: The command line arguments to parse. + :type to_parse: list(str) :param config: The config object to parse the command line into. :type config: Configuration """ - self._parser.parse_args(argv, config) + self._parser.parse_args(to_parse, config) def preprocess(self, argv, *options): """Do some guess work to get a value for the specified option. @@ -973,19 +500,34 @@ class CLIParser(ConfigParser): :returns: A config with the processed options. :rtype: Configuration """ - config = Config() - config.add_options(self._option_definitions) + config = Configuration() + config.add_options(self._option_definitions.items()) args = self._parser.parse_known_args(argv)[0] for option in options: + option = option.replace("-", "_") config.set_option(option, getattr(args, option, None)) return config + def add_help_section(self, title, description, level=0): + """Add an extra help section to the help message. + + :param title: The title of the section. + This is included as part of the help message. + :type title: str + :param description: The description of the help section. + :type description: str + :param level: The minimum level of help needed to include this + in the help message. + :type level: int + """ + self._parser.add_argument_group(title, description, level=level) + class FileParser(ConfigParser, metaclass=abc.ABCMeta): @abc.abstractmethod - def parse(self, file_path, config): + def parse(self, to_parse, config): pass @@ -997,17 +539,20 @@ class IniFileParser(FileParser): self._parser = configparser.ConfigParser(inline_comment_prefixes=("#", ";")) def add_option_definitions(self, option_definitions): + self._option_definitions.update(option_definitions) for option, definition in option_definitions: group, default = self._convert_definition(option, definition) - try: - self._parser.add_section(group) - except configparser.DuplicateSectionError: - pass - else: - self._option_groups.add(group) + if group != self._parser.default_section: + try: + self._parser.add_section(group) + except configparser.DuplicateSectionError: + pass + else: + self._option_groups.add(group) - self._parser["DEFAULT"].update(default) + if default is not None: + self._parser["DEFAULT"].update(default) @staticmethod def _convert_definition(option, definition): @@ -1021,13 +566,17 @@ class IniFileParser(FileParser): :returns: The converted definition. :rtype: tuple(str, dict) """ - default = {option: definition.get("default")} + default = None + if definition.get("default"): + unvalidator = UNVALIDATORS.get(definition.get("type"), str) + default_value = unvalidator(definition["default"]) + default = {option: default_value} group = definition.get("group", "DEFAULT").upper() return group, default - def parse(self, file_path, config): - self._parser.read(file_path) + def parse(self, to_parse, config): + self._parser.read(to_parse) for section in self._parser.sections(): # Normalise the section titles @@ -1039,8 +588,17 @@ class IniFileParser(FileParser): section = section.upper() for option, value in self._parser.items(section): + if isinstance(value, str): + definition = self._option_definitions.get(option, {}) + type_ = definition.get("type") + validator = VALIDATORS.get(type_, lambda x: x) + value = validator(value) config.set_option(option, value) + def write(self, stream=sys.stdout): + # TODO: Check if option descriptions are written out + self._parser.write(stream) + class LongHelpFormatter(argparse.HelpFormatter): output_level = None @@ -1092,6 +650,25 @@ class LongHelpAction(argparse.Action): parser.exit() +class LongHelpArgumentGroup(argparse._ArgumentGroup): + def __init__(self, *args, level=0, **kwargs): + super(LongHelpArgumentGroup, self).__init__(*args, **kwargs) + self.level = level + + def add_argument(self, *args, **kwargs): + """See :func:`argparse.ArgumentParser.add_argument`. + + Patches in the level to each created action instance. + + :returns: The created action. + :rtype: argparse.Action + """ + level = kwargs.pop("level", 0) + action = super(LongHelpArgumentGroup, self).add_argument(*args, **kwargs) + action.level = level + return action + + class LongHelpArgumentParser(argparse.ArgumentParser): def __init__(self, formatter_class=LongHelpFormatter, **kwargs): self._max_level = 0 @@ -1140,9 +717,9 @@ class LongHelpArgumentParser(argparse.ArgumentParser): action.level = level return action - def add_argument_group(self, *args, level=0, **kwargs): - group = super(LongHelpArgumentParser, self).add_argument_group(*args, **kwargs) - group.level = level + def add_argument_group(self, *args, **kwargs): + group = LongHelpArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) return group # These methods use yucky way of passing the level to the formatter class diff --git a/pylint/lint.py b/pylint/lint.py index 4721e911f..34b935064 100644 --- a/pylint/lint.py +++ b/pylint/lint.py @@ -64,11 +64,6 @@ import collections import contextlib import operator import os - -try: - import multiprocessing -except ImportError: - multiprocessing = None # type: ignore import sys import tokenize import warnings @@ -215,85 +210,9 @@ MSGS = { } -def _cpu_count() -> int: - """Use sched_affinity if available for virtualized or containerized environments.""" - sched_getaffinity = getattr(os, "sched_getaffinity", None) - # pylint: disable=not-callable,using-constant-test - if sched_getaffinity: - return len(sched_getaffinity(0)) - if multiprocessing: - return multiprocessing.cpu_count() - return 1 - - -if multiprocessing is not None: - - class ChildRunner(multiprocessing.Process): - def run(self): - # pylint: disable=no-member, unbalanced-tuple-unpacking - tasks_queue, results_queue, self._config = self._args - - self._config["jobs"] = 1 # Child does not parallelize any further. - self._python3_porting_mode = self._config.pop("python3_porting_mode", None) - self._plugins = self._config.pop("plugins", None) - - # Run linter for received files/modules. - for file_or_module in iter(tasks_queue.get, "STOP"): - try: - result = self._run_linter(file_or_module[0]) - results_queue.put(result) - except Exception as ex: - print( - "internal error with sending report for module %s" - % file_or_module, - file=sys.stderr, - ) - print(ex, file=sys.stderr) - results_queue.put({}) - - def _run_linter(self, file_or_module): - linter = PyLinter() - - # Register standard checkers. - linter.load_default_plugins() - # Load command line plugins. - if self._plugins: - linter.load_plugin_modules(self._plugins) - - linter.load_configuration_from_config(self._config) - - # Load plugin specific configuration - linter.load_plugin_configuration() - - linter.set_reporter(reporters.CollectingReporter()) - - # Enable the Python 3 checker mode. This option is - # passed down from the parent linter up to here, since - # the Python 3 porting flag belongs to the Run class, - # instead of the Linter class. - if self._python3_porting_mode: - linter.python3_porting_mode() - - # Run the checks. - linter.check(file_or_module) - - msgs = [_get_new_args(m) for m in linter.reporter.messages] - return ( - file_or_module, - linter.file_state.base_name, - linter.current_name, - msgs, - linter.stats, - linter.msg_status, - ) - - # pylint: disable=too-many-instance-attributes class PyLinter( - config.OptionsManagerMixIn, - utils.MessagesHandlerMixIn, - utils.ReportsHandlerMixIn, - checkers.BaseTokenChecker, + utils.MessagesHandlerMixIn, utils.ReportsHandlerMixIn, checkers.BaseTokenChecker ): """lint Python modules using external checkers. @@ -315,261 +234,246 @@ class PyLinter( level = 0 msgs = MSGS - @staticmethod - def make_options(): - return ( - ( - "ignore", - { - "type": "csv", - "metavar": "<file>,...", - "dest": "black_list", - "default": ("CVS",), - "help": "Add files or directories to the blacklist. " - "They should be base names, not paths.", - }, - ), - ( - "ignore-patterns", - { - "type": "regexp_csv", - "metavar": "<pattern>,...", - "dest": "black_list_re", - "default": (), - "help": "Add files or directories matching the regex patterns to the" - " blacklist. The regex matches against base names, not paths.", - }, - ), - ( - "persistent", - { - "default": True, - "type": "yn", - "metavar": "<y_or_n>", - "level": 1, - "help": "Pickle collected data for later comparisons.", - }, - ), - ( - "load-plugins", - { - "type": "csv", - "metavar": "<modules>", - "default": (), - "level": 1, - "help": "List of plugins (as comma separated values of " - "python modules names) to load, usually to register " - "additional checkers.", - }, - ), - ( - "output-format", - { - "default": "text", - "type": "string", - "metavar": "<format>", - "short": "f", - "group": "Reports", - "help": "Set the output format. Available formats are text," - " parseable, colorized, json and msvs (visual studio)." - " You can also give a reporter class, e.g. mypackage.mymodule." - "MyReporterClass.", - }, - ), - ( - "reports", - { - "default": False, - "type": "yn", - "metavar": "<y_or_n>", - "short": "r", - "group": "Reports", - "help": "Tells whether to display a full report or only the " - "messages.", - }, - ), - ( - "evaluation", - { - "type": "string", - "metavar": "<python_expression>", - "group": "Reports", - "level": 1, - "default": "10.0 - ((float(5 * error + warning + refactor + " - "convention) / statement) * 10)", - "help": "Python expression which should return a note less " - "than 10 (10 is the highest note). You have access " - "to the variables errors warning, statement which " - "respectively contain the number of errors / " - "warnings messages and the total number of " - "statements analyzed. This is used by the global " - "evaluation report (RP0004).", - }, - ), - ( - "score", - { - "default": True, - "type": "yn", - "metavar": "<y_or_n>", - "short": "s", - "group": "Reports", - "help": "Activate the evaluation score.", - }, - ), - ( - "confidence", - { - "type": "multiple_choice", - "metavar": "<levels>", - "default": "", - "choices": [c.name for c in interfaces.CONFIDENCE_LEVELS], - "group": "Messages control", - "help": "Only show warnings with the listed confidence levels." - " Leave empty to show all. Valid levels: %s." - % (", ".join(c.name for c in interfaces.CONFIDENCE_LEVELS),), - }, - ), - ( - "enable", - { - "type": "csv", - "metavar": "<msg ids>", - "short": "e", - "group": "Messages control", - "help": "Enable the message, report, category or checker with the " - "given id(s). You can either give multiple identifier " - "separated by comma (,) or put this option multiple time " - "(only on the command line, not in the configuration file " - "where it should appear only once). " - 'See also the "--disable" option for examples.', - }, - ), - ( - "disable", - { - "type": "csv", - "metavar": "<msg ids>", - "short": "d", - "group": "Messages control", - "help": "Disable the message, report, category or checker " - "with the given id(s). You can either give multiple identifiers " - "separated by comma (,) or put this option multiple times " - "(only on the command line, not in the configuration file " - "where it should appear only once). " - 'You can also use "--disable=all" to disable everything first ' - "and then reenable specific checks. For example, if you want " - "to run only the similarities checker, you can use " - '"--disable=all --enable=similarities". ' - "If you want to run only the classes checker, but have no " - "Warning level messages displayed, use " - '"--disable=all --enable=classes --disable=W".', - }, - ), - ( - "msg-template", - { - "type": "string", - "metavar": "<template>", - "group": "Reports", - "help": ( - "Template used to display messages. " - "This is a python new-style format string " - "used to format the message information. " - "See doc for all details." - ), - }, - ), - ( - "jobs", - { - "type": "int", - "metavar": "<n-processes>", - "short": "j", - "default": 1, - "help": "Use multiple processes to speed up Pylint. Specifying 0 will " - "auto-detect the number of processors available to use.", - }, - ), - ( - "unsafe-load-any-extension", - { - "type": "yn", - "metavar": "<yn>", - "default": False, - "hide": True, - "help": ( - "Allow loading of arbitrary C extensions. Extensions" - " are imported into the active Python interpreter and" - " may run arbitrary code." - ), - }, - ), - ( - "limit-inference-results", - { - "type": "int", - "metavar": "<number-of-results>", - "default": 100, - "help": ( - "Control the amount of potential inferred values when inferring " - "a single object. This can help the performance when dealing with " - "large functions or complex, nested conditions. " - ), - }, - ), - ( - "extension-pkg-whitelist", - { - "type": "csv", - "metavar": "<pkg>,...", - "default": [], - "help": ( - "A comma-separated list of package or module names" - " from where C extensions may be loaded. Extensions are" - " loading into the active Python interpreter and may run" - " arbitrary code." - ), - }, - ), - ( - "suggestion-mode", - { - "type": "yn", - "metavar": "<yn>", - "default": True, - "help": ( - "When enabled, pylint would attempt to guess common " - "misconfiguration and emit user-friendly hints instead " - "of false-positive error messages." - ), - }, - ), - ( - "exit-zero", - { - "action": "store_true", - "help": ( - "Always return a 0 (non-error) status code, even if " - "lint errors are found. This is primarily useful in " - "continuous integration scripts." - ), - }, - ), - ) + options = ( + ( + "ignore", + { + "type": "csv", + "metavar": "<file>,...", + "dest": "black_list", + "default": ("CVS",), + "help": "Add files or directories to the blacklist. " + "They should be base names, not paths.", + }, + ), + ( + "ignore-patterns", + { + "type": "regexp_csv", + "metavar": "<pattern>,...", + "dest": "black_list_re", + "default": (), + "help": "Add files or directories matching the regex patterns to the" + " blacklist. The regex matches against base names, not paths.", + }, + ), + ( + "persistent", + { + "default": True, + "type": "yn", + "metavar": "<y_or_n>", + "level": 1, + "help": "Pickle collected data for later comparisons.", + }, + ), + ( + "load-plugins", + { + "type": "csv", + "metavar": "<modules>", + "default": (), + "level": 1, + "help": "List of plugins (as comma separated values of " + "python modules names) to load, usually to register " + "additional checkers.", + }, + ), + ( + "output-format", + { + "default": "text", + "type": "string", + "metavar": "<format>", + "short": "f", + "group": "Reports", + "help": "Set the output format. Available formats are text," + " parseable, colorized, json and msvs (visual studio)." + " You can also give a reporter class, e.g. mypackage.mymodule." + "MyReporterClass.", + }, + ), + ( + "reports", + { + "default": False, + "type": "yn", + "metavar": "<y_or_n>", + "short": "r", + "group": "Reports", + "help": "Tells whether to display a full report or only the " + "messages", + }, + ), + ( + "evaluation", + { + "type": "string", + "metavar": "<python_expression>", + "group": "Reports", + "level": 1, + "default": "10.0 - ((float(5 * error + warning + refactor + " + "convention) / statement) * 10)", + "help": "Python expression which should return a note less " + "than 10 (10 is the highest note). You have access " + "to the variables errors warning, statement which " + "respectively contain the number of errors / " + "warnings messages and the total number of " + "statements analyzed. This is used by the global " + "evaluation report (RP0004).", + }, + ), + ( + "score", + { + "default": True, + "type": "yn", + "metavar": "<y_or_n>", + "short": "s", + "group": "Reports", + "help": "Activate the evaluation score.", + }, + ), + ( + "confidence", + { + "type": "multiple_choice", + "metavar": "<levels>", + "default": "", + "choices": [c.name for c in interfaces.CONFIDENCE_LEVELS], + "group": "Messages control", + "help": "Only show warnings with the listed confidence levels." + " Leave empty to show all. Valid levels: %s" + % (", ".join(c.name for c in interfaces.CONFIDENCE_LEVELS),), + }, + ), + ( + "enable", + { + "type": "csv", + "metavar": "<msg ids>", + "short": "e", + "group": "Messages control", + "help": "Enable the message, report, category or checker with the " + "given id(s). You can either give multiple identifier " + "separated by comma (,) or put this option multiple time " + "(only on the command line, not in the configuration file " + "where it should appear only once). " + 'See also the "--disable" option for examples. ', + }, + ), + ( + "disable", + { + "type": "csv", + "metavar": "<msg ids>", + "short": "d", + "group": "Messages control", + "help": "Disable the message, report, category or checker " + "with the given id(s). You can either give multiple identifiers" + " separated by comma (,) or put this option multiple times " + "(only on the command line, not in the configuration file " + "where it should appear only once)." + 'You can also use "--disable=all" to disable everything first ' + "and then reenable specific checks. For example, if you want " + "to run only the similarities checker, you can use " + '"--disable=all --enable=similarities". ' + "If you want to run only the classes checker, but have no " + "Warning level messages displayed, use" + '"--disable=all --enable=classes --disable=W"', + }, + ), + ( + "msg-template", + { + "type": "string", + "metavar": "<template>", + "default": "", + "group": "Reports", + "help": ( + "Template used to display messages. " + "This is a python new-style format string " + "used to format the message information. " + "See doc for all details" + ), + }, + ), + ( + "jobs", + { + "type": "int", + "metavar": "<n-processes>", + "short": "j", + "default": 1, + "help": """Use multiple processes to speed up Pylint.""", + }, + ), + ( + "unsafe-load-any-extension", + { + "type": "yn", + "metavar": "<yn>", + "default": False, + "hide": True, + "help": ( + "Allow loading of arbitrary C extensions. Extensions" + " are imported into the active Python interpreter and" + " may run arbitrary code." + ), + }, + ), + ( + "extension-pkg-whitelist", + { + "type": "csv", + "metavar": "<pkg>,...", + "default": [], + "help": ( + "A comma-separated list of package or module names" + " from where C extensions may be loaded. Extensions are" + " loading into the active Python interpreter and may run" + " arbitrary code" + ), + }, + ), + ( + "suggestion-mode", + { + "type": "yn", + "metavar": "<yn>", + "default": True, + "help": ( + "When enabled, pylint would attempt to guess common " + "misconfiguration and emit user-friendly hints instead " + "of false-positive error messages" + ), + }, + ), + ( + "exit-zero", + { + "action": "store_true", + "help": ( + "Always return a 0 (non-error) status code, even if " + "lint errors are found. This is primarily useful in " + "continuous integration scripts." + ), + }, + ), + ) option_groups = ( ("Messages control", "Options controlling analysis messages"), ("Reports", "Options related to output formatting and reporting"), ) - def __init__(self, options=(), reporter=None, option_groups=(), pylintrc=None): + def __init__(self, config=None): # some stuff has to be done before ancestors initialization... # # messages store / checkers / reporter / astroid manager + self.config = config self.msgs_store = utils.MessagesStore() self.reporter = None - self._reporter_name = None self._reporters = {} self._checkers = collections.defaultdict(list) self._pragma_lineno = {} @@ -579,15 +483,8 @@ class PyLinter( self.current_name = None self.current_file = None self.stats = None - # init options - self._external_opts = options - self.options = options + PyLinter.make_options() - self.option_groups = option_groups + PyLinter.option_groups - self._options_methods = {"enable": self.enable, "disable": self.disable} - self._bw_options_methods = { - "disable-msg": self.disable, - "enable-msg": self.enable, - } + + # TODO: Runner needs to give this to parser? full_version = "%%prog %s\nastroid %s\nPython %s" % ( version, astroid_version, @@ -595,9 +492,6 @@ class PyLinter( ) utils.MessagesHandlerMixIn.__init__(self) utils.ReportsHandlerMixIn.__init__(self) - super(PyLinter, self).__init__( - usage=__doc__, version=full_version, config_file=pylintrc or config.PYLINTRC - ) checkers.BaseTokenChecker.__init__(self) # provided reports self.reports = ( @@ -609,32 +503,9 @@ class PyLinter( ), ("RP0003", "Messages", report_messages_stats), ) - self.register_checker(self) self._dynamic_plugins = set() self._python3_porting_mode = False self._error_mode = False - self.load_provider_defaults() - if reporter: - self.set_reporter(reporter) - - def load_default_plugins(self): - checkers.initialize(self) - reporters.initialize(self) - # Make sure to load the default reporter, because - # the option has been set before the plugins had been loaded. - if not self.reporter: - self._load_reporter() - - def load_plugin_modules(self, modnames): - """take a list of module names which are pylint plugins and load - and register them - """ - for modname in modnames: - if modname in self._dynamic_plugins: - continue - self._dynamic_plugins.add(modname) - module = modutils.load_module_from_name(modname) - module.register(self) def load_plugin_configuration(self): """Call the configuration hook for plugins @@ -648,8 +519,8 @@ class PyLinter( if hasattr(module, "load_configuration"): module.load_configuration(self) - def _load_reporter(self): - name = self._reporter_name.lower() + def load_reporter(self): + name = self.config.output_format.lower() if name in self._reporters: self.set_reporter(self._reporters[name]()) else: @@ -661,7 +532,7 @@ class PyLinter( self.set_reporter(reporter_class()) def _load_reporter_class(self): - qname = self._reporter_name + qname = self.config.output_format module = modutils.load_module_from_name(modutils.get_module_part(qname)) class_name = qname.split(".")[-1] reporter_class = getattr(module, class_name) @@ -672,42 +543,10 @@ class PyLinter( self.reporter = reporter reporter.linter = self - def set_option(self, optname, value, action=None, optdict=None): - """overridden from config.OptionsProviderMixin to handle some - special options - """ - if optname in self._options_methods or optname in self._bw_options_methods: - if value: - try: - meth = self._options_methods[optname] - except KeyError: - meth = self._bw_options_methods[optname] - warnings.warn( - "%s is deprecated, replace it by %s" - % (optname, optname.split("-")[0]), - DeprecationWarning, - ) - value = utils._check_csv(value) - if isinstance(value, (list, tuple)): - for _id in value: - meth(_id, ignore_unknown=True) - else: - meth(value) - return # no need to call set_option, disable/enable methods do it - elif optname == "output-format": - self._reporter_name = value - # If the reporters are already available, load - # the reporter class. - if self._reporters: - self._load_reporter() - - try: - checkers.BaseTokenChecker.set_option(self, optname, value, action, optdict) - except config.UnsupportedAction: - print("option %s can't be read from config file" % optname, file=sys.stderr) - def register_reporter(self, reporter_class): self._reporters[reporter_class.name] = reporter_class + if reporter_class.name == self.config.output_format.lower(): + self.load_reporter() def report_order(self): reports = sorted(self._reports, key=lambda x: getattr(x, "name", "")) @@ -723,25 +562,6 @@ class PyLinter( # checkers manipulation methods ############################################ - def register_checker(self, checker): - """register a new checker - - checker is an object implementing IRawChecker or / and IAstroidChecker - """ - assert checker.priority <= 0, "checker priority can't be >= 0" - self._checkers[checker.name].append(checker) - for r_id, r_title, r_cb in checker.reports: - self.register_report(r_id, r_title, r_cb, checker) - self.register_options_provider(checker) - if hasattr(checker, "msgs"): - self.msgs_store.register_messages_from_checker(checker) - checker.load_defaults() - - # Register the checker, but disable all of its messages. - # TODO(cpopa): we should have a better API for this. - if not getattr(checker, "enabled", True): - self.disable(checker.name) - def disable_noerror_messages(self): for msgcat, msgids in self.msgs_store._msgs_by_category.items(): # enable only messages with 'error' severity and above ('fatal') @@ -805,6 +625,7 @@ class PyLinter( """process tokens from the current module to search for module/block level options """ + options_methods = {"enable": self.enable, "disable": self.disable} control_pragmas = {"disable", "enable"} for (tok_type, content, start, _, _) in tokens: if tok_type != tokenize.COMMENT: @@ -835,17 +656,8 @@ class PyLinter( ) continue opt = opt.strip() - if opt in self._options_methods or opt in self._bw_options_methods: - try: - meth = self._options_methods[opt] - except KeyError: - meth = self._bw_options_methods[opt] - # found a "(dis|en)able-msg" pragma deprecated suppression - self.add_message( - "deprecated-pragma", - line=start[0], - args=(opt, opt.replace("-msg", "")), - ) + if opt in options_methods: + meth = options_methods[opt] for msgid in utils._splitstrip(value): # Add the line where a control pragma was encountered. if opt in control_pragmas: @@ -928,6 +740,7 @@ class PyLinter( """main checking entry: check a list of files or modules from their name. """ + assert self.reporter, "A reporter has not been loaded" # initialize msgs_state now that all messages have been registered into # the store self._init_msg_states() @@ -1230,7 +1043,6 @@ def preprocess_options(args, search_for): @contextlib.contextmanager def fix_import_path(args): """Prepare sys.path for running the linter checks. - Within this context, each of the given arguments is importable. Paths are added to sys.path in corresponding order to the arguments. We avoid adding duplicate directories to sys.path. @@ -1251,13 +1063,207 @@ def fix_import_path(args): sys.path[:] = orig -class Run: - """helper class to use as main for pylint : +def guess_lint_path(args): + """Attempt to determine the file being linted from a list of arguments. + + :param args: The list of command line arguments to guess from. + :type args: list(str) - run(*sys.argv[1:]) + :returns: The path to file being linted if it can be guessed. + None otherwise. + :rtype: str or None """ + value = None + + # We only care if it's a path. If it's a module, + # we can't get a config from it + if args and os.path.exists(args[-1]): + value = args[-1] + + return value + + +class CheckerRegistry(object): + """A class to register checkers to.""" + + def __init__(self, linter): + super(CheckerRegistry, self).__init__() + self.register_options = lambda options: None + self._checkers = collections.defaultdict(list) + # TODO: Remove. This is needed for the MessagesHandlerMixIn for now. + linter._checkers = self._checkers + self._linter = linter + self.register_checker(linter) + + def for_all_checkers(self): + """Loop through all registered checkers. + + :returns: Each registered checker. + :rtype: iterable(BaseChecker) + """ + for checkers in self._checkers.values(): + yield from checkers + + def register_checker(self, checker): + """Register a checker. + + :param checker: The checker to register. + :type checker: BaseChecker + + :raises ValueError: If the priority of the checker is invalid. + """ + if checker.name in self._checkers: + # TODO: Raise if classes are the same + for duplicate in self._checkers[checker.name]: + msg = "A checker called {} has already been registered ({})." + msg = msg.format(checker.name, duplicate.__class__) + warnings.warn(msg) + + if checker.priority > 0: + # TODO: Use a custom exception + msg = "{}.priority must be <= 0".format(checker.__class__) + raise ValueError(msg) + + self._checkers[checker.name].append(checker) + + # TODO: Move elsewhere + for r_id, r_title, r_cb in checker.reports: + self._linter.register_report(r_id, r_title, r_cb, checker) + + self.register_options(checker.options) + + # TODO: Move elsewhere + if hasattr(checker, "msgs"): + self._linter.msgs_store.register_messages(checker) + + # Register the checker, but disable all of its messages. + # TODO(cpopa): we should have a better API for this. + if not getattr(checker, "enabled", True): + self._linter.disable(checker.name) + + # For now simply defer missing attributs to the linter, + # until we know what API we want. + def __getattr__(self, attribute): + return getattr(self._linter, attribute) + + +class Runner(object): + """A class to manager how the linter runs.""" + + option_definitions = () + """The runner specific configuration options. + + :type: set(OptionDefinition) + """ + + +class CLIRunner(Runner): + option_definitions = ( + ( + "rcfile", + { + "type": "string", + "metavar": "<file>", + "help": "Specify a configuration file.", + }, + ), + ( + "init-hook", + { + "type": "string", + "metavar": "<code>", + "level": 1, + "help": "Python code to execute, usually for sys.path " + "manipulation such as pygtk.require().", + }, + ), + ( + "help-msg", + { + "type": "string", + "metavar": "<msg-id>", + "group": "Commands", + "default": None, + "help": "Display a help message for the given message id and " + "exit. The value may be a comma separated list of message ids.", + }, + ), + ( + "list-msgs", + { + "metavar": "<msg-id>", + "group": "Commands", + "level": 1, + "default": None, + "help": "Generate pylint's messages.", + }, + ), + ( + "list-conf-levels", + { + "group": "Commands", + "level": 1, + "action": "store_true", + "default": False, + "help": "Generate pylint's confidence levels.", + }, + ), + ( + "full-documentation", + { + "metavar": "<msg-id>", + "default": None, + "group": "Commands", + "level": 1, + "help": "Generate pylint's full documentation.", + }, + ), + ( + "generate-rcfile", + { + "group": "Commands", + "action": "store_true", + "default": False, + "help": "Generate a sample configuration file according to " + "the current configuration. You can put other options " + "before this one to get them in the generated " + "configuration.", + }, + ), + ( + "generate-man", + { + "group": "Commands", + "action": "store_true", + "default": False, + "help": "Generate pylint's man page.", + "hide": True, + }, + ), + ( + "errors-only", + { + "short": "E", + "action": "store_true", + "default": False, + "help": "In error mode, checkers without error messages are " + "disabled and for others, only the ERROR messages are " + "displayed, and no reports are done by default" + "", + }, + ), + ( + "py3k", + { + "action": "store_true", + "default": False, + "help": "In Python 3 porting mode, all checkers will be " + "disabled and only messages emitted by the porting " + "checker will be displayed", + }, + ), + ) - LinterClass = PyLinter option_groups = ( ( "Commands", @@ -1266,158 +1272,34 @@ group are mutually exclusive.", ), ) - def __init__(self, args, reporter=None, do_exit=True): - self._rcfile = None - self._plugins = [] - self.verbose = None - try: - preprocess_options( - args, - { - # option: (callback, takearg) - "init-hook": (cb_init_hook, True), - "rcfile": (self.cb_set_rcfile, True), - "load-plugins": (self.cb_add_plugins, True), - "verbose": (self.cb_verbose_mode, False), - }, - ) - except ArgumentPreprocessingError as ex: - print(ex, file=sys.stderr) - sys.exit(32) + description = ( + "pylint [options] module_or_package\n" + "\n" + " Check that a module satisfies a coding standard (and more !).\n" + "\n" + " pylint --help\n" + "\n" + " Display this help message and exit.\n" + "\n" + " pylint --help-msg <msg-id>[,<msg-id>]\n" + "\n" + " Display help messages about given message identifiers and exit.\n" + ) - self.linter = linter = self.LinterClass( - ( - ( - "rcfile", - { - "action": "callback", - "callback": lambda *args: 1, - "type": "string", - "metavar": "<file>", - "help": "Specify a configuration file.", - }, - ), - ( - "init-hook", - { - "action": "callback", - "callback": lambda *args: 1, - "type": "string", - "metavar": "<code>", - "level": 1, - "help": "Python code to execute, usually for sys.path " - "manipulation such as pygtk.require().", - }, - ), - ( - "help-msg", - { - "action": "callback", - "type": "string", - "metavar": "<msg-id>", - "callback": self.cb_help_message, - "group": "Commands", - "help": "Display a help message for the given message id and " - "exit. The value may be a comma separated list of message ids.", - }, - ), - ( - "list-msgs", - { - "action": "callback", - "metavar": "<msg-id>", - "callback": self.cb_list_messages, - "group": "Commands", - "level": 1, - "help": "Generate pylint's messages.", - }, - ), - ( - "list-conf-levels", - { - "action": "callback", - "callback": cb_list_confidence_levels, - "group": "Commands", - "level": 1, - "help": "Generate pylint's confidence levels.", - }, - ), - ( - "full-documentation", - { - "action": "callback", - "metavar": "<msg-id>", - "callback": self.cb_full_documentation, - "group": "Commands", - "level": 1, - "help": "Generate pylint's full documentation.", - }, - ), - ( - "generate-rcfile", - { - "action": "callback", - "callback": self.cb_generate_config, - "group": "Commands", - "help": "Generate a sample configuration file according to " - "the current configuration. You can put other options " - "before this one to get them in the generated " - "configuration.", - }, - ), - ( - "generate-man", - { - "action": "callback", - "callback": self.cb_generate_manpage, - "group": "Commands", - "help": "Generate pylint's man page.", - "hide": True, - }, - ), - ( - "errors-only", - { - "action": "callback", - "callback": self.cb_error_mode, - "short": "E", - "help": "In error mode, checkers without error messages are " - "disabled and for others, only the ERROR messages are " - "displayed, and no reports are done by default.", - }, - ), - ( - "py3k", - { - "action": "callback", - "callback": self.cb_python3_porting_mode, - "help": "In Python 3 porting mode, all checkers will be " - "disabled and only messages emitted by the porting " - "checker will be displayed.", - }, - ), - ( - "verbose", - { - "action": "callback", - "callback": self.cb_verbose_mode, - "short": "v", - "help": "In verbose mode, extra non-checker-related info " - "will be displayed.", - }, - ), - ), - option_groups=self.option_groups, - pylintrc=self._rcfile, - ) - # register standard checkers - linter.load_default_plugins() - # load command line plugins - linter.load_plugin_modules(self._plugins) - # add some help section - linter.add_help_section("Environment variables", config.ENV_HELP, level=1) + def __init__(self): + super(CLIRunner, self).__init__() + self._linter = PyLinter() + self._checker_registry = CheckerRegistry(self._linter) + self._loaded_plugins = set() + + def run(self, args): + # Phase 1: Preprocessing + option_definitions = self.option_definitions + self._linter.options + parser = config.CLIParser(self.description) + parser.add_option_definitions(option_definitions) + parser.add_help_section("Environment variables", config.ENV_HELP, level=1) # pylint: disable=bad-continuation - linter.add_help_section( + parser.add_help_section( "Output", "Using the default text output, the message format is : \n" " \n" @@ -1432,7 +1314,7 @@ group are mutually exclusive.", "processing.\n", level=1, ) - linter.add_help_section( + parser.add_help_section( "Output status code", "Pylint should leave with following status code: \n" " * 0 if everything went fine \n" @@ -1447,244 +1329,127 @@ group are mutually exclusive.", "been issued by analysing pylint output status code\n", level=1, ) - # read configuration - linter.disable("I") - linter.enable("c-extension-no-member") - linter.read_config_file(verbose=self.verbose) - config_parser = linter.cfgfile_parser - # run init hook, if present, before loading plugins - if config_parser._parser.has_option("MASTER", "init-hook"): - cb_init_hook( - "init-hook", - utils._unquote(config_parser._parser.get("MASTER", "init-hook")), - ) - # is there some additional plugins in the file configuration, in - if config_parser._parser.has_option("MASTER", "load-plugins"): - plugins = utils._splitstrip( - config_parser._parser.get("MASTER", "load-plugins") - ) - linter.load_plugin_modules(plugins) - # now we can load file config and command line, plugins (which can - # provide options) have been registered - linter.load_config_file() - - if reporter: - # if a custom reporter is provided as argument, it may be overridden - # by file parameters, so re-set it here, but before command line - # parsing so it's still overrideable by command line option - linter.set_reporter(reporter) - try: - args = linter.load_command_line_configuration(args) - except SystemExit as exc: - if exc.code == 2: # bad options - exc.code = 32 - raise - if not args: - print(linter.help()) - sys.exit(32) - - if linter.config.jobs < 0: - print( - "Jobs number (%d) should be greater than or equal to 0" - % linter.config.jobs, - file=sys.stderr, - ) - sys.exit(32) - if linter.config.jobs > 1 or linter.config.jobs == 0: - if multiprocessing is None: - print( - "Multiprocessing library is missing, " "fallback to single process", - file=sys.stderr, - ) - linter.set_option("jobs", 1) - elif linter.config.jobs == 0: - linter.config.jobs = _cpu_count() - - # We have loaded configuration from config file and command line. Now, we can - # load plugin specific configuration. - #linter.load_plugin_configuration() - - # insert current working directory to the python path to have a correct - # behaviour - with fix_import_path(args): - if linter.config.jobs == 1: - linter.check(args) - else: - self._parallel_run(args) - linter.generate_reports() - if do_exit: - if linter.config.exit_zero: - sys.exit(0) - else: - sys.exit(self.linter.msg_status) + global_config = config.Configuration() + global_config.add_options(option_definitions) + self._linter.config = global_config - def _parallel_run(self, files_or_modules): - with _patch_sysmodules(): - self.linter._init_msg_states() - self._parallel_check(files_or_modules) + parsed = parser.preprocess( + args, "init_hook", "rcfile", "load_plugins", "ignore", "ignore_patterns" + ) - def _parallel_task(self, files_or_modules): - # Prepare configuration for child linters. - child_config = self._get_jobs_config() + # Call init-hook + if parsed.init_hook: + exec(parsed.init_hook) - children = [] - manager = multiprocessing.Manager() - tasks_queue = manager.Queue() - results_queue = manager.Queue() + # Load rcfile, else system rcfile + file_parser = config.IniFileParser() + file_parser.add_option_definitions(self._linter.options) + rcfile = parsed.rcfile or config.PYLINTRC + if rcfile: + file_parser.parse(rcfile, global_config) - for _ in range(self.linter.config.jobs): - child_linter = ChildRunner(args=(tasks_queue, results_queue, child_config)) - child_linter.start() - children.append(child_linter) + def register_options(options): + global_config.add_options(options) + parser.add_option_definitions(options) + file_parser.add_option_definitions(options) - # Send files to child linters. - expanded_files = utils.expand_files( - files_or_modules, - self.linter, - self.linter.config.black_list, - self.linter.config.black_list_re, - ) - for module_desc in expanded_files: - tasks_queue.put([module_desc.path]) + self._checker_registry.register_options = register_options - # collect results from child linters - failed = False - for _ in expanded_files: - try: - result = results_queue.get() - except Exception as ex: - print( - "internal error while receiving results from child linter", - file=sys.stderr, - ) - print(ex, file=sys.stderr) - failed = True - break - yield result - - # Stop child linters and wait for their completion. - for _ in range(self.linter.config.jobs): - tasks_queue.put("STOP") - for child in children: - child.join() - - if failed: - print("Error occured, stopping the linter.", file=sys.stderr) - sys.exit(32) - - def _parallel_check(self, files_or_modules): - # Reset stats. - self.linter.open() - - all_stats = [] - module = None - for result in self._parallel_task(files_or_modules): - if not result: - continue - ( - _, - self.linter.file_state.base_name, - module, - messages, - stats, - msg_status, - ) = result - - for msg in messages: - msg = utils.Message(*msg) - self.linter.set_current_module(module) - self.linter.reporter.handle_message(msg) - - all_stats.append(stats) - self.linter.msg_status |= msg_status - - self.linter.stats = _merge_stats(all_stats) - self.linter.current_name = module - - # Insert stats data to local checkers. - for checker in self.linter.get_checkers(): - if checker is not self.linter: - checker.stats = self.linter.stats - - def _get_jobs_config(self): - child_config = collections.OrderedDict() - filter_options = {"long-help"} - filter_options.update((opt_name for opt_name, _ in self.linter._external_opts)) - for opt_providers in six.itervalues(self.linter._all_options): - for optname, optdict, val in opt_providers.options_and_values(): - if optdict.get("deprecated"): - continue - - if optname not in filter_options: - child_config[optname] = utils._format_option_value(optdict, val) - child_config["python3_porting_mode"] = self.linter._python3_porting_mode - child_config["plugins"] = self.linter._dynamic_plugins - return child_config - - def cb_set_rcfile(self, name, value): - """callback for option preprocessing (i.e. before option parsing)""" - self._rcfile = value - - def cb_add_plugins(self, name, value): - """callback for option preprocessing (i.e. before option parsing)""" - self._plugins.extend(utils._splitstrip(value)) - - def cb_error_mode(self, *args, **kwargs): - """error mode: - * disable all but error messages - * disable the 'miscellaneous' checker which can be safely deactivated in - debug - * disable reports - * do not save execution information - """ - self.linter.error_mode() + checkers.initialize(self._checker_registry) + + # Load plugins from CLI + plugins = parsed.load_plugins or [] + for plugin in plugins: + self.load_plugin(plugin) + + # TODO: This is for per directory config support (#618) + # Phase 2: Discover more plugins found in config files + # Walk and discover config files, watching for blacklists as we go + + # Load plugins from config files - def cb_generate_config(self, *args, **kwargs): - """optik callback for sample config file generation""" - self.linter.generate_config(skipsections=("COMMANDS",)) - sys.exit(0) + # Phase 3: Full load + # Fully load config files + if rcfile: + file_parser.parse(rcfile, global_config) + # Fully load CLI into global config + parser.parse(args, global_config) - def cb_generate_manpage(self, *args, **kwargs): - """optik callback for sample config file generation""" - from pylint import __pkginfo__ + if global_config.generate_rcfile: + file_parser.write() + sys.exit(0) - self.linter.generate_manpage(__pkginfo__) - sys.exit(0) + # TODO: if global_config.generate_man - def cb_help_message(self, option, optname, value, parser): - """optik callback for printing some help about a particular message""" - self.linter.msgs_store.help_message(utils._splitstrip(value)) - sys.exit(0) + if global_config.errors_only: + self._linter.errors_mode() - def cb_full_documentation(self, option, optname, value, parser): - """optik callback for printing full documentation""" - self.linter.print_full_documentation() - sys.exit(0) + if global_config.py3k: + self._linter.python3_porting_mode() - def cb_list_messages(self, option, optname, value, parser): # FIXME - """optik callback for printing available messages""" - self.linter.msgs_store.list_messages() - sys.exit(0) + if global_config.full_documentation: + self._linter.print_full_documentation() + sys.exit(0) - def cb_python3_porting_mode(self, *args, **kwargs): - """Activate only the python3 porting checker.""" - self.linter.python3_porting_mode() + if global_config.list_conf_levels: + for level in interfaces.CONFIDENCE_LEVELS: + print("%-18s: %s" % level) + sys.exit(0) - def cb_verbose_mode(self, *args, **kwargs): - self.verbose = True + if global_config.list_msgs: + self._linter.msgs_store.list_messages() + sys.exit(0) + if global_config.help_msg: + msg = utils._splitstrip(global_config.help_msg) + self._linter.msgs_store.help_message(msg) + sys.exit(0) -def cb_list_confidence_levels(option, optname, value, parser): - for level in interfaces.CONFIDENCE_LEVELS: - print("%-18s: %s" % level) - sys.exit(0) + self.load_default_plugins() + self._linter.disable("I") + self._linter.enable("c-extension-no-member") -def cb_init_hook(optname, value): - """exec arbitrary code to set sys.path for instance""" - exec(value) # pylint: disable=exec-used + for checker in self._checker_registry.for_all_checkers(): + checker.config = global_config + + with fix_import_path(global_config.module_or_package): + assert self._linter.config.jobs == 1 + self._linter.check(global_config.module_or_package) + + self._linter.generate_reports() + + if linter.config.exit_zero: + sys.exit(0) + else: + sys.exit(self.linter.msg_status) + + def load_plugin(self, module_name): + if module_name in self._loaded_plugins: + msg = "Already loaded plugin {0}. Ignoring".format(module_name) + warnings.warn(msg) + else: + module = astroid.modutils.load_module_from_name(module_name) + module.register(self._checker_registry) + + def load_plugins(self, module_names): + """Load a plugin. + + Args: + module_names (list(str)): The name of plugin modules to load. + """ + for module_name in module_names: + self.load_plugin(module_name) + + def load_default_plugins(self): + """Load all of the default plugins.""" + reporters.initialize(self._linter) + # Make sure to load the default reporter, because + # the option has been set before the plugins had been loaded. + if not self._linter.reporter: + self._linter.load_reporter() if __name__ == "__main__": - Run(sys.argv[1:]) + CLIRunner().run(sys.argv[1:]) diff --git a/pylint/utils.py b/pylint/utils.py index 3f71cba67..513ed4666 100644 --- a/pylint/utils.py +++ b/pylint/utils.py @@ -1340,24 +1340,26 @@ class PyLintASTWalker: PY_EXTS = (".py", ".pyc", ".pyo", ".pyw", ".so", ".dll") -def register_plugins(linter, directory): - """load all module and package in the given directory, looking for a - 'register' function in each one, used to register pylint checkers +def register_plugins(registry, directory): + """Load plugins from all modules and packages in the given directory. + + Args: + registry (CheckerRegistry): The registry to register the checkers with. + directory (str): The directory to search for plugins. """ - imported = {} + imported = {"__init__", "__pycache__"} for filename in os.listdir(directory): - base, extension = splitext(filename) - if base in imported or base == "__pycache__": + base, extension = os.path.splitext(filename) + if base in imported: continue - if ( - extension in PY_EXTS - and base != "__init__" - or (not extension and isdir(join(directory, base))) - ): + + package_dir = os.path.join(directory, base) + if extension in PY_EXTS or (not extension and os.path.isdir(package_dir)): + file_path = os.path.join(directory, filename) try: - module = modutils.load_module_from_file(join(directory, filename)) + module = modutils.load_module_from_file(file_path) except ValueError: - # empty module name (usually emacs auto-save files) + # Empty module name continue except ImportError as exc: print( @@ -1365,8 +1367,8 @@ def register_plugins(linter, directory): ) else: if hasattr(module, "register"): - module.register(linter) - imported[base] = 1 + module.register(registry) + imported.add(base) def get_global_option(checker, option, default=None): |