diff options
| -rw-r--r-- | docs/cli-usage.rst | 57 | ||||
| -rw-r--r-- | gitlab/cli.py | 108 | ||||
| -rw-r--r-- | gitlab/client.py | 82 | ||||
| -rw-r--r-- | gitlab/config.py | 4 | ||||
| -rw-r--r-- | tests/functional/cli/test_cli.py | 79 | ||||
| -rw-r--r-- | tests/unit/test_config.py | 2 | ||||
| -rw-r--r-- | tests/unit/test_gitlab_auth.py | 114 |
7 files changed, 421 insertions, 25 deletions
diff --git a/docs/cli-usage.rst b/docs/cli-usage.rst index 50fac6d..6dbce5d 100644 --- a/docs/cli-usage.rst +++ b/docs/cli-usage.rst @@ -3,17 +3,60 @@ #################### ``python-gitlab`` provides a :command:`gitlab` command-line tool to interact -with GitLab servers. It uses a configuration file to define how to connect to -the servers. Without a configuration file, ``gitlab`` will default to -https://gitlab.com and unauthenticated requests. +with GitLab servers. + +This is especially convenient for running quick ad-hoc commands locally, easily +interacting with the API inside GitLab CI, or with more advanced shell scripting +when integrating with other tooling. .. _cli_configuration: Configuration ============= -Files ------ +``gitlab`` allows setting configuration options via command-line arguments, +environment variables, and configuration files. + +For a complete list of global CLI options and their environment variable +equivalents, see :doc:`/cli-objects`. + +With no configuration provided, ``gitlab`` will default to unauthenticated +requests against `GitLab.com <https://gitlab.com>`__. + +With no configuration but running inside a GitLab CI job, it will default to +authenticated requests using the current job token against the current instance +(via ``CI_SERVER_URL`` and ``CI_JOB_TOKEN`` environment variables). + +.. warning:: + Please note the job token has very limited permissions and can only be used + with certain endpoints. You may need to provide a personal access token instead. + +When you provide configuration, values are evaluated with the following precedence: + +1. Explicitly provided CLI arguments, +2. Environment variables, +3. Configuration files: + + a. explicitly defined config files: + + i. via the ``--config-file`` CLI argument, + ii. via the ``PYTHON_GITLAB_CFG`` environment variable, + + b. user-specific config file, + c. system-level config file, + +4. Environment variables always present in CI (``CI_SERVER_URL``, ``CI_JOB_TOKEN``). + +Additionally, authentication will take the following precedence +when multiple options or environment variables are present: + +1. Private token, +2. OAuth token, +3. CI job token. + + +Configuration files +------------------- ``gitlab`` looks up 3 configuration files by default: @@ -35,8 +78,8 @@ You can use a different configuration file with the ``--config-file`` option. If the environment variable is defined and the target file cannot be accessed, ``gitlab`` will fail explicitly. -Content -------- +Configuration file format +------------------------- The configuration file uses the ``INI`` format. It contains at least a ``[global]`` section, and a specific section for each GitLab server. For diff --git a/gitlab/cli.py b/gitlab/cli.py index c1a1334..a48b53b 100644 --- a/gitlab/cli.py +++ b/gitlab/cli.py @@ -19,6 +19,7 @@ import argparse import functools +import os import re import sys from types import ModuleType @@ -112,17 +113,25 @@ def _get_base_parser(add_help: bool = True) -> argparse.ArgumentParser: "-v", "--verbose", "--fancy", - help="Verbose mode (legacy format only)", + help="Verbose mode (legacy format only) [env var: GITLAB_VERBOSE]", action="store_true", + default=os.getenv("GITLAB_VERBOSE"), ) parser.add_argument( - "-d", "--debug", help="Debug mode (display HTTP requests)", action="store_true" + "-d", + "--debug", + help="Debug mode (display HTTP requests) [env var: GITLAB_DEBUG]", + action="store_true", + default=os.getenv("GITLAB_DEBUG"), ) parser.add_argument( "-c", "--config-file", action="append", - help="Configuration file to use. Can be used multiple times.", + help=( + "Configuration file to use. Can be used multiple times. " + "[env var: PYTHON_GITLAB_CFG]" + ), ) parser.add_argument( "-g", @@ -151,7 +160,86 @@ def _get_base_parser(add_help: bool = True) -> argparse.ArgumentParser: ), required=False, ) + parser.add_argument( + "--server-url", + help=("GitLab server URL [env var: GITLAB_URL]"), + required=False, + default=os.getenv("GITLAB_URL"), + ) + parser.add_argument( + "--ssl-verify", + help=( + "Whether SSL certificates should be validated. [env var: GITLAB_SSL_VERIFY]" + ), + required=False, + default=os.getenv("GITLAB_SSL_VERIFY"), + ) + parser.add_argument( + "--timeout", + help=( + "Timeout to use for requests to the GitLab server. " + "[env var: GITLAB_TIMEOUT]" + ), + required=False, + default=os.getenv("GITLAB_TIMEOUT"), + ) + parser.add_argument( + "--api-version", + help=("GitLab API version [env var: GITLAB_API_VERSION]"), + required=False, + default=os.getenv("GITLAB_API_VERSION"), + ) + parser.add_argument( + "--per-page", + help=( + "Number of entries to return per page in the response. " + "[env var: GITLAB_PER_PAGE]" + ), + required=False, + default=os.getenv("GITLAB_PER_PAGE"), + ) + parser.add_argument( + "--pagination", + help=( + "Whether to use keyset or offset pagination [env var: GITLAB_PAGINATION]" + ), + required=False, + default=os.getenv("GITLAB_PAGINATION"), + ) + parser.add_argument( + "--order-by", + help=("Set order_by globally [env var: GITLAB_ORDER_BY]"), + required=False, + default=os.getenv("GITLAB_ORDER_BY"), + ) + parser.add_argument( + "--user-agent", + help=( + "The user agent to send to GitLab with the HTTP request. " + "[env var: GITLAB_USER_AGENT]" + ), + required=False, + default=os.getenv("GITLAB_USER_AGENT"), + ) + tokens = parser.add_mutually_exclusive_group() + tokens.add_argument( + "--private-token", + help=("GitLab private access token [env var: GITLAB_PRIVATE_TOKEN]"), + required=False, + default=os.getenv("GITLAB_PRIVATE_TOKEN"), + ) + tokens.add_argument( + "--oauth-token", + help=("GitLab OAuth token [env var: GITLAB_OAUTH_TOKEN]"), + required=False, + default=os.getenv("GITLAB_OAUTH_TOKEN"), + ) + tokens.add_argument( + "--job-token", + help=("GitLab CI job token [env var: CI_JOB_TOKEN]"), + required=False, + ) return parser @@ -243,13 +331,23 @@ def main() -> None: "whaction", "version", "output", + "fields", + "server_url", + "ssl_verify", + "timeout", + "api_version", + "pagination", + "user_agent", + "private_token", + "oauth_token", + "job_token", ): args_dict.pop(item) args_dict = {k: _parse_value(v) for k, v in args_dict.items() if v is not None} try: - gl = gitlab.Gitlab.from_config(gitlab_id, config_files) - if gl.private_token or gl.oauth_token or gl.job_token: + gl = gitlab.Gitlab.merge_config(vars(options), gitlab_id, config_files) + if gl.private_token or gl.oauth_token: gl.auth() except Exception as e: die(str(e)) diff --git a/gitlab/client.py b/gitlab/client.py index c1e0825..b791c8f 100644 --- a/gitlab/client.py +++ b/gitlab/client.py @@ -16,6 +16,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. """Wrapper for the GitLab API.""" +import os import time from typing import Any, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union @@ -256,6 +257,87 @@ class Gitlab(object): retry_transient_errors=config.retry_transient_errors, ) + @classmethod + def merge_config( + cls, + options: dict, + gitlab_id: Optional[str] = None, + config_files: Optional[List[str]] = None, + ) -> "Gitlab": + """Create a Gitlab connection by merging configuration with + the following precedence: + + 1. Explicitly provided CLI arguments, + 2. Environment variables, + 3. Configuration files: + a. explicitly defined config files: + i. via the `--config-file` CLI argument, + ii. via the `PYTHON_GITLAB_CFG` environment variable, + b. user-specific config file, + c. system-level config file, + 4. Environment variables always present in CI (CI_SERVER_URL, CI_JOB_TOKEN). + + Args: + options: A dictionary of explicitly provided key-value options. + gitlab_id: ID of the configuration section. + config_files: List of paths to configuration files. + Returns: + (gitlab.Gitlab): A Gitlab connection. + + Raises: + gitlab.config.GitlabDataError: If the configuration is not correct. + """ + config = gitlab.config.GitlabConfigParser( + gitlab_id=gitlab_id, config_files=config_files + ) + url = ( + options.get("server_url") + or config.url + or os.getenv("CI_SERVER_URL") + or gitlab.const.DEFAULT_URL + ) + private_token, oauth_token, job_token = cls._merge_auth(options, config) + + return cls( + url=url, + private_token=private_token, + oauth_token=oauth_token, + job_token=job_token, + ssl_verify=options.get("ssl_verify") or config.ssl_verify, + timeout=options.get("timeout") or config.timeout, + api_version=options.get("api_version") or config.api_version, + per_page=options.get("per_page") or config.per_page, + pagination=options.get("pagination") or config.pagination, + order_by=options.get("order_by") or config.order_by, + user_agent=options.get("user_agent") or config.user_agent, + ) + + @staticmethod + def _merge_auth(options: dict, config: gitlab.config.GitlabConfigParser) -> Tuple: + """ + Return a tuple where at most one of 3 token types ever has a value. + Since multiple types of tokens may be present in the environment, + options, or config files, this precedence ensures we don't + inadvertently cause errors when initializing the client. + + This is especially relevant when executed in CI where user and + CI-provided values are both available. + """ + private_token = options.get("private_token") or config.private_token + oauth_token = options.get("oauth_token") or config.oauth_token + job_token = ( + options.get("job_token") or config.job_token or os.getenv("CI_JOB_TOKEN") + ) + + if private_token: + return (private_token, None, None) + if oauth_token: + return (None, oauth_token, None) + if job_token: + return (None, None, job_token) + + return (None, None, None) + def auth(self) -> None: """Performs an authentication using private token. diff --git a/gitlab/config.py b/gitlab/config.py index 154f063..c11a4e9 100644 --- a/gitlab/config.py +++ b/gitlab/config.py @@ -23,7 +23,7 @@ from os.path import expanduser, expandvars from pathlib import Path from typing import List, Optional, Union -from gitlab.const import DEFAULT_URL, USER_AGENT +from gitlab.const import USER_AGENT _DEFAULT_FILES: List[str] = [ "/etc/python-gitlab.cfg", @@ -119,7 +119,7 @@ class GitlabConfigParser(object): self.retry_transient_errors: bool = False self.ssl_verify: Union[bool, str] = True self.timeout: int = 60 - self.url: str = DEFAULT_URL + self.url: Optional[str] = None self.user_agent: str = USER_AGENT self._files = _get_config_files(config_files) diff --git a/tests/functional/cli/test_cli.py b/tests/functional/cli/test_cli.py index b9a0e67..eb27cb7 100644 --- a/tests/functional/cli/test_cli.py +++ b/tests/functional/cli/test_cli.py @@ -1,22 +1,31 @@ +""" +Some test cases are run in-process to intercept requests to gitlab.com +and example servers. +""" + +import copy import json import pytest import responses from gitlab import __version__, config +from gitlab.const import DEFAULT_URL + +PRIVATE_TOKEN = "glpat-abc123" +CI_JOB_TOKEN = "ci-job-token" +CI_SERVER_URL = "https://gitlab.example.com" @pytest.fixture def resp_get_project(): - with responses.RequestsMock() as rsps: - rsps.add( - method=responses.GET, - url="https://gitlab.com/api/v4/projects/1", - json={"name": "name", "path": "test-path", "id": 1}, - content_type="application/json", - status=200, - ) - yield rsps + return { + "method": responses.GET, + "url": f"{DEFAULT_URL}/api/v4/projects/1", + "json": {"name": "name", "path": "test-path", "id": 1}, + "content_type": "application/json", + "status": 200, + } def test_main_entrypoint(script_runner, gitlab_config): @@ -30,17 +39,67 @@ def test_version(script_runner): @pytest.mark.script_launch_mode("inprocess") +@responses.activate def test_defaults_to_gitlab_com(script_runner, resp_get_project, monkeypatch): + responses.add(**resp_get_project) with monkeypatch.context() as m: # Ensure we don't pick up any config files that may already exist in the local # environment. m.setattr(config, "_DEFAULT_FILES", []) - # Runs in-process to intercept requests to gitlab.com ret = script_runner.run("gitlab", "project", "get", "--id", "1") assert ret.success assert "id: 1" in ret.stdout +@pytest.mark.script_launch_mode("inprocess") +@responses.activate +def test_uses_ci_server_url(monkeypatch, script_runner, resp_get_project): + monkeypatch.setenv("CI_SERVER_URL", CI_SERVER_URL) + resp_get_project_in_ci = copy.deepcopy(resp_get_project) + resp_get_project_in_ci.update(url=f"{CI_SERVER_URL}/api/v4/projects/1") + + responses.add(**resp_get_project_in_ci) + ret = script_runner.run("gitlab", "project", "get", "--id", "1") + assert ret.success + + +@pytest.mark.script_launch_mode("inprocess") +@responses.activate +def test_uses_ci_job_token(monkeypatch, script_runner, resp_get_project): + monkeypatch.setenv("CI_JOB_TOKEN", CI_JOB_TOKEN) + resp_get_project_in_ci = copy.deepcopy(resp_get_project) + resp_get_project_in_ci.update( + match=[responses.matchers.header_matcher({"JOB-TOKEN": CI_JOB_TOKEN})], + ) + + responses.add(**resp_get_project_in_ci) + ret = script_runner.run("gitlab", "project", "get", "--id", "1") + assert ret.success + + +@pytest.mark.script_launch_mode("inprocess") +@responses.activate +def test_private_token_overrides_job_token( + monkeypatch, script_runner, resp_get_project +): + monkeypatch.setenv("GITLAB_PRIVATE_TOKEN", PRIVATE_TOKEN) + monkeypatch.setenv("CI_JOB_TOKEN", CI_JOB_TOKEN) + + resp_get_project_with_token = copy.deepcopy(resp_get_project) + resp_get_project_with_token.update( + match=[responses.matchers.header_matcher({"PRIVATE-TOKEN": PRIVATE_TOKEN})], + ) + + # CLI first calls .auth() when private token is present + resp_auth_with_token = copy.deepcopy(resp_get_project_with_token) + resp_auth_with_token.update(url=f"{DEFAULT_URL}/api/v4/user") + + responses.add(**resp_get_project_with_token) + responses.add(**resp_auth_with_token) + ret = script_runner.run("gitlab", "project", "get", "--id", "1") + assert ret.success + + def test_env_config_missing_file_raises(script_runner, monkeypatch): monkeypatch.setenv("PYTHON_GITLAB_CFG", "non-existent") ret = script_runner.run("gitlab", "project", "list") diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 6874e94..7ba312b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -150,7 +150,7 @@ def test_default_config(mock_clean_env, monkeypatch): assert cp.retry_transient_errors is False assert cp.ssl_verify is True assert cp.timeout == 60 - assert cp.url == const.DEFAULT_URL + assert cp.url is None assert cp.user_agent == const.USER_AGENT diff --git a/tests/unit/test_gitlab_auth.py b/tests/unit/test_gitlab_auth.py index 314fbed..8d6677f 100644 --- a/tests/unit/test_gitlab_auth.py +++ b/tests/unit/test_gitlab_auth.py @@ -2,6 +2,7 @@ import pytest import requests from gitlab import Gitlab +from gitlab.config import GitlabConfigParser def test_invalid_auth_args(): @@ -83,3 +84,116 @@ def test_http_auth(): assert isinstance(gl._http_auth, requests.auth.HTTPBasicAuth) assert gl.headers["PRIVATE-TOKEN"] == "private_token" assert "Authorization" not in gl.headers + + +@pytest.mark.parametrize( + "options,config,expected_private_token,expected_oauth_token,expected_job_token", + [ + ( + { + "private_token": "options-private-token", + "oauth_token": "options-oauth-token", + "job_token": "options-job-token", + }, + { + "private_token": "config-private-token", + "oauth_token": "config-oauth-token", + "job_token": "config-job-token", + }, + "options-private-token", + None, + None, + ), + ( + { + "private_token": None, + "oauth_token": "options-oauth-token", + "job_token": "options-job-token", + }, + { + "private_token": "config-private-token", + "oauth_token": "config-oauth-token", + "job_token": "config-job-token", + }, + "config-private-token", + None, + None, + ), + ( + { + "private_token": None, + "oauth_token": None, + "job_token": "options-job-token", + }, + { + "private_token": "config-private-token", + "oauth_token": "config-oauth-token", + "job_token": "config-job-token", + }, + "config-private-token", + None, + None, + ), + ( + { + "private_token": None, + "oauth_token": None, + "job_token": None, + }, + { + "private_token": "config-private-token", + "oauth_token": "config-oauth-token", + "job_token": "config-job-token", + }, + "config-private-token", + None, + None, + ), + ( + { + "private_token": None, + "oauth_token": None, + "job_token": None, + }, + { + "private_token": None, + "oauth_token": "config-oauth-token", + "job_token": "config-job-token", + }, + None, + "config-oauth-token", + None, + ), + ( + { + "private_token": None, + "oauth_token": None, + "job_token": None, + }, + { + "private_token": None, + "oauth_token": None, + "job_token": "config-job-token", + }, + None, + None, + "config-job-token", + ), + ], +) +def test_merge_auth( + options, + config, + expected_private_token, + expected_oauth_token, + expected_job_token, +): + cp = GitlabConfigParser() + cp.private_token = config["private_token"] + cp.oauth_token = config["oauth_token"] + cp.job_token = config["job_token"] + + private_token, oauth_token, job_token = Gitlab._merge_auth(options, cp) + assert private_token == expected_private_token + assert oauth_token == expected_oauth_token + assert job_token == expected_job_token |