feat(cli): allow options from args and environment variablesfeat/merge-cli-env-file-config

BREAKING-CHANGE: The gitlab CLI will now accept CLI arguments
and environment variables for its global options in addition
to configuration file options. This may change behavior for
some workflows such as running inside GitLab CI and with
certain environment variables configured.

7 files changed, 421 insertions, 25 deletions

diff --git a/docs/cli-usage.rst b/docs/cli-usage.rst
index 50fac6d..6dbce5d 100644
--- a/docs/cli-usage.rst
+++ b/docs/cli-usage.rst
@@ -3,17 +3,60 @@
 ####################
 
 ``python-gitlab`` provides a :command:`gitlab` command-line tool to interact
-with GitLab servers. It uses a configuration file to define how to connect to
-the servers. Without a configuration file, ``gitlab`` will default to
-https://gitlab.com and unauthenticated requests.
+with GitLab servers.
+
+This is especially convenient for running quick ad-hoc commands locally, easily
+interacting with the API inside GitLab CI, or with more advanced shell scripting
+when integrating with other tooling.
 
 .. _cli_configuration:
 
 Configuration
 =============
 
-Files
------
+``gitlab`` allows setting configuration options via command-line arguments,
+environment variables, and configuration files.
+
+For a complete list of global CLI options and their environment variable
+equivalents, see :doc:`/cli-objects`.
+
+With no configuration provided, ``gitlab`` will default to unauthenticated
+requests against `GitLab.com <https://gitlab.com>`__.
+
+With no configuration but running inside a GitLab CI job, it will default to
+authenticated requests using the current job token against the current instance
+(via ``CI_SERVER_URL`` and ``CI_JOB_TOKEN`` environment variables).
+
+.. warning::
+   Please note the job token has very limited permissions and can only be used
+   with certain endpoints. You may need to provide a personal access token instead.
+
+When you provide configuration, values are evaluated with the following precedence:
+
+1. Explicitly provided CLI arguments,
+2. Environment variables,
+3. Configuration files:
+
+   a. explicitly defined config files:
+
+      i. via the ``--config-file`` CLI argument,
+      ii. via the ``PYTHON_GITLAB_CFG`` environment variable,
+
+   b. user-specific config file,
+   c. system-level config file,
+
+4. Environment variables always present in CI (``CI_SERVER_URL``, ``CI_JOB_TOKEN``).
+
+Additionally, authentication will take the following precedence
+when multiple options or environment variables are present:
+
+1. Private token,
+2. OAuth token,
+3. CI job token.
+
+
+Configuration files
+-------------------
 
 ``gitlab`` looks up 3 configuration files by default:
 
@@ -35,8 +78,8 @@ You can use a different configuration file with the ``--config-file`` option.
     If the environment variable is defined and the target file cannot be accessed,
     ``gitlab`` will fail explicitly.
 
-Content
--------
+Configuration file format
+-------------------------
 
 The configuration file uses the ``INI`` format. It contains at least a
 ``[global]`` section, and a specific section for each GitLab server. For
diff --git a/gitlab/cli.py b/gitlab/cli.py
index c1a1334..a48b53b 100644
--- a/gitlab/cli.py
+++ b/gitlab/cli.py
@@ -19,6 +19,7 @@
 
 import argparse
 import functools
+import os
 import re
 import sys
 from types import ModuleType
@@ -112,17 +113,25 @@ def _get_base_parser(add_help: bool = True) -> argparse.ArgumentParser:
         "-v",
         "--verbose",
         "--fancy",
-        help="Verbose mode (legacy format only)",
+        help="Verbose mode (legacy format only) [env var: GITLAB_VERBOSE]",
         action="store_true",
+        default=os.getenv("GITLAB_VERBOSE"),
     )
     parser.add_argument(
-        "-d", "--debug", help="Debug mode (display HTTP requests)", action="store_true"
+        "-d",
+        "--debug",
+        help="Debug mode (display HTTP requests) [env var: GITLAB_DEBUG]",
+        action="store_true",
+        default=os.getenv("GITLAB_DEBUG"),
     )
     parser.add_argument(
         "-c",
         "--config-file",
         action="append",
-        help="Configuration file to use. Can be used multiple times.",
+        help=(
+            "Configuration file to use. Can be used multiple times. "
+            "[env var: PYTHON_GITLAB_CFG]"
+        ),
     )
     parser.add_argument(
         "-g",
@@ -151,7 +160,86 @@ def _get_base_parser(add_help: bool = True) -> argparse.ArgumentParser:
         ),
         required=False,
     )
+    parser.add_argument(
+        "--server-url",
+        help=("GitLab server URL [env var: GITLAB_URL]"),
+        required=False,
+        default=os.getenv("GITLAB_URL"),
+    )
+    parser.add_argument(
+        "--ssl-verify",
+        help=(
+            "Whether SSL certificates should be validated. [env var: GITLAB_SSL_VERIFY]"
+        ),
+        required=False,
+        default=os.getenv("GITLAB_SSL_VERIFY"),
+    )
+    parser.add_argument(
+        "--timeout",
+        help=(
+            "Timeout to use for requests to the GitLab server. "
+            "[env var: GITLAB_TIMEOUT]"
+        ),
+        required=False,
+        default=os.getenv("GITLAB_TIMEOUT"),
+    )
+    parser.add_argument(
+        "--api-version",
+        help=("GitLab API version [env var: GITLAB_API_VERSION]"),
+        required=False,
+        default=os.getenv("GITLAB_API_VERSION"),
+    )
+    parser.add_argument(
+        "--per-page",
+        help=(
+            "Number of entries to return per page in the response. "
+            "[env var: GITLAB_PER_PAGE]"
+        ),
+        required=False,
+        default=os.getenv("GITLAB_PER_PAGE"),
+    )
+    parser.add_argument(
+        "--pagination",
+        help=(
+            "Whether to use keyset or offset pagination [env var: GITLAB_PAGINATION]"
+        ),
+        required=False,
+        default=os.getenv("GITLAB_PAGINATION"),
+    )
+    parser.add_argument(
+        "--order-by",
+        help=("Set order_by globally [env var: GITLAB_ORDER_BY]"),
+        required=False,
+        default=os.getenv("GITLAB_ORDER_BY"),
+    )
+    parser.add_argument(
+        "--user-agent",
+        help=(
+            "The user agent to send to GitLab with the HTTP request. "
+            "[env var: GITLAB_USER_AGENT]"
+        ),
+        required=False,
+        default=os.getenv("GITLAB_USER_AGENT"),
+    )
 
+    tokens = parser.add_mutually_exclusive_group()
+    tokens.add_argument(
+        "--private-token",
+        help=("GitLab private access token [env var: GITLAB_PRIVATE_TOKEN]"),
+        required=False,
+        default=os.getenv("GITLAB_PRIVATE_TOKEN"),
+    )
+    tokens.add_argument(
+        "--oauth-token",
+        help=("GitLab OAuth token [env var: GITLAB_OAUTH_TOKEN]"),
+        required=False,
+        default=os.getenv("GITLAB_OAUTH_TOKEN"),
+    )
+    tokens.add_argument(
+        "--job-token",
+        help=("GitLab CI job token [env var: CI_JOB_TOKEN]"),
+        required=False,
+    )
     return parser
 
 
@@ -243,13 +331,23 @@ def main() -> None:
         "whaction",
         "version",
         "output",
+        "fields",
+        "server_url",
+        "ssl_verify",
+        "timeout",
+        "api_version",
+        "pagination",
+        "user_agent",
+        "private_token",
+        "oauth_token",
+        "job_token",
     ):
         args_dict.pop(item)
     args_dict = {k: _parse_value(v) for k, v in args_dict.items() if v is not None}
 
     try:
-        gl = gitlab.Gitlab.from_config(gitlab_id, config_files)
-        if gl.private_token or gl.oauth_token or gl.job_token:
+        gl = gitlab.Gitlab.merge_config(vars(options), gitlab_id, config_files)
+        if gl.private_token or gl.oauth_token:
             gl.auth()
     except Exception as e:
         die(str(e))
diff --git a/gitlab/client.py b/gitlab/client.py
index c1e0825..b791c8f 100644
--- a/gitlab/client.py
+++ b/gitlab/client.py
@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Wrapper for the GitLab API."""
 
+import os
 import time
 from typing import Any, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union
 
@@ -256,6 +257,87 @@ class Gitlab(object):
             retry_transient_errors=config.retry_transient_errors,
         )
 
+    @classmethod
+    def merge_config(
+        cls,
+        options: dict,
+        gitlab_id: Optional[str] = None,
+        config_files: Optional[List[str]] = None,
+    ) -> "Gitlab":
+        """Create a Gitlab connection by merging configuration with
+        the following precedence:
+
+        1. Explicitly provided CLI arguments,
+        2. Environment variables,
+        3. Configuration files:
+            a. explicitly defined config files:
+                i. via the `--config-file` CLI argument,
+                ii. via the `PYTHON_GITLAB_CFG` environment variable,
+            b. user-specific config file,
+            c. system-level config file,
+        4. Environment variables always present in CI (CI_SERVER_URL, CI_JOB_TOKEN).
+
+        Args:
+            options: A dictionary of explicitly provided key-value options.
+            gitlab_id: ID of the configuration section.
+            config_files: List of paths to configuration files.
+        Returns:
+            (gitlab.Gitlab): A Gitlab connection.
+
+        Raises:
+            gitlab.config.GitlabDataError: If the configuration is not correct.
+        """
+        config = gitlab.config.GitlabConfigParser(
+            gitlab_id=gitlab_id, config_files=config_files
+        )
+        url = (
+            options.get("server_url")
+            or config.url
+            or os.getenv("CI_SERVER_URL")
+            or gitlab.const.DEFAULT_URL
+        )
+        private_token, oauth_token, job_token = cls._merge_auth(options, config)
+
+        return cls(
+            url=url,
+            private_token=private_token,
+            oauth_token=oauth_token,
+            job_token=job_token,
+            ssl_verify=options.get("ssl_verify") or config.ssl_verify,
+            timeout=options.get("timeout") or config.timeout,
+            api_version=options.get("api_version") or config.api_version,
+            per_page=options.get("per_page") or config.per_page,
+            pagination=options.get("pagination") or config.pagination,
+            order_by=options.get("order_by") or config.order_by,
+            user_agent=options.get("user_agent") or config.user_agent,
+        )
+
+    @staticmethod
+    def _merge_auth(options: dict, config: gitlab.config.GitlabConfigParser) -> Tuple:
+        """
+        Return a tuple where at most one of 3 token types ever has a value.
+        Since multiple types of tokens may be present in the environment,
+        options, or config files, this precedence ensures we don't
+        inadvertently cause errors when initializing the client.
+
+        This is especially relevant when executed in CI where user and
+        CI-provided values are both available.
+        """
+        private_token = options.get("private_token") or config.private_token
+        oauth_token = options.get("oauth_token") or config.oauth_token
+        job_token = (
+            options.get("job_token") or config.job_token or os.getenv("CI_JOB_TOKEN")
+        )
+
+        if private_token:
+            return (private_token, None, None)
+        if oauth_token:
+            return (None, oauth_token, None)
+        if job_token:
+            return (None, None, job_token)
+
+        return (None, None, None)
+
     def auth(self) -> None:
         """Performs an authentication using private token.
 
diff --git a/gitlab/config.py b/gitlab/config.py
index 154f063..c11a4e9 100644
--- a/gitlab/config.py
+++ b/gitlab/config.py
@@ -23,7 +23,7 @@ from os.path import expanduser, expandvars
 from pathlib import Path
 from typing import List, Optional, Union
 
-from gitlab.const import DEFAULT_URL, USER_AGENT
+from gitlab.const import USER_AGENT
 
 _DEFAULT_FILES: List[str] = [
     "/etc/python-gitlab.cfg",
@@ -119,7 +119,7 @@ class GitlabConfigParser(object):
         self.retry_transient_errors: bool = False
         self.ssl_verify: Union[bool, str] = True
         self.timeout: int = 60
-        self.url: str = DEFAULT_URL
+        self.url: Optional[str] = None
         self.user_agent: str = USER_AGENT
 
         self._files = _get_config_files(config_files)
diff --git a/tests/functional/cli/test_cli.py b/tests/functional/cli/test_cli.py
index b9a0e67..eb27cb7 100644
--- a/tests/functional/cli/test_cli.py
+++ b/tests/functional/cli/test_cli.py
@@ -1,22 +1,31 @@
+"""
+Some test cases are run in-process to intercept requests to gitlab.com
+and example servers.
+"""
+
+import copy
 import json
 
 import pytest
 import responses
 
 from gitlab import __version__, config
+from gitlab.const import DEFAULT_URL
+
+PRIVATE_TOKEN = "glpat-abc123"
+CI_JOB_TOKEN = "ci-job-token"
+CI_SERVER_URL = "https://gitlab.example.com"
 
 
 @pytest.fixture
 def resp_get_project():
-    with responses.RequestsMock() as rsps:
-        rsps.add(
-            method=responses.GET,
-            url="https://gitlab.com/api/v4/projects/1",
-            json={"name": "name", "path": "test-path", "id": 1},
-            content_type="application/json",
-            status=200,
-        )
-        yield rsps
+    return {
+        "method": responses.GET,
+        "url": f"{DEFAULT_URL}/api/v4/projects/1",
+        "json": {"name": "name", "path": "test-path", "id": 1},
+        "content_type": "application/json",
+        "status": 200,
+    }
 
 
 def test_main_entrypoint(script_runner, gitlab_config):
@@ -30,17 +39,67 @@ def test_version(script_runner):
 
 
 @pytest.mark.script_launch_mode("inprocess")
+@responses.activate
 def test_defaults_to_gitlab_com(script_runner, resp_get_project, monkeypatch):
+    responses.add(**resp_get_project)
     with monkeypatch.context() as m:
         # Ensure we don't pick up any config files that may already exist in the local
         # environment.
         m.setattr(config, "_DEFAULT_FILES", [])
-        # Runs in-process to intercept requests to gitlab.com
         ret = script_runner.run("gitlab", "project", "get", "--id", "1")
     assert ret.success
     assert "id: 1" in ret.stdout
 
 
+@pytest.mark.script_launch_mode("inprocess")
+@responses.activate
+def test_uses_ci_server_url(monkeypatch, script_runner, resp_get_project):
+    monkeypatch.setenv("CI_SERVER_URL", CI_SERVER_URL)
+    resp_get_project_in_ci = copy.deepcopy(resp_get_project)
+    resp_get_project_in_ci.update(url=f"{CI_SERVER_URL}/api/v4/projects/1")
+
+    responses.add(**resp_get_project_in_ci)
+    ret = script_runner.run("gitlab", "project", "get", "--id", "1")
+    assert ret.success
+
+
+@pytest.mark.script_launch_mode("inprocess")
+@responses.activate
+def test_uses_ci_job_token(monkeypatch, script_runner, resp_get_project):
+    monkeypatch.setenv("CI_JOB_TOKEN", CI_JOB_TOKEN)
+    resp_get_project_in_ci = copy.deepcopy(resp_get_project)
+    resp_get_project_in_ci.update(
+        match=[responses.matchers.header_matcher({"JOB-TOKEN": CI_JOB_TOKEN})],
+    )
+
+    responses.add(**resp_get_project_in_ci)
+    ret = script_runner.run("gitlab", "project", "get", "--id", "1")
+    assert ret.success
+
+
+@pytest.mark.script_launch_mode("inprocess")
+@responses.activate
+def test_private_token_overrides_job_token(
+    monkeypatch, script_runner, resp_get_project
+):
+    monkeypatch.setenv("GITLAB_PRIVATE_TOKEN", PRIVATE_TOKEN)
+    monkeypatch.setenv("CI_JOB_TOKEN", CI_JOB_TOKEN)
+
+    resp_get_project_with_token = copy.deepcopy(resp_get_project)
+    resp_get_project_with_token.update(
+        match=[responses.matchers.header_matcher({"PRIVATE-TOKEN": PRIVATE_TOKEN})],
+    )
+
+    # CLI first calls .auth() when private token is present
+    resp_auth_with_token = copy.deepcopy(resp_get_project_with_token)
+    resp_auth_with_token.update(url=f"{DEFAULT_URL}/api/v4/user")
+
+    responses.add(**resp_get_project_with_token)
+    responses.add(**resp_auth_with_token)
+    ret = script_runner.run("gitlab", "project", "get", "--id", "1")
+    assert ret.success
+
+
 def test_env_config_missing_file_raises(script_runner, monkeypatch):
     monkeypatch.setenv("PYTHON_GITLAB_CFG", "non-existent")
     ret = script_runner.run("gitlab", "project", "list")
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 6874e94..7ba312b 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -150,7 +150,7 @@ def test_default_config(mock_clean_env, monkeypatch):
     assert cp.retry_transient_errors is False
     assert cp.ssl_verify is True
     assert cp.timeout == 60
-    assert cp.url == const.DEFAULT_URL
+    assert cp.url is None
     assert cp.user_agent == const.USER_AGENT
 
 
diff --git a/tests/unit/test_gitlab_auth.py b/tests/unit/test_gitlab_auth.py
index 314fbed..8d6677f 100644
--- a/tests/unit/test_gitlab_auth.py
+++ b/tests/unit/test_gitlab_auth.py
@@ -2,6 +2,7 @@ import pytest
 import requests
 
 from gitlab import Gitlab
+from gitlab.config import GitlabConfigParser
 
 
 def test_invalid_auth_args():
@@ -83,3 +84,116 @@ def test_http_auth():
     assert isinstance(gl._http_auth, requests.auth.HTTPBasicAuth)
     assert gl.headers["PRIVATE-TOKEN"] == "private_token"
     assert "Authorization" not in gl.headers
+
+
+@pytest.mark.parametrize(
+    "options,config,expected_private_token,expected_oauth_token,expected_job_token",
+    [
+        (
+            {
+                "private_token": "options-private-token",
+                "oauth_token": "options-oauth-token",
+                "job_token": "options-job-token",
+            },
+            {
+                "private_token": "config-private-token",
+                "oauth_token": "config-oauth-token",
+                "job_token": "config-job-token",
+            },
+            "options-private-token",
+            None,
+            None,
+        ),
+        (
+            {
+                "private_token": None,
+                "oauth_token": "options-oauth-token",
+                "job_token": "options-job-token",
+            },
+            {
+                "private_token": "config-private-token",
+                "oauth_token": "config-oauth-token",
+                "job_token": "config-job-token",
+            },
+            "config-private-token",
+            None,
+            None,
+        ),
+        (
+            {
+                "private_token": None,
+                "oauth_token": None,
+                "job_token": "options-job-token",
+            },
+            {
+                "private_token": "config-private-token",
+                "oauth_token": "config-oauth-token",
+                "job_token": "config-job-token",
+            },
+            "config-private-token",
+            None,
+            None,
+        ),
+        (
+            {
+                "private_token": None,
+                "oauth_token": None,
+                "job_token": None,
+            },
+            {
+                "private_token": "config-private-token",
+                "oauth_token": "config-oauth-token",
+                "job_token": "config-job-token",
+            },
+            "config-private-token",
+            None,
+            None,
+        ),
+        (
+            {
+                "private_token": None,
+                "oauth_token": None,
+                "job_token": None,
+            },
+            {
+                "private_token": None,
+                "oauth_token": "config-oauth-token",
+                "job_token": "config-job-token",
+            },
+            None,
+            "config-oauth-token",
+            None,
+        ),
+        (
+            {
+                "private_token": None,
+                "oauth_token": None,
+                "job_token": None,
+            },
+            {
+                "private_token": None,
+                "oauth_token": None,
+                "job_token": "config-job-token",
+            },
+            None,
+            None,
+            "config-job-token",
+        ),
+    ],
+)
+def test_merge_auth(
+    options,
+    config,
+    expected_private_token,
+    expected_oauth_token,
+    expected_job_token,
+):
+    cp = GitlabConfigParser()
+    cp.private_token = config["private_token"]
+    cp.oauth_token = config["oauth_token"]
+    cp.job_token = config["job_token"]
+
+    private_token, oauth_token, job_token = Gitlab._merge_auth(options, cp)
+    assert private_token == expected_private_token
+    assert oauth_token == expected_oauth_token
+    assert job_token == expected_job_token