diff options
author | Valentin Grégoire <32612304+valentingregoire@users.noreply.github.com> | 2023-02-01 17:58:28 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-01 17:58:28 +0100 |
commit | 3cfd3903757bf61386972a18f3225665145324eb (patch) | |
tree | 0c5ef9c929db30986b32fc5814ae7bda18446d12 | |
parent | ee5f444b16e4d2f645499ac06f5d81f22867f050 (diff) | |
parent | a208276c83006ddddfbd38bc46883af078e6b215 (diff) | |
download | gitlab-3cfd3903757bf61386972a18f3225665145324eb.tar.gz |
Merge branch 'main' into typos
44 files changed, 1082 insertions, 160 deletions
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 47e6c34..0a6df72 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,9 +22,9 @@ jobs: sphinx: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 - name: Set up Python - uses: actions/setup-python@v4.3.0 + uses: actions/setup-python@v4.5.0 with: python-version: "3.11" - name: Install dependencies @@ -34,7 +34,7 @@ jobs: TOXENV: docs run: tox - name: Archive generated docs - uses: actions/upload-artifact@v3.1.1 + uses: actions/upload-artifact@v3.1.2 with: name: html-docs path: build/sphinx/html/ @@ -42,9 +42,9 @@ jobs: twine-check: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 - name: Set up Python - uses: actions/setup-python@v4.3.0 + uses: actions/setup-python@v4.5.0 with: python-version: "3.11" - name: Install dependencies diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d36ae89..1d68d55 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,10 +22,10 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 with: fetch-depth: 0 - - uses: actions/setup-python@v4.3.0 + - uses: actions/setup-python@v4.5.0 with: python-version: "3.11" - run: pip install --upgrade tox diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 4389c44..202f439 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -15,6 +15,6 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v3 + - uses: dessant/lock-threads@v4.0.0 with: process-only: 'issues' diff --git a/.github/workflows/pre_commit.yml b/.github/workflows/pre_commit.yml index 70aa361..ab88414 100644 --- a/.github/workflows/pre_commit.yml +++ b/.github/workflows/pre_commit.yml @@ -29,8 +29,8 @@ jobs: pre_commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.1.0 - - uses: actions/setup-python@v4.3.0 + - uses: actions/checkout@v3.3.0 + - uses: actions/setup-python@v4.5.0 with: python-version: "3.11" - name: install tox diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35b6701..c53e065 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,12 +10,12 @@ jobs: if: github.repository == 'python-gitlab/python-gitlab' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 with: fetch-depth: 0 token: ${{ secrets.RELEASE_GITHUB_TOKEN }} - name: Python Semantic Release - uses: relekang/python-semantic-release@v7.32.2 + uses: relekang/python-semantic-release@v7.33.0 with: github_token: ${{ secrets.RELEASE_GITHUB_TOKEN }} pypi_token: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0dbcf48..a6bd1e4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -15,7 +15,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v6.0.1 + - uses: actions/stale@v7.0.0 with: any-of-labels: 'need info,Waiting for response' stale-issue-message: > diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be264a4..c00584e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,6 +22,7 @@ jobs: unit: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest] python: @@ -45,9 +46,9 @@ jobs: version: "3.11" toxenv: py310,smoke steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 - name: Set up Python ${{ matrix.python.version }} - uses: actions/setup-python@v4.3.0 + uses: actions/setup-python@v4.5.0 with: python-version: ${{ matrix.python.version }} - name: Install dependencies @@ -63,9 +64,9 @@ jobs: matrix: toxenv: [api_func_v4, cli_func_v4] steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 - name: Set up Python - uses: actions/setup-python@v4.3.0 + uses: actions/setup-python@v4.5.0 with: python-version: "3.11" - name: Install dependencies @@ -84,9 +85,9 @@ jobs: coverage: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4.3.0 + uses: actions/setup-python@v4.5.0 with: python-version: "3.11" - name: Install dependencies @@ -125,12 +126,12 @@ jobs: runs-on: ubuntu-latest needs: [dist] steps: - - uses: actions/checkout@v3.1.0 + - uses: actions/checkout@v3.3.0 - name: Set up Python - uses: actions/setup-python@v4.3.0 + uses: actions/setup-python@v4.5.0 with: python-version: '3.11' - - uses: actions/download-artifact@v3.0.1 + - uses: actions/download-artifact@v3.0.2 with: name: dist path: dist diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67f1807..08e0b9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,24 +3,24 @@ default_language_version: repos: - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 22.12.0 hooks: - id: black - repo: https://github.com/commitizen-tools/commitizen - rev: v2.37.1 + rev: v2.40.0 hooks: - id: commitizen stages: [commit-msg] - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 + rev: 6.0.0 hooks: - id: flake8 - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pycqa/pylint - rev: v2.15.7 + rev: v2.15.10 hooks: - id: pylint additional_dependencies: @@ -41,12 +41,12 @@ repos: - types-requests==2.28.11.2 - types-setuptools==64.0.1 - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 + rev: v1.10.0 hooks: - id: rst-backticks - id: rst-directive-colons - id: rst-inline-touching-normal - repo: https://github.com/maxbrunet/pre-commit-renovate - rev: 34.48.0 + rev: 34.117.1 hooks: - id: renovate-config-validator diff --git a/CHANGELOG.md b/CHANGELOG.md index 27c81ad..1a28de6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ <!--next-version-placeholder--> +## v3.13.0 (2023-01-30) +### Feature +* **group:** Add support for group restore API ([`9322db6`](https://github.com/python-gitlab/python-gitlab/commit/9322db663ecdaecf399e3192810d973c6a9a4020)) +* **client:** Automatically retry on HTTP 409 Resource lock ([`dced76a`](https://github.com/python-gitlab/python-gitlab/commit/dced76a9900c626c9f0b90b85a5e371101a24fb4)) +* **api:** Add support for bulk imports API ([`043de2d`](https://github.com/python-gitlab/python-gitlab/commit/043de2d265e0e5114d1cd901f82869c003413d9b)) +* **api:** Add support for resource groups ([`5f8b8f5`](https://github.com/python-gitlab/python-gitlab/commit/5f8b8f5be901e944dfab2257f9e0cc4b2b1d2cd5)) +* **api:** Support listing pipelines triggered by pipeline schedules ([`865fa41`](https://github.com/python-gitlab/python-gitlab/commit/865fa417a20163b526596549b9afbce679fc2817)) +* Allow filtering pipelines by source ([`b6c0872`](https://github.com/python-gitlab/python-gitlab/commit/b6c08725042380d20ef5f09979bc29f2f6c1ab6f)) +* **client:** Bootstrap the http backends concept ([#2391](https://github.com/python-gitlab/python-gitlab/issues/2391)) ([`91a665f`](https://github.com/python-gitlab/python-gitlab/commit/91a665f331c3ffc260db3470ad71fde0d3b56aa2)) +* Add resource iteration events (see https://docs.gitlab.com/ee/api/resource_iteration_events.html) ([`ef5feb4`](https://github.com/python-gitlab/python-gitlab/commit/ef5feb4d07951230452a2974da729a958bdb9d6a)) +* Allow passing kwargs to Gitlab class when instantiating with `from_config` ([#2392](https://github.com/python-gitlab/python-gitlab/issues/2392)) ([`e88d34e`](https://github.com/python-gitlab/python-gitlab/commit/e88d34e38dd930b00d7bb48f0e1c39420e09fa0f)) +* Add keep_base_url when getting configuration from file ([`50a0301`](https://github.com/python-gitlab/python-gitlab/commit/50a03017f2ba8ec3252911dd1cf0ed7df42cfe50)) + +### Fix +* **client:** Regression - do not automatically get_next if page=# and ([`585e3a8`](https://github.com/python-gitlab/python-gitlab/commit/585e3a86c4cafa9ee73ed38676a78f3c34dbe6b2)) +* Change return value to "None" in case getattr returns None to prevent error ([`3f86d36`](https://github.com/python-gitlab/python-gitlab/commit/3f86d36218d80b293b346b37f8be5efa6455d10c)) +* **deps:** Bump requests-toolbelt to fix deprecation warning ([`faf842e`](https://github.com/python-gitlab/python-gitlab/commit/faf842e97d4858ff5ebd8ae6996e0cb3ca29881c)) +* Use the ProjectIterationManager within the Project object ([`44f05dc`](https://github.com/python-gitlab/python-gitlab/commit/44f05dc017c5496e14db82d9650c6a0110b95cf9)) +* **api:** Make description optional for releases ([`5579750`](https://github.com/python-gitlab/python-gitlab/commit/5579750335245011a3acb9456cb488f0fa1cda61)) + +### Documentation +* **faq:** Describe and group common errors ([`4c9a072`](https://github.com/python-gitlab/python-gitlab/commit/4c9a072b053f12f8098e4ea6fc47e3f6ab4f8b07)) + ## v3.12.0 (2022-11-28) ### Feature * Add support for SAML group links ([#2367](https://github.com/python-gitlab/python-gitlab/issues/2367)) ([`1020ce9`](https://github.com/python-gitlab/python-gitlab/commit/1020ce965ff0cd3bfc283d4f0ad40e41e4d1bcee)) diff --git a/docs/api-objects.rst b/docs/api-objects.rst index cccf1c6..8c7bb19 100644 --- a/docs/api-objects.rst +++ b/docs/api-objects.rst @@ -11,6 +11,7 @@ API examples gl_objects/emojis gl_objects/badges gl_objects/branches + gl_objects/bulk_imports gl_objects/messages gl_objects/ci_lint gl_objects/commits @@ -51,6 +52,7 @@ API examples gl_objects/remote_mirrors gl_objects/repositories gl_objects/repository_tags + gl_objects/resource_groups gl_objects/search gl_objects/secure_files gl_objects/settings diff --git a/docs/api-usage-advanced.rst b/docs/api-usage-advanced.rst index 2b069b8..dcc27ca 100644 --- a/docs/api-usage-advanced.rst +++ b/docs/api-usage-advanced.rst @@ -123,9 +123,16 @@ GitLab server can sometimes return a transient HTTP error. python-gitlab can automatically retry in such case, when ``retry_transient_errors`` argument is set to ``True``. When enabled, HTTP error codes 500 (Internal Server Error), 502 (502 Bad Gateway), -503 (Service Unavailable), and 504 (Gateway Timeout) are retried. It will retry until reaching -the ``max_retries`` value. By default, ``retry_transient_errors`` is set to ``False`` and an exception -is raised for these errors. +503 (Service Unavailable), 504 (Gateway Timeout), and Cloudflare +errors (520-530) are retried. + +Additionally, HTTP error code 409 (Conflict) is retried if the reason +is a +`Resource lock <https://gitlab.com/gitlab-org/gitlab/-/blob/443c12cf3b238385db728f03b2cdbb4f17c70292/lib/api/api.rb#L111>`__. + +It will retry until reaching the ``max_retries`` +value. By default, ``retry_transient_errors`` is set to ``False`` and an +exception is raised for these errors. .. code-block:: python diff --git a/docs/api-usage.rst b/docs/api-usage.rst index 444064b..2e7f5c6 100644 --- a/docs/api-usage.rst +++ b/docs/api-usage.rst @@ -171,6 +171,8 @@ conflict with python or python-gitlab when using them as kwargs: gl.user_activities.list(query_parameters={'from': '2019-01-01'}, iterator=True) # OK +.. _objects: + Gitlab Objects ============== @@ -220,21 +222,16 @@ the value on the object is accepted: issue.my_super_awesome_feature_flag = "random_value" issue.save() +As a dictionary +--------------- + You can get a dictionary representation copy of the Gitlab Object. Modifications made to the dictionary will have no impact on the GitLab Object. - * ``asdict()`` method. Returns a dictionary representation of the Gitlab object. - * ``attributes`` property. Returns a dictionary representation of the Gitlab +* ``asdict()`` method. Returns a dictionary representation of the Gitlab object. +* ``attributes`` property. Returns a dictionary representation of the Gitlab object. Also returns any relevant parent object attributes. -.. note:: - - ``attributes`` returns the parent object attributes that are defined in - ``object._from_parent_attrs``. What this can mean is that for example a ``ProjectIssue`` - object will have a ``project_id`` key in the dictionary returned from ``attributes`` but - ``asdict()`` will not. - - .. code-block:: python project = gl.projects.get(1) @@ -244,6 +241,22 @@ the dictionary will have no impact on the GitLab Object. issue = project.issues.get(1) attribute_dict = issue.attributes + # The following will return the same value + title = issue.title + title = issue.attributes["title"] + +.. hint:: + + This can be used to access attributes that clash with python-gitlab's own methods or managers. + Note that: + + ``attributes`` returns the parent object attributes that are defined in + ``object._from_parent_attrs``. For example, a ``ProjectIssue`` object will have a + ``project_id`` key in the dictionary returned from ``attributes`` but ``asdict()`` will not. + +As JSON +------- + You can get a JSON string represenation of the Gitlab Object. For example: .. code-block:: python @@ -380,6 +393,9 @@ The generator exposes extra listing information as received from the server: Prior to python-gitlab 3.6.0 the argument ``as_list`` was used instead of ``iterator``. ``as_list=False`` is the equivalent of ``iterator=True``. +.. note:: + If ``page`` and ``iterator=True`` are used together, the latter is ignored. + Sudo ==== diff --git a/docs/faq.rst b/docs/faq.rst index 3f2ee6c..cd4734a 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -2,52 +2,79 @@ FAQ ### -I cannot edit the merge request / issue I've just retrieved - It is likely that you used a ``MergeRequest``, ``GroupMergeRequest``, - ``Issue`` or ``GroupIssue`` object. These objects cannot be edited. But you - can create a new ``ProjectMergeRequest`` or ``ProjectIssue`` object to - apply changes. For example:: +General +------- - issue = gl.issues.list()[0] - project = gl.projects.get(issue.project_id, lazy=True) - editable_issue = project.issues.get(issue.iid, lazy=True) - # you can now edit the object +I cannot edit the merge request / issue I've just retrieved. +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - See the :ref:`merge requests example <merge_requests_examples>` and the - :ref:`issues examples <issues_examples>`. +It is likely that you used a ``MergeRequest``, ``GroupMergeRequest``, +``Issue`` or ``GroupIssue`` object. These objects cannot be edited. But you +can create a new ``ProjectMergeRequest`` or ``ProjectIssue`` object to +apply changes. For example:: -.. _attribute_error_list: + issue = gl.issues.list()[0] + project = gl.projects.get(issue.project_id, lazy=True) + editable_issue = project.issues.get(issue.iid, lazy=True) + # you can now edit the object + +See the :ref:`merge requests example <merge_requests_examples>` and the +:ref:`issues examples <issues_examples>`. -I get an ``AttributeError`` when accessing attributes of an object retrieved via a ``list()`` call. - Fetching a list of objects, doesn’t always include all attributes in the - objects. To retrieve an object with all attributes use a ``get()`` call. +How can I clone the repository of a project? +"""""""""""""""""""""""""""""""""""""""""""" - Example with projects:: +python-gitlab does not provide an API to clone a project. You have to use a +git library or call the ``git`` command. - for projects in gl.projects.list(): - # Retrieve project object with all attributes - project = gl.projects.get(project.id) +The git URI is exposed in the ``ssh_url_to_repo`` attribute of ``Project`` +objects. -How can I clone the repository of a project? - python-gitlab doesn't provide an API to clone a project. You have to use a - git library or call the ``git`` command. +Example:: + + import subprocess + + project = gl.projects.create(data) # or gl.projects.get(project_id) + print(project.attributes) # displays all the attributes + git_url = project.ssh_url_to_repo + subprocess.call(['git', 'clone', git_url]) + +Not all items are returned from the API +""""""""""""""""""""""""""""""""""""""" + +If you've passed ``all=True`` (or ``--all`` via the CLI) to the API and still cannot see all items returned, +use ``get_all=True`` (or ``--get-all`` via the CLI) instead. See :ref:`pagination` for more details. + +Common errors +------------- + +.. _attribute_error_list: + +``AttributeError`` when accessing object attributes retrieved via ``list()`` +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Fetching a list of objects does not always include all attributes in the objects. +To retrieve an object with all attributes, use a ``get()`` call. + +Example with projects:: - The git URI is exposed in the ``ssh_url_to_repo`` attribute of ``Project`` - objects. + for projects in gl.projects.list(): + # Retrieve project object with all attributes + project = gl.projects.get(project.id) - Example:: +``AttributeError`` when accessing attributes after ``save()`` or ``refresh()`` +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - import subprocess +You are most likely trying to access an attribute that was not returned +by the server on the second request. Please look at the documentation in +:ref:`object_attributes` to see how to avoid this. - project = gl.projects.create(data) # or gl.projects.get(project_id) - print(project.attributes) # displays all the attributes - git_url = project.ssh_url_to_repo - subprocess.call(['git', 'clone', git_url]) +``TypeError`` when accessing object attributes +"""""""""""""""""""""""""""""""""""""""""""""" -I get an ``AttributeError`` when accessing attributes after ``save()`` or ``refresh()``. - You are most likely trying to access an attribute that was not returned - by the server on the second request. Please look at the documentation in - :ref:`object_attributes` to see how to avoid this. +When you encounter errors such as ``object is not iterable`` or ``object is not subscriptable`` +when trying to access object attributes returned from the server, you are most likely trying to +access an attribute that is shadowed by python-gitlab's own methods or managers. -I passed ``all=True`` (or ``--all`` via the CLI) to the API and I still cannot see all items returned. - Use ``get_all=True`` (or ``--get-all`` via the CLI). See :ref:`pagination` for more details. +You can use the object's ``attributes`` dictionary to access it directly instead. +See the :ref:`objects` section for more details on how attributes are exposed. diff --git a/docs/gl_objects/bulk_imports.rst b/docs/gl_objects/bulk_imports.rst new file mode 100644 index 0000000..fa386bd --- /dev/null +++ b/docs/gl_objects/bulk_imports.rst @@ -0,0 +1,82 @@ +######################### +Migrations (Bulk Imports) +######################### + +References +---------- + +* v4 API: + + + :class:`gitlab.v4.objects.BulkImport` + + :class:`gitlab.v4.objects.BulkImportManager` + + :attr:`gitlab.Gitlab.bulk_imports` + + :class:`gitlab.v4.objects.BulkImportAllEntity` + + :class:`gitlab.v4.objects.BulkImportAllEntityManager` + + :attr:`gitlab.Gitlab.bulk_import_entities` + + :class:`gitlab.v4.objects.BulkImportEntity` + + :class:`gitlab.v4.objects.BulkImportEntityManager` + + :attr:`gitlab.v4.objects.BulkImport.entities` + +* GitLab API: https://docs.gitlab.com/ee/api/bulk_imports.html + +Examples +-------- + +.. note:: + + Like the project/group imports and exports, this is an asynchronous operation and you + will need to refresh the state from the server to get an accurate migration status. See + :ref:`project_import_export` in the import/export section for more details and examples. + +Start a bulk import/migration of a group and wait for completion:: + + # Create the migration + configuration = { + "url": "https://gitlab.example.com", + "access_token": private_token, + } + entity = { + "source_full_path": "source_group", + "source_type": "group_entity", + "destination_slug": "imported-group", + "destination_namespace": "imported-namespace", + } + migration = gl.bulk_imports.create( + { + "configuration": configuration, + "entities": [entity], + } + ) + + # Wait for the 'finished' status + while migration.status != "finished": + time.sleep(1) + migration.refresh() + +List all migrations:: + + gl.bulk_imports.list() + +List the entities of all migrations:: + + gl.bulk_import_entities.list() + +Get a single migration by ID:: + + migration = gl.bulk_imports.get(123) + +List the entities of a single migration:: + + entities = migration.entities.list() + +Get a single entity of a migration by ID:: + + entity = migration.entities.get(123) + +Refresh the state of a migration or entity from the server:: + + migration.refresh() + entity.refresh() + + print(migration.status) + print(entity.status) diff --git a/docs/gl_objects/groups.rst b/docs/gl_objects/groups.rst index fafa40a..37977da 100644 --- a/docs/gl_objects/groups.rst +++ b/docs/gl_objects/groups.rst @@ -88,6 +88,11 @@ Remove a group:: # or group.delete() +Restore a Group marked for deletion (Premium only)::: + + group.restore() + + Share/unshare the group with a group:: group.share(group2.id, gitlab.const.AccessLevel.DEVELOPER) diff --git a/docs/gl_objects/pipelines_and_jobs.rst b/docs/gl_objects/pipelines_and_jobs.rst index fd293a9..1c33682 100644 --- a/docs/gl_objects/pipelines_and_jobs.rst +++ b/docs/gl_objects/pipelines_and_jobs.rst @@ -112,8 +112,8 @@ objects to get the associated project:: Reference: https://docs.gitlab.com/ee/ci/triggers/#trigger-token -Pipeline schedule -================= +Pipeline schedules +================== You can schedule pipeline runs using a cron-like syntax. Variables can be associated with the scheduled pipelines. @@ -128,7 +128,10 @@ Reference + :attr:`gitlab.v4.objects.Project.pipelineschedules` + :class:`gitlab.v4.objects.ProjectPipelineScheduleVariable` + :class:`gitlab.v4.objects.ProjectPipelineScheduleVariableManager` - + :attr:`gitlab.v4.objects.Project.pipelineschedules` + + :attr:`gitlab.v4.objects.ProjectPipelineSchedule.variables` + + :class:`gitlab.v4.objects.ProjectPipelineSchedulePipeline` + + :class:`gitlab.v4.objects.ProjectPipelineSchedulePipelineManager` + + :attr:`gitlab.v4.objects.ProjectPipelineSchedule.pipelines` * GitLab API: https://docs.gitlab.com/ce/api/pipeline_schedules.html @@ -188,6 +191,9 @@ Delete a schedule variable:: var.delete() +List all pipelines triggered by a pipeline schedule:: + + pipelines = sched.pipelines.list() Jobs ==== diff --git a/docs/gl_objects/projects.rst b/docs/gl_objects/projects.rst index 7fc0ab9..b80df0d 100644 --- a/docs/gl_objects/projects.rst +++ b/docs/gl_objects/projects.rst @@ -275,6 +275,8 @@ Reference * GitLab API: https://docs.gitlab.com/ce/api/project_import_export.html +.. _project_import_export: + Examples -------- diff --git a/docs/gl_objects/resource_groups.rst b/docs/gl_objects/resource_groups.rst new file mode 100644 index 0000000..3fa0f92 --- /dev/null +++ b/docs/gl_objects/resource_groups.rst @@ -0,0 +1,38 @@ +############### +Resource Groups +############### + +Reference +--------- + +* v4 API: + + + :class:`gitlab.v4.objects.ProjectResourceGroup` + + :class:`gitlab.v4.objects.ProjectResourceGroupManager` + + :attr:`gitlab.v4.objects.Project.resource_groups` + + :class:`gitlab.v4.objects.ProjectResourceGroupUpcomingJob` + + :class:`gitlab.v4.objects.ProjectResourceGroupUpcomingJobManager` + + :attr:`gitlab.v4.objects.ProjectResourceGroup.upcoming_jobs` + +* Gitlab API: https://docs.gitlab.com/ee/api/resource_groups.html + +Examples +-------- + +List resource groups for a project:: + + project = gl.projects.get(project_id, lazy=True) + resource_group = project.resource_groups.list() + +Get a single resource group:: + + resource_group = project.resource_groups.get("production") + +Edit a resource group:: + + resource_group.process_mode = "oldest_first" + resource_group.save() + +List upcoming jobs for a resource group:: + + upcoming_jobs = resource_group.upcoming_jobs.list() diff --git a/gitlab/_backends/__init__.py b/gitlab/_backends/__init__.py new file mode 100644 index 0000000..aa53d0a --- /dev/null +++ b/gitlab/_backends/__init__.py @@ -0,0 +1,8 @@ +""" +Defines http backends for processing http requests +""" + +from .requests_backend import RequestsBackend, RequestsResponse + +DefaultBackend = RequestsBackend +DefaultResponse = RequestsResponse diff --git a/gitlab/_backends/requests_backend.py b/gitlab/_backends/requests_backend.py new file mode 100644 index 0000000..4ea23a7 --- /dev/null +++ b/gitlab/_backends/requests_backend.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING, Union + +import requests +from requests.structures import CaseInsensitiveDict +from requests_toolbelt.multipart.encoder import MultipartEncoder # type: ignore + + +class RequestsResponse: + def __init__(self, response: requests.Response) -> None: + self._response: requests.Response = response + + @property + def response(self) -> requests.Response: + return self._response + + @property + def status_code(self) -> int: + return self._response.status_code + + @property + def headers(self) -> CaseInsensitiveDict[str]: + return self._response.headers + + @property + def content(self) -> bytes: + return self._response.content + + @property + def reason(self) -> str: + return self._response.reason + + def json(self) -> Any: + return self._response.json() + + +class RequestsBackend: + def __init__(self, session: Optional[requests.Session] = None) -> None: + self._client: requests.Session = session or requests.Session() + + @property + def client(self) -> requests.Session: + return self._client + + @staticmethod + def prepare_send_data( + files: Optional[Dict[str, Any]] = None, + post_data: Optional[Union[Dict[str, Any], bytes]] = None, + raw: bool = False, + ) -> Tuple[ + Optional[Union[Dict[str, Any], bytes]], + Optional[Union[Dict[str, Any], MultipartEncoder]], + str, + ]: + if files: + if post_data is None: + post_data = {} + else: + # booleans does not exists for data (neither for MultipartEncoder): + # cast to string int to avoid: 'bool' object has no attribute 'encode' + if TYPE_CHECKING: + assert isinstance(post_data, dict) + for k, v in post_data.items(): + if isinstance(v, bool): + post_data[k] = str(int(v)) + post_data["file"] = files.get("file") + post_data["avatar"] = files.get("avatar") + + data = MultipartEncoder(post_data) + return (None, data, data.content_type) + + if raw and post_data: + return (None, post_data, "application/octet-stream") + + return (post_data, None, "application/json") + + def http_request( + self, + method: str, + url: str, + json: Optional[Union[Dict[str, Any], bytes]] = None, + data: Optional[Union[Dict[str, Any], MultipartEncoder]] = None, + params: Optional[Any] = None, + timeout: Optional[float] = None, + verify: Optional[Union[bool, str]] = True, + stream: Optional[bool] = False, + **kwargs: Any, + ) -> RequestsResponse: + """Make HTTP request + + Args: + method: The HTTP method to call ('get', 'post', 'put', 'delete', etc.) + url: The full URL + data: The data to send to the server in the body of the request + json: Data to send in the body in json by default + timeout: The timeout, in seconds, for the request + verify: Whether SSL certificates should be validated. If + the value is a string, it is the path to a CA file used for + certificate validation. + stream: Whether the data should be streamed + + Returns: + A requests Response object. + """ + response: requests.Response = self._client.request( + method=method, + url=url, + params=params, + data=data, + timeout=timeout, + stream=stream, + verify=verify, + json=json, + **kwargs, + ) + return RequestsResponse(response=response) diff --git a/gitlab/_version.py b/gitlab/_version.py index 48a24d3..9c1e050 100644 --- a/gitlab/_version.py +++ b/gitlab/_version.py @@ -3,4 +3,4 @@ __copyright__ = "Copyright 2013-2019 Gauvain Pocentek, 2019-2022 python-gitlab t __email__ = "gauvainpocentek@gmail.com" __license__ = "LGPL3" __title__ = "python-gitlab" -__version__ = "3.12.0" +__version__ = "3.13.0" diff --git a/gitlab/client.py b/gitlab/client.py index 49882d6..5e6c71a 100644 --- a/gitlab/client.py +++ b/gitlab/client.py @@ -3,18 +3,16 @@ import os import re import time -from typing import Any, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union +from typing import Any, cast, Dict, List, Optional, Tuple, Type, TYPE_CHECKING, Union from urllib import parse import requests -import requests.utils -from requests_toolbelt.multipart.encoder import MultipartEncoder # type: ignore import gitlab import gitlab.config import gitlab.const import gitlab.exceptions -from gitlab import utils +from gitlab import _backends, utils REDIRECT_MSG = ( "python-gitlab detected a {status_code} ({reason!r}) redirection. You must update " @@ -22,7 +20,6 @@ REDIRECT_MSG = ( "{source!r} to {target!r}" ) -RETRYABLE_TRANSIENT_ERROR_CODES = [500, 502, 503, 504] + list(range(520, 531)) # https://docs.gitlab.com/ee/api/#offset-based-pagination _PAGINATION_URL = ( @@ -32,6 +29,7 @@ _PAGINATION_URL = ( class Gitlab: + """Represents a GitLab server connection. Args: @@ -53,6 +51,10 @@ class Gitlab: or 52x responses. Defaults to False. keep_base_url: keep user-provided base URL for pagination if it differs from response headers + + Keyword Args: + requests.Session session: HTTP Requests Session + RequestsBackend backend: Backend that will be used to make http requests """ def __init__( @@ -66,15 +68,14 @@ class Gitlab: http_password: Optional[str] = None, timeout: Optional[float] = None, api_version: str = "4", - session: Optional[requests.Session] = None, per_page: Optional[int] = None, pagination: Optional[str] = None, order_by: Optional[str] = None, user_agent: str = gitlab.const.USER_AGENT, retry_transient_errors: bool = False, keep_base_url: bool = False, + **kwargs: Any, ) -> None: - self._api_version = str(api_version) self._server_version: Optional[str] = None self._server_revision: Optional[str] = None @@ -98,7 +99,11 @@ class Gitlab: self._set_auth_info() #: Create a session object for requests - self.session = session or requests.Session() + _backend: Type[_backends.DefaultBackend] = kwargs.pop( + "backend", _backends.DefaultBackend + ) + self._backend = _backend(**kwargs) + self.session = self._backend.client self.per_page = per_page self.pagination = pagination @@ -116,6 +121,10 @@ class Gitlab: self.broadcastmessages = objects.BroadcastMessageManager(self) """See :class:`~gitlab.v4.objects.BroadcastMessageManager`""" + self.bulk_imports = objects.BulkImportManager(self) + """See :class:`~gitlab.v4.objects.BulkImportManager`""" + self.bulk_import_entities = objects.BulkImportAllEntityManager(self) + """See :class:`~gitlab.v4.objects.BulkImportAllEntityManager`""" self.ci_lint = objects.CiLintManager(self) """See :class:`~gitlab.v4.objects.CiLintManager`""" self.deploykeys = objects.DeployKeyManager(self) @@ -630,38 +639,6 @@ class Gitlab: ) ) - @staticmethod - def _prepare_send_data( - files: Optional[Dict[str, Any]] = None, - post_data: Optional[Union[Dict[str, Any], bytes]] = None, - raw: bool = False, - ) -> Tuple[ - Optional[Union[Dict[str, Any], bytes]], - Optional[Union[Dict[str, Any], MultipartEncoder]], - str, - ]: - if files: - if post_data is None: - post_data = {} - else: - # booleans does not exists for data (neither for MultipartEncoder): - # cast to string int to avoid: 'bool' object has no attribute 'encode' - if TYPE_CHECKING: - assert isinstance(post_data, dict) - for k, v in post_data.items(): - if isinstance(v, bool): - post_data[k] = str(int(v)) - post_data["file"] = files.get("file") - post_data["avatar"] = files.get("avatar") - - data = MultipartEncoder(post_data) - return (None, data, data.content_type) - - if raw and post_data: - return (None, post_data, "application/octet-stream") - - return (post_data, None, "application/json") - def http_request( self, verb: str, @@ -739,13 +716,15 @@ class Gitlab: retry_transient_errors = self.retry_transient_errors # We need to deal with json vs. data when uploading files - json, data, content_type = self._prepare_send_data(files, post_data, raw) + json, data, content_type = self._backend.prepare_send_data( + files, post_data, raw + ) opts["headers"]["Content-type"] = content_type cur_retries = 0 while True: try: - result = self.session.request( + result = self._backend.http_request( method=verb, url=url, json=json, @@ -767,15 +746,25 @@ class Gitlab: raise - self._check_redirects(result) + self._check_redirects(result.response) if 200 <= result.status_code < 300: - return result + return result.response + + def should_retry() -> bool: + if result.status_code == 429 and obey_rate_limit: + return True + + if not retry_transient_errors: + return False + if result.status_code in gitlab.const.RETRYABLE_TRANSIENT_ERROR_CODES: + return True + if result.status_code == 409 and "Resource lock" in result.reason: + return True + + return False - if (429 == result.status_code and obey_rate_limit) or ( - result.status_code in RETRYABLE_TRANSIENT_ERROR_CODES - and retry_transient_errors - ): + if should_retry(): # Response headers documentation: # https://docs.gitlab.com/ee/user/admin_area/settings/user_and_ip_rate_limits.html#response-headers if max_retries == -1 or cur_retries < max_retries: @@ -940,7 +929,20 @@ class Gitlab: page = kwargs.get("page") - if iterator: + if iterator and page is not None: + arg_used_message = f"iterator={iterator}" + if as_list is not None: + arg_used_message = f"as_list={as_list}" + utils.warn( + message=( + f"`{arg_used_message}` and `page={page}` were both specified. " + f"`{arg_used_message}` will be ignored and a `list` will be " + f"returned." + ), + category=UserWarning, + ) + + if iterator and page is None: # Generator requested return GitlabList(self, url, query_data, **kwargs) diff --git a/gitlab/const.py b/gitlab/const.py index 1dab752..5d3602b 100644 --- a/gitlab/const.py +++ b/gitlab/const.py @@ -131,6 +131,8 @@ SEARCH_SCOPE_PROJECT_NOTES = SearchScope.PROJECT_NOTES.value USER_AGENT: str = f"{__title__}/{__version__}" +RETRYABLE_TRANSIENT_ERROR_CODES = [500, 502, 503, 504] + list(range(520, 531)) + __all__ = [ "AccessLevel", "Visibility", diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py index 1a6ea39..41ffa1e 100644 --- a/gitlab/v4/cli.py +++ b/gitlab/v4/cli.py @@ -490,7 +490,7 @@ class LegacyPrinter: id = getattr(obj, obj._id_attr) print(f"{obj._id_attr.replace('_', '-')}: {id}") if obj._repr_attr: - value = getattr(obj, obj._repr_attr, "None") + value = getattr(obj, obj._repr_attr, "None") or "None" value = value.replace("\r", "").replace("\n", " ") # If the attribute is a note (ProjectCommitComment) then we do # some modifications to fit everything on one line diff --git a/gitlab/v4/objects/__init__.py b/gitlab/v4/objects/__init__.py index 56c17d5..bd3ae82 100644 --- a/gitlab/v4/objects/__init__.py +++ b/gitlab/v4/objects/__init__.py @@ -8,6 +8,7 @@ from .badges import * from .boards import * from .branches import * from .broadcast_messages import * +from .bulk_imports import * from .ci_lint import * from .clusters import * from .commits import * @@ -52,6 +53,7 @@ from .projects import * from .push_rules import * from .releases import * from .repositories import * +from .resource_groups import * from .runners import * from .secure_files import * from .settings import * diff --git a/gitlab/v4/objects/bulk_imports.py b/gitlab/v4/objects/bulk_imports.py new file mode 100644 index 0000000..e8ef74f --- /dev/null +++ b/gitlab/v4/objects/bulk_imports.py @@ -0,0 +1,54 @@ +from typing import Any, cast, Union + +from gitlab.base import RESTManager, RESTObject +from gitlab.mixins import CreateMixin, ListMixin, RefreshMixin, RetrieveMixin +from gitlab.types import RequiredOptional + +__all__ = [ + "BulkImport", + "BulkImportManager", + "BulkImportAllEntity", + "BulkImportAllEntityManager", + "BulkImportEntity", + "BulkImportEntityManager", +] + + +class BulkImport(RefreshMixin, RESTObject): + entities: "BulkImportEntityManager" + + +class BulkImportManager(CreateMixin, RetrieveMixin, RESTManager): + _path = "/bulk_imports" + _obj_cls = BulkImport + _create_attrs = RequiredOptional(required=("configuration", "entities")) + _list_filters = ("sort", "status") + + def get(self, id: Union[str, int], lazy: bool = False, **kwargs: Any) -> BulkImport: + return cast(BulkImport, super().get(id=id, lazy=lazy, **kwargs)) + + +class BulkImportEntity(RefreshMixin, RESTObject): + pass + + +class BulkImportEntityManager(RetrieveMixin, RESTManager): + _path = "/bulk_imports/{bulk_import_id}/entities" + _obj_cls = BulkImportEntity + _from_parent_attrs = {"bulk_import_id": "id"} + _list_filters = ("sort", "status") + + def get( + self, id: Union[str, int], lazy: bool = False, **kwargs: Any + ) -> BulkImportEntity: + return cast(BulkImportEntity, super().get(id=id, lazy=lazy, **kwargs)) + + +class BulkImportAllEntity(RESTObject): + pass + + +class BulkImportAllEntityManager(ListMixin, RESTManager): + _path = "/bulk_imports/entities" + _obj_cls = BulkImportAllEntity + _list_filters = ("sort", "status") diff --git a/gitlab/v4/objects/groups.py b/gitlab/v4/objects/groups.py index d03eb38..0eb516f 100644 --- a/gitlab/v4/objects/groups.py +++ b/gitlab/v4/objects/groups.py @@ -287,6 +287,21 @@ class Group(SaveMixin, ObjectDeleteMixin, RESTObject): path = f"/groups/{self.encoded_id}/share/{group_id}" self.manager.gitlab.http_delete(path, **kwargs) + @cli.register_custom_action("Group") + @exc.on_http_error(exc.GitlabRestoreError) + def restore(self, **kwargs: Any) -> None: + """Restore a group marked for deletion.. + + Args: + **kwargs: Extra options to send to the server (e.g. sudo) + + Raises: + GitlabAuthenticationError: If authentication is not correct + GitlabRestoreError: If the server failed to perform the request + """ + path = f"/groups/{self.encoded_id}/restore" + self.manager.gitlab.http_post(path, **kwargs) + class GroupManager(CRUDMixin, RESTManager): _path = "/groups" diff --git a/gitlab/v4/objects/pipelines.py b/gitlab/v4/objects/pipelines.py index eec46a1..7530611 100644 --- a/gitlab/v4/objects/pipelines.py +++ b/gitlab/v4/objects/pipelines.py @@ -32,6 +32,8 @@ __all__ = [ "ProjectPipelineVariableManager", "ProjectPipelineScheduleVariable", "ProjectPipelineScheduleVariableManager", + "ProjectPipelineSchedulePipeline", + "ProjectPipelineSchedulePipelineManager", "ProjectPipelineSchedule", "ProjectPipelineScheduleManager", "ProjectPipelineTestReport", @@ -96,6 +98,7 @@ class ProjectPipelineManager(RetrieveMixin, CreateMixin, DeleteMixin, RESTManage _list_filters = ( "scope", "status", + "source", "ref", "sha", "yaml_errors", @@ -183,8 +186,19 @@ class ProjectPipelineScheduleVariableManager( _update_attrs = RequiredOptional(required=("key", "value")) +class ProjectPipelineSchedulePipeline(RESTObject): + pass + + +class ProjectPipelineSchedulePipelineManager(ListMixin, RESTManager): + _path = "/projects/{project_id}/pipeline_schedules/{pipeline_schedule_id}/pipelines" + _obj_cls = ProjectPipelineSchedulePipeline + _from_parent_attrs = {"project_id": "project_id", "pipeline_schedule_id": "id"} + + class ProjectPipelineSchedule(SaveMixin, ObjectDeleteMixin, RESTObject): variables: ProjectPipelineScheduleVariableManager + pipelines: ProjectPipelineSchedulePipelineManager @cli.register_custom_action("ProjectPipelineSchedule") @exc.on_http_error(exc.GitlabOwnershipError) diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index f19b845..6330cc5 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -81,6 +81,7 @@ from .project_access_tokens import ProjectAccessTokenManager # noqa: F401 from .push_rules import ProjectPushRulesManager # noqa: F401 from .releases import ProjectReleaseManager # noqa: F401 from .repositories import RepositoryMixin +from .resource_groups import ProjectResourceGroupManager from .runners import ProjectRunnerManager # noqa: F401 from .secure_files import ProjectSecureFileManager # noqa: F401 from .snippets import ProjectSnippetManager # noqa: F401 @@ -207,6 +208,7 @@ class Project(RefreshMixin, SaveMixin, ObjectDeleteMixin, RepositoryMixin, RESTO protectedtags: ProjectProtectedTagManager pushrules: ProjectPushRulesManager releases: ProjectReleaseManager + resource_groups: ProjectResourceGroupManager remote_mirrors: "ProjectRemoteMirrorManager" repositories: ProjectRegistryRepositoryManager runners: ProjectRunnerManager @@ -665,6 +667,7 @@ class ProjectManager(CRUDMixin, RESTManager): "name", "path", "allow_merge_on_skipped_pipeline", + "only_allow_merge_if_all_status_checks_passed", "analytics_access_level", "approvals_before_merge", "auto_cancel_pending_pipelines", @@ -678,6 +681,7 @@ class ProjectManager(CRUDMixin, RESTManager): "builds_access_level", "ci_config_path", "container_expiration_policy_attributes", + "container_registry_access_level", "container_registry_enabled", "default_branch", "description", @@ -706,11 +710,17 @@ class ProjectManager(CRUDMixin, RESTManager): "requirements_access_level", "printing_merge_request_link_enabled", "public_builds", + "releases_access_level", + "environments_access_level", + "feature_flags_access_level", + "infrastructure_access_level", + "monitor_access_level", "remove_source_branch_after_merge", "repository_access_level", "repository_storage", "request_access_enabled", "resolve_outdated_diff_discussions", + "security_and_compliance_access_level", "shared_runners_enabled", "show_default_award_emojis", "snippets_access_level", @@ -731,6 +741,7 @@ class ProjectManager(CRUDMixin, RESTManager): _update_attrs = RequiredOptional( optional=( "allow_merge_on_skipped_pipeline", + "only_allow_merge_if_all_status_checks_passed", "analytics_access_level", "approvals_before_merge", "auto_cancel_pending_pipelines", @@ -745,22 +756,31 @@ class ProjectManager(CRUDMixin, RESTManager): "ci_config_path", "ci_default_git_depth", "ci_forward_deployment_enabled", + "ci_allow_fork_pipelines_to_run_in_parent_project", + "ci_separated_caches", "container_expiration_policy_attributes", + "container_registry_access_level", "container_registry_enabled", "default_branch", "description", "emails_disabled", + "enforce_auth_checks_on_uploads", "external_authorization_classification_label", "forking_access_level", "import_url", "issues_access_level", "issues_enabled", + "issues_template", "jobs_enabled", + "keep_latest_artifact", "lfs_enabled", + "merge_commit_template", "merge_method", "merge_pipelines_enabled", "merge_requests_access_level", "merge_requests_enabled", + "merge_requests_template", + "merge_trains_enabled", "mirror_overwrites_diverged_branches", "mirror_trigger_builds", "mirror_user_id", @@ -777,16 +797,24 @@ class ProjectManager(CRUDMixin, RESTManager): "restrict_user_defined_variables", "path", "public_builds", + "releases_access_level", + "environments_access_level", + "feature_flags_access_level", + "infrastructure_access_level", + "monitor_access_level", "remove_source_branch_after_merge", "repository_access_level", "repository_storage", "request_access_enabled", "resolve_outdated_diff_discussions", + "security_and_compliance_access_level", "service_desk_enabled", "shared_runners_enabled", "show_default_award_emojis", "snippets_access_level", "snippets_enabled", + "issue_branch_template", + "squash_commit_template", "squash_option", "suggestion_commit_message", "tag_list", @@ -794,8 +822,6 @@ class ProjectManager(CRUDMixin, RESTManager): "visibility", "wiki_access_level", "wiki_enabled", - "issues_template", - "merge_requests_template", ), ) _list_filters = ( diff --git a/gitlab/v4/objects/resource_groups.py b/gitlab/v4/objects/resource_groups.py new file mode 100644 index 0000000..1ca34f6 --- /dev/null +++ b/gitlab/v4/objects/resource_groups.py @@ -0,0 +1,45 @@ +from typing import Any, cast, Union + +from gitlab.base import RESTManager, RESTObject +from gitlab.mixins import ListMixin, RetrieveMixin, SaveMixin, UpdateMixin +from gitlab.types import RequiredOptional + +__all__ = [ + "ProjectResourceGroup", + "ProjectResourceGroupManager", + "ProjectResourceGroupUpcomingJob", + "ProjectResourceGroupUpcomingJobManager", +] + + +class ProjectResourceGroup(SaveMixin, RESTObject): + _id_attr = "key" + + upcoming_jobs: "ProjectResourceGroupUpcomingJobManager" + + +class ProjectResourceGroupManager(RetrieveMixin, UpdateMixin, RESTManager): + _path = "/projects/{project_id}/resource_groups" + _obj_cls = ProjectResourceGroup + _from_parent_attrs = {"project_id": "id"} + _list_filters = ( + "order_by", + "sort", + "include_html_description", + ) + _update_attrs = RequiredOptional(optional=("process_mode",)) + + def get( + self, id: Union[str, int], lazy: bool = False, **kwargs: Any + ) -> ProjectResourceGroup: + return cast(ProjectResourceGroup, super().get(id=id, lazy=lazy, **kwargs)) + + +class ProjectResourceGroupUpcomingJob(RESTObject): + pass + + +class ProjectResourceGroupUpcomingJobManager(ListMixin, RESTManager): + _path = "/projects/{project_id}/resource_groups/{resource_group_key}/upcoming_jobs" + _obj_cls = ProjectResourceGroupUpcomingJob + _from_parent_attrs = {"project_id": "project_id", "resource_group_key": "key"} diff --git a/requirements-lint.txt b/requirements-lint.txt index 1ea9dfa..86a8270 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -1,3 +1,4 @@ +-r requirements.txt argcomplete==2.0.0 black==22.10.0 commitizen==2.35.0 diff --git a/requirements-test.txt b/requirements-test.txt index bc68ef1..6d190db 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ +-r requirements.txt build==0.9.0 coverage==6.5.0 pytest-console-scripts==1.3.1 diff --git a/requirements.txt b/requirements.txt index bb79bc4..e658331 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -requests==2.28.1 +requests==2.28.2 requests-toolbelt==0.10.1 @@ -28,7 +28,7 @@ setup( license="LGPL-3.0-or-later", url="https://github.com/python-gitlab/python-gitlab", packages=find_packages(exclude=["docs*", "tests*"]), - install_requires=["requests>=2.25.0", "requests-toolbelt>=0.9.1"], + install_requires=["requests>=2.25.0", "requests-toolbelt>=0.10.1"], package_data={ "gitlab": ["py.typed"], }, diff --git a/tests/functional/api/test_bulk_imports.py b/tests/functional/api/test_bulk_imports.py new file mode 100644 index 0000000..899d358 --- /dev/null +++ b/tests/functional/api/test_bulk_imports.py @@ -0,0 +1,41 @@ +def test_bulk_imports(gl, group): + destination = f"{group.full_path}-import" + configuration = { + "url": gl.url, + "access_token": gl.private_token, + } + migration_entity = { + "source_full_path": group.full_path, + "source_type": "group_entity", + "destination_slug": destination, + "destination_namespace": destination, + } + created_migration = gl.bulk_imports.create( + { + "configuration": configuration, + "entities": [migration_entity], + } + ) + + assert created_migration.source_type == "gitlab" + assert created_migration.status == "created" + + migration = gl.bulk_imports.get(created_migration.id) + assert migration == created_migration + + migration.refresh() + assert migration == created_migration + + migrations = gl.bulk_imports.list() + assert migration in migrations + + all_entities = gl.bulk_import_entities.list() + entities = migration.entities.list() + assert isinstance(entities, list) + assert entities[0] in all_entities + + entity = migration.entities.get(entities[0].id) + assert entity == entities[0] + + entity.refresh() + assert entity.created_at == entities[0].created_at diff --git a/tests/functional/api/test_gitlab.py b/tests/functional/api/test_gitlab.py index c2aba09..ced77c2 100644 --- a/tests/functional/api/test_gitlab.py +++ b/tests/functional/api/test_gitlab.py @@ -15,11 +15,9 @@ def get_all_kwargs(request): return request.param -def test_auth_from_config(gl, temp_dir): +def test_auth_from_config(gl, gitlab_config, temp_dir): """Test token authentication from config file""" - test_gitlab = gitlab.Gitlab.from_config( - config_files=[temp_dir / "python-gitlab.cfg"] - ) + test_gitlab = gitlab.Gitlab.from_config(config_files=[gitlab_config]) test_gitlab.auth() assert isinstance(test_gitlab.user, gitlab.v4.objects.CurrentUser) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index d2ff5e0..c85b172 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -236,15 +236,13 @@ def wait_for_sidekiq(gl): @pytest.fixture(scope="session") -def gitlab_config( +def gitlab_token( check_is_alive, gitlab_container_name: str, gitlab_url: str, docker_services, - temp_dir: pathlib.Path, fixture_dir: pathlib.Path, -): - config_file = temp_dir / "python-gitlab.cfg" +) -> str: start_time = time.perf_counter() logging.info("Waiting for GitLab container to become ready.") @@ -263,7 +261,12 @@ def gitlab_config( f"GitLab container is now ready after {minutes} minute(s), {seconds} seconds" ) - token = set_token(gitlab_container_name, fixture_dir=fixture_dir) + return set_token(gitlab_container_name, fixture_dir=fixture_dir) + + +@pytest.fixture(scope="session") +def gitlab_config(gitlab_url: str, gitlab_token: str, temp_dir: pathlib.Path): + config_file = temp_dir / "python-gitlab.cfg" config = f"""[global] default = local @@ -271,7 +274,7 @@ timeout = 60 [local] url = {gitlab_url} -private_token = {token} +private_token = {gitlab_token} api_version = 4""" with open(config_file, "w", encoding="utf-8") as f: @@ -281,11 +284,11 @@ api_version = 4""" @pytest.fixture(scope="session") -def gl(gitlab_config): +def gl(gitlab_url: str, gitlab_token: str) -> gitlab.Gitlab: """Helper instance to make fixtures and asserts directly via the API.""" logging.info("Instantiating python-gitlab gitlab.Gitlab instance") - instance = gitlab.Gitlab.from_config("local", [gitlab_config]) + instance = gitlab.Gitlab(gitlab_url, private_token=gitlab_token) logging.info("Reset GitLab") reset_gitlab(instance) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 5d3bfd5..476e41c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -90,5 +90,15 @@ def release(project, tag_name): @pytest.fixture +def schedule(project): + return project.pipelineschedules.get(1, lazy=True) + + +@pytest.fixture def user(gl): return gl.users.get(1, lazy=True) + + +@pytest.fixture +def migration(gl): + return gl.bulk_imports.get(1, lazy=True) diff --git a/tests/unit/objects/test_bulk_imports.py b/tests/unit/objects/test_bulk_imports.py new file mode 100644 index 0000000..5effcdc --- /dev/null +++ b/tests/unit/objects/test_bulk_imports.py @@ -0,0 +1,159 @@ +""" +GitLab API: https://docs.gitlab.com/ce/api/bulk_imports.html +""" + +import pytest +import responses + +from gitlab.v4.objects import BulkImport, BulkImportAllEntity, BulkImportEntity + +migration_content = { + "id": 1, + "status": "finished", + "source_type": "gitlab", + "created_at": "2021-06-18T09:45:55.358Z", + "updated_at": "2021-06-18T09:46:27.003Z", +} +entity_content = { + "id": 1, + "bulk_import_id": 1, + "status": "finished", + "source_full_path": "source_group", + "destination_slug": "destination_slug", + "destination_namespace": "destination_path", + "parent_id": None, + "namespace_id": 1, + "project_id": None, + "created_at": "2021-06-18T09:47:37.390Z", + "updated_at": "2021-06-18T09:47:51.867Z", + "failures": [], +} + + +@pytest.fixture +def resp_create_bulk_import(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.POST, + url="http://localhost/api/v4/bulk_imports", + json=migration_content, + content_type="application/json", + status=201, + ) + yield rsps + + +@pytest.fixture +def resp_list_bulk_imports(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/bulk_imports", + json=[migration_content], + content_type="application/json", + status=200, + ) + yield rsps + + +@pytest.fixture +def resp_get_bulk_import(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/bulk_imports/1", + json=migration_content, + content_type="application/json", + status=200, + ) + yield rsps + + +@pytest.fixture +def resp_list_all_bulk_import_entities(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/bulk_imports/entities", + json=[entity_content], + content_type="application/json", + status=200, + ) + yield rsps + + +@pytest.fixture +def resp_list_bulk_import_entities(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/bulk_imports/1/entities", + json=[entity_content], + content_type="application/json", + status=200, + ) + yield rsps + + +@pytest.fixture +def resp_get_bulk_import_entity(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/bulk_imports/1/entities/1", + json=entity_content, + content_type="application/json", + status=200, + ) + yield rsps + + +def test_create_bulk_import(gl, resp_create_bulk_import): + configuration = { + "url": gl.url, + "access_token": "test-token", + } + migration_entity = { + "source_full_path": "source", + "source_type": "group_entity", + "destination_slug": "destination", + "destination_namespace": "destination", + } + migration = gl.bulk_imports.create( + { + "configuration": configuration, + "entities": [migration_entity], + } + ) + assert isinstance(migration, BulkImport) + assert migration.status == "finished" + + +def test_list_bulk_imports(gl, resp_list_bulk_imports): + migrations = gl.bulk_imports.list() + assert isinstance(migrations[0], BulkImport) + assert migrations[0].status == "finished" + + +def test_get_bulk_import(gl, resp_get_bulk_import): + migration = gl.bulk_imports.get(1) + assert isinstance(migration, BulkImport) + assert migration.status == "finished" + + +def test_list_all_bulk_import_entities(gl, resp_list_all_bulk_import_entities): + entities = gl.bulk_import_entities.list() + assert isinstance(entities[0], BulkImportAllEntity) + assert entities[0].bulk_import_id == 1 + + +def test_list_bulk_import_entities(gl, migration, resp_list_bulk_import_entities): + entities = migration.entities.list() + assert isinstance(entities[0], BulkImportEntity) + assert entities[0].bulk_import_id == 1 + + +def test_get_bulk_import_entity(gl, migration, resp_get_bulk_import_entity): + entity = migration.entities.get(1) + assert isinstance(entity, BulkImportEntity) + assert entity.bulk_import_id == 1 diff --git a/tests/unit/objects/test_groups.py b/tests/unit/objects/test_groups.py index 58c3508..1f25821 100644 --- a/tests/unit/objects/test_groups.py +++ b/tests/unit/objects/test_groups.py @@ -316,6 +316,19 @@ def resp_delete_saml_group_link(no_content): yield rsps +@pytest.fixture +def resp_restore_group(created_content): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.POST, + url="http://localhost/api/v4/groups/1/restore", + json=created_content, + content_type="application/json", + status=201, + ) + yield rsps + + def test_get_group(gl, resp_groups): data = gl.groups.get(1) assert isinstance(data, gitlab.v4.objects.Group) @@ -453,3 +466,7 @@ def test_create_saml_group_link(group, resp_create_saml_group_link): def test_delete_saml_group_link(group, resp_delete_saml_group_link): saml_group_link = group.saml_group_links.create(create_saml_group_link_request_body) saml_group_link.delete() + + +def test_group_restore(group, resp_restore_group): + group.restore() diff --git a/tests/unit/objects/test_pipeline_schedules.py b/tests/unit/objects/test_pipeline_schedules.py index f038756..8b8dab8 100644 --- a/tests/unit/objects/test_pipeline_schedules.py +++ b/tests/unit/objects/test_pipeline_schedules.py @@ -4,9 +4,24 @@ GitLab API: https://docs.gitlab.com/ce/api/pipeline_schedules.html import pytest import responses +from gitlab.v4.objects import ProjectPipelineSchedulePipeline + +pipeline_content = { + "id": 48, + "iid": 13, + "project_id": 29, + "status": "pending", + "source": "scheduled", + "ref": "new-pipeline", + "sha": "eb94b618fb5865b26e80fdd8ae531b7a63ad851a", + "web_url": "https://example.com/foo/bar/pipelines/48", + "created_at": "2016-08-12T10:06:04.561Z", + "updated_at": "2016-08-12T10:09:56.223Z", +} + @pytest.fixture -def resp_project_pipeline_schedule(created_content): +def resp_create_pipeline_schedule(): content = { "id": 14, "description": "Build packages", @@ -36,9 +51,15 @@ def resp_project_pipeline_schedule(created_content): content_type="application/json", status=200, ) + yield rsps + + +@pytest.fixture +def resp_play_pipeline_schedule(created_content): + with responses.RequestsMock() as rsps: rsps.add( method=responses.POST, - url="http://localhost/api/v4/projects/1/pipeline_schedules/14/play", + url="http://localhost/api/v4/projects/1/pipeline_schedules/1/play", json=created_content, content_type="application/json", status=201, @@ -46,7 +67,20 @@ def resp_project_pipeline_schedule(created_content): yield rsps -def test_project_pipeline_schedule_play(project, resp_project_pipeline_schedule): +@pytest.fixture +def resp_list_schedule_pipelines(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/projects/1/pipeline_schedules/1/pipelines", + json=[pipeline_content], + content_type="application/json", + status=200, + ) + yield rsps + + +def test_create_project_pipeline_schedule(project, resp_create_pipeline_schedule): description = "Build packages" cronline = "0 1 * * 5" sched = project.pipelineschedules.create( @@ -56,7 +90,18 @@ def test_project_pipeline_schedule_play(project, resp_project_pipeline_schedule) assert description == sched.description assert cronline == sched.cron - play_result = sched.play() + +def test_play_project_pipeline_schedule(schedule, resp_play_pipeline_schedule): + play_result = schedule.play() assert play_result is not None assert "message" in play_result assert play_result["message"] == "201 Created" + + +def test_list_project_pipeline_schedule_pipelines( + schedule, resp_list_schedule_pipelines +): + pipelines = schedule.pipelines.list() + assert isinstance(pipelines, list) + assert isinstance(pipelines[0], ProjectPipelineSchedulePipeline) + assert pipelines[0].source == "scheduled" diff --git a/tests/unit/objects/test_resource_groups.py b/tests/unit/objects/test_resource_groups.py new file mode 100644 index 0000000..dd579ac --- /dev/null +++ b/tests/unit/objects/test_resource_groups.py @@ -0,0 +1,79 @@ +""" +GitLab API: +https://docs.gitlab.com/ee/api/resource_groups.html +""" +import pytest +import responses + +from gitlab.v4.objects import ProjectResourceGroup, ProjectResourceGroupUpcomingJob + +from .test_jobs import job_content + +resource_group_content = { + "id": 3, + "key": "production", + "process_mode": "unordered", + "created_at": "2021-09-01T08:04:59.650Z", + "updated_at": "2021-09-01T08:04:59.650Z", +} + + +@pytest.fixture +def resp_list_resource_groups(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/projects/1/resource_groups", + json=[resource_group_content], + content_type="application/json", + status=200, + ) + yield rsps + + +@pytest.fixture +def resp_get_resource_group(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/projects/1/resource_groups/production", + json=resource_group_content, + content_type="application/json", + status=200, + ) + yield rsps + + +@pytest.fixture +def resp_list_upcoming_jobs(): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url="http://localhost/api/v4/projects/1/resource_groups/production/upcoming_jobs", + json=[job_content], + content_type="application/json", + status=200, + ) + yield rsps + + +def test_list_project_resource_groups(project, resp_list_resource_groups): + resource_groups = project.resource_groups.list() + assert isinstance(resource_groups, list) + assert isinstance(resource_groups[0], ProjectResourceGroup) + assert resource_groups[0].process_mode == "unordered" + + +def test_get_project_resource_group(project, resp_get_resource_group): + resource_group = project.resource_groups.get("production") + assert isinstance(resource_group, ProjectResourceGroup) + assert resource_group.process_mode == "unordered" + + +def test_list_resource_group_upcoming_jobs(project, resp_list_upcoming_jobs): + resource_group = project.resource_groups.get("production", lazy=True) + upcoming_jobs = resource_group.upcoming_jobs.list() + + assert isinstance(upcoming_jobs, list) + assert isinstance(upcoming_jobs[0], ProjectResourceGroupUpcomingJob) + assert upcoming_jobs[0].ref == "main" diff --git a/tests/unit/test_gitlab_http_methods.py b/tests/unit/test_gitlab_http_methods.py index 252ecb6..569df81 100644 --- a/tests/unit/test_gitlab_http_methods.py +++ b/tests/unit/test_gitlab_http_methods.py @@ -6,7 +6,7 @@ import requests import responses from gitlab import GitlabHttpError, GitlabList, GitlabParsingError, RedirectError -from gitlab.client import RETRYABLE_TRANSIENT_ERROR_CODES +from gitlab.const import RETRYABLE_TRANSIENT_ERROR_CODES from tests.unit import helpers @@ -372,6 +372,63 @@ def test_http_request_302_put_raises_redirect_error(gl): assert "http://example.com/api/v4/user/status" in error_message +def test_http_request_on_409_resource_lock_retries(gl_retry): + url = "http://localhost/api/v4/user" + retried = False + + def response_callback( + response: requests.models.Response, + ) -> requests.models.Response: + """We need a callback that adds a resource lock reason only on first call""" + nonlocal retried + + if not retried: + response.reason = "Resource lock" + + retried = True + return response + + with responses.RequestsMock(response_callback=response_callback) as rsps: + rsps.add( + method=responses.GET, + url=url, + status=409, + match=helpers.MATCH_EMPTY_QUERY_PARAMS, + ) + rsps.add( + method=responses.GET, + url=url, + status=200, + match=helpers.MATCH_EMPTY_QUERY_PARAMS, + ) + response = gl_retry.http_request("get", "/user") + + assert response.status_code == 200 + + +def test_http_request_on_409_resource_lock_without_retry_raises(gl): + url = "http://localhost/api/v4/user" + + def response_callback( + response: requests.models.Response, + ) -> requests.models.Response: + """Without retry, this will fail on the first call""" + response.reason = "Resource lock" + return response + + with responses.RequestsMock(response_callback=response_callback) as req_mock: + req_mock.add( + method=responses.GET, + url=url, + status=409, + match=helpers.MATCH_EMPTY_QUERY_PARAMS, + ) + with pytest.raises(GitlabHttpError) as excinfo: + gl.http_request("get", "/user") + + assert excinfo.value.response_code == 409 + + @responses.activate def test_get_request(gl): url = "http://localhost/api/v4/projects" @@ -485,6 +542,29 @@ def test_list_request(gl): assert responses.assert_call_count(url, 3) is True +@responses.activate +def test_list_request_page_and_iterator(gl): + response_dict = copy.deepcopy(large_list_response) + response_dict["match"] = [responses.matchers.query_param_matcher({"page": "1"})] + responses.add(**response_dict) + + with pytest.warns( + UserWarning, match="`iterator=True` and `page=1` were both specified" + ): + result = gl.http_list("/projects", iterator=True, page=1) + assert isinstance(result, list) + assert len(result) == 20 + assert len(responses.calls) == 1 + + with pytest.warns( + UserWarning, match="`as_list=False` and `page=1` were both specified" + ): + result = gl.http_list("/projects", as_list=False, page=1) + assert isinstance(result, list) + assert len(result) == 20 + assert len(responses.calls) == 2 + + large_list_response = { "method": responses.GET, "url": "http://localhost/api/v4/projects", @@ -18,7 +18,7 @@ passenv = setenv = VIRTUAL_ENV={envdir} whitelist_externals = true usedevelop = True -install_command = pip install {opts} {packages} +install_command = pip install {opts} {packages} -e . isolated_build = True deps = -r{toxinidir}/requirements.txt @@ -28,42 +28,36 @@ commands = [testenv:black] basepython = python3 -envdir={toxworkdir}/lint deps = -r{toxinidir}/requirements-lint.txt commands = black {posargs} . [testenv:isort] basepython = python3 -envdir={toxworkdir}/lint deps = -r{toxinidir}/requirements-lint.txt commands = isort {posargs} {toxinidir} [testenv:mypy] basepython = python3 -envdir={toxworkdir}/lint deps = -r{toxinidir}/requirements-lint.txt commands = mypy {posargs} [testenv:flake8] basepython = python3 -envdir={toxworkdir}/lint deps = -r{toxinidir}/requirements-lint.txt commands = flake8 {posargs} . [testenv:pylint] basepython = python3 -envdir={toxworkdir}/lint deps = -r{toxinidir}/requirements-lint.txt commands = pylint {posargs} gitlab/ [testenv:cz] basepython = python3 -envdir={toxworkdir}/lint deps = -r{toxinidir}/requirements-lint.txt commands = cz check --rev-range 65ecadc..HEAD # cz is fast, check from first valid commit |