fix: handle situation where GitLab does not return valuesjlvillal/pagination

If a query returns more than 10,000 records than the following values
are NOT returned:
  x.total_pages
  x.total

Modify the code to allow no value to be set for these values. If there
is not a value returned the functions will now return None.

Update unit test so no longer `xfail`

https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers

Closes #1686

5 files changed, 35 insertions, 23 deletions

diff --git a/docs/api-usage.rst b/docs/api-usage.rst
index f30ed03..66e5887 100644
--- a/docs/api-usage.rst
+++ b/docs/api-usage.rst
@@ -265,8 +265,17 @@ The generator exposes extra listing information as received from the server:
 * ``prev_page``: if ``None`` the current page is the first one
 * ``next_page``: if ``None`` the current page is the last one
 * ``per_page``: number of items per page
-* ``total_pages``: total number of pages available
-* ``total``: total number of items in the list
+* ``total_pages``: total number of pages available. This may be a ``None`` value.
+* ``total``: total number of items in the list. This may be a ``None`` value.
+
+.. note::
+
+   For performance reasons, if a query returns more than 10,000 records, GitLab
+   does not return the ``total_pages`` or ``total`` headers.  In this case,
+   ``total_pages`` and ``total`` will have a value of ``None``.
+
+   For more information see:
+   https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
 
 Sudo
 ====
diff --git a/gitlab/base.py b/gitlab/base.py
index 64604b4..50f09c5 100644
--- a/gitlab/base.py
+++ b/gitlab/base.py
@@ -288,12 +288,12 @@ class RESTObjectList(object):
         return self._list.per_page
 
     @property
-    def total_pages(self) -> int:
+    def total_pages(self) -> Optional[int]:
         """The total number of pages."""
         return self._list.total_pages
 
     @property
-    def total(self) -> int:
+    def total(self) -> Optional[int]:
         """The total number of items."""
         return self._list.total
 
diff --git a/gitlab/client.py b/gitlab/client.py
index 84fd40f..c1e0825 100644
--- a/gitlab/client.py
+++ b/gitlab/client.py
@@ -917,14 +917,12 @@ class GitlabList(object):
             self._next_url = next_url
         except KeyError:
             self._next_url = None
-        self._current_page: Optional[Union[str, int]] = result.headers.get("X-Page")
-        self._prev_page: Optional[Union[str, int]] = result.headers.get("X-Prev-Page")
-        self._next_page: Optional[Union[str, int]] = result.headers.get("X-Next-Page")
-        self._per_page: Optional[Union[str, int]] = result.headers.get("X-Per-Page")
-        self._total_pages: Optional[Union[str, int]] = result.headers.get(
-            "X-Total-Pages"
-        )
-        self._total: Optional[Union[str, int]] = result.headers.get("X-Total")
+        self._current_page: Optional[str] = result.headers.get("X-Page")
+        self._prev_page: Optional[str] = result.headers.get("X-Prev-Page")
+        self._next_page: Optional[str] = result.headers.get("X-Next-Page")
+        self._per_page: Optional[str] = result.headers.get("X-Per-Page")
+        self._total_pages: Optional[str] = result.headers.get("X-Total-Pages")
+        self._total: Optional[str] = result.headers.get("X-Total")
 
         try:
             self._data: List[Dict[str, Any]] = result.json()
@@ -965,19 +963,22 @@ class GitlabList(object):
             assert self._per_page is not None
         return int(self._per_page)
 
+    # NOTE(jlvillal): When a query returns more than 10,000 items, GitLab doesn't return
+    # the headers 'x-total-pages' and 'x-total'. In those cases we return None.
+    # https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
     @property
-    def total_pages(self) -> int:
+    def total_pages(self) -> Optional[int]:
         """The total number of pages."""
-        if TYPE_CHECKING:
-            assert self._total_pages is not None
-        return int(self._total_pages)
+        if self._total_pages is not None:
+            return int(self._total_pages)
+        return None
 
     @property
-    def total(self) -> int:
+    def total(self) -> Optional[int]:
         """The total number of items."""
-        if TYPE_CHECKING:
-            assert self._total is not None
-        return int(self._total)
+        if self._total is not None:
+            return int(self._total)
+        return None
 
     def __iter__(self) -> "GitlabList":
         return self
diff --git a/pyproject.toml b/pyproject.toml
index 2aa5b1d..bc0530a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -87,3 +87,6 @@ disable = [
     "useless-object-inheritance",
 
 ]
+
+[tool.pytest.ini_options]
+xfail_strict = true
diff --git a/tests/unit/test_gitlab.py b/tests/unit/test_gitlab.py
index 2981ebb..4d742d3 100644
--- a/tests/unit/test_gitlab.py
+++ b/tests/unit/test_gitlab.py
@@ -16,9 +16,9 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import pickle
 import warnings
-from copy import deepcopy
 
 import pytest
 import responses
@@ -109,7 +109,7 @@ def _strip_pagination_headers(response):
     """
     https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
     """
-    stripped = deepcopy(response)
+    stripped = copy.deepcopy(response)
 
     del stripped["headers"]["X-Total-Pages"]
     del stripped["headers"]["X-Total"]
@@ -117,7 +117,6 @@ def _strip_pagination_headers(response):
     return stripped
 
 
-@pytest.mark.xfail(reason="See #1686")
 @responses.activate
 def test_gitlab_build_list_missing_headers(gl, resp_page_1, resp_page_2):
     stripped_page_1 = _strip_pagination_headers(resp_page_1)