fix: handle situation where GitLab does not return values

JohnVillalovos · JohnVillalovos · commit 4690dc2c1112 · 2021-12-25T09:26:38.000-08:00
If a query returns more than 10,000 records than the following values are NOT returned: x.total_pages x.total Modify the code to allow no value to be set for these values. If there is not a value returned the functions will now return None. https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers Closes #1686
diff --git a/docs/api-usage.rst b/docs/api-usage.rst
@@ -265,8 +265,17 @@ The generator exposes extra listing information as received from the server:
 * ``prev_page``: if ``None`` the current page is the first one
 * ``next_page``: if ``None`` the current page is the last one
 * ``per_page``: number of items per page
-* ``total_pages``: total number of pages available
-* ``total``: total number of items in the list
+* ``total_pages``: total number of pages available. This may be a ``None`` value.
+* ``total``: total number of items in the list. This may be a ``None`` value.
+
+.. note::
+
+   For performance reasons, if a query returns more than 10,000 records, GitLab
+   does not return the ``total_pages`` or ``total`` headers.  In this case,
+   ``total_pages`` and ``total`` will have a value of ``None``.
+
+   For more information see:
+   https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
 
 Sudo
 ====
diff --git a/gitlab/base.py b/gitlab/base.py
@@ -288,12 +288,12 @@ def per_page(self) -> int:
         return self._list.per_page
 
     @property
-    def total_pages(self) -> int:
+    def total_pages(self) -> Optional[int]:
         """The total number of pages."""
         return self._list.total_pages
 
     @property
-    def total(self) -> int:
+    def total(self) -> Optional[int]:
         """The total number of items."""
         return self._list.total
 
diff --git a/gitlab/client.py b/gitlab/client.py
@@ -917,14 +917,12 @@ def _query(
             self._next_url = next_url
         except KeyError:
             self._next_url = None
-        self._current_page: Optional[Union[str, int]] = result.headers.get("X-Page")
-        self._prev_page: Optional[Union[str, int]] = result.headers.get("X-Prev-Page")
-        self._next_page: Optional[Union[str, int]] = result.headers.get("X-Next-Page")
-        self._per_page: Optional[Union[str, int]] = result.headers.get("X-Per-Page")
-        self._total_pages: Optional[Union[str, int]] = result.headers.get(
-            "X-Total-Pages"
-        )
-        self._total: Optional[Union[str, int]] = result.headers.get("X-Total")
+        self._current_page: Optional[str] = result.headers.get("X-Page")
+        self._prev_page: Optional[str] = result.headers.get("X-Prev-Page")
+        self._next_page: Optional[str] = result.headers.get("X-Next-Page")
+        self._per_page: Optional[str] = result.headers.get("X-Per-Page")
+        self._total_pages: Optional[str] = result.headers.get("X-Total-Pages")
+        self._total: Optional[str] = result.headers.get("X-Total")
 
         try:
             self._data: List[Dict[str, Any]] = result.json()
@@ -965,19 +963,22 @@ def per_page(self) -> int:
             assert self._per_page is not None
         return int(self._per_page)
 
+    # NOTE(jlvillal): When a query returns more than 10,000 items, GitLab doesn't return
+    # the headers 'x-total-pages' and 'x-total'. In those cases we return None.
+    # https://docs.gitlab.com/ee/user/gitlab_com/index.html#pagination-response-headers
     @property
-    def total_pages(self) -> int:
+    def total_pages(self) -> Optional[int]:
         """The total number of pages."""
-        if TYPE_CHECKING:
-            assert self._total_pages is not None
-        return int(self._total_pages)
+        if self._total_pages is not None:
+            return int(self._total_pages)
+        return None
 
     @property
-    def total(self) -> int:
+    def total(self) -> Optional[int]:
         """The total number of items."""
-        if TYPE_CHECKING:
-            assert self._total is not None
-        return int(self._total)
+        if self._total is not None:
+            return int(self._total)
+        return None
 
     def __iter__(self) -> "GitlabList":
         return self
