Merge pull request #1001 from python-gitlab/feat/keyset-pagination

bufferoverflow · web-flow · commit df485a92b713 · 2020-01-26T17:49:22.000+01:00
Feat/keyset pagination
diff --git a/docs/api-usage.rst b/docs/api-usage.rst
@@ -219,6 +219,18 @@ You can define the ``per_page`` value globally to avoid passing it to every
 
    gl = gitlab.Gitlab(url, token, per_page=50)
 
+Gitlab allows to also use keyset pagination. You can supply it to your project listing,
+but you can also do so globally. Be aware that GitLab then also requires you to only use supported
+order options. At the time of writing, only ``order_by="id"`` works.
+
+.. code-block:: python
+
+   gl = gitlab.Gitlab(url, token, pagination="keyset", order_by="id", per_page=100)
+   gl.projects.list()
+
+Reference:
+https://docs.gitlab.com/ce/api/README.html#keyset-based-pagination
+
 ``list()`` methods can also return a generator object which will handle the
 next calls to the API when required. This is the recommended way to iterate
 through a large number of items:
diff --git a/gitlab/__init__.py b/gitlab/__init__.py
@@ -69,6 +69,8 @@ class Gitlab(object):
         http_username (str): Username for HTTP authentication
         http_password (str): Password for HTTP authentication
         api_version (str): Gitlab API version to use (support for 4 only)
+        pagination (str): Can be set to 'keyset' to use keyset pagination
+        order_by (str): Set order_by globally
     """
 
     def __init__(
@@ -84,6 +86,8 @@ def __init__(
         api_version="4",
         session=None,
         per_page=None,
+        pagination=None,
+        order_by=None,
     ):
 
         self._api_version = str(api_version)
@@ -109,6 +113,8 @@ def __init__(
         self.session = session or requests.Session()
 
         self.per_page = per_page
+        self.pagination = pagination
+        self.order_by = order_by
 
         objects = importlib.import_module("gitlab.v%s.objects" % self._api_version)
         self._objects = objects
@@ -200,6 +206,8 @@ def from_config(cls, gitlab_id=None, config_files=None):
             http_password=config.http_password,
             api_version=config.api_version,
             per_page=config.per_page,
+            pagination=config.pagination,
+            order_by=config.order_by,
         )
 
     def auth(self):
diff --git a/gitlab/config.py b/gitlab/config.py
@@ -163,3 +163,15 @@ def __init__(self, gitlab_id=None, config_files=None):
                 pass
         if self.per_page is not None and not 0 <= self.per_page <= 100:
             raise GitlabDataError("Unsupported per_page number: %s" % self.per_page)
+
+        self.pagination = None
+        try:
+            self.pagination = self._config.get(self.gitlab_id, "pagination")
+        except Exception:
+            pass
+
+        self.order_by = None
+        try:
+            self.order_by = self._config.get(self.gitlab_id, "order_by")
+        except Exception:
+            pass
diff --git a/gitlab/mixins.py b/gitlab/mixins.py
@@ -120,6 +120,13 @@ def list(self, **kwargs):
         if self.gitlab.per_page:
             data.setdefault("per_page", self.gitlab.per_page)
 
+        # global keyset pagination
+        if self.gitlab.pagination:
+            data.setdefault("pagination", self.gitlab.pagination)
+
+        if self.gitlab.order_by:
+            data.setdefault("order_by", self.gitlab.order_by)
+
         # We get the attributes that need some special transformation
         types = getattr(self, "_types", {})
         if types:
diff --git a/gitlab/v4/objects.py b/gitlab/v4/objects.py
@@ -268,6 +268,13 @@ class UserProjectManager(ListMixin, CreateMixin, RESTManager):
         "statistics",
         "with_issues_enabled",
         "with_merge_requests_enabled",
+        "with_custom_attributes",
+        "with_programming_language",
+        "wiki_checksum_failed",
+        "repository_checksum_failed",
+        "min_access_level",
+        "id_after",
+        "id_before",
     )
 
     def list(self, **kwargs):
@@ -1192,12 +1199,16 @@ class GroupProjectManager(ListMixin, RESTManager):
         "order_by",
         "sort",
         "search",
-        "ci_enabled_first",
         "simple",
         "owned",
         "starred",
         "with_custom_attributes",
         "include_subgroups",
+        "with_issues_enabled",
+        "with_merge_requests_enabled",
+        "with_shared",
+        "min_access_level",
+        "with_security_reports",
     )
 
 
