From bded2de51951902444bc62aa016a3ad34aab799e Mon Sep 17 00:00:00 2001 From: Max Wittig Date: Sun, 26 Jan 2020 17:20:40 +0100 Subject: [PATCH 1/3] refactor: support new list filters This is most likely only useful for the CLI --- gitlab/v4/objects.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gitlab/v4/objects.py b/gitlab/v4/objects.py index c38a4bf7c..1750a3641 100644 --- a/gitlab/v4/objects.py +++ b/gitlab/v4/objects.py @@ -268,6 +268,13 @@ class UserProjectManager(ListMixin, CreateMixin, RESTManager): "statistics", "with_issues_enabled", "with_merge_requests_enabled", + "with_custom_attributes", + "with_programming_language", + "wiki_checksum_failed", + "repository_checksum_failed", + "min_access_level", + "id_after", + "id_before", ) def list(self, **kwargs): @@ -1192,12 +1199,16 @@ class GroupProjectManager(ListMixin, RESTManager): "order_by", "sort", "search", - "ci_enabled_first", "simple", "owned", "starred", "with_custom_attributes", "include_subgroups", + "with_issues_enabled", + "with_merge_requests_enabled", + "with_shared", + "min_access_level", + "with_security_reports", ) From 0b71ba4d2965658389b705c1bb0d83d1ff2ee8f2 Mon Sep 17 00:00:00 2001 From: Max Wittig Date: Sun, 26 Jan 2020 17:28:30 +0100 Subject: [PATCH 2/3] feat: support keyset pagination globally --- docs/api-usage.rst | 12 ++++++++++++ gitlab/__init__.py | 4 ++++ gitlab/config.py | 6 ++++++ gitlab/mixins.py | 4 ++++ 4 files changed, 26 insertions(+) diff --git a/docs/api-usage.rst b/docs/api-usage.rst index d211e25ab..5b1bd93ec 100644 --- a/docs/api-usage.rst +++ b/docs/api-usage.rst @@ -219,6 +219,18 @@ You can define the ``per_page`` value globally to avoid passing it to every gl = gitlab.Gitlab(url, token, per_page=50) +Gitlab allows to also use keyset pagination. You can supply it to your project listing, +but you can also do so globally. Be aware that GitLab then also requires you to only use supported +order options. At the time of writing, only ``order_by="id"`` works. + +.. code-block:: python + + gl = gitlab.Gitlab(url, token, pagination="keyset", per_page=100) + gl.projects.list(order_by="id") + +Reference: +https://docs.gitlab.com/ce/api/README.html#keyset-based-pagination + ``list()`` methods can also return a generator object which will handle the next calls to the API when required. This is the recommended way to iterate through a large number of items: diff --git a/gitlab/__init__.py b/gitlab/__init__.py index 9cb027b5b..166e00fa3 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -69,6 +69,7 @@ class Gitlab(object): http_username (str): Username for HTTP authentication http_password (str): Password for HTTP authentication api_version (str): Gitlab API version to use (support for 4 only) + pagination (str): Can be set to 'keyset' to use keyset pagination """ def __init__( @@ -84,6 +85,7 @@ def __init__( api_version="4", session=None, per_page=None, + pagination=None, ): self._api_version = str(api_version) @@ -109,6 +111,7 @@ def __init__( self.session = session or requests.Session() self.per_page = per_page + self.pagination = pagination objects = importlib.import_module("gitlab.v%s.objects" % self._api_version) self._objects = objects @@ -200,6 +203,7 @@ def from_config(cls, gitlab_id=None, config_files=None): http_password=config.http_password, api_version=config.api_version, per_page=config.per_page, + pagination=config.pagination, ) def auth(self): diff --git a/gitlab/config.py b/gitlab/config.py index b2c0dbf84..95a1245c5 100644 --- a/gitlab/config.py +++ b/gitlab/config.py @@ -163,3 +163,9 @@ def __init__(self, gitlab_id=None, config_files=None): pass if self.per_page is not None and not 0 <= self.per_page <= 100: raise GitlabDataError("Unsupported per_page number: %s" % self.per_page) + + self.pagination = None + try: + self.pagination = self._config.get(self.gitlab_id, "pagination") + except Exception: + pass diff --git a/gitlab/mixins.py b/gitlab/mixins.py index c812d66b7..2437a6f3d 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -120,6 +120,10 @@ def list(self, **kwargs): if self.gitlab.per_page: data.setdefault("per_page", self.gitlab.per_page) + # global keyset pagination + if self.gitlab.pagination: + data.setdefault("pagination", self.gitlab.pagination) + # We get the attributes that need some special transformation types = getattr(self, "_types", {}) if types: From d1879253dae93e182710fe22b0a6452296e2b532 Mon Sep 17 00:00:00 2001 From: Max Wittig Date: Sun, 26 Jan 2020 17:33:53 +0100 Subject: [PATCH 3/3] feat: add global order_by option to ease pagination --- docs/api-usage.rst | 4 ++-- gitlab/__init__.py | 4 ++++ gitlab/config.py | 6 ++++++ gitlab/mixins.py | 3 +++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/api-usage.rst b/docs/api-usage.rst index 5b1bd93ec..19fdea08b 100644 --- a/docs/api-usage.rst +++ b/docs/api-usage.rst @@ -225,8 +225,8 @@ order options. At the time of writing, only ``order_by="id"`` works. .. code-block:: python - gl = gitlab.Gitlab(url, token, pagination="keyset", per_page=100) - gl.projects.list(order_by="id") + gl = gitlab.Gitlab(url, token, pagination="keyset", order_by="id", per_page=100) + gl.projects.list() Reference: https://docs.gitlab.com/ce/api/README.html#keyset-based-pagination diff --git a/gitlab/__init__.py b/gitlab/__init__.py index 166e00fa3..85fc5e0c3 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -70,6 +70,7 @@ class Gitlab(object): http_password (str): Password for HTTP authentication api_version (str): Gitlab API version to use (support for 4 only) pagination (str): Can be set to 'keyset' to use keyset pagination + order_by (str): Set order_by globally """ def __init__( @@ -86,6 +87,7 @@ def __init__( session=None, per_page=None, pagination=None, + order_by=None, ): self._api_version = str(api_version) @@ -112,6 +114,7 @@ def __init__( self.per_page = per_page self.pagination = pagination + self.order_by = order_by objects = importlib.import_module("gitlab.v%s.objects" % self._api_version) self._objects = objects @@ -204,6 +207,7 @@ def from_config(cls, gitlab_id=None, config_files=None): api_version=config.api_version, per_page=config.per_page, pagination=config.pagination, + order_by=config.order_by, ) def auth(self): diff --git a/gitlab/config.py b/gitlab/config.py index 95a1245c5..2272dd3c5 100644 --- a/gitlab/config.py +++ b/gitlab/config.py @@ -169,3 +169,9 @@ def __init__(self, gitlab_id=None, config_files=None): self.pagination = self._config.get(self.gitlab_id, "pagination") except Exception: pass + + self.order_by = None + try: + self.order_by = self._config.get(self.gitlab_id, "order_by") + except Exception: + pass diff --git a/gitlab/mixins.py b/gitlab/mixins.py index 2437a6f3d..854449949 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -124,6 +124,9 @@ def list(self, **kwargs): if self.gitlab.pagination: data.setdefault("pagination", self.gitlab.pagination) + if self.gitlab.order_by: + data.setdefault("order_by", self.gitlab.order_by) + # We get the attributes that need some special transformation types = getattr(self, "_types", {}) if types: