From 29e2efeae22ce5fa82e3541360b234e0053a65c2 Mon Sep 17 00:00:00 2001 From: Gauvain Pocentek Date: Tue, 30 Aug 2016 08:45:42 +0200 Subject: [PATCH 1/5] docs: improve the pagination section --- docs/api-usage.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/api-usage.rst b/docs/api-usage.rst index 976a3a08a..b33913dca 100644 --- a/docs/api-usage.rst +++ b/docs/api-usage.rst @@ -106,19 +106,28 @@ actions on the GitLab resources. For example: Pagination ========== -You can use pagination to go throught long lists: +You can use pagination to iterate over long lists. All the Gitlab objects +listing methods support the ``page`` and ``per_page`` parameters: .. code-block:: python ten_first_groups = gl.groups.list(page=0, per_page=10) -Use the ``all`` parameter to get all the items when using listing methods: +By default GitLab does not return the complete list of items. Use the ``all`` +parameter to get all the items when using listing methods: .. code-block:: python all_groups = gl.groups.list(all=True) all_owned_projects = gl.projects.owned(all=True) +.. note:: + + python-gitlab will iterate over the list by calling the correspnding API + multiple times. This might take some time if you have a lot of items to + retrieve. This might also consume a lot of memory as all the items will be + stored in RAM. + Sudo ==== From c08c913b82bf7421600b248d055db497d627802a Mon Sep 17 00:00:00 2001 From: Gauvain Pocentek Date: Sat, 3 Sep 2016 18:35:21 +0200 Subject: [PATCH 2/5] Fix and test pagination Fixes #140 --- gitlab/__init__.py | 6 +++--- tools/python_test.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/gitlab/__init__.py b/gitlab/__init__.py index 5f72d0d04..cbe90e331 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -365,14 +365,14 @@ def _raw_list(self, path, cls, extra_attrs={}, **kwargs): get_all_results = kwargs.get('all', False) + r = self._raw_get(path, **params) + raise_error_from_response(r, GitlabListError) + # Remove parameters from kwargs before passing it to constructor for key in ['all', 'page', 'per_page', 'sudo', 'next_url']: if key in params: del params[key] - r = self._raw_get(path, **params) - raise_error_from_response(r, GitlabListError) - # Add _from_api manually, because we are not creating objects # through normal path params['_from_api'] = True diff --git a/tools/python_test.py b/tools/python_test.py index 920486274..f9f5bb823 100644 --- a/tools/python_test.py +++ b/tools/python_test.py @@ -136,6 +136,13 @@ assert(len(gl.projects.owned()) == 2) assert(len(gl.projects.search("admin")) == 1) +# test pagination +l1 = gl.projects.list(per_page=1, page=1) +l2 = gl.projects.list(per_page=1, page=2) +assert(len(l1) == 1) +assert(len(l2) == 1) +assert(l1[0].id != l2[0].id) + # project content (files) admin_project.files.create({'file_path': 'README', 'branch_name': 'master', From b815f3a1f58cd697b4b95f6f0b24883282e09f77 Mon Sep 17 00:00:00 2001 From: Gauvain Pocentek Date: Sat, 3 Sep 2016 18:46:57 +0200 Subject: [PATCH 3/5] 'path' is an existing gitlab attr, don't use it Use path_ instead of path as parameter name for methods using it. Otherwise it might conflict with GitlabObject attributes. --- gitlab/__init__.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/gitlab/__init__.py b/gitlab/__init__.py index cbe90e331..48f483933 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -338,11 +338,11 @@ def enable_debug(self): requests_log.setLevel(logging.DEBUG) requests_log.propagate = True - def _raw_get(self, path, content_type=None, streamed=False, **kwargs): - if path.startswith('http://') or path.startswith('https://'): - url = path + def _raw_get(self, path_, content_type=None, streamed=False, **kwargs): + if path_.startswith('http://') or path_.startswith('https://'): + url = path_ else: - url = '%s%s' % (self._url, path) + url = '%s%s' % (self._url, path_) headers = self._create_headers(content_type) try: @@ -359,13 +359,13 @@ def _raw_get(self, path, content_type=None, streamed=False, **kwargs): raise GitlabConnectionError( "Can't connect to GitLab server (%s)" % e) - def _raw_list(self, path, cls, extra_attrs={}, **kwargs): + def _raw_list(self, path_, cls, extra_attrs={}, **kwargs): params = extra_attrs.copy() params.update(kwargs.copy()) get_all_results = kwargs.get('all', False) - r = self._raw_get(path, **params) + r = self._raw_get(path_, **params) raise_error_from_response(r, GitlabListError) # Remove parameters from kwargs before passing it to constructor @@ -374,7 +374,7 @@ def _raw_list(self, path, cls, extra_attrs={}, **kwargs): del params[key] # Add _from_api manually, because we are not creating objects - # through normal path + # through normal path_ params['_from_api'] = True results = [cls(self, item, **params) for item in r.json() @@ -386,8 +386,8 @@ def _raw_list(self, path, cls, extra_attrs={}, **kwargs): results.extend(self.list(cls, **args)) return results - def _raw_post(self, path, data=None, content_type=None, **kwargs): - url = '%s%s' % (self._url, path) + def _raw_post(self, path_, data=None, content_type=None, **kwargs): + url = '%s%s' % (self._url, path_) headers = self._create_headers(content_type) try: return self.session.post(url, params=kwargs, data=data, @@ -401,8 +401,8 @@ def _raw_post(self, path, data=None, content_type=None, **kwargs): raise GitlabConnectionError( "Can't connect to GitLab server (%s)" % e) - def _raw_put(self, path, data=None, content_type=None, **kwargs): - url = '%s%s' % (self._url, path) + def _raw_put(self, path_, data=None, content_type=None, **kwargs): + url = '%s%s' % (self._url, path_) headers = self._create_headers(content_type) try: @@ -417,8 +417,8 @@ def _raw_put(self, path, data=None, content_type=None, **kwargs): raise GitlabConnectionError( "Can't connect to GitLab server (%s)" % e) - def _raw_delete(self, path, content_type=None, **kwargs): - url = '%s%s' % (self._url, path) + def _raw_delete(self, path_, content_type=None, **kwargs): + url = '%s%s' % (self._url, path_) headers = self._create_headers(content_type) try: From dc3dcd11f3921929cc13260fbfb13aa3ae5117ce Mon Sep 17 00:00:00 2001 From: Gauvain Pocentek Date: Sat, 3 Sep 2016 19:14:45 +0200 Subject: [PATCH 4/5] Properly fix _raw_list --- gitlab/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gitlab/__init__.py b/gitlab/__init__.py index 48f483933..aa0ec1caa 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -365,11 +365,17 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs): get_all_results = kwargs.get('all', False) + # Remove these keys to avoid breaking the listing (urls will get too + # long otherwise) + for key in ['all', 'next_url']: + if key in params: + del params[key] + r = self._raw_get(path_, **params) raise_error_from_response(r, GitlabListError) - # Remove parameters from kwargs before passing it to constructor - for key in ['all', 'page', 'per_page', 'sudo', 'next_url']: + # These attributes are not needed in the object + for key in ['page', 'per_page', 'sudo']: if key in params: del params[key] From 90ad2dec7a0b158b2e77ae0cb403b01f2e1498d4 Mon Sep 17 00:00:00 2001 From: Gauvain Pocentek Date: Sat, 3 Sep 2016 20:25:41 +0200 Subject: [PATCH 5/5] 0.15.1 release --- ChangeLog | 6 ++++++ gitlab/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 04736cd1b..b6d8efa7e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +Version 0.15.1 + + * docs: improve the pagination section + * Fix and test pagination + * 'path' is an existing gitlab attr, don't use it as method argument + Version 0.15 * Add a basic HTTP debug method diff --git a/gitlab/__init__.py b/gitlab/__init__.py index aa0ec1caa..fa1927d80 100644 --- a/gitlab/__init__.py +++ b/gitlab/__init__.py @@ -33,7 +33,7 @@ from gitlab.objects import * # noqa __title__ = 'python-gitlab' -__version__ = '0.15' +__version__ = '0.15.1' __author__ = 'Gauvain Pocentek' __email__ = 'gauvain@pocentek.net' __license__ = 'LGPL3'