python-gitlab · gpocentek · Mar 23, 2017 · Mar 15, 2017 · Mar 21, 2017 · Mar 21, 2017
diff --git a/docs/api-usage.rst b/docs/api-usage.rst
@@ -142,7 +142,9 @@ parameter to get all the items when using listing methods:
    python-gitlab will iterate over the list by calling the correspnding API
    multiple times. This might take some time if you have a lot of items to
    retrieve. This might also consume a lot of memory as all the items will be
-   stored in RAM.
+   stored in RAM. If you're encountering the python recursion limit exception,
+   use ``safe_all=True`` instead to stop pagination automatically if the
+   recursion limit is hit.
 
 Sudo
 ====

diff --git a/gitlab/__init__.py b/gitlab/__init__.py
@@ -112,8 +112,8 @@ def __init__(self, url, private_token=None, email=None, password=None,
         # build the "submanagers"
         for parent_cls in six.itervalues(globals()):
             if (not inspect.isclass(parent_cls)
-               or not issubclass(parent_cls, GitlabObject)
-               or parent_cls == CurrentUser):
+                or not issubclass(parent_cls, GitlabObject)
+                    or parent_cls == CurrentUser):
                 continue
 
             if not parent_cls.managers:
@@ -312,11 +312,13 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
         params = extra_attrs.copy()
         params.update(kwargs.copy())
 
-        get_all_results = kwargs.get('all', False)
+        catch_recursion_limit = kwargs.get('safe_all', False)
+        get_all_results = (kwargs.get('all', False) is True
+                           or catch_recursion_limit)
 
         # Remove these keys to avoid breaking the listing (urls will get too
         # long otherwise)
-        for key in ['all', 'next_url']:
+        for key in ['all', 'next_url', 'safe_all']:
             if key in params:
                 del params[key]
 
@@ -334,12 +336,20 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
 
         results = [cls(self, item, **params) for item in r.json()
                    if item is not None]
-        if ('next' in r.links and 'url' in r.links['next']
-           and get_all_results is True):
-            args = kwargs.copy()
-            args.update(extra_attrs)
-            args['next_url'] = r.links['next']['url']
-            results.extend(self.list(cls, **args))
+        try:
+            if ('next' in r.links and 'url' in r.links['next']
+                    and get_all_results):
+                args = kwargs.copy()
+                args.update(extra_attrs)
+                args['next_url'] = r.links['next']['url']
+                results.extend(self.list(cls, **args))
+        except Exception as e:
+            # Catch the recursion limit exception if the 'safe_all'
+            # kwarg was provided
+            if not (catch_recursion_limit and
+                    "maximum recursion depth exceeded" in str(e)):
+                raise e
+
         return results
 
     def _raw_post(self, path_, data=None, content_type=None, **kwargs):

diff --git a/gitlab/tests/test_gitlab.py b/gitlab/tests/test_gitlab.py
@@ -26,6 +26,7 @@
 from httmock import HTTMock  # noqa
 from httmock import response  # noqa
 from httmock import urlmatch  # noqa
+import six
 
 import gitlab
 from gitlab import *  # noqa
@@ -243,6 +244,75 @@ def resp_two(url, request):
             self.assertEqual(data[0].ref, "b")
             self.assertEqual(len(data), 2)
 
+    def test_list_recursion_limit_caught(self):
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get")
+        def resp_one(url, request):
+            """First request:
+
+            http://localhost/api/v3/projects/1/repository/branches?per_page=1
+            """
+            headers = {
+                'content-type': 'application/json',
+                'link': '<http://localhost/api/v3/projects/1/repository/branc'
+                'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
+                '/projects/1/repository/branches?page=2&per_page=0>; rel="las'
+                't", <http://localhost/api/v3/projects/1/repository/branches?'
+                'page=1&per_page=0>; rel="first"'
+            }
+            content = ('[{"branch_name": "otherbranch", '
+                       '"project_id": 1, "ref": "b"}]').encode("utf-8")
+            resp = response(200, content, headers, None, 5, request)
+            return resp
+
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get",
+                  query=r'.*page=2.*')
+        def resp_two(url, request):
+            # Mock a runtime error
+            raise RuntimeError("maximum recursion depth exceeded")
+
+        with HTTMock(resp_two, resp_one):
+            data = self.gl.list(ProjectBranch, project_id=1, per_page=1,
+                                safe_all=True)
+            self.assertEqual(data[0].branch_name, "otherbranch")
+            self.assertEqual(data[0].project_id, 1)
+            self.assertEqual(data[0].ref, "b")
+            self.assertEqual(len(data), 1)
+
+    def test_list_recursion_limit_not_caught(self):
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get")
+        def resp_one(url, request):
+            """First request:
+
+            http://localhost/api/v3/projects/1/repository/branches?per_page=1
+            """
+            headers = {
+                'content-type': 'application/json',
+                'link': '<http://localhost/api/v3/projects/1/repository/branc'
+                'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
+                '/projects/1/repository/branches?page=2&per_page=0>; rel="las'
+                't", <http://localhost/api/v3/projects/1/repository/branches?'
+                'page=1&per_page=0>; rel="first"'
+            }
+            content = ('[{"branch_name": "otherbranch", '
+                       '"project_id": 1, "ref": "b"}]').encode("utf-8")
+            resp = response(200, content, headers, None, 5, request)
+            return resp
+
+        @urlmatch(scheme="http", netloc="localhost",
+                  path='/api/v3/projects/1/repository/branches', method="get",
+                  query=r'.*page=2.*')
+        def resp_two(url, request):
+            # Mock a runtime error
+            raise RuntimeError("maximum recursion depth exceeded")
+
+        with HTTMock(resp_two, resp_one):
+            with six.assertRaisesRegex(self, GitlabError,
+                                       "(maximum recursion depth exceeded)"):
+                self.gl.list(ProjectBranch, project_id=1, per_page=1, all=True)
+
     def test_list_401(self):
         @urlmatch(scheme="http", netloc="localhost",
                   path="/api/v3/projects/1/repository/branches", method="get")