Skip to content

Commit 989f3b7

Browse files
Johan BrandhorstGauvain Pocentek
Johan Brandhorst
authored and
Gauvain Pocentek
committed
Stop listing if recursion limit is hit (#234)
1 parent 22bf128 commit 989f3b7

File tree

3 files changed

+93
-11
lines changed

3 files changed

+93
-11
lines changed

‎docs/api-usage.rst

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,9 @@ parameter to get all the items when using listing methods:
142142
python-gitlab will iterate over the list by calling the correspnding API
143143
multiple times. This might take some time if you have a lot of items to
144144
retrieve. This might also consume a lot of memory as all the items will be
145-
stored in RAM.
145+
stored in RAM. If you're encountering the python recursion limit exception,
146+
use ``safe_all=True`` instead to stop pagination automatically if the
147+
recursion limit is hit.
146148

147149
Sudo
148150
====

‎gitlab/__init__.py

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ def __init__(self, url, private_token=None, email=None, password=None,
112112
# build the "submanagers"
113113
for parent_cls in six.itervalues(globals()):
114114
if (not inspect.isclass(parent_cls)
115-
or not issubclass(parent_cls, GitlabObject)
116-
or parent_cls == CurrentUser):
115+
or not issubclass(parent_cls, GitlabObject)
116+
or parent_cls == CurrentUser):
117117
continue
118118

119119
if not parent_cls.managers:
@@ -312,11 +312,13 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
312312
params = extra_attrs.copy()
313313
params.update(kwargs.copy())
314314

315-
get_all_results = kwargs.get('all', False)
315+
catch_recursion_limit = kwargs.get('safe_all', False)
316+
get_all_results = (kwargs.get('all', False) is True
317+
or catch_recursion_limit)
316318

317319
# Remove these keys to avoid breaking the listing (urls will get too
318320
# long otherwise)
319-
for key in ['all', 'next_url']:
321+
for key in ['all', 'next_url', 'safe_all']:
320322
if key in params:
321323
del params[key]
322324

@@ -334,12 +336,20 @@ def _raw_list(self, path_, cls, extra_attrs={}, **kwargs):
334336

335337
results = [cls(self, item, **params) for item in r.json()
336338
if item is not None]
337-
if ('next' in r.links and 'url' in r.links['next']
338-
and get_all_results is True):
339-
args = kwargs.copy()
340-
args.update(extra_attrs)
341-
args['next_url'] = r.links['next']['url']
342-
results.extend(self.list(cls, **args))
339+
try:
340+
if ('next' in r.links and 'url' in r.links['next']
341+
and get_all_results):
342+
args = kwargs.copy()
343+
args.update(extra_attrs)
344+
args['next_url'] = r.links['next']['url']
345+
results.extend(self.list(cls, **args))
346+
except Exception as e:
347+
# Catch the recursion limit exception if the 'safe_all'
348+
# kwarg was provided
349+
if not (catch_recursion_limit and
350+
"maximum recursion depth exceeded" in str(e)):
351+
raise e
352+
343353
return results
344354

345355
def _raw_post(self, path_, data=None, content_type=None, **kwargs):

‎gitlab/tests/test_gitlab.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
from httmock import HTTMock # noqa
2727
from httmock import response # noqa
2828
from httmock import urlmatch # noqa
29+
import six
2930

3031
import gitlab
3132
from gitlab import * # noqa
@@ -243,6 +244,75 @@ def resp_two(url, request):
243244
self.assertEqual(data[0].ref, "b")
244245
self.assertEqual(len(data), 2)
245246

247+
def test_list_recursion_limit_caught(self):
248+
@urlmatch(scheme="http", netloc="localhost",
249+
path='/api/v3/projects/1/repository/branches', method="get")
250+
def resp_one(url, request):
251+
"""First request:
252+
253+
http://localhost/api/v3/projects/1/repository/branches?per_page=1
254+
"""
255+
headers = {
256+
'content-type': 'application/json',
257+
'link': '<http://localhost/api/v3/projects/1/repository/branc'
258+
'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
259+
'/projects/1/repository/branches?page=2&per_page=0>; rel="las'
260+
't", <http://localhost/api/v3/projects/1/repository/branches?'
261+
'page=1&per_page=0>; rel="first"'
262+
}
263+
content = ('[{"branch_name": "otherbranch", '
264+
'"project_id": 1, "ref": "b"}]').encode("utf-8")
265+
resp = response(200, content, headers, None, 5, request)
266+
return resp
267+
268+
@urlmatch(scheme="http", netloc="localhost",
269+
path='/api/v3/projects/1/repository/branches', method="get",
270+
query=r'.*page=2.*')
271+
def resp_two(url, request):
272+
# Mock a runtime error
273+
raise RuntimeError("maximum recursion depth exceeded")
274+
275+
with HTTMock(resp_two, resp_one):
276+
data = self.gl.list(ProjectBranch, project_id=1, per_page=1,
277+
safe_all=True)
278+
self.assertEqual(data[0].branch_name, "otherbranch")
279+
self.assertEqual(data[0].project_id, 1)
280+
self.assertEqual(data[0].ref, "b")
281+
self.assertEqual(len(data), 1)
282+
283+
def test_list_recursion_limit_not_caught(self):
284+
@urlmatch(scheme="http", netloc="localhost",
285+
path='/api/v3/projects/1/repository/branches', method="get")
286+
def resp_one(url, request):
287+
"""First request:
288+
289+
http://localhost/api/v3/projects/1/repository/branches?per_page=1
290+
"""
291+
headers = {
292+
'content-type': 'application/json',
293+
'link': '<http://localhost/api/v3/projects/1/repository/branc'
294+
'hes?page=2&per_page=0>; rel="next", <http://localhost/api/v3'
295+
'/projects/1/repository/branches?page=2&per_page=0>; rel="las'
296+
't", <http://localhost/api/v3/projects/1/repository/branches?'
297+
'page=1&per_page=0>; rel="first"'
298+
}
299+
content = ('[{"branch_name": "otherbranch", '
300+
'"project_id": 1, "ref": "b"}]').encode("utf-8")
301+
resp = response(200, content, headers, None, 5, request)
302+
return resp
303+
304+
@urlmatch(scheme="http", netloc="localhost",
305+
path='/api/v3/projects/1/repository/branches', method="get",
306+
query=r'.*page=2.*')
307+
def resp_two(url, request):
308+
# Mock a runtime error
309+
raise RuntimeError("maximum recursion depth exceeded")
310+
311+
with HTTMock(resp_two, resp_one):
312+
with six.assertRaisesRegex(self, GitlabError,
313+
"(maximum recursion depth exceeded)"):
314+
self.gl.list(ProjectBranch, project_id=1, per_page=1, all=True)
315+
246316
def test_list_401(self):
247317
@urlmatch(scheme="http", netloc="localhost",
248318
path="/api/v3/projects/1/repository/branches", method="get")

0 commit comments

Comments
 (0)