Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit3bd6d8d

Browse files
feat: emit a warning when using alist() method returns max
A common cause of issues filed and questions raised is that a userwill call a `list()` method and only get 20 items. As this is thedefault maximum of items that will be returned from a `list()` method.To help with this we now emit a warning when the result from a`list()` method is greater-than or equal to 20 (or the specified`per_page` value) and the user is not using either `all=True`,`all=False`, `as_list=False`, or `page=X`.
1 parent5beda3b commit3bd6d8d

File tree

3 files changed

+195
-10
lines changed

3 files changed

+195
-10
lines changed

‎gitlab/client.py‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
importrequests.utils
2525
fromrequests_toolbelt.multipart.encoderimportMultipartEncoder# type: ignore
2626

27+
importgitlab
2728
importgitlab.config
2829
importgitlab.const
2930
importgitlab.exceptions
@@ -35,6 +36,12 @@
3536
"{source!r} to {target!r}"
3637
)
3738

39+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
40+
_PAGINATION_URL= (
41+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
42+
f"api-usage.html#pagination"
43+
)
44+
3845

3946
classGitlab:
4047
"""Represents a GitLab server connection.
@@ -803,25 +810,60 @@ def http_list(
803810
GitlabHttpError: When the return code is not 2xx
804811
GitlabParsingError: If the json data could not be parsed
805812
"""
813+
# pylint: disable=too-many-return-statements
806814
query_data=query_dataor {}
807815

808816
# In case we want to change the default behavior at some point
809817
as_list=Trueifas_listisNoneelseas_list
810818

811-
get_all=kwargs.pop("all",False)
819+
get_all=kwargs.pop("all",None)
812820
url=self._build_url(path)
813821

814822
page=kwargs.get("page")
815823

816-
ifget_allisTrueandas_listisTrue:
817-
returnlist(GitlabList(self,url,query_data,**kwargs))
824+
ifas_listisFalse:
825+
# Generator requested
826+
returnGitlabList(self,url,query_data,**kwargs)
818827

819-
ifpageoras_listisTrue:
820-
# pagination requested, we return a list
821-
returnlist(GitlabList(self,url,query_data,get_next=False,**kwargs))
828+
ifget_allisTrue:
829+
returnlist(GitlabList(self,url,query_data,**kwargs))
822830

823-
# No pagination, generator requested
824-
returnGitlabList(self,url,query_data,**kwargs)
831+
# pagination requested, we return a list
832+
gl_list=GitlabList(self,url,query_data,get_next=False,**kwargs)
833+
items=list(gl_list)
834+
835+
# No warning is emitted if any of the following conditions apply:
836+
# * `all=False` was set in the `list()` call.
837+
# * `page` was set in the `list()` call.
838+
# * GitLab did not return the `x-per-page` header.
839+
# * Number of items received is less than per-page value.
840+
# * Number of items received is >= total available.
841+
ifget_allisFalse:
842+
returnitems
843+
ifpageisnotNone:
844+
returnitems
845+
ifgl_list.per_pageisNone:
846+
returnitems
847+
iflen(items)<gl_list.per_page:
848+
returnitems
849+
ifgl_list.totalisnotNoneandlen(items)>=gl_list.total:
850+
returnitems
851+
852+
# Warn the user that they are only going to retrieve `per_page`
853+
# maximum items. This is a common cause of issues filed.
854+
total_items="many"ifgl_list.totalisNoneelsegl_list.total
855+
utils.warn(
856+
message=(
857+
f"Calling a `list()` method without specifying `all=True` or "
858+
f"`as_list=False` will return a maximum of{gl_list.per_page} items. "
859+
f"Your query returned{len(items)} of{total_items} items. See "
860+
f"{_PAGINATION_URL} for more details. If this was done intentionally, "
861+
f"then this warning can be supressed by adding the argument "
862+
f"`all=False` to the `list()` call."
863+
),
864+
category=UserWarning,
865+
)
866+
returnitems
825867

826868
defhttp_post(
827869
self,

‎tests/functional/api/test_gitlab.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
importwarnings
2+
13
importpytest
24

35
importgitlab
@@ -181,3 +183,46 @@ def test_rate_limits(gl):
181183
settings.throttle_authenticated_api_enabled=False
182184
settings.save()
183185
[project.delete()forprojectinprojects]
186+
187+
188+
deftest_list_default_warning(gl):
189+
"""When there are more than 20 items and use default `list()` then warning is
190+
generated"""
191+
withwarnings.catch_warnings(record=True)ascaught_warnings:
192+
gl.gitlabciymls.list()
193+
assertlen(caught_warnings)==1
194+
warning=caught_warnings[0]
195+
assertisinstance(warning.message,UserWarning)
196+
message=str(warning.message)
197+
assert"python-gitlab.readthedocs.io"inmessage
198+
assert__file__==warning.filename
199+
200+
201+
deftest_list_page_nowarning(gl):
202+
"""Using `page=X` will disable the warning"""
203+
withwarnings.catch_warnings(record=True)ascaught_warnings:
204+
gl.gitlabciymls.list(page=1)
205+
assertlen(caught_warnings)==0
206+
207+
208+
deftest_list_all_false_nowarning(gl):
209+
"""Using `all=False` will disable the warning"""
210+
withwarnings.catch_warnings(record=True)ascaught_warnings:
211+
gl.gitlabciymls.list(all=False)
212+
assertlen(caught_warnings)==0
213+
214+
215+
deftest_list_all_true_nowarning(gl):
216+
"""Using `all=True` will disable the warning"""
217+
withwarnings.catch_warnings(record=True)ascaught_warnings:
218+
items=gl.gitlabciymls.list(all=True)
219+
assertlen(caught_warnings)==0
220+
assertlen(items)>20
221+
222+
223+
deftest_list_as_list_false_nowarning(gl):
224+
"""Using `as_list=False` will disable the warning"""
225+
withwarnings.catch_warnings(record=True)ascaught_warnings:
226+
items=gl.gitlabciymls.list(as_list=False)
227+
assertlen(caught_warnings)==0
228+
assertlen(list(items))>20

‎tests/unit/test_gitlab_http_methods.py‎

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
importcopy
2+
importwarnings
3+
14
importpytest
25
importrequests
36
importresponses
@@ -329,20 +332,115 @@ def test_list_request(gl):
329332
match=MATCH_EMPTY_QUERY_PARAMS,
330333
)
331334

332-
result=gl.http_list("/projects",as_list=True)
335+
withwarnings.catch_warnings(record=True)ascaught_warnings:
336+
result=gl.http_list("/projects",as_list=True)
337+
assertlen(caught_warnings)==0
333338
assertisinstance(result,list)
334339
assertlen(result)==1
335340

336341
result=gl.http_list("/projects",as_list=False)
337342
assertisinstance(result,GitlabList)
338-
assertlen(result)==1
343+
assertlen(list(result))==1
339344

340345
result=gl.http_list("/projects",all=True)
341346
assertisinstance(result,list)
342347
assertlen(result)==1
343348
assertresponses.assert_call_count(url,3)isTrue
344349

345350

351+
large_list_response= {
352+
"method":responses.GET,
353+
"url":"http://localhost/api/v4/projects",
354+
"json": [
355+
{"name":"project01"},
356+
{"name":"project02"},
357+
{"name":"project03"},
358+
{"name":"project04"},
359+
{"name":"project05"},
360+
{"name":"project06"},
361+
{"name":"project07"},
362+
{"name":"project08"},
363+
{"name":"project09"},
364+
{"name":"project10"},
365+
{"name":"project11"},
366+
{"name":"project12"},
367+
{"name":"project13"},
368+
{"name":"project14"},
369+
{"name":"project15"},
370+
{"name":"project16"},
371+
{"name":"project17"},
372+
{"name":"project18"},
373+
{"name":"project19"},
374+
{"name":"project20"},
375+
],
376+
"headers": {"X-Total":"30","x-per-page":"20"},
377+
"status":200,
378+
"match":MATCH_EMPTY_QUERY_PARAMS,
379+
}
380+
381+
382+
@responses.activate
383+
deftest_list_request_pagination_warning(gl):
384+
responses.add(**large_list_response)
385+
386+
withwarnings.catch_warnings(record=True)ascaught_warnings:
387+
result=gl.http_list("/projects",as_list=True)
388+
assertlen(caught_warnings)==1
389+
warning=caught_warnings[0]
390+
assertisinstance(warning.message,UserWarning)
391+
message=str(warning.message)
392+
assert"Calling a `list()` method"inmessage
393+
assert"python-gitlab.readthedocs.io"inmessage
394+
assert__file__==warning.filename
395+
assertisinstance(result,list)
396+
assertlen(result)==20
397+
assertlen(responses.calls)==1
398+
399+
400+
@responses.activate
401+
deftest_list_request_as_list_false_nowarning(gl):
402+
responses.add(**large_list_response)
403+
withwarnings.catch_warnings(record=True)ascaught_warnings:
404+
result=gl.http_list("/projects",as_list=False)
405+
assertlen(caught_warnings)==0
406+
assertisinstance(result,GitlabList)
407+
assertlen(list(result))==20
408+
assertlen(responses.calls)==1
409+
410+
411+
@responses.activate
412+
deftest_list_request_all_true_nowarning(gl):
413+
responses.add(**large_list_response)
414+
withwarnings.catch_warnings(record=True)ascaught_warnings:
415+
result=gl.http_list("/projects",all=True)
416+
assertlen(caught_warnings)==0
417+
assertisinstance(result,list)
418+
assertlen(result)==20
419+
assertlen(responses.calls)==1
420+
421+
422+
@responses.activate
423+
deftest_list_request_all_false_nowarning(gl):
424+
responses.add(**large_list_response)
425+
withwarnings.catch_warnings(record=True)ascaught_warnings:
426+
result=gl.http_list("/projects",all=False)
427+
assertlen(caught_warnings)==0
428+
assertisinstance(result,list)
429+
assertlen(result)==20
430+
assertlen(responses.calls)==1
431+
432+
433+
@responses.activate
434+
deftest_list_request_page_nowarning(gl):
435+
response_dict=copy.deepcopy(large_list_response)
436+
response_dict["match"]= [responses.matchers.query_param_matcher({"page":"1"})]
437+
responses.add(**response_dict)
438+
withwarnings.catch_warnings(record=True)ascaught_warnings:
439+
gl.http_list("/projects",page=1)
440+
assertlen(caught_warnings)==0
441+
assertlen(responses.calls)==1
442+
443+
346444
@responses.activate
347445
deftest_list_request_404(gl):
348446
url="http://localhost/api/v4/not_there"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp