Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit9117111

Browse files
feat: emit a warning when using alist() method returns max
A common cause of issues filed and questions raised is that a userwill call a `list()` method and only get 20 items. As this is thedefault maximum of items that will be returned from a `list()` method.To help with this we now emit a warning when the result from a`list()` method is greater-than or equal to 20 (or the specified`per_page` value) and the user is not using either `all=True`,`all=False`, `as_list=False`, or `page=X`.
1 parent5370979 commit9117111

File tree

3 files changed

+200
-10
lines changed

3 files changed

+200
-10
lines changed

‎gitlab/client.py‎

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
importrequests.utils
2525
fromrequests_toolbelt.multipart.encoderimportMultipartEncoder# type: ignore
2626

27+
importgitlab
2728
importgitlab.config
2829
importgitlab.const
2930
importgitlab.exceptions
@@ -37,6 +38,12 @@
3738

3839
RETRYABLE_TRANSIENT_ERROR_CODES= [500,502,503,504]+list(range(520,531))
3940

41+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
42+
_PAGINATION_URL= (
43+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
44+
f"api-usage.html#pagination"
45+
)
46+
4047

4148
classGitlab:
4249
"""Represents a GitLab server connection.
@@ -821,25 +828,65 @@ def http_list(
821828
GitlabHttpError: When the return code is not 2xx
822829
GitlabParsingError: If the json data could not be parsed
823830
"""
831+
# pylint: disable=too-many-return-statements
824832
query_data=query_dataor {}
825833

826834
# In case we want to change the default behavior at some point
827835
as_list=Trueifas_listisNoneelseas_list
828836

829-
get_all=kwargs.pop("all",False)
837+
get_all=kwargs.pop("all",None)
830838
url=self._build_url(path)
831839

832840
page=kwargs.get("page")
833841

834-
ifget_allisTrueandas_listisTrue:
835-
returnlist(GitlabList(self,url,query_data,**kwargs))
842+
ifas_listisFalse:
843+
# Generator requested
844+
returnGitlabList(self,url,query_data,**kwargs)
836845

837-
ifpageoras_listisTrue:
838-
# pagination requested, we return a list
839-
returnlist(GitlabList(self,url,query_data,get_next=False,**kwargs))
846+
ifget_allisTrue:
847+
returnlist(GitlabList(self,url,query_data,**kwargs))
840848

841-
# No pagination, generator requested
842-
returnGitlabList(self,url,query_data,**kwargs)
849+
# pagination requested, we return a list
850+
gl_list=GitlabList(self,url,query_data,get_next=False,**kwargs)
851+
items=list(gl_list)
852+
853+
defshould_emit_warning()->bool:
854+
# No warning is emitted if any of the following conditions apply:
855+
# * `all=False` was set in the `list()` call.
856+
# * `page` was set in the `list()` call.
857+
# * GitLab did not return the `x-per-page` header.
858+
# * Number of items received is less than per-page value.
859+
# * Number of items received is >= total available.
860+
ifget_allisFalse:
861+
returnFalse
862+
ifpageisnotNone:
863+
returnFalse
864+
ifgl_list.per_pageisNone:
865+
returnFalse
866+
iflen(items)<gl_list.per_page:
867+
returnFalse
868+
ifgl_list.totalisnotNoneandlen(items)>=gl_list.total:
869+
returnFalse
870+
returnTrue
871+
872+
ifnotshould_emit_warning():
873+
returnitems
874+
875+
# Warn the user that they are only going to retrieve `per_page`
876+
# maximum items. This is a common cause of issues filed.
877+
total_items="many"ifgl_list.totalisNoneelsegl_list.total
878+
utils.warn(
879+
message=(
880+
f"Calling a `list()` method without specifying `all=True` or "
881+
f"`as_list=False` will return a maximum of{gl_list.per_page} items. "
882+
f"Your query returned{len(items)} of{total_items} items. See "
883+
f"{_PAGINATION_URL} for more details. If this was done intentionally, "
884+
f"then this warning can be supressed by adding the argument "
885+
f"`all=False` to the `list()` call."
886+
),
887+
category=UserWarning,
888+
)
889+
returnitems
843890

844891
defhttp_post(
845892
self,

‎tests/functional/api/test_gitlab.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
importwarnings
2+
13
importpytest
24

35
importgitlab
@@ -181,3 +183,46 @@ def test_rate_limits(gl):
181183
settings.throttle_authenticated_api_enabled=False
182184
settings.save()
183185
[project.delete()forprojectinprojects]
186+
187+
188+
deftest_list_default_warning(gl):
189+
"""When there are more than 20 items and use default `list()` then warning is
190+
generated"""
191+
withwarnings.catch_warnings(record=True)ascaught_warnings:
192+
gl.gitlabciymls.list()
193+
assertlen(caught_warnings)==1
194+
warning=caught_warnings[0]
195+
assertisinstance(warning.message,UserWarning)
196+
message=str(warning.message)
197+
assert"python-gitlab.readthedocs.io"inmessage
198+
assert__file__==warning.filename
199+
200+
201+
deftest_list_page_nowarning(gl):
202+
"""Using `page=X` will disable the warning"""
203+
withwarnings.catch_warnings(record=True)ascaught_warnings:
204+
gl.gitlabciymls.list(page=1)
205+
assertlen(caught_warnings)==0
206+
207+
208+
deftest_list_all_false_nowarning(gl):
209+
"""Using `all=False` will disable the warning"""
210+
withwarnings.catch_warnings(record=True)ascaught_warnings:
211+
gl.gitlabciymls.list(all=False)
212+
assertlen(caught_warnings)==0
213+
214+
215+
deftest_list_all_true_nowarning(gl):
216+
"""Using `all=True` will disable the warning"""
217+
withwarnings.catch_warnings(record=True)ascaught_warnings:
218+
items=gl.gitlabciymls.list(all=True)
219+
assertlen(caught_warnings)==0
220+
assertlen(items)>20
221+
222+
223+
deftest_list_as_list_false_nowarning(gl):
224+
"""Using `as_list=False` will disable the warning"""
225+
withwarnings.catch_warnings(record=True)ascaught_warnings:
226+
items=gl.gitlabciymls.list(as_list=False)
227+
assertlen(caught_warnings)==0
228+
assertlen(list(items))>20

‎tests/unit/test_gitlab_http_methods.py‎

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
importcopy
2+
importwarnings
3+
14
importpytest
25
importrequests
36
importresponses
@@ -425,20 +428,115 @@ def test_list_request(gl):
425428
match=MATCH_EMPTY_QUERY_PARAMS,
426429
)
427430

428-
result=gl.http_list("/projects",as_list=True)
431+
withwarnings.catch_warnings(record=True)ascaught_warnings:
432+
result=gl.http_list("/projects",as_list=True)
433+
assertlen(caught_warnings)==0
429434
assertisinstance(result,list)
430435
assertlen(result)==1
431436

432437
result=gl.http_list("/projects",as_list=False)
433438
assertisinstance(result,GitlabList)
434-
assertlen(result)==1
439+
assertlen(list(result))==1
435440

436441
result=gl.http_list("/projects",all=True)
437442
assertisinstance(result,list)
438443
assertlen(result)==1
439444
assertresponses.assert_call_count(url,3)isTrue
440445

441446

447+
large_list_response= {
448+
"method":responses.GET,
449+
"url":"http://localhost/api/v4/projects",
450+
"json": [
451+
{"name":"project01"},
452+
{"name":"project02"},
453+
{"name":"project03"},
454+
{"name":"project04"},
455+
{"name":"project05"},
456+
{"name":"project06"},
457+
{"name":"project07"},
458+
{"name":"project08"},
459+
{"name":"project09"},
460+
{"name":"project10"},
461+
{"name":"project11"},
462+
{"name":"project12"},
463+
{"name":"project13"},
464+
{"name":"project14"},
465+
{"name":"project15"},
466+
{"name":"project16"},
467+
{"name":"project17"},
468+
{"name":"project18"},
469+
{"name":"project19"},
470+
{"name":"project20"},
471+
],
472+
"headers": {"X-Total":"30","x-per-page":"20"},
473+
"status":200,
474+
"match":MATCH_EMPTY_QUERY_PARAMS,
475+
}
476+
477+
478+
@responses.activate
479+
deftest_list_request_pagination_warning(gl):
480+
responses.add(**large_list_response)
481+
482+
withwarnings.catch_warnings(record=True)ascaught_warnings:
483+
result=gl.http_list("/projects",as_list=True)
484+
assertlen(caught_warnings)==1
485+
warning=caught_warnings[0]
486+
assertisinstance(warning.message,UserWarning)
487+
message=str(warning.message)
488+
assert"Calling a `list()` method"inmessage
489+
assert"python-gitlab.readthedocs.io"inmessage
490+
assert__file__==warning.filename
491+
assertisinstance(result,list)
492+
assertlen(result)==20
493+
assertlen(responses.calls)==1
494+
495+
496+
@responses.activate
497+
deftest_list_request_as_list_false_nowarning(gl):
498+
responses.add(**large_list_response)
499+
withwarnings.catch_warnings(record=True)ascaught_warnings:
500+
result=gl.http_list("/projects",as_list=False)
501+
assertlen(caught_warnings)==0
502+
assertisinstance(result,GitlabList)
503+
assertlen(list(result))==20
504+
assertlen(responses.calls)==1
505+
506+
507+
@responses.activate
508+
deftest_list_request_all_true_nowarning(gl):
509+
responses.add(**large_list_response)
510+
withwarnings.catch_warnings(record=True)ascaught_warnings:
511+
result=gl.http_list("/projects",all=True)
512+
assertlen(caught_warnings)==0
513+
assertisinstance(result,list)
514+
assertlen(result)==20
515+
assertlen(responses.calls)==1
516+
517+
518+
@responses.activate
519+
deftest_list_request_all_false_nowarning(gl):
520+
responses.add(**large_list_response)
521+
withwarnings.catch_warnings(record=True)ascaught_warnings:
522+
result=gl.http_list("/projects",all=False)
523+
assertlen(caught_warnings)==0
524+
assertisinstance(result,list)
525+
assertlen(result)==20
526+
assertlen(responses.calls)==1
527+
528+
529+
@responses.activate
530+
deftest_list_request_page_nowarning(gl):
531+
response_dict=copy.deepcopy(large_list_response)
532+
response_dict["match"]= [responses.matchers.query_param_matcher({"page":"1"})]
533+
responses.add(**response_dict)
534+
withwarnings.catch_warnings(record=True)ascaught_warnings:
535+
gl.http_list("/projects",page=1)
536+
assertlen(caught_warnings)==0
537+
assertlen(responses.calls)==1
538+
539+
442540
@responses.activate
443541
deftest_list_request_404(gl):
444542
url="http://localhost/api/v4/not_there"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp