Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit4d6f125

Browse files
authored
Merge pull request#1875 from python-gitlab/jlvillal/list_warning
feat: emit a warning when using a `list()` method returns max
2 parents5370979 +1339d64 commit4d6f125

File tree

3 files changed

+199
-10
lines changed

3 files changed

+199
-10
lines changed

‎gitlab/client.py‎

Lines changed: 54 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
importrequests.utils
2525
fromrequests_toolbelt.multipart.encoderimportMultipartEncoder# type: ignore
2626

27+
importgitlab
2728
importgitlab.config
2829
importgitlab.const
2930
importgitlab.exceptions
@@ -37,6 +38,12 @@
3738

3839
RETRYABLE_TRANSIENT_ERROR_CODES= [500,502,503,504]+list(range(520,531))
3940

41+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
42+
_PAGINATION_URL= (
43+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
44+
f"api-usage.html#pagination"
45+
)
46+
4047

4148
classGitlab:
4249
"""Represents a GitLab server connection.
@@ -826,20 +833,59 @@ def http_list(
826833
# In case we want to change the default behavior at some point
827834
as_list=Trueifas_listisNoneelseas_list
828835

829-
get_all=kwargs.pop("all",False)
836+
get_all=kwargs.pop("all",None)
830837
url=self._build_url(path)
831838

832839
page=kwargs.get("page")
833840

834-
ifget_allisTrueandas_listisTrue:
835-
returnlist(GitlabList(self,url,query_data,**kwargs))
841+
ifas_listisFalse:
842+
# Generator requested
843+
returnGitlabList(self,url,query_data,**kwargs)
836844

837-
ifpageoras_listisTrue:
838-
# pagination requested, we return a list
839-
returnlist(GitlabList(self,url,query_data,get_next=False,**kwargs))
845+
ifget_allisTrue:
846+
returnlist(GitlabList(self,url,query_data,**kwargs))
840847

841-
# No pagination, generator requested
842-
returnGitlabList(self,url,query_data,**kwargs)
848+
# pagination requested, we return a list
849+
gl_list=GitlabList(self,url,query_data,get_next=False,**kwargs)
850+
items=list(gl_list)
851+
852+
defshould_emit_warning()->bool:
853+
# No warning is emitted if any of the following conditions apply:
854+
# * `all=False` was set in the `list()` call.
855+
# * `page` was set in the `list()` call.
856+
# * GitLab did not return the `x-per-page` header.
857+
# * Number of items received is less than per-page value.
858+
# * Number of items received is >= total available.
859+
ifget_allisFalse:
860+
returnFalse
861+
ifpageisnotNone:
862+
returnFalse
863+
ifgl_list.per_pageisNone:
864+
returnFalse
865+
iflen(items)<gl_list.per_page:
866+
returnFalse
867+
ifgl_list.totalisnotNoneandlen(items)>=gl_list.total:
868+
returnFalse
869+
returnTrue
870+
871+
ifnotshould_emit_warning():
872+
returnitems
873+
874+
# Warn the user that they are only going to retrieve `per_page`
875+
# maximum items. This is a common cause of issues filed.
876+
total_items="many"ifgl_list.totalisNoneelsegl_list.total
877+
utils.warn(
878+
message=(
879+
f"Calling a `list()` method without specifying `all=True` or "
880+
f"`as_list=False` will return a maximum of{gl_list.per_page} items. "
881+
f"Your query returned{len(items)} of{total_items} items. See "
882+
f"{_PAGINATION_URL} for more details. If this was done intentionally, "
883+
f"then this warning can be supressed by adding the argument "
884+
f"`all=False` to the `list()` call."
885+
),
886+
category=UserWarning,
887+
)
888+
returnitems
843889

844890
defhttp_post(
845891
self,

‎tests/functional/api/test_gitlab.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
importwarnings
2+
13
importpytest
24

35
importgitlab
@@ -181,3 +183,46 @@ def test_rate_limits(gl):
181183
settings.throttle_authenticated_api_enabled=False
182184
settings.save()
183185
[project.delete()forprojectinprojects]
186+
187+
188+
deftest_list_default_warning(gl):
189+
"""When there are more than 20 items and use default `list()` then warning is
190+
generated"""
191+
withwarnings.catch_warnings(record=True)ascaught_warnings:
192+
gl.gitlabciymls.list()
193+
assertlen(caught_warnings)==1
194+
warning=caught_warnings[0]
195+
assertisinstance(warning.message,UserWarning)
196+
message=str(warning.message)
197+
assert"python-gitlab.readthedocs.io"inmessage
198+
assert__file__==warning.filename
199+
200+
201+
deftest_list_page_nowarning(gl):
202+
"""Using `page=X` will disable the warning"""
203+
withwarnings.catch_warnings(record=True)ascaught_warnings:
204+
gl.gitlabciymls.list(page=1)
205+
assertlen(caught_warnings)==0
206+
207+
208+
deftest_list_all_false_nowarning(gl):
209+
"""Using `all=False` will disable the warning"""
210+
withwarnings.catch_warnings(record=True)ascaught_warnings:
211+
gl.gitlabciymls.list(all=False)
212+
assertlen(caught_warnings)==0
213+
214+
215+
deftest_list_all_true_nowarning(gl):
216+
"""Using `all=True` will disable the warning"""
217+
withwarnings.catch_warnings(record=True)ascaught_warnings:
218+
items=gl.gitlabciymls.list(all=True)
219+
assertlen(caught_warnings)==0
220+
assertlen(items)>20
221+
222+
223+
deftest_list_as_list_false_nowarning(gl):
224+
"""Using `as_list=False` will disable the warning"""
225+
withwarnings.catch_warnings(record=True)ascaught_warnings:
226+
items=gl.gitlabciymls.list(as_list=False)
227+
assertlen(caught_warnings)==0
228+
assertlen(list(items))>20

‎tests/unit/test_gitlab_http_methods.py‎

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
importcopy
2+
importwarnings
3+
14
importpytest
25
importrequests
36
importresponses
@@ -425,20 +428,115 @@ def test_list_request(gl):
425428
match=MATCH_EMPTY_QUERY_PARAMS,
426429
)
427430

428-
result=gl.http_list("/projects",as_list=True)
431+
withwarnings.catch_warnings(record=True)ascaught_warnings:
432+
result=gl.http_list("/projects",as_list=True)
433+
assertlen(caught_warnings)==0
429434
assertisinstance(result,list)
430435
assertlen(result)==1
431436

432437
result=gl.http_list("/projects",as_list=False)
433438
assertisinstance(result,GitlabList)
434-
assertlen(result)==1
439+
assertlen(list(result))==1
435440

436441
result=gl.http_list("/projects",all=True)
437442
assertisinstance(result,list)
438443
assertlen(result)==1
439444
assertresponses.assert_call_count(url,3)isTrue
440445

441446

447+
large_list_response= {
448+
"method":responses.GET,
449+
"url":"http://localhost/api/v4/projects",
450+
"json": [
451+
{"name":"project01"},
452+
{"name":"project02"},
453+
{"name":"project03"},
454+
{"name":"project04"},
455+
{"name":"project05"},
456+
{"name":"project06"},
457+
{"name":"project07"},
458+
{"name":"project08"},
459+
{"name":"project09"},
460+
{"name":"project10"},
461+
{"name":"project11"},
462+
{"name":"project12"},
463+
{"name":"project13"},
464+
{"name":"project14"},
465+
{"name":"project15"},
466+
{"name":"project16"},
467+
{"name":"project17"},
468+
{"name":"project18"},
469+
{"name":"project19"},
470+
{"name":"project20"},
471+
],
472+
"headers": {"X-Total":"30","x-per-page":"20"},
473+
"status":200,
474+
"match":MATCH_EMPTY_QUERY_PARAMS,
475+
}
476+
477+
478+
@responses.activate
479+
deftest_list_request_pagination_warning(gl):
480+
responses.add(**large_list_response)
481+
482+
withwarnings.catch_warnings(record=True)ascaught_warnings:
483+
result=gl.http_list("/projects",as_list=True)
484+
assertlen(caught_warnings)==1
485+
warning=caught_warnings[0]
486+
assertisinstance(warning.message,UserWarning)
487+
message=str(warning.message)
488+
assert"Calling a `list()` method"inmessage
489+
assert"python-gitlab.readthedocs.io"inmessage
490+
assert__file__==warning.filename
491+
assertisinstance(result,list)
492+
assertlen(result)==20
493+
assertlen(responses.calls)==1
494+
495+
496+
@responses.activate
497+
deftest_list_request_as_list_false_nowarning(gl):
498+
responses.add(**large_list_response)
499+
withwarnings.catch_warnings(record=True)ascaught_warnings:
500+
result=gl.http_list("/projects",as_list=False)
501+
assertlen(caught_warnings)==0
502+
assertisinstance(result,GitlabList)
503+
assertlen(list(result))==20
504+
assertlen(responses.calls)==1
505+
506+
507+
@responses.activate
508+
deftest_list_request_all_true_nowarning(gl):
509+
responses.add(**large_list_response)
510+
withwarnings.catch_warnings(record=True)ascaught_warnings:
511+
result=gl.http_list("/projects",all=True)
512+
assertlen(caught_warnings)==0
513+
assertisinstance(result,list)
514+
assertlen(result)==20
515+
assertlen(responses.calls)==1
516+
517+
518+
@responses.activate
519+
deftest_list_request_all_false_nowarning(gl):
520+
responses.add(**large_list_response)
521+
withwarnings.catch_warnings(record=True)ascaught_warnings:
522+
result=gl.http_list("/projects",all=False)
523+
assertlen(caught_warnings)==0
524+
assertisinstance(result,list)
525+
assertlen(result)==20
526+
assertlen(responses.calls)==1
527+
528+
529+
@responses.activate
530+
deftest_list_request_page_nowarning(gl):
531+
response_dict=copy.deepcopy(large_list_response)
532+
response_dict["match"]= [responses.matchers.query_param_matcher({"page":"1"})]
533+
responses.add(**response_dict)
534+
withwarnings.catch_warnings(record=True)ascaught_warnings:
535+
gl.http_list("/projects",page=1)
536+
assertlen(caught_warnings)==0
537+
assertlen(responses.calls)==1
538+
539+
442540
@responses.activate
443541
deftest_list_request_404(gl):
444542
url="http://localhost/api/v4/not_there"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp