Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitf99cf74

Browse files
feat: emit a warning when using alist() method returns max
A common cause of issues filed and questions raised is that a userwill call a `list()` method and only get 20 items. As this is thedefault maximum of items that will be returned from a `list()` method.To help with this we now emit a warning when the result from a`list()` method is greater-than or equal to 20 (or the specified`per_page` value) and the user is not using either `all=True`,`all=False`, `as_list=False`, or `page=X`.
1 parent9fe60f7 commitf99cf74

File tree

3 files changed

+195
-10
lines changed

3 files changed

+195
-10
lines changed

‎gitlab/client.py‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
importrequests.utils
2525
fromrequests_toolbelt.multipart.encoderimportMultipartEncoder# type: ignore
2626

27+
importgitlab
2728
importgitlab.config
2829
importgitlab.const
2930
importgitlab.exceptions
@@ -35,6 +36,12 @@
3536
"{source!r} to {target!r}"
3637
)
3738

39+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
40+
_PAGINATION_URL= (
41+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
42+
f"api-usage.html#pagination"
43+
)
44+
3845

3946
classGitlab:
4047
"""Represents a GitLab server connection.
@@ -807,25 +814,60 @@ def http_list(
807814
GitlabHttpError: When the return code is not 2xx
808815
GitlabParsingError: If the json data could not be parsed
809816
"""
817+
# pylint: disable=too-many-return-statements
810818
query_data=query_dataor {}
811819

812820
# In case we want to change the default behavior at some point
813821
as_list=Trueifas_listisNoneelseas_list
814822

815-
get_all=kwargs.pop("all",False)
823+
get_all=kwargs.pop("all",None)
816824
url=self._build_url(path)
817825

818826
page=kwargs.get("page")
819827

820-
ifget_allisTrueandas_listisTrue:
821-
returnlist(GitlabList(self,url,query_data,**kwargs))
828+
ifas_listisFalse:
829+
# Generator requested
830+
returnGitlabList(self,url,query_data,**kwargs)
822831

823-
ifpageoras_listisTrue:
824-
# pagination requested, we return a list
825-
returnlist(GitlabList(self,url,query_data,get_next=False,**kwargs))
832+
ifget_allisTrue:
833+
returnlist(GitlabList(self,url,query_data,**kwargs))
826834

827-
# No pagination, generator requested
828-
returnGitlabList(self,url,query_data,**kwargs)
835+
# pagination requested, we return a list
836+
gl_list=GitlabList(self,url,query_data,get_next=False,**kwargs)
837+
items=list(gl_list)
838+
839+
# No warning is emitted if any of the following conditions apply:
840+
# * `all=False` was set in the `list()` call.
841+
# * `page` was set in the `list()` call.
842+
# * GitLab did not return the `x-per-page` header.
843+
# * Number of items received is less than per-page value.
844+
# * Number of items received is >= total available.
845+
ifget_allisFalse:
846+
returnitems
847+
ifpageisnotNone:
848+
returnitems
849+
ifgl_list.per_pageisNone:
850+
returnitems
851+
iflen(items)<gl_list.per_page:
852+
returnitems
853+
ifgl_list.totalisnotNoneandlen(items)>=gl_list.total:
854+
returnitems
855+
856+
# Warn the user that they are only going to retrieve `per_page`
857+
# maximum items. This is a common cause of issues filed.
858+
total_items="many"ifgl_list.totalisNoneelsegl_list.total
859+
utils.warn(
860+
message=(
861+
f"Calling a `list()` method without specifying `all=True` or "
862+
f"`as_list=False` will return a maximum of{gl_list.per_page} items. "
863+
f"Your query returned{len(items)} of{total_items} items. See "
864+
f"{_PAGINATION_URL} for more details. If this was done intentionally, "
865+
f"then this warning can be supressed by adding the argument "
866+
f"`all=False` to the `list()` call."
867+
),
868+
category=UserWarning,
869+
)
870+
returnitems
829871

830872
defhttp_post(
831873
self,

‎tests/functional/api/test_gitlab.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
importwarnings
2+
13
importpytest
24

35
importgitlab
@@ -181,3 +183,46 @@ def test_rate_limits(gl):
181183
settings.throttle_authenticated_api_enabled=False
182184
settings.save()
183185
[project.delete()forprojectinprojects]
186+
187+
188+
deftest_list_default_warning(gl):
189+
"""When there are more than 20 items and use default `list()` then warning is
190+
generated"""
191+
withwarnings.catch_warnings(record=True)ascaught_warnings:
192+
gl.gitlabciymls.list()
193+
assertlen(caught_warnings)==1
194+
warning=caught_warnings[0]
195+
assertisinstance(warning.message,UserWarning)
196+
message=str(warning.message)
197+
assert"python-gitlab.readthedocs.io"inmessage
198+
assert__file__==warning.filename
199+
200+
201+
deftest_list_page_nowarning(gl):
202+
"""Using `page=X` will disable the warning"""
203+
withwarnings.catch_warnings(record=True)ascaught_warnings:
204+
gl.gitlabciymls.list(page=1)
205+
assertlen(caught_warnings)==0
206+
207+
208+
deftest_list_all_false_nowarning(gl):
209+
"""Using `all=False` will disable the warning"""
210+
withwarnings.catch_warnings(record=True)ascaught_warnings:
211+
gl.gitlabciymls.list(all=False)
212+
assertlen(caught_warnings)==0
213+
214+
215+
deftest_list_all_true_nowarning(gl):
216+
"""Using `all=True` will disable the warning"""
217+
withwarnings.catch_warnings(record=True)ascaught_warnings:
218+
items=gl.gitlabciymls.list(all=True)
219+
assertlen(caught_warnings)==0
220+
assertlen(items)>20
221+
222+
223+
deftest_list_as_list_false_nowarning(gl):
224+
"""Using `as_list=False` will disable the warning"""
225+
withwarnings.catch_warnings(record=True)ascaught_warnings:
226+
items=gl.gitlabciymls.list(as_list=False)
227+
assertlen(caught_warnings)==0
228+
assertlen(list(items))>20

‎tests/unit/test_gitlab_http_methods.py‎

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
importcopy
2+
importwarnings
3+
14
importpytest
25
importrequests
36
importresponses
@@ -329,20 +332,115 @@ def test_list_request(gl):
329332
match=MATCH_EMPTY_QUERY_PARAMS,
330333
)
331334

332-
result=gl.http_list("/projects",as_list=True)
335+
withwarnings.catch_warnings(record=True)ascaught_warnings:
336+
result=gl.http_list("/projects",as_list=True)
337+
assertlen(caught_warnings)==0
333338
assertisinstance(result,list)
334339
assertlen(result)==1
335340

336341
result=gl.http_list("/projects",as_list=False)
337342
assertisinstance(result,GitlabList)
338-
assertlen(result)==1
343+
assertlen(list(result))==1
339344

340345
result=gl.http_list("/projects",all=True)
341346
assertisinstance(result,list)
342347
assertlen(result)==1
343348
assertresponses.assert_call_count(url,3)isTrue
344349

345350

351+
large_list_response= {
352+
"method":responses.GET,
353+
"url":"http://localhost/api/v4/projects",
354+
"json": [
355+
{"name":"project01"},
356+
{"name":"project02"},
357+
{"name":"project03"},
358+
{"name":"project04"},
359+
{"name":"project05"},
360+
{"name":"project06"},
361+
{"name":"project07"},
362+
{"name":"project08"},
363+
{"name":"project09"},
364+
{"name":"project10"},
365+
{"name":"project11"},
366+
{"name":"project12"},
367+
{"name":"project13"},
368+
{"name":"project14"},
369+
{"name":"project15"},
370+
{"name":"project16"},
371+
{"name":"project17"},
372+
{"name":"project18"},
373+
{"name":"project19"},
374+
{"name":"project20"},
375+
],
376+
"headers": {"X-Total":"30","x-per-page":"20"},
377+
"status":200,
378+
"match":MATCH_EMPTY_QUERY_PARAMS,
379+
}
380+
381+
382+
@responses.activate
383+
deftest_list_request_pagination_warning(gl):
384+
responses.add(**large_list_response)
385+
386+
withwarnings.catch_warnings(record=True)ascaught_warnings:
387+
result=gl.http_list("/projects",as_list=True)
388+
assertlen(caught_warnings)==1
389+
warning=caught_warnings[0]
390+
assertisinstance(warning.message,UserWarning)
391+
message=str(warning.message)
392+
assert"Calling a `list()` method"inmessage
393+
assert"python-gitlab.readthedocs.io"inmessage
394+
assert__file__==warning.filename
395+
assertisinstance(result,list)
396+
assertlen(result)==20
397+
assertlen(responses.calls)==1
398+
399+
400+
@responses.activate
401+
deftest_list_request_as_list_false_nowarning(gl):
402+
responses.add(**large_list_response)
403+
withwarnings.catch_warnings(record=True)ascaught_warnings:
404+
result=gl.http_list("/projects",as_list=False)
405+
assertlen(caught_warnings)==0
406+
assertisinstance(result,GitlabList)
407+
assertlen(list(result))==20
408+
assertlen(responses.calls)==1
409+
410+
411+
@responses.activate
412+
deftest_list_request_all_true_nowarning(gl):
413+
responses.add(**large_list_response)
414+
withwarnings.catch_warnings(record=True)ascaught_warnings:
415+
result=gl.http_list("/projects",all=True)
416+
assertlen(caught_warnings)==0
417+
assertisinstance(result,list)
418+
assertlen(result)==20
419+
assertlen(responses.calls)==1
420+
421+
422+
@responses.activate
423+
deftest_list_request_all_false_nowarning(gl):
424+
responses.add(**large_list_response)
425+
withwarnings.catch_warnings(record=True)ascaught_warnings:
426+
result=gl.http_list("/projects",all=False)
427+
assertlen(caught_warnings)==0
428+
assertisinstance(result,list)
429+
assertlen(result)==20
430+
assertlen(responses.calls)==1
431+
432+
433+
@responses.activate
434+
deftest_list_request_page_nowarning(gl):
435+
response_dict=copy.deepcopy(large_list_response)
436+
response_dict["match"]= [responses.matchers.query_param_matcher({"page":"1"})]
437+
responses.add(**response_dict)
438+
withwarnings.catch_warnings(record=True)ascaught_warnings:
439+
gl.http_list("/projects",page=1)
440+
assertlen(caught_warnings)==0
441+
assertlen(responses.calls)==1
442+
443+
346444
@responses.activate
347445
deftest_list_request_404(gl):
348446
url="http://localhost/api/v4/not_there"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp