Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitfb933c5

Browse files
feat: emit a warning when using alist() method returns max
A common cause of issues filed and questions raised is that a userwill call a `list()` method and only get 20 items. As this is thedefault maximum of items that will be returned from a `list()` method.To help with this we now emit a warning when the result from a`list()` method is greater-than or equal to 20 (or the specified`per_page` value) and the user is not using either `all=True`,`all=False`, `as_list=False`, or `page=X`.
1 parent0bb0626 commitfb933c5

File tree

4 files changed

+190
-11
lines changed

4 files changed

+190
-11
lines changed

‎docs/api-usage.rst‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,20 +93,26 @@ Examples:
9393
..code-block::python
9494
9595
# list all the projects
96-
projects= gl.projects.list()
96+
projects= gl.projects.list(as_list=False)
9797
for projectin projects:
9898
print(project)
9999
100100
# get the group with id == 2
101101
group= gl.groups.get(2)
102-
for projectin group.projects.list():
102+
for projectin group.projects.list(as_list=False):
103103
print(project)
104104
105105
# create a new user
106106
user_data= {'email':'jen@foo.com','username':'jen','name':'Jen'}
107107
user= gl.users.create(user_data)
108108
print(user)
109109
110+
..warning::
111+
Calling `list()` without any arguments will by default not return the complete list
112+
of items. Use either the `all=True` or `as_list=False` parameters to get all the
113+
items when using listing methods. See the:ref:`pagination` section for more
114+
information.
115+
110116
You can list the mandatory and optional attributes for object creation and
111117
update with the manager's ``get_create_attrs()`` and ``get_update_attrs()``
112118
methods. They return 2 tuples, the first one is the list of mandatory
@@ -133,7 +139,7 @@ Some objects also provide managers to access related GitLab resources:
133139
134140
# list the issues for a project
135141
project= gl.projects.get(1)
136-
issues= project.issues.list()
142+
issues= project.issues.list(all=True)
137143
138144
python-gitlab allows to send any data to the GitLab server when making queries.
139145
In case of invalid or missing arguments python-gitlab will raise an exception
@@ -150,9 +156,9 @@ conflict with python or python-gitlab when using them as kwargs:
150156

151157
..code-block::python
152158
153-
gl.user_activities.list(from='2019-01-01')## invalid
159+
gl.user_activities.list(from='2019-01-01',as_list=False)## invalid
154160
155-
gl.user_activities.list(query_parameters={'from':'2019-01-01'})# OK
161+
gl.user_activities.list(query_parameters={'from':'2019-01-01'},as_list=False)# OK
156162
157163
Gitlab Objects
158164
==============
@@ -222,6 +228,8 @@ a project (the previous example used 2 API calls):
222228
project= gl.projects.get(1,lazy=True)# no API call
223229
project.star()# API call
224230
231+
.. _pagination:
232+
225233
Pagination
226234
==========
227235

‎gitlab/client.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
importrequests.utils
2525
fromrequests_toolbelt.multipart.encoderimportMultipartEncoder# type: ignore
2626

27+
importgitlab
2728
importgitlab.config
2829
importgitlab.const
2930
importgitlab.exceptions
@@ -35,6 +36,12 @@
3536
"{source!r} to {target!r}"
3637
)
3738

39+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
40+
_PAGINATION_URL= (
41+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
42+
f"api-usage.html#pagination"
43+
)
44+
3845

3946
classGitlab:
4047
"""Represents a GitLab server connection.
@@ -808,7 +815,7 @@ def http_list(
808815
# In case we want to change the default behavior at some point
809816
as_list=Trueifas_listisNoneelseas_list
810817

811-
get_all=kwargs.pop("all",False)
818+
get_all=kwargs.pop("all",None)
812819
url=self._build_url(path)
813820

814821
page=kwargs.get("page")
@@ -818,7 +825,28 @@ def http_list(
818825

819826
ifpageoras_listisTrue:
820827
# pagination requested, we return a list
821-
returnlist(GitlabList(self,url,query_data,get_next=False,**kwargs))
828+
gl_list=GitlabList(self,url,query_data,get_next=False,**kwargs)
829+
items=list(gl_list)
830+
ifpageisNoneandget_allisNoneandgl_list.per_pageisnotNone:
831+
iflen(items)>=gl_list.per_pageand (
832+
gl_list.totalisNoneorlen(items)<gl_list.total
833+
):
834+
total_items="10,000+"ifgl_list.totalisNoneelsegl_list.total
835+
# Warn the user that they are only going to retrieve `per_page`
836+
# maximum items. This is a common cause of issues filed.
837+
utils.warn(
838+
message=(
839+
f"Calling a `list()` method without specifying `all=True` "
840+
f"or `as_list=False` will return a maximum of "
841+
f"{gl_list.per_page} items. Your query returned "
842+
f"{len(items)} of{total_items} items. See "
843+
f"{_PAGINATION_URL} for more details. If this was done "
844+
f"intentionally, then this warning can be supressed by "
845+
f"adding the argument `all=False` to the `list()` call."
846+
),
847+
category=UserWarning,
848+
)
849+
returnitems
822850

823851
# No pagination, generator requested
824852
returnGitlabList(self,url,query_data,**kwargs)

‎tests/functional/api/test_gitlab.py‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
importwarnings
2+
13
importpytest
24

35
importgitlab
@@ -81,13 +83,13 @@ def test_template_dockerfile(gl):
8183

8284

8385
deftest_template_gitignore(gl):
84-
assertgl.gitignores.list()
86+
assertgl.gitignores.list(all=True)
8587
gitignore=gl.gitignores.get("Node")
8688
assertgitignore.contentisnotNone
8789

8890

8991
deftest_template_gitlabciyml(gl):
90-
assertgl.gitlabciymls.list()
92+
assertgl.gitlabciymls.list(all=True)
9193
gitlabciyml=gl.gitlabciymls.get("Nodejs")
9294
assertgitlabciyml.contentisnotNone
9395

@@ -181,3 +183,46 @@ def test_rate_limits(gl):
181183
settings.throttle_authenticated_api_enabled=False
182184
settings.save()
183185
[project.delete()forprojectinprojects]
186+
187+
188+
deftest_list_default_warning(gl):
189+
"""When there are more than 20 items and use default `list()` then warning is
190+
generated"""
191+
withwarnings.catch_warnings(record=True)ascaught_warnings:
192+
gl.gitlabciymls.list()
193+
assertlen(caught_warnings)==1
194+
warning=caught_warnings[0]
195+
assertisinstance(warning.message,UserWarning)
196+
message=str(warning.message)
197+
assert"python-gitlab.readthedocs.io"inmessage
198+
assert__file__==warning.filename
199+
200+
201+
deftest_list_page_nowarning(gl):
202+
"""Using `page=X` will disable the warning"""
203+
withwarnings.catch_warnings(record=True)ascaught_warnings:
204+
gl.gitlabciymls.list(page=1)
205+
assertlen(caught_warnings)==0
206+
207+
208+
deftest_list_all_false_nowarning(gl):
209+
"""Using `all=False` will disable the warning"""
210+
withwarnings.catch_warnings(record=True)ascaught_warnings:
211+
gl.gitlabciymls.list(all=False)
212+
assertlen(caught_warnings)==0
213+
214+
215+
deftest_list_all_true_nowarning(gl):
216+
"""Using `all=True` will disable the warning"""
217+
withwarnings.catch_warnings(record=True)ascaught_warnings:
218+
items=gl.gitlabciymls.list(all=True)
219+
assertlen(caught_warnings)==0
220+
assertlen(items)>20
221+
222+
223+
deftest_list_as_list_false_nowarning(gl):
224+
"""Using `as_list=False` will disable the warning"""
225+
withwarnings.catch_warnings(record=True)ascaught_warnings:
226+
items=gl.gitlabciymls.list(as_list=False)
227+
assertlen(caught_warnings)==0
228+
assertlen(list(items))>20

‎tests/unit/test_gitlab_http_methods.py‎

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
importcopy
2+
importwarnings
3+
14
importpytest
25
importrequests
36
importresponses
@@ -329,20 +332,115 @@ def test_list_request(gl):
329332
match=MATCH_EMPTY_QUERY_PARAMS,
330333
)
331334

332-
result=gl.http_list("/projects",as_list=True)
335+
withwarnings.catch_warnings(record=True)ascaught_warnings:
336+
result=gl.http_list("/projects",as_list=True)
337+
assertlen(caught_warnings)==0
333338
assertisinstance(result,list)
334339
assertlen(result)==1
335340

336341
result=gl.http_list("/projects",as_list=False)
337342
assertisinstance(result,GitlabList)
338-
assertlen(result)==1
343+
assertlen(list(result))==1
339344

340345
result=gl.http_list("/projects",all=True)
341346
assertisinstance(result,list)
342347
assertlen(result)==1
343348
assertresponses.assert_call_count(url,3)isTrue
344349

345350

351+
large_list_response= {
352+
"method":responses.GET,
353+
"url":"http://localhost/api/v4/projects",
354+
"json": [
355+
{"name":"project01"},
356+
{"name":"project02"},
357+
{"name":"project03"},
358+
{"name":"project04"},
359+
{"name":"project05"},
360+
{"name":"project06"},
361+
{"name":"project07"},
362+
{"name":"project08"},
363+
{"name":"project09"},
364+
{"name":"project10"},
365+
{"name":"project11"},
366+
{"name":"project12"},
367+
{"name":"project13"},
368+
{"name":"project14"},
369+
{"name":"project15"},
370+
{"name":"project16"},
371+
{"name":"project17"},
372+
{"name":"project18"},
373+
{"name":"project19"},
374+
{"name":"project20"},
375+
],
376+
"headers": {"X-Total":"30","x-per-page":"20"},
377+
"status":200,
378+
"match":MATCH_EMPTY_QUERY_PARAMS,
379+
}
380+
381+
382+
@responses.activate
383+
deftest_list_request_pagination_warning(gl):
384+
responses.add(**large_list_response)
385+
386+
withwarnings.catch_warnings(record=True)ascaught_warnings:
387+
result=gl.http_list("/projects",as_list=True)
388+
assertlen(caught_warnings)==1
389+
warning=caught_warnings[0]
390+
assertisinstance(warning.message,UserWarning)
391+
message=str(warning.message)
392+
assert"Calling a `list()` method"inmessage
393+
assert"python-gitlab.readthedocs.io"inmessage
394+
assert__file__==warning.filename
395+
assertisinstance(result,list)
396+
assertlen(result)==20
397+
assertlen(responses.calls)==1
398+
399+
400+
@responses.activate
401+
deftest_list_request_as_list_false_nowarning(gl):
402+
responses.add(**large_list_response)
403+
withwarnings.catch_warnings(record=True)ascaught_warnings:
404+
result=gl.http_list("/projects",as_list=False)
405+
assertlen(caught_warnings)==0
406+
assertisinstance(result,GitlabList)
407+
assertlen(list(result))==20
408+
assertlen(responses.calls)==1
409+
410+
411+
@responses.activate
412+
deftest_list_request_all_true_nowarning(gl):
413+
responses.add(**large_list_response)
414+
withwarnings.catch_warnings(record=True)ascaught_warnings:
415+
result=gl.http_list("/projects",all=True)
416+
assertlen(caught_warnings)==0
417+
assertisinstance(result,list)
418+
assertlen(result)==20
419+
assertlen(responses.calls)==1
420+
421+
422+
@responses.activate
423+
deftest_list_request_all_false_nowarning(gl):
424+
responses.add(**large_list_response)
425+
withwarnings.catch_warnings(record=True)ascaught_warnings:
426+
result=gl.http_list("/projects",all=False)
427+
assertlen(caught_warnings)==0
428+
assertisinstance(result,list)
429+
assertlen(result)==20
430+
assertlen(responses.calls)==1
431+
432+
433+
@responses.activate
434+
deftest_list_request_page_nowarning(gl):
435+
response_dict=copy.deepcopy(large_list_response)
436+
response_dict["match"]= [responses.matchers.query_param_matcher({"page":"1"})]
437+
responses.add(**response_dict)
438+
withwarnings.catch_warnings(record=True)ascaught_warnings:
439+
gl.http_list("/projects",page=1)
440+
assertlen(caught_warnings)==0
441+
assertlen(responses.calls)==1
442+
443+
346444
@responses.activate
347445
deftest_list_request_404(gl):
348446
url="http://localhost/api/v4/not_there"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp