Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit0223989

Browse files
feat: emit a warning when using alist() method returns max
A common cause of issues filed and questions raised is that a userwill call a `list()` method and only get 20 items. As this is thedefault maximum of items that will be returned from a `list()` method.To help with this we now emit a warning when the result from a`list()` method is greater-than or equal to 20 (or the specified`per_page` value) and the user is not using either `all=True`,`as_list=False`, or `page=X`.
1 parentaa32ea5 commit0223989

File tree

4 files changed

+194
-7
lines changed

4 files changed

+194
-7
lines changed

‎gitlab/client.py‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@
1818

1919
importos
2020
importtime
21+
importtraceback
22+
importwarnings
2123
fromtypingimportAny,cast,Dict,List,Optional,Tuple,TYPE_CHECKING,Union
2224

2325
importrequests
2426
importrequests.utils
2527
fromrequests_toolbelt.multipart.encoderimportMultipartEncoder# type: ignore
2628

29+
importgitlab
2730
importgitlab.config
2831
importgitlab.const
2932
importgitlab.exceptions
@@ -35,6 +38,12 @@
3538
"{source!r} to {target!r}"
3639
)
3740

41+
# https://docs.gitlab.com/ee/api/#offset-based-pagination
42+
_PAGINATION_URL= (
43+
f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
44+
f"api-usage.html#pagination"
45+
)
46+
3847

3948
classGitlab:
4049
"""Represents a GitLab server connection.
@@ -808,7 +817,7 @@ def http_list(
808817
# In case we want to change the default behavior at some point
809818
as_list=Trueifas_listisNoneelseas_list
810819

811-
get_all=kwargs.pop("all",False)
820+
get_all=kwargs.pop("all",None)
812821
url=self._build_url(path)
813822

814823
page=kwargs.get("page")
@@ -818,7 +827,39 @@ def http_list(
818827

819828
ifpageoras_listisTrue:
820829
# pagination requested, we return a list
821-
returnlist(GitlabList(self,url,query_data,get_next=False,**kwargs))
830+
gl_list=GitlabList(self,url,query_data,get_next=False,**kwargs)
831+
items=list(gl_list)
832+
if (
833+
pageisNone
834+
andget_allisNone
835+
and (gl_list.per_pageisnotNoneandlen(items)>=gl_list.per_page)
836+
and (gl_list.totalisNoneorlen(items)<gl_list.total)
837+
):
838+
# Get `stacklevel` for user code so we indicate where issue is in their
839+
# code.
840+
pg_dir=os.path.abspath(os.path.dirname(__file__))
841+
stack=traceback.extract_stack()
842+
forstacklevel,frameinenumerate(reversed(stack),start=1):
843+
frame_dir=os.path.abspath(os.path.dirname(frame.filename))
844+
ifnotframe_dir.startswith(pg_dir):
845+
break
846+
847+
total_items="10,000+"ifgl_list.totalisNoneelsegl_list.total
848+
# Warn the user that they are only going to retrieve `per_page` maximum
849+
# items. This is a common cause of issues filed.
850+
warnings.warn(
851+
(
852+
f"Calling a `list()` method without specifying `all=True` or "
853+
f"`as_list=False` will return a maximum of{gl_list.per_page} "
854+
f"items. Your query returned{len(items)} of{total_items} "
855+
f"items. See{_PAGINATION_URL} for more details. If this was "
856+
f"done intentionally, then this warning can be supressed by "
857+
f"adding the argument `all=False` to the `list()` call."
858+
),
859+
category=UserWarning,
860+
stacklevel=stacklevel,
861+
)
862+
returnitems
822863

823864
# No pagination, generator requested
824865
returnGitlabList(self,url,query_data,**kwargs)

‎tests/functional/api/test_gitlab.py‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
importwarnings
2+
13
importpytest
24

35
importgitlab
@@ -81,13 +83,13 @@ def test_template_dockerfile(gl):
8183

8284

8385
deftest_template_gitignore(gl):
84-
assertgl.gitignores.list()
86+
assertgl.gitignores.list(all=True)
8587
gitignore=gl.gitignores.get("Node")
8688
assertgitignore.contentisnotNone
8789

8890

8991
deftest_template_gitlabciyml(gl):
90-
assertgl.gitlabciymls.list()
92+
assertgl.gitlabciymls.list(all=True)
9193
gitlabciyml=gl.gitlabciymls.get("Nodejs")
9294
assertgitlabciyml.contentisnotNone
9395

@@ -181,3 +183,48 @@ def test_rate_limits(gl):
181183
settings.throttle_authenticated_api_enabled=False
182184
settings.save()
183185
[project.delete()forprojectinprojects]
186+
187+
188+
deftest_list_default_warning(gl):
189+
"""When there are more than 20 items and use default `list()` then warning is
190+
generated"""
191+
withwarnings.catch_warnings(record=True)ascaught_warnings:
192+
gl.gitlabciymls.list()
193+
assertlen(caught_warnings)==1
194+
warning=caught_warnings[0]
195+
assertisinstance(warning.message,UserWarning)
196+
message=str(caught_warnings[0].message)
197+
assert"Calling"inmessage
198+
assert"return a maximum of"inmessage
199+
assert"readthedocs"inmessage
200+
assert__file__==warning.filename
201+
202+
203+
deftest_list_page_nowarning(gl):
204+
"""Using `page=X` will disable the warning"""
205+
withwarnings.catch_warnings(record=True)ascaught_warnings:
206+
gl.gitlabciymls.list(page=1)
207+
assertlen(caught_warnings)==0
208+
209+
210+
deftest_list_all_false_nowarning(gl):
211+
"""Using `all=False` will disable the warning"""
212+
withwarnings.catch_warnings(record=True)ascaught_warnings:
213+
gl.gitlabciymls.list(all=False)
214+
assertlen(caught_warnings)==0
215+
216+
217+
deftest_list_all_true_nowarning(gl):
218+
"""Using `all=True` will disable the warning"""
219+
withwarnings.catch_warnings(record=True)ascaught_warnings:
220+
items=gl.gitlabciymls.list(all=True)
221+
assertlen(caught_warnings)==0
222+
assertlen(items)>20
223+
224+
225+
deftest_list_as_list_false_nowarning(gl):
226+
"""Using `as_list=False` will disable the warning"""
227+
withwarnings.catch_warnings(record=True)ascaught_warnings:
228+
items=gl.gitlabciymls.list(as_list=False)
229+
assertlen(caught_warnings)==0
230+
assertlen(list(items))>20

‎tests/functional/fixtures/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ services:
1414
GITLAB_ROOT_PASSWORD:5iveL!fe
1515
GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN:registration-token
1616
GITLAB_OMNIBUS_CONFIG:|
17-
external_url 'http://127.0.0.1:8080'
17+
external_url 'http://localhost:8080'
1818
registry['enable'] =false
1919
nginx['redirect_http_to_https'] =false
2020
nginx['listen_port'] = 80

‎tests/unit/test_gitlab_http_methods.py‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
importcopy
2+
importwarnings
3+
14
importpytest
25
importrequests
36
importresponses
@@ -329,20 +332,116 @@ def test_list_request(gl):
329332
match=MATCH_EMPTY_QUERY_PARAMS,
330333
)
331334

332-
result=gl.http_list("/projects",as_list=True)
335+
withwarnings.catch_warnings(record=True)ascaught_warnings:
336+
result=gl.http_list("/projects",as_list=True)
337+
assertlen(caught_warnings)==0
333338
assertisinstance(result,list)
334339
assertlen(result)==1
335340

336341
result=gl.http_list("/projects",as_list=False)
337342
assertisinstance(result,GitlabList)
338-
assertlen(result)==1
343+
assertlen(list(result))==1
339344

340345
result=gl.http_list("/projects",all=True)
341346
assertisinstance(result,list)
342347
assertlen(result)==1
343348
assertresponses.assert_call_count(url,3)isTrue
344349

345350

351+
large_list_response= {
352+
"method":responses.GET,
353+
"url":"http://localhost/api/v4/projects",
354+
"json": [
355+
{"name":"project01"},
356+
{"name":"project02"},
357+
{"name":"project03"},
358+
{"name":"project04"},
359+
{"name":"project05"},
360+
{"name":"project06"},
361+
{"name":"project07"},
362+
{"name":"project08"},
363+
{"name":"project09"},
364+
{"name":"project10"},
365+
{"name":"project11"},
366+
{"name":"project12"},
367+
{"name":"project13"},
368+
{"name":"project14"},
369+
{"name":"project15"},
370+
{"name":"project16"},
371+
{"name":"project17"},
372+
{"name":"project18"},
373+
{"name":"project19"},
374+
{"name":"project20"},
375+
],
376+
"headers": {"X-Total":"30","x-per-page":"20"},
377+
"status":200,
378+
"match":MATCH_EMPTY_QUERY_PARAMS,
379+
}
380+
381+
382+
@responses.activate
383+
deftest_list_request_pagination_warning(gl):
384+
responses.add(**large_list_response)
385+
386+
withwarnings.catch_warnings(record=True)ascaught_warnings:
387+
result=gl.http_list("/projects",as_list=True)
388+
assertlen(caught_warnings)==1
389+
warning=caught_warnings[0]
390+
assertisinstance(warning.message,UserWarning)
391+
message=str(caught_warnings[0].message)
392+
assert"Calling"inmessage
393+
assert"return a maximum of"inmessage
394+
assert"readthedocs"inmessage
395+
assert__file__==warning.filename
396+
assertisinstance(result,list)
397+
assertlen(result)==20
398+
assertlen(responses.calls)==1
399+
400+
401+
@responses.activate
402+
deftest_list_request_as_list_false_nowarning(gl):
403+
responses.add(**large_list_response)
404+
withwarnings.catch_warnings(record=True)ascaught_warnings:
405+
result=gl.http_list("/projects",as_list=False)
406+
assertlen(caught_warnings)==0
407+
assertisinstance(result,GitlabList)
408+
assertlen(list(result))==20
409+
assertlen(responses.calls)==1
410+
411+
412+
@responses.activate
413+
deftest_list_request_all_true_nowarning(gl):
414+
responses.add(**large_list_response)
415+
withwarnings.catch_warnings(record=True)ascaught_warnings:
416+
result=gl.http_list("/projects",all=True)
417+
assertlen(caught_warnings)==0
418+
assertisinstance(result,list)
419+
assertlen(result)==20
420+
assertlen(responses.calls)==1
421+
422+
423+
@responses.activate
424+
deftest_list_request_all_false_nowarning(gl):
425+
responses.add(**large_list_response)
426+
withwarnings.catch_warnings(record=True)ascaught_warnings:
427+
result=gl.http_list("/projects",all=False)
428+
assertlen(caught_warnings)==0
429+
assertisinstance(result,list)
430+
assertlen(result)==20
431+
assertlen(responses.calls)==1
432+
433+
434+
@responses.activate
435+
deftest_list_request_page_nowarning(gl):
436+
response_dict=copy.deepcopy(large_list_response)
437+
response_dict["match"]= [responses.matchers.query_param_matcher({"page":"1"})]
438+
responses.add(**response_dict)
439+
withwarnings.catch_warnings(record=True)ascaught_warnings:
440+
gl.http_list("/projects",page=1)
441+
assertlen(caught_warnings)==0
442+
assertlen(responses.calls)==1
443+
444+
346445
@responses.activate
347446
deftest_list_request_404(gl):
348447
url="http://localhost/api/v4/not_there"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp