Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb644721

Browse files
authored
feat(downloads): allow streaming downloads access to response iterator (#1956)
* feat(downloads): allow streaming downloads access to response iteratorAllow access to the underlying response iterator when downloading instreaming mode by specifying `iterator=True`.Update type annotations to support this change.* docs(api-docs): add iterator example to artifact downloadDocument the usage of the `iterator=True` option when downloadingartifacts* test(packages): add tests for streaming downloads
1 parent0f2a602 commitb644721

File tree

13 files changed

+165
-37
lines changed

13 files changed

+165
-37
lines changed

‎docs/gl_objects/pipelines_and_jobs.rst‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,19 @@ You can also directly stream the output into a file, and unzip it afterwards::
274274
subprocess.run(["unzip", "-bo", zipfn])
275275
os.unlink(zipfn)
276276

277+
Or, you can also use the underlying response iterator directly::
278+
279+
artifact_bytes_iterator = build_or_job.artifacts(iterator=True)
280+
281+
This can be used with frameworks that expect an iterator (such as FastAPI/Starlette's
282+
``StreamingResponse``) to forward a download from GitLab without having to download
283+
the entire content server-side first::
284+
285+
@app.get("/download_artifact")
286+
def download_artifact():
287+
artifact_bytes_iterator = build_or_job.artifacts(iterator=True)
288+
return StreamingResponse(artifact_bytes_iterator, media_type="application/zip")
289+
277290
Delete all artifacts of a project that can be deleted::
278291

279292
project.artifacts.delete()

‎gitlab/mixins.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
Any,
2121
Callable,
2222
Dict,
23+
Iterator,
2324
List,
2425
Optional,
2526
Tuple,
@@ -612,16 +613,19 @@ class DownloadMixin(_RestObjectBase):
612613
defdownload(
613614
self,
614615
streamed:bool=False,
616+
iterator:bool=False,
615617
action:Optional[Callable]=None,
616618
chunk_size:int=1024,
617619
**kwargs:Any,
618-
)->Optional[bytes]:
620+
)->Optional[Union[bytes,Iterator[Any]]]:
619621
"""Download the archive of a resource export.
620622
621623
Args:
622624
streamed: If True the data will be processed by chunks of
623625
`chunk_size` and each chunk is passed to `action` for
624626
treatment
627+
iterator: If True directly return the underlying response
628+
iterator
625629
action: Callable responsible of dealing with chunk of
626630
data
627631
chunk_size: Size of each chunk
@@ -640,7 +644,7 @@ def download(
640644
)
641645
ifTYPE_CHECKING:
642646
assertisinstance(result,requests.Response)
643-
returnutils.response_content(result,streamed,action,chunk_size)
647+
returnutils.response_content(result,streamed,iterator,action,chunk_size)
644648

645649

646650
classSubscribableMixin(_RestObjectBase):

‎gitlab/utils.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
importtraceback
2020
importurllib.parse
2121
importwarnings
22-
fromtypingimportAny,Callable,Dict,Optional,Tuple,Type,Union
22+
fromtypingimportAny,Callable,Dict,Iterator,Optional,Tuple,Type,Union
2323

2424
importrequests
2525

@@ -34,9 +34,13 @@ def __call__(self, chunk: Any) -> None:
3434
defresponse_content(
3535
response:requests.Response,
3636
streamed:bool,
37+
iterator:bool,
3738
action:Optional[Callable],
3839
chunk_size:int,
39-
)->Optional[bytes]:
40+
)->Optional[Union[bytes,Iterator[Any]]]:
41+
ifiterator:
42+
returnresponse.iter_content(chunk_size=chunk_size)
43+
4044
ifstreamedisFalse:
4145
returnresponse.content
4246

‎gitlab/v4/cli.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ def do_project_export_download(self) -> None:
127127
data=export_status.download()
128128
ifTYPE_CHECKING:
129129
assertdataisnotNone
130+
assertisinstance(data,bytes)
130131
sys.stdout.buffer.write(data)
131132

132133
exceptExceptionase:# pragma: no cover, cli.die is unit-tested

‎gitlab/v4/objects/artifacts.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
GitLab API:
33
https://docs.gitlab.com/ee/api/job_artifacts.html
44
"""
5-
fromtypingimportAny,Callable,Optional,TYPE_CHECKING
5+
fromtypingimportAny,Callable,Iterator,Optional,TYPE_CHECKING,Union
66

77
importrequests
88

@@ -40,10 +40,14 @@ def __call__(
4040
),
4141
category=DeprecationWarning,
4242
)
43-
returnself.download(
43+
data=self.download(
4444
*args,
4545
**kwargs,
4646
)
47+
ifTYPE_CHECKING:
48+
assertdataisnotNone
49+
assertisinstance(data,bytes)
50+
returndata
4751

4852
@exc.on_http_error(exc.GitlabDeleteError)
4953
defdelete(self,**kwargs:Any)->None:
@@ -71,10 +75,11 @@ def download(
7175
ref_name:str,
7276
job:str,
7377
streamed:bool=False,
78+
iterator:bool=False,
7479
action:Optional[Callable]=None,
7580
chunk_size:int=1024,
7681
**kwargs:Any,
77-
)->Optional[bytes]:
82+
)->Optional[Union[bytes,Iterator[Any]]]:
7883
"""Get the job artifacts archive from a specific tag or branch.
7984
8085
Args:
@@ -85,6 +90,8 @@ def download(
8590
streamed: If True the data will be processed by chunks of
8691
`chunk_size` and each chunk is passed to `action` for
8792
treatment
93+
iterator: If True directly return the underlying response
94+
iterator
8895
action: Callable responsible of dealing with chunk of
8996
data
9097
chunk_size: Size of each chunk
@@ -103,7 +110,7 @@ def download(
103110
)
104111
ifTYPE_CHECKING:
105112
assertisinstance(result,requests.Response)
106-
returnutils.response_content(result,streamed,action,chunk_size)
113+
returnutils.response_content(result,streamed,iterator,action,chunk_size)
107114

108115
@cli.register_custom_action(
109116
"ProjectArtifactManager", ("ref_name","artifact_path","job")
@@ -115,10 +122,11 @@ def raw(
115122
artifact_path:str,
116123
job:str,
117124
streamed:bool=False,
125+
iterator:bool=False,
118126
action:Optional[Callable]=None,
119127
chunk_size:int=1024,
120128
**kwargs:Any,
121-
)->Optional[bytes]:
129+
)->Optional[Union[bytes,Iterator[Any]]]:
122130
"""Download a single artifact file from a specific tag or branch from
123131
within the job's artifacts archive.
124132
@@ -130,6 +138,8 @@ def raw(
130138
streamed: If True the data will be processed by chunks of
131139
`chunk_size` and each chunk is passed to `action` for
132140
treatment
141+
iterator: If True directly return the underlying response
142+
iterator
133143
action: Callable responsible of dealing with chunk of
134144
data
135145
chunk_size: Size of each chunk
@@ -148,4 +158,4 @@ def raw(
148158
)
149159
ifTYPE_CHECKING:
150160
assertisinstance(result,requests.Response)
151-
returnutils.response_content(result,streamed,action,chunk_size)
161+
returnutils.response_content(result,streamed,iterator,action,chunk_size)

‎gitlab/v4/objects/files.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
11
importbase64
2-
fromtypingimportAny,Callable,cast,Dict,List,Optional,TYPE_CHECKING
2+
fromtypingimport (
3+
Any,
4+
Callable,
5+
cast,
6+
Dict,
7+
Iterator,
8+
List,
9+
Optional,
10+
TYPE_CHECKING,
11+
Union,
12+
)
313

414
importrequests
515

@@ -220,10 +230,11 @@ def raw(
220230
file_path:str,
221231
ref:str,
222232
streamed:bool=False,
233+
iterator:bool=False,
223234
action:Optional[Callable[...,Any]]=None,
224235
chunk_size:int=1024,
225236
**kwargs:Any,
226-
)->Optional[bytes]:
237+
)->Optional[Union[bytes,Iterator[Any]]]:
227238
"""Return the content of a file for a commit.
228239
229240
Args:
@@ -232,6 +243,8 @@ def raw(
232243
streamed: If True the data will be processed by chunks of
233244
`chunk_size` and each chunk is passed to `action` for
234245
treatment
246+
iterator: If True directly return the underlying response
247+
iterator
235248
action: Callable responsible of dealing with chunk of
236249
data
237250
chunk_size: Size of each chunk
@@ -252,7 +265,7 @@ def raw(
252265
)
253266
ifTYPE_CHECKING:
254267
assertisinstance(result,requests.Response)
255-
returnutils.response_content(result,streamed,action,chunk_size)
268+
returnutils.response_content(result,streamed,iterator,action,chunk_size)
256269

257270
@cli.register_custom_action("ProjectFileManager", ("file_path","ref"))
258271
@exc.on_http_error(exc.GitlabListError)

‎gitlab/v4/objects/jobs.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
fromtypingimportAny,Callable,cast,Dict,Optional,TYPE_CHECKING,Union
1+
fromtypingimportAny,Callable,cast,Dict,Iterator,Optional,TYPE_CHECKING,Union
22

33
importrequests
44

@@ -116,16 +116,19 @@ def delete_artifacts(self, **kwargs: Any) -> None:
116116
defartifacts(
117117
self,
118118
streamed:bool=False,
119+
iterator:bool=False,
119120
action:Optional[Callable[...,Any]]=None,
120121
chunk_size:int=1024,
121122
**kwargs:Any,
122-
)->Optional[bytes]:
123+
)->Optional[Union[bytes,Iterator[Any]]]:
123124
"""Get the job artifacts.
124125
125126
Args:
126127
streamed: If True the data will be processed by chunks of
127128
`chunk_size` and each chunk is passed to `action` for
128129
treatment
130+
iterator: If True directly return the underlying response
131+
iterator
129132
action: Callable responsible of dealing with chunk of
130133
data
131134
chunk_size: Size of each chunk
@@ -144,25 +147,28 @@ def artifacts(
144147
)
145148
ifTYPE_CHECKING:
146149
assertisinstance(result,requests.Response)
147-
returnutils.response_content(result,streamed,action,chunk_size)
150+
returnutils.response_content(result,streamed,iterator,action,chunk_size)
148151

149152
@cli.register_custom_action("ProjectJob")
150153
@exc.on_http_error(exc.GitlabGetError)
151154
defartifact(
152155
self,
153156
path:str,
154157
streamed:bool=False,
158+
iterator:bool=False,
155159
action:Optional[Callable[...,Any]]=None,
156160
chunk_size:int=1024,
157161
**kwargs:Any,
158-
)->Optional[bytes]:
162+
)->Optional[Union[bytes,Iterator[Any]]]:
159163
"""Get a single artifact file from within the job's artifacts archive.
160164
161165
Args:
162166
path: Path of the artifact
163167
streamed: If True the data will be processed by chunks of
164168
`chunk_size` and each chunk is passed to `action` for
165169
treatment
170+
iterator: If True directly return the underlying response
171+
iterator
166172
action: Callable responsible of dealing with chunk of
167173
data
168174
chunk_size: Size of each chunk
@@ -181,13 +187,14 @@ def artifact(
181187
)
182188
ifTYPE_CHECKING:
183189
assertisinstance(result,requests.Response)
184-
returnutils.response_content(result,streamed,action,chunk_size)
190+
returnutils.response_content(result,streamed,iterator,action,chunk_size)
185191

186192
@cli.register_custom_action("ProjectJob")
187193
@exc.on_http_error(exc.GitlabGetError)
188194
deftrace(
189195
self,
190196
streamed:bool=False,
197+
iterator:bool=False,
191198
action:Optional[Callable[...,Any]]=None,
192199
chunk_size:int=1024,
193200
**kwargs:Any,
@@ -198,6 +205,8 @@ def trace(
198205
streamed: If True the data will be processed by chunks of
199206
`chunk_size` and each chunk is passed to `action` for
200207
treatment
208+
iterator: If True directly return the underlying response
209+
iterator
201210
action: Callable responsible of dealing with chunk of
202211
data
203212
chunk_size: Size of each chunk
@@ -216,7 +225,9 @@ def trace(
216225
)
217226
ifTYPE_CHECKING:
218227
assertisinstance(result,requests.Response)
219-
return_value=utils.response_content(result,streamed,action,chunk_size)
228+
return_value=utils.response_content(
229+
result,streamed,iterator,action,chunk_size
230+
)
220231
ifTYPE_CHECKING:
221232
assertisinstance(return_value,dict)
222233
returnreturn_value

‎gitlab/v4/objects/packages.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"""
66

77
frompathlibimportPath
8-
fromtypingimportAny,Callable,cast,Optional,TYPE_CHECKING,Union
8+
fromtypingimportAny,Callable,cast,Iterator,Optional,TYPE_CHECKING,Union
99

1010
importrequests
1111

@@ -103,10 +103,11 @@ def download(
103103
package_version:str,
104104
file_name:str,
105105
streamed:bool=False,
106+
iterator:bool=False,
106107
action:Optional[Callable]=None,
107108
chunk_size:int=1024,
108109
**kwargs:Any,
109-
)->Optional[bytes]:
110+
)->Optional[Union[bytes,Iterator[Any]]]:
110111
"""Download a generic package.
111112
112113
Args:
@@ -116,6 +117,8 @@ def download(
116117
streamed: If True the data will be processed by chunks of
117118
`chunk_size` and each chunk is passed to `action` for
118119
treatment
120+
iterator: If True directly return the underlying response
121+
iterator
119122
action: Callable responsible of dealing with chunk of
120123
data
121124
chunk_size: Size of each chunk
@@ -132,7 +135,7 @@ def download(
132135
result=self.gitlab.http_get(path,streamed=streamed,raw=True,**kwargs)
133136
ifTYPE_CHECKING:
134137
assertisinstance(result,requests.Response)
135-
returnutils.response_content(result,streamed,action,chunk_size)
138+
returnutils.response_content(result,streamed,iterator,action,chunk_size)
136139

137140

138141
classGroupPackage(RESTObject):

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp