Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit1271686

Browse files
feat: support configurable retries in upload_chunks_concurrently (#1120)
* feat: support configurable retries in upload_chunks_concurrently* lint* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md---------Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent14a1909 commit1271686

File tree

2 files changed

+46
-1
lines changed

2 files changed

+46
-1
lines changed

‎google/cloud/storage/transfer_manager.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
fromgoogle.cloud.storageimportBlob
2929
fromgoogle.cloud.storage.blobimport_get_host_name
3030
fromgoogle.cloud.storage.constantsimport_DEFAULT_TIMEOUT
31+
fromgoogle.cloud.storage._helpersimport_api_core_retry_to_resumable_media_retry
32+
fromgoogle.cloud.storage.retryimportDEFAULT_RETRY
3133

3234
fromgoogle.resumable_media.requests.uploadimportXMLMPUContainer
3335
fromgoogle.resumable_media.requests.uploadimportXMLMPUPart
@@ -871,6 +873,7 @@ def upload_chunks_concurrently(
871873
*,
872874
checksum="md5",
873875
timeout=_DEFAULT_TIMEOUT,
876+
retry=DEFAULT_RETRY,
874877
):
875878
"""Upload a single file in chunks, concurrently.
876879
@@ -966,6 +969,20 @@ def upload_chunks_concurrently(
966969
(Optional) The amount of time, in seconds, to wait
967970
for the server response. See: :ref:`configuring_timeouts`
968971
972+
:type retry: google.api_core.retry.Retry
973+
:param retry: (Optional) How to retry the RPC. A None value will disable
974+
retries. A google.api_core.retry.Retry value will enable retries,
975+
and the object will configure backoff and timeout options. Custom
976+
predicates (customizable error codes) are not supported for media
977+
operations such as this one.
978+
979+
This function does not accept ConditionalRetryPolicy values because
980+
preconditions are not supported by the underlying API call.
981+
982+
See the retry.py source code and docstrings in this package
983+
(google.cloud.storage.retry) for information on retry types and how
984+
to configure them.
985+
969986
:raises: :exc:`concurrent.futures.TimeoutError` if deadline is exceeded.
970987
"""
971988

@@ -995,6 +1012,8 @@ def upload_chunks_concurrently(
9951012
headers["x-goog-encryption-kms-key-name"]=blob.kms_key_name
9961013

9971014
container=XMLMPUContainer(url,filename,headers=headers)
1015+
container._retry_strategy=_api_core_retry_to_resumable_media_retry(retry)
1016+
9981017
container.initiate(transport=transport,content_type=content_type)
9991018
upload_id=container.upload_id
10001019

@@ -1025,6 +1044,7 @@ def upload_chunks_concurrently(
10251044
part_number=part_number,
10261045
checksum=checksum,
10271046
headers=headers,
1047+
retry=retry,
10281048
)
10291049
)
10301050

@@ -1054,6 +1074,7 @@ def _upload_part(
10541074
part_number,
10551075
checksum,
10561076
headers,
1077+
retry,
10571078
):
10581079
"""Helper function that runs inside a thread or subprocess to upload a part.
10591080
@@ -1075,6 +1096,7 @@ def _upload_part(
10751096
checksum=checksum,
10761097
headers=headers,
10771098
)
1099+
part._retry_strategy=_api_core_retry_to_resumable_media_retry(retry)
10781100
part.upload(client._http)
10791101
return (part_number,part.etag)
10801102

‎tests/unit/test_transfer_manager.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,11 @@ def test_upload_chunks_concurrently():
658658
container_mock.register_part.assert_any_call(1,ETAG)
659659
container_mock.register_part.assert_any_call(2,ETAG)
660660
container_mock.finalize.assert_called_once_with(bucket.client._http)
661+
662+
assertcontainer_mock._retry_strategy.max_sleep==60.0
663+
assertcontainer_mock._retry_strategy.max_cumulative_retry==120.0
664+
assertcontainer_mock._retry_strategy.max_retriesisNone
665+
661666
part_mock.upload.assert_called_with(transport)
662667

663668

@@ -693,12 +698,15 @@ def test_upload_chunks_concurrently_passes_concurrency_options():
693698
worker_type=transfer_manager.THREAD,
694699
max_workers=MAX_WORKERS,
695700
deadline=DEADLINE,
701+
retry=None,
696702
)
697703
exceptValueError:
698704
pass# The futures don't actually work, so we expect this to abort.
699705
# Conveniently, that gives us a chance to test the auto-delete
700706
# exception handling feature.
701707
container_mock.cancel.assert_called_once_with(transport)
708+
assertcontainer_mock._retry_strategy.max_retries==0
709+
702710
pool_patch.assert_called_with(max_workers=MAX_WORKERS)
703711
wait_patch.assert_called_with(mock.ANY,timeout=DEADLINE,return_when=mock.ANY)
704712

@@ -905,6 +913,8 @@ def test__download_and_write_chunk_in_place():
905913

906914

907915
deftest__upload_part():
916+
fromgoogle.cloud.storage.retryimportDEFAULT_RETRY
917+
908918
pickled_mock=pickle.dumps(_PickleableMockClient())
909919
FILENAME="file_a.txt"
910920
UPLOAD_ID="abcd"
@@ -916,9 +926,22 @@ def test__upload_part():
916926
"google.cloud.storage.transfer_manager.XMLMPUPart",return_value=part
917927
):
918928
result=transfer_manager._upload_part(
919-
pickled_mock,URL,UPLOAD_ID,FILENAME,0,256,1,None, {"key","value"}
929+
pickled_mock,
930+
URL,
931+
UPLOAD_ID,
932+
FILENAME,
933+
0,
934+
256,
935+
1,
936+
None,
937+
{"key","value"},
938+
retry=DEFAULT_RETRY,
920939
)
921940
part.upload.assert_called_once()
941+
assertpart._retry_strategy.max_sleep==60.0
942+
assertpart._retry_strategy.max_cumulative_retry==120.0
943+
assertpart._retry_strategy.max_retriesisNone
944+
922945
assertresult== (1,ETAG)
923946

924947

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp