Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitbcde0ec

Browse files
authored
feat: allow no project in client methods using storage emulator (#703)
* feat: allow no project in client methods using storage emulator and remove client side validation* changes from comments
1 parent126d94c commitbcde0ec

File tree

3 files changed

+217
-25
lines changed

3 files changed

+217
-25
lines changed

‎google/cloud/storage/_helpers.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,11 @@
3232
STORAGE_EMULATOR_ENV_VAR="STORAGE_EMULATOR_HOST"
3333
"""Environment variable defining host for Storage emulator."""
3434

35-
_DEFAULT_STORAGE_HOST=u"https://storage.googleapis.com"
35+
_DEFAULT_STORAGE_HOST="https://storage.googleapis.com"
36+
"""Default storage host for JSON API."""
37+
38+
_BASE_STORAGE_URI="storage.googleapis.com"
39+
"""Base request endpoint URI for JSON API."""
3640

3741
# etag match parameters in snake case and equivalent header
3842
_ETAG_MATCH_PARAMETERS= (

‎google/cloud/storage/client.py‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
fromgoogle.cloud.exceptionsimportNotFound
3434
fromgoogle.cloud.storage._helpersimport_get_environ_project
3535
fromgoogle.cloud.storage._helpersimport_get_storage_host
36+
fromgoogle.cloud.storage._helpersimport_BASE_STORAGE_URI
3637
fromgoogle.cloud.storage._helpersimport_DEFAULT_STORAGE_HOST
3738
fromgoogle.cloud.storage._helpersimport_bucket_bound_hostname_url
3839
fromgoogle.cloud.storage._helpersimport_add_etag_match_headers
@@ -146,7 +147,7 @@ def __init__(
146147
# STORAGE_EMULATOR_HOST or a non-default api_endpoint is set.
147148
if (
148149
kw_args["api_endpoint"]isnotNone
149-
andkw_args["api_endpoint"].find("storage.googleapis.com")<0
150+
and_BASE_STORAGE_URInotinkw_args["api_endpoint"]
150151
):
151152
ifcredentialsisNone:
152153
credentials=AnonymousCredentials()
@@ -932,12 +933,22 @@ def create_bucket(
932933
933934
"""
934935
bucket=self._bucket_arg_to_bucket(bucket_or_name)
936+
query_params= {}
935937

936938
ifprojectisNone:
937939
project=self.project
938940

939-
ifprojectisNone:
940-
raiseValueError("Client project not set: pass an explicit project.")
941+
# Use no project if STORAGE_EMULATOR_HOST is set
942+
if_BASE_STORAGE_URInotin_get_storage_host():
943+
ifprojectisNone:
944+
project=_get_environ_project()
945+
ifprojectisNone:
946+
project="<none>"
947+
948+
# Only include the project parameter if a project is set.
949+
# If a project is not set, falls back to API validation (BadRequest).
950+
ifprojectisnotNone:
951+
query_params= {"project":project}
941952

942953
ifrequester_paysisnotNone:
943954
warnings.warn(
@@ -947,8 +958,6 @@ def create_bucket(
947958
)
948959
bucket.requester_pays=requester_pays
949960

950-
query_params= {"project":project}
951-
952961
ifpredefined_aclisnotNone:
953962
predefined_acl=BucketACL.validate_predefined(predefined_acl)
954963
query_params["predefinedAcl"]=predefined_acl
@@ -1375,13 +1384,22 @@ def list_buckets(
13751384
:returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket`
13761385
belonging to this project.
13771386
"""
1387+
extra_params= {}
1388+
13781389
ifprojectisNone:
13791390
project=self.project
13801391

1381-
ifprojectisNone:
1382-
raiseValueError("Client project not set: pass an explicit project.")
1392+
# Use no project if STORAGE_EMULATOR_HOST is set
1393+
if_BASE_STORAGE_URInotin_get_storage_host():
1394+
ifprojectisNone:
1395+
project=_get_environ_project()
1396+
ifprojectisNone:
1397+
project="<none>"
13831398

1384-
extra_params= {"project":project}
1399+
# Only include the project parameter if a project is set.
1400+
# If a project is not set, falls back to API validation (BadRequest).
1401+
ifprojectisnotNone:
1402+
extra_params= {"project":project}
13851403

13861404
ifprefixisnotNone:
13871405
extra_params["prefix"]=prefix

‎tests/unit/test_client.py‎

Lines changed: 186 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@
2323
importunittest
2424
importurllib
2525

26-
2726
fromgoogle.api_coreimportexceptions
28-
27+
fromgoogle.auth.credentialsimportAnonymousCredentials
2928
fromgoogle.oauth2.service_accountimportCredentials
30-
from .import_read_local_json
3129

30+
fromgoogle.cloud.storage._helpersimportSTORAGE_EMULATOR_ENV_VAR
3231
fromgoogle.cloud.storage.retryimportDEFAULT_RETRY
3332
fromgoogle.cloud.storage.retryimportDEFAULT_RETRY_IF_GENERATION_SPECIFIED
3433

34+
from .import_read_local_json
3535

3636
_SERVICE_ACCOUNT_JSON=_read_local_json("url_signer_v4_test_account.json")
3737
_CONFORMANCE_TESTS=_read_local_json("url_signer_v4_test_data.json")[
@@ -237,9 +237,6 @@ def test_ctor_mtls(self):
237237
self.assertEqual(client._connection.API_BASE_URL,"http://foo")
238238

239239
deftest_ctor_w_emulator_wo_project(self):
240-
fromgoogle.auth.credentialsimportAnonymousCredentials
241-
fromgoogle.cloud.storage._helpersimportSTORAGE_EMULATOR_ENV_VAR
242-
243240
# avoids authentication if STORAGE_EMULATOR_ENV_VAR is set
244241
host="http://localhost:8080"
245242
environ= {STORAGE_EMULATOR_ENV_VAR:host}
@@ -259,9 +256,6 @@ def test_ctor_w_emulator_wo_project(self):
259256
self.assertIsInstance(client._connection.credentials,AnonymousCredentials)
260257

261258
deftest_ctor_w_emulator_w_environ_project(self):
262-
fromgoogle.auth.credentialsimportAnonymousCredentials
263-
fromgoogle.cloud.storage._helpersimportSTORAGE_EMULATOR_ENV_VAR
264-
265259
# avoids authentication and infers the project from the environment
266260
host="http://localhost:8080"
267261
environ_project="environ-project"
@@ -277,9 +271,6 @@ def test_ctor_w_emulator_w_environ_project(self):
277271
self.assertIsInstance(client._connection.credentials,AnonymousCredentials)
278272

279273
deftest_ctor_w_emulator_w_project_arg(self):
280-
fromgoogle.auth.credentialsimportAnonymousCredentials
281-
fromgoogle.cloud.storage._helpersimportSTORAGE_EMULATOR_ENV_VAR
282-
283274
# project argument overrides project set in the enviroment
284275
host="http://localhost:8080"
285276
environ_project="environ-project"
@@ -296,7 +287,6 @@ def test_ctor_w_emulator_w_project_arg(self):
296287
self.assertIsInstance(client._connection.credentials,AnonymousCredentials)
297288

298289
deftest_create_anonymous_client(self):
299-
fromgoogle.auth.credentialsimportAnonymousCredentials
300290
fromgoogle.cloud.storage._httpimportConnection
301291

302292
klass=self._get_target_class()
@@ -1187,11 +1177,91 @@ def test_lookup_bucket_hit_w_retry(self):
11871177
)
11881178

11891179
deftest_create_bucket_w_missing_client_project(self):
1180+
fromgoogle.cloud.exceptionsimportBadRequest
1181+
11901182
credentials=_make_credentials()
11911183
client=self._make_one(project=None,credentials=credentials)
11921184

1193-
withself.assertRaises(ValueError):
1194-
client.create_bucket("bucket")
1185+
client._post_resource=mock.Mock()
1186+
client._post_resource.side_effect=BadRequest("Required parameter: project")
1187+
1188+
bucket_name="bucket-name"
1189+
1190+
withself.assertRaises(BadRequest):
1191+
client.create_bucket(bucket_name)
1192+
1193+
expected_path="/b"
1194+
expected_data= {"name":bucket_name}
1195+
# no required parameter: project
1196+
expected_query_params= {}
1197+
client._post_resource.assert_called_once_with(
1198+
expected_path,
1199+
expected_data,
1200+
query_params=expected_query_params,
1201+
timeout=self._get_default_timeout(),
1202+
retry=DEFAULT_RETRY,
1203+
_target_object=mock.ANY,
1204+
)
1205+
1206+
deftest_create_bucket_w_missing_client_project_w_emulator(self):
1207+
# mock STORAGE_EMULATOR_ENV_VAR is set
1208+
host="http://localhost:8080"
1209+
environ= {STORAGE_EMULATOR_ENV_VAR:host}
1210+
withmock.patch("os.environ",environ):
1211+
client=self._make_one()
1212+
1213+
bucket_name="bucket-name"
1214+
api_response= {"name":bucket_name}
1215+
client._post_resource=mock.Mock()
1216+
client._post_resource.return_value=api_response
1217+
1218+
# mock STORAGE_EMULATOR_ENV_VAR is set
1219+
withmock.patch("os.environ",environ):
1220+
bucket=client.create_bucket(bucket_name)
1221+
1222+
expected_path="/b"
1223+
expected_data=api_response
1224+
expected_query_params= {"project":"<none>"}
1225+
client._post_resource.assert_called_once_with(
1226+
expected_path,
1227+
expected_data,
1228+
query_params=expected_query_params,
1229+
timeout=self._get_default_timeout(),
1230+
retry=DEFAULT_RETRY,
1231+
_target_object=bucket,
1232+
)
1233+
1234+
deftest_create_bucket_w_environ_project_w_emulator(self):
1235+
# mock STORAGE_EMULATOR_ENV_VAR is set
1236+
host="http://localhost:8080"
1237+
environ_project="environ-project"
1238+
environ= {
1239+
STORAGE_EMULATOR_ENV_VAR:host,
1240+
"GOOGLE_CLOUD_PROJECT":environ_project,
1241+
}
1242+
withmock.patch("os.environ",environ):
1243+
client=self._make_one()
1244+
1245+
bucket_name="bucket-name"
1246+
api_response= {"name":bucket_name}
1247+
client._post_resource=mock.Mock()
1248+
client._post_resource.return_value=api_response
1249+
1250+
# mock STORAGE_EMULATOR_ENV_VAR is set
1251+
withmock.patch("os.environ",environ):
1252+
bucket=client.create_bucket(bucket_name)
1253+
1254+
expected_path="/b"
1255+
expected_data=api_response
1256+
expected_query_params= {"project":environ_project}
1257+
client._post_resource.assert_called_once_with(
1258+
expected_path,
1259+
expected_data,
1260+
query_params=expected_query_params,
1261+
timeout=self._get_default_timeout(),
1262+
retry=DEFAULT_RETRY,
1263+
_target_object=bucket,
1264+
)
11951265

11961266
deftest_create_bucket_w_conflict_w_user_project(self):
11971267
fromgoogle.cloud.exceptionsimportConflict
@@ -1787,12 +1857,112 @@ def test_list_blobs_w_explicit_w_user_project(self):
17871857
)
17881858

17891859
deftest_list_buckets_wo_project(self):
1860+
fromgoogle.cloud.exceptionsimportBadRequest
1861+
fromgoogle.cloud.storage.clientimport_item_to_bucket
1862+
17901863
credentials=_make_credentials()
17911864
client=self._make_one(project=None,credentials=credentials)
17921865

1793-
withself.assertRaises(ValueError):
1866+
client._list_resource=mock.Mock()
1867+
client._list_resource.side_effect=BadRequest("Required parameter: project")
1868+
1869+
withself.assertRaises(BadRequest):
17941870
client.list_buckets()
17951871

1872+
expected_path="/b"
1873+
expected_item_to_value=_item_to_bucket
1874+
expected_page_token=None
1875+
expected_max_results=None
1876+
expected_page_size=None
1877+
# no required parameter: project
1878+
expected_extra_params= {
1879+
"projection":"noAcl",
1880+
}
1881+
client._list_resource.assert_called_once_with(
1882+
expected_path,
1883+
expected_item_to_value,
1884+
page_token=expected_page_token,
1885+
max_results=expected_max_results,
1886+
extra_params=expected_extra_params,
1887+
page_size=expected_page_size,
1888+
timeout=self._get_default_timeout(),
1889+
retry=DEFAULT_RETRY,
1890+
)
1891+
1892+
deftest_list_buckets_wo_project_w_emulator(self):
1893+
fromgoogle.cloud.storage.clientimport_item_to_bucket
1894+
1895+
# mock STORAGE_EMULATOR_ENV_VAR is set
1896+
host="http://localhost:8080"
1897+
environ= {STORAGE_EMULATOR_ENV_VAR:host}
1898+
withmock.patch("os.environ",environ):
1899+
client=self._make_one()
1900+
1901+
client._list_resource=mock.Mock(spec=[])
1902+
1903+
# mock STORAGE_EMULATOR_ENV_VAR is set
1904+
withmock.patch("os.environ",environ):
1905+
client.list_buckets()
1906+
1907+
expected_path="/b"
1908+
expected_item_to_value=_item_to_bucket
1909+
expected_page_token=None
1910+
expected_max_results=None
1911+
expected_page_size=None
1912+
expected_extra_params= {
1913+
"project":"<none>",
1914+
"projection":"noAcl",
1915+
}
1916+
client._list_resource.assert_called_once_with(
1917+
expected_path,
1918+
expected_item_to_value,
1919+
page_token=expected_page_token,
1920+
max_results=expected_max_results,
1921+
extra_params=expected_extra_params,
1922+
page_size=expected_page_size,
1923+
timeout=self._get_default_timeout(),
1924+
retry=DEFAULT_RETRY,
1925+
)
1926+
1927+
deftest_list_buckets_w_environ_project_w_emulator(self):
1928+
fromgoogle.cloud.storage.clientimport_item_to_bucket
1929+
1930+
# mock STORAGE_EMULATOR_ENV_VAR is set
1931+
host="http://localhost:8080"
1932+
environ_project="environ-project"
1933+
environ= {
1934+
STORAGE_EMULATOR_ENV_VAR:host,
1935+
"GOOGLE_CLOUD_PROJECT":environ_project,
1936+
}
1937+
withmock.patch("os.environ",environ):
1938+
client=self._make_one()
1939+
1940+
client._list_resource=mock.Mock(spec=[])
1941+
1942+
# mock STORAGE_EMULATOR_ENV_VAR is set
1943+
withmock.patch("os.environ",environ):
1944+
client.list_buckets()
1945+
1946+
expected_path="/b"
1947+
expected_item_to_value=_item_to_bucket
1948+
expected_page_token=None
1949+
expected_max_results=None
1950+
expected_page_size=None
1951+
expected_extra_params= {
1952+
"project":environ_project,
1953+
"projection":"noAcl",
1954+
}
1955+
client._list_resource.assert_called_once_with(
1956+
expected_path,
1957+
expected_item_to_value,
1958+
page_token=expected_page_token,
1959+
max_results=expected_max_results,
1960+
extra_params=expected_extra_params,
1961+
page_size=expected_page_size,
1962+
timeout=self._get_default_timeout(),
1963+
retry=DEFAULT_RETRY,
1964+
)
1965+
17961966
deftest_list_buckets_w_defaults(self):
17971967
fromgoogle.cloud.storage.clientimport_item_to_bucket
17981968

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp