Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit22fd848

Browse files
authored
docs: add loading data from Firestore backup sample (#737)
Follow-up to#736To be included here:https://cloud.google.com/bigquery/docs/loading-data-cloud-firestoreAlso* Use `google-cloud-testutils` for cleanup as described ingoogleapis/python-test-utils#39
1 parent4ff8bed commit22fd848

File tree

5 files changed

+94
-26
lines changed

5 files changed

+94
-26
lines changed

‎samples/snippets/conftest.py‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -12,38 +12,18 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
importdatetime
16-
importrandom
17-
1815
fromgoogle.cloudimportbigquery
1916
importpytest
17+
importtest_utils.prefixer
2018

2119

22-
RESOURCE_PREFIX="python_bigquery_samples_snippets"
23-
RESOURCE_DATE_FORMAT="%Y%m%d_%H%M%S"
24-
RESOURCE_DATE_LENGTH=4+2+2+1+2+2+2
25-
26-
27-
defresource_prefix()->str:
28-
timestamp=datetime.datetime.utcnow().strftime(RESOURCE_DATE_FORMAT)
29-
random_string=hex(random.randrange(1000000))[2:]
30-
returnf"{RESOURCE_PREFIX}_{timestamp}_{random_string}"
31-
32-
33-
defresource_name_to_date(resource_name:str):
34-
start_date=len(RESOURCE_PREFIX)+1
35-
date_string=resource_name[start_date :start_date+RESOURCE_DATE_LENGTH]
36-
returndatetime.datetime.strptime(date_string,RESOURCE_DATE_FORMAT)
20+
prefixer=test_utils.prefixer.Prefixer("python-bigquery","samples/snippets")
3721

3822

3923
@pytest.fixture(scope="session",autouse=True)
4024
defcleanup_datasets(bigquery_client:bigquery.Client):
41-
yesterday=datetime.datetime.utcnow()-datetime.timedelta(days=1)
4225
fordatasetinbigquery_client.list_datasets():
43-
if (
44-
dataset.dataset_id.startswith(RESOURCE_PREFIX)
45-
andresource_name_to_date(dataset.dataset_id)<yesterday
46-
):
26+
ifprefixer.should_cleanup(dataset.dataset_id):
4727
bigquery_client.delete_dataset(
4828
dataset,delete_contents=True,not_found_ok=True
4929
)
@@ -62,14 +42,25 @@ def project_id(bigquery_client):
6242

6343
@pytest.fixture(scope="session")
6444
defdataset_id(bigquery_client:bigquery.Client,project_id:str):
65-
dataset_id=resource_prefix()
45+
dataset_id=prefixer.create_prefix()
6646
full_dataset_id=f"{project_id}.{dataset_id}"
6747
dataset=bigquery.Dataset(full_dataset_id)
6848
bigquery_client.create_dataset(dataset)
6949
yielddataset_id
7050
bigquery_client.delete_dataset(dataset,delete_contents=True,not_found_ok=True)
7151

7252

53+
@pytest.fixture
54+
defrandom_table_id(bigquery_client:bigquery.Client,project_id:str,dataset_id:str):
55+
"""Create a new table ID each time, so random_table_id can be used as
56+
target for load jobs.
57+
"""
58+
random_table_id=prefixer.create_prefix()
59+
full_table_id=f"{project_id}.{dataset_id}.{random_table_id}"
60+
yieldfull_table_id
61+
bigquery_client.delete_table(full_table_id,not_found_ok=True)
62+
63+
7364
@pytest.fixture
7465
defbigquery_client_patch(monkeypatch,bigquery_client):
7566
monkeypatch.setattr(bigquery,"Client",lambda:bigquery_client)
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
defload_table_uri_firestore(table_id):
17+
orig_table_id=table_id
18+
# [START bigquery_load_table_gcs_firestore]
19+
# TODO(developer): Set table_id to the ID of the table to create.
20+
table_id="your-project.your_dataset.your_table_name"
21+
22+
# TODO(developer): Set uri to the path of the kind export metadata
23+
uri= (
24+
"gs://cloud-samples-data/bigquery/us-states"
25+
"/2021-07-02T16:04:48_70344/all_namespaces/kind_us-states"
26+
"/all_namespaces_kind_us-states.export_metadata"
27+
)
28+
29+
# TODO(developer): Set projection_fields to a list of document properties
30+
# to import. Leave unset or set to `None` for all fields.
31+
projection_fields= ["name","post_abbr"]
32+
33+
# [END bigquery_load_table_gcs_firestore]
34+
table_id=orig_table_id
35+
36+
# [START bigquery_load_table_gcs_firestore]
37+
fromgoogle.cloudimportbigquery
38+
39+
# Construct a BigQuery client object.
40+
client=bigquery.Client()
41+
42+
job_config=bigquery.LoadJobConfig(
43+
source_format=bigquery.SourceFormat.DATASTORE_BACKUP,
44+
projection_fields=projection_fields,
45+
)
46+
47+
load_job=client.load_table_from_uri(
48+
uri,table_id,job_config=job_config
49+
)# Make an API request.
50+
51+
load_job.result()# Waits for the job to complete.
52+
53+
destination_table=client.get_table(table_id)
54+
print("Loaded {} rows.".format(destination_table.num_rows))
55+
# [END bigquery_load_table_gcs_firestore]
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
importload_table_uri_firestore
16+
17+
18+
deftest_load_table_uri_firestore(capsys,random_table_id):
19+
load_table_uri_firestore.load_table_uri_firestore(random_table_id)
20+
out,_=capsys.readouterr()
21+
assert"Loaded 50 rows."inout
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1+
google-cloud-testutils==0.3.0
12
pytest==6.2.4
23
mock==4.0.3

‎samples/snippets/test_update_with_dml.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@
1515
fromgoogle.cloudimportbigquery
1616
importpytest
1717

18-
fromconftestimportresource_prefix
18+
fromconftestimportprefixer
1919
importupdate_with_dml
2020

2121

2222
@pytest.fixture
2323
deftable_id(bigquery_client:bigquery.Client,project_id:str,dataset_id:str):
24-
table_id=f"{resource_prefix()}_update_with_dml"
24+
table_id=f"{prefixer.create_prefix()}_update_with_dml"
2525
yieldtable_id
2626
full_table_id=f"{project_id}.{dataset_id}.{table_id}"
2727
bigquery_client.delete_table(full_table_id,not_found_ok=True)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp