Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit176fb2a

Browse files
abecerrilsalasleahecolesteffnay
authored
feat: add support for table clones (#1235)
* feat: add support for table clones* feat: clone test* feat: debugging* feat: more debugging* feat: more debugging* feat: even more debugging* feat: debugging test* feat: even more test debugging* feat: check* feat: modify test* feat: deleting print statement* feat: testing* feat: test update* feat: change table name* feat: debugging table name* feat: cleaning up test* feat: degubbing test* feat: add properties check to test* feat: test change* feat: added more properties* Update samples/snippets/requirements.txtCo-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com>Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com>Co-authored-by: Steffany Brown <30247553+steffnay@users.noreply.github.com>
1 parent6573f67 commit176fb2a

File tree

7 files changed

+172
-1
lines changed

7 files changed

+172
-1
lines changed

‎docs/reference.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ Table
9797
table.Row
9898
table.RowIterator
9999
table.SnapshotDefinition
100+
table.CloneDefinition
100101
table.Table
101102
table.TableListItem
102103
table.TableReference

‎google/cloud/bigquery/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
fromgoogle.cloud.bigquery.tableimportRangePartitioning
102102
fromgoogle.cloud.bigquery.tableimportRow
103103
fromgoogle.cloud.bigquery.tableimportSnapshotDefinition
104+
fromgoogle.cloud.bigquery.tableimportCloneDefinition
104105
fromgoogle.cloud.bigquery.tableimportTable
105106
fromgoogle.cloud.bigquery.tableimportTableReference
106107
fromgoogle.cloud.bigquery.tableimportTimePartitioningType
@@ -132,6 +133,7 @@
132133
"RangePartitioning",
133134
"Row",
134135
"SnapshotDefinition",
136+
"CloneDefinition",
135137
"TimePartitioning",
136138
"TimePartitioningType",
137139
# Jobs

‎google/cloud/bigquery/job/copy_.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ class OperationType:
4040
SNAPSHOT="SNAPSHOT"
4141
"""The source table type is TABLE and the destination table type is SNAPSHOT."""
4242

43+
CLONE="CLONE"
44+
"""The source table type is TABLE and the destination table type is CLONE."""
45+
4346
RESTORE="RESTORE"
4447
"""The source table type is SNAPSHOT and the destination table type is TABLE."""
4548

‎google/cloud/bigquery/table.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,7 @@ class Table(_TableBase):
356356
"time_partitioning":"timePartitioning",
357357
"schema":"schema",
358358
"snapshot_definition":"snapshotDefinition",
359+
"clone_definition":"cloneDefinition",
359360
"streaming_buffer":"streamingBuffer",
360361
"self_link":"selfLink",
361362
"time_partitioning":"timePartitioning",
@@ -929,6 +930,19 @@ def snapshot_definition(self) -> Optional["SnapshotDefinition"]:
929930
snapshot_info=SnapshotDefinition(snapshot_info)
930931
returnsnapshot_info
931932

933+
@property
934+
defclone_definition(self)->Optional["CloneDefinition"]:
935+
"""Information about the clone. This value is set via clone creation.
936+
937+
See: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#Table.FIELDS.clone_definition
938+
"""
939+
clone_info=self._properties.get(
940+
self._PROPERTY_TO_API_FIELD["clone_definition"]
941+
)
942+
ifclone_infoisnotNone:
943+
clone_info=CloneDefinition(clone_info)
944+
returnclone_info
945+
932946
@classmethod
933947
deffrom_string(cls,full_table_id:str)->"Table":
934948
"""Construct a table from fully-qualified table ID.
@@ -1304,6 +1318,29 @@ def __init__(self, resource: Dict[str, Any]):
13041318
)
13051319

13061320

1321+
classCloneDefinition:
1322+
"""Information about base table and clone time of the clone.
1323+
1324+
See https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#clonedefinition
1325+
1326+
Args:
1327+
resource: Clone definition representation returned from the API.
1328+
"""
1329+
1330+
def__init__(self,resource:Dict[str,Any]):
1331+
self.base_table_reference=None
1332+
if"baseTableReference"inresource:
1333+
self.base_table_reference=TableReference.from_api_repr(
1334+
resource["baseTableReference"]
1335+
)
1336+
1337+
self.clone_time=None
1338+
if"cloneTime"inresource:
1339+
self.clone_time=google.cloud._helpers._rfc3339_to_datetime(
1340+
resource["cloneTime"]
1341+
)
1342+
1343+
13071344
classRow(object):
13081345
"""A BigQuery row.
13091346

‎samples/magics/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ grpcio==1.46.1
55
ipython===7.31.1;python_version=='3.7'
66
ipython===8.0.1;python_version=='3.8'
77
ipython==8.3.0;python_version>='3.9'
8-
matplotlib==3.5.2
8+
matplotlib==3.5.1
99
pandas===1.3.5;python_version=='3.7'
1010
pandas==1.4.2;python_version>='3.8'
1111
pyarrow==8.0.0

‎tests/system/test_client.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2190,3 +2190,57 @@ def test_table_snapshots(dataset_id):
21902190
rows_iter=client.list_rows(source_table_path)
21912191
rows=sorted(row.values()forrowinrows_iter)
21922192
assertrows== [(1,"one"), (2,"two")]
2193+
2194+
2195+
deftest_table_clones(dataset_id):
2196+
fromgoogle.cloud.bigqueryimportCopyJobConfig
2197+
fromgoogle.cloud.bigqueryimportOperationType
2198+
2199+
client=Config.CLIENT
2200+
2201+
table_path_source=f"{client.project}.{dataset_id}.test_table_clone"
2202+
clone_table_path=f"{table_path_source}_clone"
2203+
2204+
# Create the table before loading so that the column order is predictable.
2205+
schema= [
2206+
bigquery.SchemaField("foo","INTEGER"),
2207+
bigquery.SchemaField("bar","STRING"),
2208+
]
2209+
source_table=helpers.retry_403(Config.CLIENT.create_table)(
2210+
Table(table_path_source,schema=schema)
2211+
)
2212+
2213+
# Populate the table with initial data.
2214+
rows= [{"foo":1,"bar":"one"}, {"foo":2,"bar":"two"}]
2215+
load_job=Config.CLIENT.load_table_from_json(rows,source_table)
2216+
load_job.result()
2217+
2218+
# Now create a clone before modifying the original table data.
2219+
copy_config=CopyJobConfig()
2220+
copy_config.operation_type=OperationType.CLONE
2221+
copy_config.write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE
2222+
2223+
copy_job=client.copy_table(
2224+
sources=table_path_source,
2225+
destination=clone_table_path,
2226+
job_config=copy_config,
2227+
)
2228+
copy_job.result()
2229+
2230+
# List rows from the source table and compare them to rows from the clone.
2231+
rows_iter=client.list_rows(table_path_source)
2232+
rows=sorted(row.values()forrowinrows_iter)
2233+
assertrows== [(1,"one"), (2,"two")]
2234+
2235+
rows_iter=client.list_rows(clone_table_path)
2236+
rows=sorted(row.values()forrowinrows_iter)
2237+
assertrows== [(1,"one"), (2,"two")]
2238+
2239+
# Compare properties of the source and clone table.
2240+
source_table_props=client.get_table(table_path_source)
2241+
clone_table_props=client.get_table(clone_table_path)
2242+
2243+
assertsource_table_props.schema==clone_table_props.schema
2244+
assertsource_table_props.num_bytes==clone_table_props.num_bytes
2245+
assertsource_table_props.num_rows==clone_table_props.num_rows
2246+
assertsource_table_props.description==clone_table_props.description

‎tests/unit/test_table.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,40 @@ def test_snapshot_definition_set(self):
841841
2010,9,28,10,20,30,123000,tzinfo=UTC
842842
)
843843

844+
deftest_clone_definition_not_set(self):
845+
dataset=DatasetReference(self.PROJECT,self.DS_ID)
846+
table_ref=dataset.table(self.TABLE_NAME)
847+
table=self._make_one(table_ref)
848+
849+
asserttable.clone_definitionisNone
850+
851+
deftest_clone_definition_set(self):
852+
fromgoogle.cloud._helpersimportUTC
853+
fromgoogle.cloud.bigquery.tableimportCloneDefinition
854+
855+
dataset=DatasetReference(self.PROJECT,self.DS_ID)
856+
table_ref=dataset.table(self.TABLE_NAME)
857+
table=self._make_one(table_ref)
858+
859+
table._properties["cloneDefinition"]= {
860+
"baseTableReference": {
861+
"projectId":"project_x",
862+
"datasetId":"dataset_y",
863+
"tableId":"table_z",
864+
},
865+
"cloneTime":"2010-09-28T10:20:30.123Z",
866+
}
867+
868+
clone=table.clone_definition
869+
870+
assertisinstance(clone,CloneDefinition)
871+
assertclone.base_table_reference.path== (
872+
"/projects/project_x/datasets/dataset_y/tables/table_z"
873+
)
874+
assertclone.clone_time==datetime.datetime(
875+
2010,9,28,10,20,30,123000,tzinfo=UTC
876+
)
877+
844878
deftest_description_setter_bad_value(self):
845879
dataset=DatasetReference(self.PROJECT,self.DS_ID)
846880
table_ref=dataset.table(self.TABLE_NAME)
@@ -1789,6 +1823,46 @@ def test_ctor_full_resource(self):
17891823
assertinstance.snapshot_time==expected_time
17901824

17911825

1826+
classTestCloneDefinition:
1827+
@staticmethod
1828+
def_get_target_class():
1829+
fromgoogle.cloud.bigquery.tableimportCloneDefinition
1830+
1831+
returnCloneDefinition
1832+
1833+
@classmethod
1834+
def_make_one(cls,*args,**kwargs):
1835+
klass=cls._get_target_class()
1836+
returnklass(*args,**kwargs)
1837+
1838+
deftest_ctor_empty_resource(self):
1839+
instance=self._make_one(resource={})
1840+
assertinstance.base_table_referenceisNone
1841+
assertinstance.clone_timeisNone
1842+
1843+
deftest_ctor_full_resource(self):
1844+
fromgoogle.cloud._helpersimportUTC
1845+
fromgoogle.cloud.bigquery.tableimportTableReference
1846+
1847+
resource= {
1848+
"baseTableReference": {
1849+
"projectId":"my-project",
1850+
"datasetId":"your-dataset",
1851+
"tableId":"our-table",
1852+
},
1853+
"cloneTime":"2005-06-07T19:35:02.123Z",
1854+
}
1855+
instance=self._make_one(resource)
1856+
1857+
expected_table_ref=TableReference.from_string(
1858+
"my-project.your-dataset.our-table"
1859+
)
1860+
assertinstance.base_table_reference==expected_table_ref
1861+
1862+
expected_time=datetime.datetime(2005,6,7,19,35,2,123000,tzinfo=UTC)
1863+
assertinstance.clone_time==expected_time
1864+
1865+
17921866
classTestRow(unittest.TestCase):
17931867
deftest_row(self):
17941868
fromgoogle.cloud.bigquery.tableimportRow

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp