Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit2c19681

Browse files
authored
Feat: Adds foreign_type_info attribute to table class and adds unit tests. (#2126)
* adds foreign_type_info attribute to table* feat: Adds foreign_type_info attribute and tests* updates docstrings for foreign_type_info* Updates property handling, especially as regards set/get_sub_prop* Removes extraneous comments and debug expressions* Refactors build_resource_from_properties w get/set_sub_prop* updates to foreign_type_info, tests and wiring* Adds logic to detect non-Sequence schema.fields value* updates assorted tests and logic
1 parent7603bd7 commit2c19681

File tree

7 files changed

+398
-104
lines changed

7 files changed

+398
-104
lines changed

‎google/cloud/bigquery/_helpers.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -978,11 +978,11 @@ def _build_resource_from_properties(obj, filter_fields):
978978
"""
979979
partial= {}
980980
forfilter_fieldinfilter_fields:
981-
api_field=obj._PROPERTY_TO_API_FIELD.get(filter_field)
981+
api_field=_get_sub_prop(obj._PROPERTY_TO_API_FIELD,filter_field)
982982
ifapi_fieldisNoneandfilter_fieldnotinobj._properties:
983983
raiseValueError("No property %s"%filter_field)
984984
elifapi_fieldisnotNone:
985-
partial[api_field]=obj._properties.get(api_field)
985+
_set_sub_prop(partial,api_field,_get_sub_prop(obj._properties,api_field))
986986
else:
987987
# allows properties that are not defined in the library
988988
# and properties that have the same name as API resource key

‎google/cloud/bigquery/schema.py‎

Lines changed: 32 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,9 @@
1515
"""Schemas for BigQuery tables / queries."""
1616

1717
from __future__importannotations
18-
importcollections
1918
importenum
2019
importtyping
21-
fromtypingimportAny,cast,Dict,Iterable,Optional,Union
20+
fromtypingimportAny,cast,Dict,Iterable,Optional,Union,Sequence
2221

2322
fromgoogle.cloud.bigqueryimport_helpers
2423
fromgoogle.cloud.bigqueryimportstandard_sql
@@ -489,6 +488,8 @@ def _parse_schema_resource(info):
489488
Optional[Sequence[google.cloud.bigquery.schema.SchemaField`]:
490489
A list of parsed fields, or ``None`` if no "fields" key found.
491490
"""
491+
ifisinstance(info,list):
492+
return [SchemaField.from_api_repr(f)forfininfo]
492493
return [SchemaField.from_api_repr(f)forfininfo.get("fields", ())]
493494

494495

@@ -501,40 +502,46 @@ def _build_schema_resource(fields):
501502
Returns:
502503
Sequence[Dict]: Mappings describing the schema of the supplied fields.
503504
"""
504-
return [field.to_api_repr()forfieldinfields]
505+
ifisinstance(fields,Sequence):
506+
# Input is a Sequence (e.g. a list): Process and return a list of SchemaFields
507+
return [field.to_api_repr()forfieldinfields]
508+
509+
else:
510+
raiseTypeError("Schema must be a Sequence (e.g. a list) or None.")
505511

506512

507513
def_to_schema_fields(schema):
508-
"""Coerce `schema` to a list of schema field instances.
514+
"""Coerces schema to a list of SchemaField instances while
515+
preserving the original structure as much as possible.
509516
510517
Args:
511-
schema(Sequence[Union[\
512-
:class:`~google.cloud.bigquery.schema.SchemaField`,\
513-
Mapping[str, Any]\
514-
]]):
515-
Table schema to convert. If some items are passed as mappings,
516-
their content must be compatible with
517-
:meth:`~google.cloud.bigquery.schema.SchemaField.from_api_repr`.
518+
schema (Sequence[Union[\
519+
:class:`~google.cloud.bigquery.schema.SchemaField`,\
520+
Mapping[str, Any]\
521+
]
522+
]
523+
)::
524+
Table schema to convert. Can be a list of SchemaField
525+
objects or mappings.
518526
519527
Returns:
520-
Sequence[:class:`~google.cloud.bigquery.schema.SchemaField`]
528+
A list ofSchemaField objects.
521529
522530
Raises:
523-
Exception: If ``schema`` is not a sequence, or if any item in the
524-
sequence is not a :class:`~google.cloud.bigquery.schema.SchemaField`
525-
instance or a compatible mapping representation of the field.
531+
TypeError: If schema is not a Sequence.
526532
"""
527-
forfieldinschema:
528-
ifnotisinstance(field, (SchemaField,collections.abc.Mapping)):
529-
raiseValueError(
530-
"Schema items must either be fields or compatible "
531-
"mapping representations."
532-
)
533533

534-
return [
535-
fieldifisinstance(field,SchemaField)elseSchemaField.from_api_repr(field)
536-
forfieldinschema
537-
]
534+
ifisinstance(schema,Sequence):
535+
# Input is a Sequence (e.g. a list): Process and return a list of SchemaFields
536+
return [
537+
field
538+
ifisinstance(field,SchemaField)
539+
elseSchemaField.from_api_repr(field)
540+
forfieldinschema
541+
]
542+
543+
else:
544+
raiseTypeError("Schema must be a Sequence (e.g. a list) or None.")
538545

539546

540547
classPolicyTagList(object):

‎google/cloud/bigquery/table.py‎

Lines changed: 69 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
importfunctools
2222
importoperator
2323
importtyping
24-
fromtypingimportAny,Dict,Iterable,Iterator,List,Optional,Tuple,Union
24+
fromtypingimportAny,Dict,Iterable,Iterator,List,Optional,Tuple,Union,Sequence
25+
2526
importwarnings
2627

2728
try:
@@ -66,6 +67,7 @@
6667
fromgoogle.cloud.bigquery.encryption_configurationimportEncryptionConfiguration
6768
fromgoogle.cloud.bigquery.enumsimportDefaultPandasDTypes
6869
fromgoogle.cloud.bigquery.external_configimportExternalConfig
70+
fromgoogle.cloud.bigqueryimportschemaas_schema
6971
fromgoogle.cloud.bigquery.schemaimport_build_schema_resource
7072
fromgoogle.cloud.bigquery.schemaimport_parse_schema_resource
7173
fromgoogle.cloud.bigquery.schemaimport_to_schema_fields
@@ -398,7 +400,7 @@ class Table(_TableBase):
398400
"partitioning_type":"timePartitioning",
399401
"range_partitioning":"rangePartitioning",
400402
"time_partitioning":"timePartitioning",
401-
"schema":"schema",
403+
"schema":["schema","fields"],
402404
"snapshot_definition":"snapshotDefinition",
403405
"clone_definition":"cloneDefinition",
404406
"streaming_buffer":"streamingBuffer",
@@ -411,6 +413,7 @@ class Table(_TableBase):
411413
"max_staleness":"maxStaleness",
412414
"resource_tags":"resourceTags",
413415
"external_catalog_table_options":"externalCatalogTableOptions",
416+
"foreign_type_info": ["schema","foreignTypeInfo"],
414417
}
415418

416419
def__init__(self,table_ref,schema=None)->None:
@@ -451,8 +454,20 @@ def schema(self):
451454
If ``schema`` is not a sequence, or if any item in the sequence
452455
is not a :class:`~google.cloud.bigquery.schema.SchemaField`
453456
instance or a compatible mapping representation of the field.
457+
458+
.. Note::
459+
If you are referencing a schema for an external catalog table such
460+
as a Hive table, it will also be necessary to populate the foreign_type_info
461+
attribute. This is not necessary if defining the schema for a BigQuery table.
462+
463+
For details, see:
464+
https://cloud.google.com/bigquery/docs/external-tables
465+
https://cloud.google.com/bigquery/docs/datasets-intro#external_datasets
466+
454467
"""
455-
prop=self._properties.get(self._PROPERTY_TO_API_FIELD["schema"])
468+
prop=_helpers._get_sub_prop(
469+
self._properties,self._PROPERTY_TO_API_FIELD["schema"]
470+
)
456471
ifnotprop:
457472
return []
458473
else:
@@ -463,10 +478,21 @@ def schema(self, value):
463478
api_field=self._PROPERTY_TO_API_FIELD["schema"]
464479

465480
ifvalueisNone:
466-
self._properties[api_field]=None
467-
else:
481+
_helpers._set_sub_prop(
482+
self._properties,
483+
api_field,
484+
None,
485+
)
486+
elifisinstance(value,Sequence):
468487
value=_to_schema_fields(value)
469-
self._properties[api_field]= {"fields":_build_schema_resource(value)}
488+
value=_build_schema_resource(value)
489+
_helpers._set_sub_prop(
490+
self._properties,
491+
api_field,
492+
value,
493+
)
494+
else:
495+
raiseTypeError("Schema must be a Sequence (e.g. a list) or None.")
470496

471497
@property
472498
deflabels(self):
@@ -1075,6 +1101,43 @@ def external_catalog_table_options(
10751101
self._PROPERTY_TO_API_FIELD["external_catalog_table_options"]
10761102
]=value
10771103

1104+
@property
1105+
defforeign_type_info(self)->Optional[_schema.ForeignTypeInfo]:
1106+
"""Optional. Specifies metadata of the foreign data type definition in
1107+
field schema (TableFieldSchema.foreign_type_definition).
1108+
1109+
Returns:
1110+
Optional[schema.ForeignTypeInfo]:
1111+
Foreign type information, or :data:`None` if not set.
1112+
1113+
.. Note::
1114+
foreign_type_info is only required if you are referencing an
1115+
external catalog such as a Hive table.
1116+
For details, see:
1117+
https://cloud.google.com/bigquery/docs/external-tables
1118+
https://cloud.google.com/bigquery/docs/datasets-intro#external_datasets
1119+
"""
1120+
1121+
prop=_helpers._get_sub_prop(
1122+
self._properties,self._PROPERTY_TO_API_FIELD["foreign_type_info"]
1123+
)
1124+
ifpropisnotNone:
1125+
return_schema.ForeignTypeInfo.from_api_repr(prop)
1126+
returnNone
1127+
1128+
@foreign_type_info.setter
1129+
defforeign_type_info(self,value:Union[_schema.ForeignTypeInfo,dict,None]):
1130+
value=_helpers._isinstance_or_raise(
1131+
value,
1132+
(_schema.ForeignTypeInfo,dict),
1133+
none_allowed=True,
1134+
)
1135+
ifisinstance(value,_schema.ForeignTypeInfo):
1136+
value=value.to_api_repr()
1137+
_helpers._set_sub_prop(
1138+
self._properties,self._PROPERTY_TO_API_FIELD["foreign_type_info"],value
1139+
)
1140+
10781141
@classmethod
10791142
deffrom_string(cls,full_table_id:str)->"Table":
10801143
"""Construct a table from fully-qualified table ID.

‎tests/unit/job/test_load.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ def test_schema_setter_invalid_field(self):
272272

273273
config=LoadJobConfig()
274274
full_name=SchemaField("full_name","STRING",mode="REQUIRED")
275-
withself.assertRaises(ValueError):
275+
withself.assertRaises(TypeError):
276276
config.schema= [full_name,object()]
277277

278278
deftest_schema_setter(self):

‎tests/unit/test_client.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2051,7 +2051,7 @@ def test_update_dataset(self):
20512051
ds.labels=LABELS
20522052
ds.access_entries= [AccessEntry("OWNER","userByEmail","phred@example.com")]
20532053
ds.resource_tags=RESOURCE_TAGS
2054-
fields= [
2054+
filter_fields= [
20552055
"description",
20562056
"friendly_name",
20572057
"location",
@@ -2065,12 +2065,12 @@ def test_update_dataset(self):
20652065
)asfinal_attributes:
20662066
ds2=client.update_dataset(
20672067
ds,
2068-
fields=fields,
2068+
fields=filter_fields,
20692069
timeout=7.5,
20702070
)
20712071

20722072
final_attributes.assert_called_once_with(
2073-
{"path":"/%s"%PATH,"fields":fields},client,None
2073+
{"path":"/%s"%PATH,"fields":filter_fields},client,None
20742074
)
20752075

20762076
conn.api_request.assert_called_once_with(
@@ -2615,7 +2615,7 @@ def test_update_table_w_schema_None(self):
26152615
self.assertEqual(len(conn.api_request.call_args_list),2)
26162616
req=conn.api_request.call_args_list[1]
26172617
self.assertEqual(req[1]["method"],"PATCH")
2618-
sent= {"schema":None}
2618+
sent= {"schema":{"fields":None}}
26192619
self.assertEqual(req[1]["data"],sent)
26202620
self.assertEqual(req[1]["path"],"/%s"%path)
26212621
self.assertEqual(len(updated_table.schema),0)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp