Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit1a9431d

Browse files
authored
feat: addAvroOptions to configure AVRO external data (#994)
* feat: add `AvroOptions` to configure AVRO external dataAlso:* Unify `ExternalConfig` class to use `_properties` for everything. This does result in more code, but it should make maintenance easier as it aligns with our other mutable resource classes.* Adds `bigtable_options`, `csv_options`, and `google_sheets_options` properties. This aligns with `parquet_options`.* remove unnecessary check for options in to_api_repr* add missing tests for to_api_repr* remove redundant type identifiers
1 parentd9a03b4 commit1a9431d

File tree

7 files changed

+518
-48
lines changed

7 files changed

+518
-48
lines changed

‎docs/format_options.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
BigQuery Format Options
2+
=======================
3+
4+
..automodule::google.cloud.bigquery.format_options
5+
:members:
6+
:undoc-members:

‎docs/reference.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,11 @@ External Configuration
167167
external_config.CSVOptions
168168
external_config.GoogleSheetsOptions
169169

170+
..toctree::
171+
:maxdepth:2
172+
173+
format_options
174+
170175

171176
Magics
172177
======

‎google/cloud/bigquery/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
fromgoogle.cloud.bigquery.external_configimportCSVOptions
5151
fromgoogle.cloud.bigquery.external_configimportGoogleSheetsOptions
5252
fromgoogle.cloud.bigquery.external_configimportExternalSourceFormat
53+
fromgoogle.cloud.bigquery.format_optionsimportAvroOptions
5354
fromgoogle.cloud.bigquery.format_optionsimportParquetOptions
5455
fromgoogle.cloud.bigquery.jobimportCompression
5556
fromgoogle.cloud.bigquery.jobimportCopyJob
@@ -144,6 +145,7 @@
144145
"PolicyTagList",
145146
"UDFResource",
146147
"ExternalConfig",
148+
"AvroOptions",
147149
"BigtableOptions",
148150
"BigtableColumnFamily",
149151
"BigtableColumn",

‎google/cloud/bigquery/external_config.py‎

Lines changed: 123 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@
2222

2323
importbase64
2424
importcopy
25-
fromtypingimportFrozenSet,Iterable,Optional
25+
fromtypingimportFrozenSet,Iterable,Optional,Union
2626

2727
fromgoogle.cloud.bigquery._helpersimport_to_bytes
2828
fromgoogle.cloud.bigquery._helpersimport_bytes_to_json
2929
fromgoogle.cloud.bigquery._helpersimport_int_or_none
3030
fromgoogle.cloud.bigquery._helpersimport_str_or_none
31-
fromgoogle.cloud.bigquery.format_optionsimportParquetOptions
31+
fromgoogle.cloud.bigquery.format_optionsimportAvroOptions,ParquetOptions
3232
fromgoogle.cloud.bigquery.schemaimportSchemaField
3333

3434

@@ -548,7 +548,13 @@ def from_api_repr(cls, resource: dict) -> "GoogleSheetsOptions":
548548
returnconfig
549549

550550

551-
_OPTION_CLASSES= (BigtableOptions,CSVOptions,GoogleSheetsOptions,ParquetOptions)
551+
_OPTION_CLASSES= (
552+
AvroOptions,
553+
BigtableOptions,
554+
CSVOptions,
555+
GoogleSheetsOptions,
556+
ParquetOptions,
557+
)
552558

553559

554560
classHivePartitioningOptions(object):
@@ -646,11 +652,6 @@ class ExternalConfig(object):
646652

647653
def__init__(self,source_format):
648654
self._properties= {"sourceFormat":source_format}
649-
self._options=None
650-
foroptclsin_OPTION_CLASSES:
651-
ifsource_format==optcls._SOURCE_FORMAT:
652-
self._options=optcls()
653-
break
654655

655656
@property
656657
defsource_format(self):
@@ -663,9 +664,17 @@ def source_format(self):
663664
returnself._properties["sourceFormat"]
664665

665666
@property
666-
defoptions(self):
667-
"""Optional[Dict[str, Any]]: Source-specific options."""
668-
returnself._options
667+
defoptions(self)->Optional[Union[_OPTION_CLASSES]]:
668+
"""Source-specific options."""
669+
foroptclsin_OPTION_CLASSES:
670+
ifself.source_format==optcls._SOURCE_FORMAT:
671+
options=optcls()
672+
self._properties.setdefault(optcls._RESOURCE_NAME, {})
673+
options._properties=self._properties[optcls._RESOURCE_NAME]
674+
returnoptions
675+
676+
# No matching source format found.
677+
returnNone
669678

670679
@property
671680
defautodetect(self):
@@ -815,23 +824,120 @@ def schema(self, value):
815824
self._properties["schema"]=prop
816825

817826
@property
818-
defparquet_options(self):
819-
"""Optional[google.cloud.bigquery.format_options.ParquetOptions]: Additional
820-
properties to set if ``sourceFormat`` is set to PARQUET.
827+
defavro_options(self)->Optional[AvroOptions]:
828+
"""Additional properties to set if ``sourceFormat`` is set to AVRO.
829+
830+
See:
831+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.avro_options
832+
"""
833+
ifself.source_format==ExternalSourceFormat.AVRO:
834+
self._properties.setdefault(AvroOptions._RESOURCE_NAME, {})
835+
resource=self._properties.get(AvroOptions._RESOURCE_NAME)
836+
ifresourceisNone:
837+
returnNone
838+
options=AvroOptions()
839+
options._properties=resource
840+
returnoptions
841+
842+
@avro_options.setter
843+
defavro_options(self,value):
844+
ifself.source_format!=ExternalSourceFormat.AVRO:
845+
msg=f"Cannot set Avro options, source format is{self.source_format}"
846+
raiseTypeError(msg)
847+
self._properties[AvroOptions._RESOURCE_NAME]=value._properties
848+
849+
@property
850+
defbigtable_options(self)->Optional[BigtableOptions]:
851+
"""Additional properties to set if ``sourceFormat`` is set to BIGTABLE.
852+
853+
See:
854+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.bigtable_options
855+
"""
856+
ifself.source_format==ExternalSourceFormat.BIGTABLE:
857+
self._properties.setdefault(BigtableOptions._RESOURCE_NAME, {})
858+
resource=self._properties.get(BigtableOptions._RESOURCE_NAME)
859+
ifresourceisNone:
860+
returnNone
861+
options=BigtableOptions()
862+
options._properties=resource
863+
returnoptions
864+
865+
@bigtable_options.setter
866+
defbigtable_options(self,value):
867+
ifself.source_format!=ExternalSourceFormat.BIGTABLE:
868+
msg=f"Cannot set Bigtable options, source format is{self.source_format}"
869+
raiseTypeError(msg)
870+
self._properties[BigtableOptions._RESOURCE_NAME]=value._properties
871+
872+
@property
873+
defcsv_options(self)->Optional[CSVOptions]:
874+
"""Additional properties to set if ``sourceFormat`` is set to CSV.
875+
876+
See:
877+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.csv_options
878+
"""
879+
ifself.source_format==ExternalSourceFormat.CSV:
880+
self._properties.setdefault(CSVOptions._RESOURCE_NAME, {})
881+
resource=self._properties.get(CSVOptions._RESOURCE_NAME)
882+
ifresourceisNone:
883+
returnNone
884+
options=CSVOptions()
885+
options._properties=resource
886+
returnoptions
887+
888+
@csv_options.setter
889+
defcsv_options(self,value):
890+
ifself.source_format!=ExternalSourceFormat.CSV:
891+
msg=f"Cannot set CSV options, source format is{self.source_format}"
892+
raiseTypeError(msg)
893+
self._properties[CSVOptions._RESOURCE_NAME]=value._properties
894+
895+
@property
896+
defgoogle_sheets_options(self)->Optional[GoogleSheetsOptions]:
897+
"""Additional properties to set if ``sourceFormat`` is set to
898+
GOOGLE_SHEETS.
899+
900+
See:
901+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.google_sheets_options
902+
"""
903+
ifself.source_format==ExternalSourceFormat.GOOGLE_SHEETS:
904+
self._properties.setdefault(GoogleSheetsOptions._RESOURCE_NAME, {})
905+
resource=self._properties.get(GoogleSheetsOptions._RESOURCE_NAME)
906+
ifresourceisNone:
907+
returnNone
908+
options=GoogleSheetsOptions()
909+
options._properties=resource
910+
returnoptions
911+
912+
@google_sheets_options.setter
913+
defgoogle_sheets_options(self,value):
914+
ifself.source_format!=ExternalSourceFormat.GOOGLE_SHEETS:
915+
msg=f"Cannot set Google Sheets options, source format is{self.source_format}"
916+
raiseTypeError(msg)
917+
self._properties[GoogleSheetsOptions._RESOURCE_NAME]=value._properties
918+
919+
@property
920+
defparquet_options(self)->Optional[ParquetOptions]:
921+
"""Additional properties to set if ``sourceFormat`` is set to PARQUET.
821922
822923
See:
823924
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.parquet_options
824925
"""
825-
ifself.source_format!=ExternalSourceFormat.PARQUET:
926+
ifself.source_format==ExternalSourceFormat.PARQUET:
927+
self._properties.setdefault(ParquetOptions._RESOURCE_NAME, {})
928+
resource=self._properties.get(ParquetOptions._RESOURCE_NAME)
929+
ifresourceisNone:
826930
returnNone
827-
returnself._options
931+
options=ParquetOptions()
932+
options._properties=resource
933+
returnoptions
828934

829935
@parquet_options.setter
830936
defparquet_options(self,value):
831937
ifself.source_format!=ExternalSourceFormat.PARQUET:
832938
msg=f"Cannot set Parquet options, source format is{self.source_format}"
833939
raiseTypeError(msg)
834-
self._options=value
940+
self._properties[ParquetOptions._RESOURCE_NAME]=value._properties
835941

836942
defto_api_repr(self)->dict:
837943
"""Build an API representation of this object.
@@ -841,10 +947,6 @@ def to_api_repr(self) -> dict:
841947
A dictionary in the format used by the BigQuery API.
842948
"""
843949
config=copy.deepcopy(self._properties)
844-
ifself.optionsisnotNone:
845-
r=self.options.to_api_repr()
846-
ifr!= {}:
847-
config[self.options._RESOURCE_NAME]=r
848950
returnconfig
849951

850952
@classmethod
@@ -862,10 +964,5 @@ def from_api_repr(cls, resource: dict) -> "ExternalConfig":
862964
ExternalConfig: Configuration parsed from ``resource``.
863965
"""
864966
config=cls(resource["sourceFormat"])
865-
foroptclsin_OPTION_CLASSES:
866-
opts=resource.get(optcls._RESOURCE_NAME)
867-
ifoptsisnotNone:
868-
config._options=optcls.from_api_repr(opts)
869-
break
870967
config._properties=copy.deepcopy(resource)
871968
returnconfig

‎google/cloud/bigquery/format_options.py‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,59 @@
1313
# limitations under the License.
1414

1515
importcopy
16-
fromtypingimportDict
16+
fromtypingimportDict,Optional
17+
18+
19+
classAvroOptions:
20+
"""Options if source format is set to AVRO."""
21+
22+
_SOURCE_FORMAT="AVRO"
23+
_RESOURCE_NAME="avroOptions"
24+
25+
def__init__(self):
26+
self._properties= {}
27+
28+
@property
29+
defuse_avro_logical_types(self)->Optional[bool]:
30+
"""[Optional] If sourceFormat is set to 'AVRO', indicates whether to
31+
interpret logical types as the corresponding BigQuery data type (for
32+
example, TIMESTAMP), instead of using the raw type (for example,
33+
INTEGER).
34+
35+
See
36+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#AvroOptions.FIELDS.use_avro_logical_types
37+
"""
38+
returnself._properties.get("useAvroLogicalTypes")
39+
40+
@use_avro_logical_types.setter
41+
defuse_avro_logical_types(self,value):
42+
self._properties["useAvroLogicalTypes"]=value
43+
44+
@classmethod
45+
deffrom_api_repr(cls,resource:Dict[str,bool])->"AvroOptions":
46+
"""Factory: construct an instance from a resource dict.
47+
48+
Args:
49+
resource (Dict[str, bool]):
50+
Definition of a :class:`~.format_options.AvroOptions` instance in
51+
the same representation as is returned from the API.
52+
53+
Returns:
54+
:class:`~.format_options.AvroOptions`:
55+
Configuration parsed from ``resource``.
56+
"""
57+
config=cls()
58+
config._properties=copy.deepcopy(resource)
59+
returnconfig
60+
61+
defto_api_repr(self)->dict:
62+
"""Build an API representation of this object.
63+
64+
Returns:
65+
Dict[str, bool]:
66+
A dictionary in the format used by the BigQuery API.
67+
"""
68+
returncopy.deepcopy(self._properties)
1769

1870

1971
classParquetOptions:

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp