Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit7d2d3e9

Browse files
authored
feat: add support for decimal target types (#735)
* feat: add support for decimal target types* Add decimal target types support to ExternalConfig* Remove ambiguous parts of DecimalTargetType docs.
1 parentcd2f09e commit7d2d3e9

File tree

8 files changed

+217
-0
lines changed

8 files changed

+217
-0
lines changed

‎google/cloud/bigquery/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
fromgoogle.cloud.bigquery.datasetimportDatasetReference
3939
fromgoogle.cloud.bigqueryimportenums
4040
fromgoogle.cloud.bigquery.enumsimportAutoRowIDs
41+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
4142
fromgoogle.cloud.bigquery.enumsimportKeyResultStatementKind
4243
fromgoogle.cloud.bigquery.enumsimportSqlTypeNames
4344
fromgoogle.cloud.bigquery.enumsimportStandardSqlDataTypes
@@ -148,6 +149,7 @@
148149
"AutoRowIDs",
149150
"Compression",
150151
"CreateDisposition",
152+
"DecimalTargetType",
151153
"DestinationFormat",
152154
"DeterminismLevel",
153155
"ExternalSourceFormat",

‎google/cloud/bigquery/enums.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,24 @@ class Compression(object):
4949
"""Specifies no compression."""
5050

5151

52+
classDecimalTargetType:
53+
"""The data types that could be used as a target type when converting decimal values.
54+
55+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#DecimalTargetType
56+
57+
.. versionadded:: 2.21.0
58+
"""
59+
60+
NUMERIC="NUMERIC"
61+
"""Decimal values could be converted to NUMERIC type."""
62+
63+
BIGNUMERIC="BIGNUMERIC"
64+
"""Decimal values could be converted to BIGNUMERIC type."""
65+
66+
STRING="STRING"
67+
"""Decimal values could be converted to STRING type."""
68+
69+
5270
classCreateDisposition(object):
5371
"""Specifies whether the job is allowed to create new tables. The default
5472
value is :attr:`CREATE_IF_NEEDED`.

‎google/cloud/bigquery/external_config.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
importbase64
2424
importcopy
25+
fromtypingimportFrozenSet,Iterable,Optional
2526

2627
fromgoogle.cloud.bigquery._helpersimport_to_bytes
2728
fromgoogle.cloud.bigquery._helpersimport_bytes_to_json
@@ -693,6 +694,28 @@ def compression(self):
693694
defcompression(self,value):
694695
self._properties["compression"]=value
695696

697+
@property
698+
defdecimal_target_types(self)->Optional[FrozenSet[str]]:
699+
"""Possible SQL data types to which the source decimal values are converted.
700+
701+
See:
702+
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.decimal_target_types
703+
704+
.. versionadded:: 2.21.0
705+
"""
706+
prop=self._properties.get("decimalTargetTypes")
707+
ifpropisnotNone:
708+
prop=frozenset(prop)
709+
returnprop
710+
711+
@decimal_target_types.setter
712+
defdecimal_target_types(self,value:Optional[Iterable[str]]):
713+
ifvalueisnotNone:
714+
self._properties["decimalTargetTypes"]=list(value)
715+
else:
716+
if"decimalTargetTypes"inself._properties:
717+
delself._properties["decimalTargetTypes"]
718+
696719
@property
697720
defhive_partitioning(self):
698721
"""Optional[:class:`~.external_config.HivePartitioningOptions`]: [Beta] When set,\

‎google/cloud/bigquery/job/load.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
"""Classes for load jobs."""
1616

17+
fromtypingimportFrozenSet,Iterable,Optional
18+
1719
fromgoogle.cloud.bigquery.encryption_configurationimportEncryptionConfiguration
1820
fromgoogle.cloud.bigquery.external_configimportHivePartitioningOptions
1921
fromgoogle.cloud.bigquery.format_optionsimportParquetOptions
@@ -121,6 +123,27 @@ def create_disposition(self):
121123
defcreate_disposition(self,value):
122124
self._set_sub_prop("createDisposition",value)
123125

126+
@property
127+
defdecimal_target_types(self)->Optional[FrozenSet[str]]:
128+
"""Possible SQL data types to which the source decimal values are converted.
129+
130+
See:
131+
https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.decimal_target_types
132+
133+
.. versionadded:: 2.21.0
134+
"""
135+
prop=self._get_sub_prop("decimalTargetTypes")
136+
ifpropisnotNone:
137+
prop=frozenset(prop)
138+
returnprop
139+
140+
@decimal_target_types.setter
141+
defdecimal_target_types(self,value:Optional[Iterable[str]]):
142+
ifvalueisnotNone:
143+
self._set_sub_prop("decimalTargetTypes",list(value))
144+
else:
145+
self._del_sub_prop("decimalTargetTypes")
146+
124147
@property
125148
defdestination_encryption_configuration(self):
126149
"""Optional[google.cloud.bigquery.encryption_configuration.EncryptionConfiguration]: Custom

‎tests/data/numeric_38_12.parquet‎

307 Bytes
Binary file not shown.

‎tests/system/test_client.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,60 @@ def test_load_table_from_local_avro_file_then_dump_table(self):
864864
sorted(row_tuples,key=by_wavelength),sorted(ROWS,key=by_wavelength)
865865
)
866866

867+
deftest_load_table_from_local_parquet_file_decimal_types(self):
868+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
869+
fromgoogle.cloud.bigquery.jobimportSourceFormat
870+
fromgoogle.cloud.bigquery.jobimportWriteDisposition
871+
872+
TABLE_NAME="test_table_parquet"
873+
874+
expected_rows= [
875+
(decimal.Decimal("123.999999999999"),),
876+
(decimal.Decimal("99999999999999999999999999.999999999999"),),
877+
]
878+
879+
dataset=self.temp_dataset(_make_dataset_id("load_local_parquet_then_dump"))
880+
table_ref=dataset.table(TABLE_NAME)
881+
table=Table(table_ref)
882+
self.to_delete.insert(0,table)
883+
884+
job_config=bigquery.LoadJobConfig()
885+
job_config.source_format=SourceFormat.PARQUET
886+
job_config.write_disposition=WriteDisposition.WRITE_TRUNCATE
887+
job_config.decimal_target_types= [
888+
DecimalTargetType.NUMERIC,
889+
DecimalTargetType.BIGNUMERIC,
890+
DecimalTargetType.STRING,
891+
]
892+
893+
withopen(DATA_PATH/"numeric_38_12.parquet","rb")asparquet_file:
894+
job=Config.CLIENT.load_table_from_file(
895+
parquet_file,table_ref,job_config=job_config
896+
)
897+
898+
job.result(timeout=JOB_TIMEOUT)# Retry until done.
899+
900+
self.assertEqual(job.output_rows,len(expected_rows))
901+
902+
table=Config.CLIENT.get_table(table)
903+
rows=self._fetch_single_page(table)
904+
row_tuples= [r.values()forrinrows]
905+
self.assertEqual(sorted(row_tuples),sorted(expected_rows))
906+
907+
# Forcing the NUMERIC type, however, should result in an error.
908+
job_config.decimal_target_types= [DecimalTargetType.NUMERIC]
909+
910+
withopen(DATA_PATH/"numeric_38_12.parquet","rb")asparquet_file:
911+
job=Config.CLIENT.load_table_from_file(
912+
parquet_file,table_ref,job_config=job_config
913+
)
914+
915+
withself.assertRaises(BadRequest)asexc_info:
916+
job.result(timeout=JOB_TIMEOUT)
917+
918+
exc_msg=str(exc_info.exception)
919+
self.assertIn("out of valid NUMERIC range",exc_msg)
920+
867921
deftest_load_table_from_json_basic_use(self):
868922
table_schema= (
869923
bigquery.SchemaField("name","STRING",mode="REQUIRED"),

‎tests/unit/job/test_load_config.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,45 @@ def test_create_disposition_setter(self):
122122
config.create_disposition=disposition
123123
self.assertEqual(config._properties["load"]["createDisposition"],disposition)
124124

125+
deftest_decimal_target_types_miss(self):
126+
config=self._get_target_class()()
127+
self.assertIsNone(config.decimal_target_types)
128+
129+
deftest_decimal_target_types_hit(self):
130+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
131+
132+
config=self._get_target_class()()
133+
decimal_target_types= [DecimalTargetType.NUMERIC,DecimalTargetType.STRING]
134+
config._properties["load"]["decimalTargetTypes"]=decimal_target_types
135+
136+
expected=frozenset(decimal_target_types)
137+
self.assertEqual(config.decimal_target_types,expected)
138+
139+
deftest_decimal_target_types_setter(self):
140+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
141+
142+
decimal_target_types= (DecimalTargetType.NUMERIC,DecimalTargetType.BIGNUMERIC)
143+
config=self._get_target_class()()
144+
config.decimal_target_types=decimal_target_types
145+
self.assertEqual(
146+
config._properties["load"]["decimalTargetTypes"],
147+
list(decimal_target_types),
148+
)
149+
150+
deftest_decimal_target_types_setter_w_none(self):
151+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
152+
153+
config=self._get_target_class()()
154+
decimal_target_types= [DecimalTargetType.BIGNUMERIC]
155+
config._properties["load"]["decimalTargetTypes"]=decimal_target_types
156+
157+
config.decimal_target_types=None
158+
159+
self.assertIsNone(config.decimal_target_types)
160+
self.assertNotIn("decimalTargetTypes",config._properties["load"])
161+
162+
config.decimal_target_types=None# No error if unsetting an unset property.
163+
125164
deftest_destination_encryption_configuration_missing(self):
126165
config=self._get_target_class()()
127166
self.assertIsNone(config.destination_encryption_configuration)

‎tests/unit/test_external_config.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,64 @@ def test_to_api_repr_parquet(self):
532532

533533
self.assertEqual(got_resource,exp_resource)
534534

535+
deftest_from_api_repr_decimal_target_types(self):
536+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
537+
538+
resource=_copy_and_update(
539+
self.BASE_RESOURCE,
540+
{
541+
"sourceFormat":"FORMAT_FOO",
542+
"decimalTargetTypes": [DecimalTargetType.NUMERIC],
543+
},
544+
)
545+
546+
ec=external_config.ExternalConfig.from_api_repr(resource)
547+
548+
self._verify_base(ec)
549+
self.assertEqual(ec.source_format,"FORMAT_FOO")
550+
self.assertEqual(
551+
ec.decimal_target_types,frozenset([DecimalTargetType.NUMERIC])
552+
)
553+
554+
# converting back to API representation should yield the same result
555+
got_resource=ec.to_api_repr()
556+
self.assertEqual(got_resource,resource)
557+
558+
delresource["decimalTargetTypes"]
559+
ec=external_config.ExternalConfig.from_api_repr(resource)
560+
self.assertIsNone(ec.decimal_target_types)
561+
562+
got_resource=ec.to_api_repr()
563+
self.assertEqual(got_resource,resource)
564+
565+
deftest_to_api_repr_decimal_target_types(self):
566+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
567+
568+
ec=external_config.ExternalConfig("FORMAT_FOO")
569+
ec.decimal_target_types= [DecimalTargetType.NUMERIC,DecimalTargetType.STRING]
570+
571+
got_resource=ec.to_api_repr()
572+
573+
expected_resource= {
574+
"sourceFormat":"FORMAT_FOO",
575+
"decimalTargetTypes": [DecimalTargetType.NUMERIC,DecimalTargetType.STRING],
576+
}
577+
self.assertEqual(got_resource,expected_resource)
578+
579+
deftest_to_api_repr_decimal_target_types_unset(self):
580+
fromgoogle.cloud.bigquery.enumsimportDecimalTargetType
581+
582+
ec=external_config.ExternalConfig("FORMAT_FOO")
583+
ec._properties["decimalTargetTypes"]= [DecimalTargetType.NUMERIC]
584+
ec.decimal_target_types=None
585+
586+
got_resource=ec.to_api_repr()
587+
588+
expected_resource= {"sourceFormat":"FORMAT_FOO"}
589+
self.assertEqual(got_resource,expected_resource)
590+
591+
ec.decimal_target_types=None# No error if unsetting when already unset.
592+
535593

536594
def_copy_and_update(d,u):
537595
d=copy.deepcopy(d)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp