Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit1b946ba

Browse files
authored
fix: remove DB-API dependency on pyarrow with decimal query parameters (#551)
* fix: DB API pyarrow dependency with decimal valuesDB API should gracefully handle the case when the optional pyarrowdependency is not installed.* Blacken DB API helpers tests* Refine the logic for recognizing NUMERIC Decimals
1 parenta460f93 commit1b946ba

File tree

2 files changed

+60
-29
lines changed

2 files changed

+60
-29
lines changed

‎google/cloud/bigquery/dbapi/_helpers.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,15 @@
1919
importfunctools
2020
importnumbers
2121

22-
try:
23-
importpyarrow
24-
exceptImportError:# pragma: NO COVER
25-
pyarrow=None
26-
2722
fromgoogle.cloudimportbigquery
2823
fromgoogle.cloud.bigqueryimporttable
2924
fromgoogle.cloud.bigquery.dbapiimportexceptions
3025

3126

27+
_NUMERIC_SERVER_MIN=decimal.Decimal("-9.9999999999999999999999999999999999999E+28")
28+
_NUMERIC_SERVER_MAX=decimal.Decimal("9.9999999999999999999999999999999999999E+28")
29+
30+
3231
defscalar_to_query_parameter(value,name=None):
3332
"""Convert a scalar value into a query parameter.
3433
@@ -189,12 +188,20 @@ def bigquery_scalar_type(value):
189188
elifisinstance(value,numbers.Real):
190189
return"FLOAT64"
191190
elifisinstance(value,decimal.Decimal):
192-
# We check for NUMERIC before BIGNUMERIC in order to support pyarrow < 3.0.
193-
scalar_object=pyarrow.scalar(value)
194-
ifisinstance(scalar_object,pyarrow.Decimal128Scalar):
191+
vtuple=value.as_tuple()
192+
# NUMERIC values have precision of 38 (number of digits) and scale of 9 (number
193+
# of fractional digits), and their max absolute value must be strictly smaller
194+
# than 1.0E+29.
195+
# https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#decimal_types
196+
if (
197+
len(vtuple.digits)<=38# max precision: 38
198+
andvtuple.exponent>=-9# max scale: 9
199+
and_NUMERIC_SERVER_MIN<=value<=_NUMERIC_SERVER_MAX
200+
):
195201
return"NUMERIC"
196202
else:
197203
return"BIGNUMERIC"
204+
198205
elifisinstance(value,str):
199206
return"STRING"
200207
elifisinstance(value,bytes):

‎tests/unit/test_dbapi__helpers.py‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525

2626
importgoogle.cloud._helpers
2727
fromgoogle.cloud.bigqueryimporttable
28-
fromgoogle.cloud.bigquery._pandas_helpersimport_BIGNUMERIC_SUPPORT
2928
fromgoogle.cloud.bigquery.dbapiimport_helpers
3029
fromgoogle.cloud.bigquery.dbapiimportexceptions
3130
fromtests.unit.helpersimport_to_pyarrow
@@ -39,9 +38,8 @@ def test_scalar_to_query_parameter(self):
3938
(123,"INT64"),
4039
(-123456789,"INT64"),
4140
(1.25,"FLOAT64"),
42-
(decimal.Decimal("1.25"),"NUMERIC"),
4341
(b"I am some bytes","BYTES"),
44-
(u"I am a string","STRING"),
42+
("I am a string","STRING"),
4543
(datetime.date(2017,4,1),"DATE"),
4644
(datetime.time(12,34,56),"TIME"),
4745
(datetime.datetime(2012,3,4,5,6,7),"DATETIME"),
@@ -51,14 +49,17 @@ def test_scalar_to_query_parameter(self):
5149
),
5250
"TIMESTAMP",
5351
),
52+
(decimal.Decimal("1.25"),"NUMERIC"),
53+
(decimal.Decimal("9.9999999999999999999999999999999999999E+28"),"NUMERIC"),
54+
(decimal.Decimal("1.0E+29"),"BIGNUMERIC"),# more than max NUMERIC value
55+
(decimal.Decimal("1.123456789"),"NUMERIC"),
56+
(decimal.Decimal("1.1234567891"),"BIGNUMERIC"),# scale > 9
57+
(decimal.Decimal("12345678901234567890123456789.012345678"),"NUMERIC"),
58+
(
59+
decimal.Decimal("12345678901234567890123456789012345678"),
60+
"BIGNUMERIC",# larger than max NUMERIC value, despite precision <=38
61+
),
5462
]
55-
if_BIGNUMERIC_SUPPORT:
56-
expected_types.append(
57-
(
58-
decimal.Decimal("1.1234567890123456789012345678901234567890"),
59-
"BIGNUMERIC",
60-
)
61-
)
6263

6364
forvalue,expected_typeinexpected_types:
6465
msg="value: {} expected_type: {}".format(value,expected_type)
@@ -71,6 +72,33 @@ def test_scalar_to_query_parameter(self):
7172
self.assertEqual(named_parameter.type_,expected_type,msg=msg)
7273
self.assertEqual(named_parameter.value,value,msg=msg)
7374

75+
deftest_decimal_to_query_parameter(self):# TODO: merge with previous test
76+
77+
expected_types= [
78+
(decimal.Decimal("9.9999999999999999999999999999999999999E+28"),"NUMERIC"),
79+
(decimal.Decimal("1.0E+29"),"BIGNUMERIC"),# more than max value
80+
(decimal.Decimal("1.123456789"),"NUMERIC"),
81+
(decimal.Decimal("1.1234567891"),"BIGNUMERIC"),# scale > 9
82+
(decimal.Decimal("12345678901234567890123456789.012345678"),"NUMERIC"),
83+
(
84+
decimal.Decimal("12345678901234567890123456789012345678"),
85+
"BIGNUMERIC",# larger than max size, even if precision <=38
86+
),
87+
]
88+
89+
forvalue,expected_typeinexpected_types:
90+
msg=f"value:{value} expected_type:{expected_type}"
91+
92+
parameter=_helpers.scalar_to_query_parameter(value)
93+
self.assertIsNone(parameter.name,msg=msg)
94+
self.assertEqual(parameter.type_,expected_type,msg=msg)
95+
self.assertEqual(parameter.value,value,msg=msg)
96+
97+
named_parameter=_helpers.scalar_to_query_parameter(value,name="myvar")
98+
self.assertEqual(named_parameter.name,"myvar",msg=msg)
99+
self.assertEqual(named_parameter.type_,expected_type,msg=msg)
100+
self.assertEqual(named_parameter.value,value,msg=msg)
101+
74102
deftest_scalar_to_query_parameter_w_unexpected_type(self):
75103
withself.assertRaises(exceptions.ProgrammingError):
76104
_helpers.scalar_to_query_parameter(value={"a":"dictionary"})
@@ -89,8 +117,9 @@ def test_array_to_query_parameter_valid_argument(self):
89117
([123,-456,0],"INT64"),
90118
([1.25,2.50],"FLOAT64"),
91119
([decimal.Decimal("1.25")],"NUMERIC"),
120+
([decimal.Decimal("{d38}.{d38}".format(d38="9"*38))],"BIGNUMERIC"),
92121
([b"foo",b"bar"],"BYTES"),
93-
([u"foo",u"bar"],"STRING"),
122+
(["foo","bar"],"STRING"),
94123
([datetime.date(2017,4,1),datetime.date(2018,4,1)],"DATE"),
95124
([datetime.time(12,34,56),datetime.time(10,20,30)],"TIME"),
96125
(
@@ -113,11 +142,6 @@ def test_array_to_query_parameter_valid_argument(self):
113142
),
114143
]
115144

116-
if_BIGNUMERIC_SUPPORT:
117-
expected_types.append(
118-
([decimal.Decimal("{d38}.{d38}".format(d38="9"*38))],"BIGNUMERIC")
119-
)
120-
121145
forvalues,expected_typeinexpected_types:
122146
msg="value: {} expected_type: {}".format(values,expected_type)
123147
parameter=_helpers.array_to_query_parameter(values)
@@ -134,7 +158,7 @@ def test_array_to_query_parameter_empty_argument(self):
134158
_helpers.array_to_query_parameter([])
135159

136160
deftest_array_to_query_parameter_unsupported_sequence(self):
137-
unsupported_iterables= [{10,20,30},u"foo",b"bar",bytearray([65,75,85])]
161+
unsupported_iterables= [{10,20,30},"foo",b"bar",bytearray([65,75,85])]
138162
foriterableinunsupported_iterables:
139163
withself.assertRaises(exceptions.ProgrammingError):
140164
_helpers.array_to_query_parameter(iterable)
@@ -144,7 +168,7 @@ def test_array_to_query_parameter_sequence_w_invalid_elements(self):
144168
_helpers.array_to_query_parameter([object(),2,7])
145169

146170
deftest_to_query_parameters_w_dict(self):
147-
parameters= {"somebool":True,"somestring":u"a-string-value"}
171+
parameters= {"somebool":True,"somestring":"a-string-value"}
148172
query_parameters=_helpers.to_query_parameters(parameters)
149173
query_parameter_tuples= []
150174
forparaminquery_parameters:
@@ -154,7 +178,7 @@ def test_to_query_parameters_w_dict(self):
154178
sorted(
155179
[
156180
("somebool","BOOL",True),
157-
("somestring","STRING",u"a-string-value"),
181+
("somestring","STRING","a-string-value"),
158182
]
159183
),
160184
)
@@ -177,14 +201,14 @@ def test_to_query_parameters_w_dict_dict_param(self):
177201
_helpers.to_query_parameters(parameters)
178202

179203
deftest_to_query_parameters_w_list(self):
180-
parameters= [True,u"a-string-value"]
204+
parameters= [True,"a-string-value"]
181205
query_parameters=_helpers.to_query_parameters(parameters)
182206
query_parameter_tuples= []
183207
forparaminquery_parameters:
184208
query_parameter_tuples.append((param.name,param.type_,param.value))
185209
self.assertSequenceEqual(
186210
sorted(query_parameter_tuples),
187-
sorted([(None,"BOOL",True), (None,"STRING",u"a-string-value")]),
211+
sorted([(None,"BOOL",True), (None,"STRING","a-string-value")]),
188212
)
189213

190214
deftest_to_query_parameters_w_list_array_param(self):

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp