Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitb289076

Browse files
authored
fix: guard imports against unsupported pyarrow versions (#934)
* fix: guard imports against unsupported pyarrow versions* add unit tests* fix pytype* second try at fixing pytype
1 parent10fee52 commitb289076

File tree

9 files changed

+184
-39
lines changed

9 files changed

+184
-39
lines changed

‎google/cloud/bigquery/_helpers.py‎

Lines changed: 69 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
importdecimal
2020
importmath
2121
importre
22-
fromtypingimportUnion
22+
fromtypingimportAny,Union
2323

2424
fromgoogle.cloud._helpersimportUTC
2525
fromgoogle.cloud._helpersimport_date_from_iso8601_date
@@ -29,7 +29,10 @@
2929
fromgoogle.cloud._helpersimport_to_bytes
3030
importpackaging.version
3131

32-
fromgoogle.cloud.bigquery.exceptionsimportLegacyBigQueryStorageError
32+
fromgoogle.cloud.bigquery.exceptionsimport (
33+
LegacyBigQueryStorageError,
34+
LegacyPyarrowError,
35+
)
3336

3437

3538
_RFC3339_MICROS_NO_ZULU="%Y-%m-%dT%H:%M:%S.%f"
@@ -42,6 +45,7 @@
4245
re.VERBOSE,
4346
)
4447

48+
_MIN_PYARROW_VERSION=packaging.version.Version("3.0.0")
4549
_MIN_BQ_STORAGE_VERSION=packaging.version.Version("2.0.0")
4650
_BQ_STORAGE_OPTIONAL_READ_SESSION_VERSION=packaging.version.Version("2.6.0")
4751

@@ -95,12 +99,74 @@ def verify_version(self):
9599
ifself.installed_version<_MIN_BQ_STORAGE_VERSION:
96100
msg= (
97101
"Dependency google-cloud-bigquery-storage is outdated, please upgrade "
98-
f"it to version >=2.0.0 (version found:{self.installed_version})."
102+
f"it to version >={_MIN_BQ_STORAGE_VERSION} (version found:{self.installed_version})."
99103
)
100104
raiseLegacyBigQueryStorageError(msg)
101105

102106

107+
classPyarrowVersions:
108+
"""Version comparisons for pyarrow package."""
109+
110+
def__init__(self):
111+
self._installed_version=None
112+
113+
@property
114+
definstalled_version(self)->packaging.version.Version:
115+
"""Return the parsed version of pyarrow."""
116+
ifself._installed_versionisNone:
117+
importpyarrow
118+
119+
self._installed_version=packaging.version.parse(
120+
# Use 0.0.0, since it is earlier than any released version.
121+
# Legacy versions also have the same property, but
122+
# creating a LegacyVersion has been deprecated.
123+
# https://github.com/pypa/packaging/issues/321
124+
getattr(pyarrow,"__version__","0.0.0")
125+
)
126+
127+
returnself._installed_version
128+
129+
deftry_import(self,raise_if_error:bool=False)->Any:
130+
"""Verify that a recent enough version of pyarrow extra is
131+
installed.
132+
133+
The function assumes that pyarrow extra is installed, and should thus
134+
be used in places where this assumption holds.
135+
136+
Because `pip` can install an outdated version of this extra despite the
137+
constraints in `setup.py`, the calling code can use this helper to
138+
verify the version compatibility at runtime.
139+
140+
Returns:
141+
The ``pyarrow`` module or ``None``.
142+
143+
Raises:
144+
LegacyPyarrowError:
145+
If the pyarrow package is outdated and ``raise_if_error`` is ``True``.
146+
"""
147+
try:
148+
importpyarrow
149+
exceptImportErrorasexc:# pragma: NO COVER
150+
ifraise_if_error:
151+
raiseLegacyPyarrowError(
152+
f"pyarrow package not found. Install pyarrow version >={_MIN_PYARROW_VERSION}."
153+
)fromexc
154+
returnNone
155+
156+
ifself.installed_version<_MIN_PYARROW_VERSION:
157+
ifraise_if_error:
158+
msg= (
159+
"Dependency pyarrow is outdated, please upgrade "
160+
f"it to version >={_MIN_PYARROW_VERSION} (version found:{self.installed_version})."
161+
)
162+
raiseLegacyPyarrowError(msg)
163+
returnNone
164+
165+
returnpyarrow
166+
167+
103168
BQ_STORAGE_VERSIONS=BQStorageVersions()
169+
PYARROW_VERSIONS=PyarrowVersions()
104170

105171

106172
def_not_null(value,field):

‎google/cloud/bigquery/_pandas_helpers.py‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,6 @@ def _to_wkb(v):
5555

5656
_to_wkb=_to_wkb()
5757

58-
try:
59-
importpyarrow
60-
importpyarrow.parquet
61-
exceptImportError:# pragma: NO COVER
62-
pyarrow=None
63-
6458
try:
6559
fromgoogle.cloud.bigquery_storageimportArrowSerializationOptions
6660
exceptImportError:
@@ -73,12 +67,10 @@ def _to_wkb(v):
7367
fromgoogle.cloud.bigqueryimportschema
7468

7569

76-
_LOGGER=logging.getLogger(__name__)
70+
pyarrow=_helpers.PYARROW_VERSIONS.try_import()
7771

78-
_NO_BQSTORAGE_ERROR= (
79-
"The google-cloud-bigquery-storage library is not installed, "
80-
"please install google-cloud-bigquery-storage to use bqstorage features."
81-
)
72+
73+
_LOGGER=logging.getLogger(__name__)
8274

8375
_PROGRESS_INTERVAL=0.2# Maximum time between download status checks, in seconds.
8476

@@ -548,8 +540,9 @@ def dataframe_to_parquet(dataframe, bq_schema, filepath, parquet_compression="SN
548540
serializing method. Defaults to "SNAPPY".
549541
https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html#pyarrow-parquet-write-table
550542
"""
551-
ifpyarrowisNone:
552-
raiseValueError("pyarrow is required for BigQuery schema conversion.")
543+
pyarrow=_helpers.PYARROW_VERSIONS.try_import(raise_if_error=True)
544+
545+
importpyarrow.parquet
553546

554547
bq_schema=schema._to_schema_fields(bq_schema)
555548
arrow_table=dataframe_to_arrow(dataframe,bq_schema)

‎google/cloud/bigquery/exceptions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,7 @@ class BigQueryError(Exception):
1919

2020
classLegacyBigQueryStorageError(BigQueryError):
2121
"""Raised when too old a version of BigQuery Storage extra is detected at runtime."""
22+
23+
24+
classLegacyPyarrowError(BigQueryError):
25+
"""Raised when too old a version of pyarrow package is detected at runtime."""

‎noxfile.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,16 @@ def unit(session):
9494
default(session)
9595

9696

97-
@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
97+
@nox.session(python=[UNIT_TEST_PYTHON_VERSIONS[0],UNIT_TEST_PYTHON_VERSIONS[-1]])
9898
defunit_noextras(session):
9999
"""Run the unit test suite."""
100+
101+
# Install optional dependencies that are out-of-date.
102+
# https://github.com/googleapis/python-bigquery/issues/933
103+
# There is no pyarrow 1.0.0 package for Python 3.9.
104+
ifsession.python==UNIT_TEST_PYTHON_VERSIONS[0]:
105+
session.install("pyarrow==1.0.0")
106+
100107
default(session,install_extras=False)
101108

102109

‎testing/constraints-3.6.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@ proto-plus==1.10.0
1919
protobuf==3.12.0
2020
pyarrow==3.0.0
2121
requests==2.18.0
22-
shapely==1.6.0
22+
Shapely==1.6.0
2323
six==1.13.0
2424
tqdm==4.7.4

‎tests/unit/job/test_query_pandas.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,6 @@
3131
importgeopandas
3232
except (ImportError,AttributeError):# pragma: NO COVER
3333
geopandas=None
34-
try:
35-
importpyarrow
36-
except (ImportError,AttributeError):# pragma: NO COVER
37-
pyarrow=None
3834
try:
3935
fromgoogle.cloudimportbigquery_storage
4036
except (ImportError,AttributeError):# pragma: NO COVER
@@ -44,11 +40,15 @@
4440
except (ImportError,AttributeError):# pragma: NO COVER
4541
tqdm=None
4642

43+
fromgoogle.cloud.bigqueryimport_helpers
4744
from .helpersimport_make_client
4845
from .helpersimport_make_connection
4946
from .helpersimport_make_job_resource
5047

5148

49+
pyarrow=_helpers.PYARROW_VERSIONS.try_import()
50+
51+
5252
@pytest.fixture
5353
deftable_read_options_kwarg():
5454
# Create a BigQuery Storage table read options object with pyarrow compression

‎tests/unit/test__helpers.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,20 @@
2424
exceptImportError:# pragma: NO COVER
2525
bigquery_storage=None
2626

27+
try:
28+
importpyarrow
29+
exceptImportError:# pragma: NO COVER
30+
pyarrow=None
31+
2732

2833
@unittest.skipIf(bigquery_storageisNone,"Requires `google-cloud-bigquery-storage`")
2934
classTestBQStorageVersions(unittest.TestCase):
35+
deftearDown(self):
36+
fromgoogle.cloud.bigqueryimport_helpers
37+
38+
# Reset any cached versions since it may not match reality.
39+
_helpers.BQ_STORAGE_VERSIONS._installed_version=None
40+
3041
def_object_under_test(self):
3142
fromgoogle.cloud.bigqueryimport_helpers
3243

@@ -89,6 +100,63 @@ def test_is_read_session_optional_false(self):
89100
assertnotversions.is_read_session_optional
90101

91102

103+
@unittest.skipIf(pyarrowisNone,"Requires `pyarrow`")
104+
classTestPyarrowVersions(unittest.TestCase):
105+
deftearDown(self):
106+
fromgoogle.cloud.bigqueryimport_helpers
107+
108+
# Reset any cached versions since it may not match reality.
109+
_helpers.PYARROW_VERSIONS._installed_version=None
110+
111+
def_object_under_test(self):
112+
fromgoogle.cloud.bigqueryimport_helpers
113+
114+
return_helpers.PyarrowVersions()
115+
116+
def_call_try_import(self,**kwargs):
117+
fromgoogle.cloud.bigqueryimport_helpers
118+
119+
_helpers.PYARROW_VERSIONS._installed_version=None
120+
return_helpers.PYARROW_VERSIONS.try_import(**kwargs)
121+
122+
deftest_try_import_raises_no_error_w_recent_pyarrow(self):
123+
fromgoogle.cloud.bigquery.exceptionsimportLegacyPyarrowError
124+
125+
withmock.patch("pyarrow.__version__",new="5.0.0"):
126+
try:
127+
pyarrow=self._call_try_import(raise_if_error=True)
128+
self.assertIsNotNone(pyarrow)
129+
exceptLegacyPyarrowError:# pragma: NO COVER
130+
self.fail("Legacy error raised with a non-legacy dependency version.")
131+
132+
deftest_try_import_returns_none_w_legacy_pyarrow(self):
133+
withmock.patch("pyarrow.__version__",new="2.0.0"):
134+
pyarrow=self._call_try_import()
135+
self.assertIsNone(pyarrow)
136+
137+
deftest_try_import_raises_error_w_legacy_pyarrow(self):
138+
fromgoogle.cloud.bigquery.exceptionsimportLegacyPyarrowError
139+
140+
withmock.patch("pyarrow.__version__",new="2.0.0"):
141+
withself.assertRaises(LegacyPyarrowError):
142+
self._call_try_import(raise_if_error=True)
143+
144+
deftest_installed_version_returns_cached(self):
145+
versions=self._object_under_test()
146+
versions._installed_version=object()
147+
assertversions.installed_versionisversions._installed_version
148+
149+
deftest_installed_version_returns_parsed_version(self):
150+
versions=self._object_under_test()
151+
152+
withmock.patch("pyarrow.__version__",new="1.2.3"):
153+
version=versions.installed_version
154+
155+
assertversion.major==1
156+
assertversion.minor==2
157+
assertversion.micro==3
158+
159+
92160
classTest_not_null(unittest.TestCase):
93161
def_call_fut(self,value,field):
94162
fromgoogle.cloud.bigquery._helpersimport_not_null

‎tests/unit/test__pandas_helpers.py‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,6 @@
2929
importpandas.testing
3030
exceptImportError:# pragma: NO COVER
3131
pandas=None
32-
try:
33-
importpyarrow
34-
importpyarrow.types
35-
exceptImportError:# pragma: NO COVER
36-
# Mock out pyarrow when missing, because methods from pyarrow.types are
37-
# used in test parameterization.
38-
pyarrow=mock.Mock()
3932
try:
4033
importgeopandas
4134
exceptImportError:# pragma: NO COVER
@@ -44,9 +37,19 @@
4437
importpytest
4538

4639
fromgoogleimportapi_core
40+
fromgoogle.cloud.bigqueryimportexceptions
4741
fromgoogle.cloud.bigqueryimport_helpers
4842
fromgoogle.cloud.bigqueryimportschema
4943

44+
45+
pyarrow=_helpers.PYARROW_VERSIONS.try_import()
46+
ifpyarrow:
47+
importpyarrow.types
48+
else:# pragma: NO COVER
49+
# Mock out pyarrow when missing, because methods from pyarrow.types are
50+
# used in test parameterization.
51+
pyarrow=mock.Mock()
52+
5053
try:
5154
fromgoogle.cloudimportbigquery_storage
5255

@@ -1120,15 +1123,19 @@ def test_dataframe_to_arrow_dict_sequence_schema(module_under_test):
11201123

11211124
@pytest.mark.skipif(pandasisNone,reason="Requires `pandas`")
11221125
deftest_dataframe_to_parquet_without_pyarrow(module_under_test,monkeypatch):
1123-
monkeypatch.setattr(module_under_test,"pyarrow",None)
1124-
withpytest.raises(ValueError)asexc_context:
1126+
mock_pyarrow_import=mock.Mock()
1127+
mock_pyarrow_import.side_effect=exceptions.LegacyPyarrowError(
1128+
"pyarrow not installed"
1129+
)
1130+
monkeypatch.setattr(_helpers.PYARROW_VERSIONS,"try_import",mock_pyarrow_import)
1131+
1132+
withpytest.raises(exceptions.LegacyPyarrowError):
11251133
module_under_test.dataframe_to_parquet(pandas.DataFrame(), (),None)
1126-
assert"pyarrow is required"instr(exc_context.value)
11271134

11281135

11291136
@pytest.mark.skipif(pandasisNone,reason="Requires `pandas`")
11301137
@pytest.mark.skipif(isinstance(pyarrow,mock.Mock),reason="Requires `pyarrow`")
1131-
deftest_dataframe_to_parquet_w_extra_fields(module_under_test,monkeypatch):
1138+
deftest_dataframe_to_parquet_w_extra_fields(module_under_test):
11321139
withpytest.raises(ValueError)asexc_context:
11331140
module_under_test.dataframe_to_parquet(
11341141
pandas.DataFrame(), (schema.SchemaField("not_in_df","STRING"),),None

‎tests/unit/test_table.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,18 +45,18 @@
4545
except (ImportError,AttributeError):# pragma: NO COVER
4646
geopandas=None
4747

48-
try:
49-
importpyarrow
50-
importpyarrow.types
51-
exceptImportError:# pragma: NO COVER
52-
pyarrow=None
53-
5448
try:
5549
fromtqdmimporttqdm
5650
except (ImportError,AttributeError):# pragma: NO COVER
5751
tqdm=None
5852

5953
fromgoogle.cloud.bigquery.datasetimportDatasetReference
54+
fromgoogle.cloud.bigqueryimport_helpers
55+
56+
57+
pyarrow=_helpers.PYARROW_VERSIONS.try_import()
58+
ifpyarrow:
59+
importpyarrow.types
6060

6161

6262
def_mock_client():

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp