Sep 9, 2021 · Jul 27, 2021 · Jul 27, 2021 · Jul 27, 2021 · Jul 27, 2021 · Jul 28, 2021
diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml
 docker:
  image: gcr.io/repo-automation-bots/owlbot-python:latest
  digest: sha256:50e35228649c47b6ca82aa0be3ff9eb2afce51c82b66c4a03fe4afeb5ff6c0fc
  digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d
diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml
 # https://github.com/googleapis/repo-automation-bots/tree/master/packages/sync-repo-settings
 # Rules formaster branch protection
 # https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings
 # Rules formain branch protection
 branchProtectionRules:
 # Identifies the protection rule pattern. Name of the branch to be protected.
 # Defaults to `master`
 - pattern:master
 # Defaults to `main`
 - pattern:main
  requiresCodeOwnerReviews: true
  requiresStrictStatusChecks: true
  requiredStatusCheckContexts:
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
 python3 -m nox --version

 # If this is a continuous build, send the test log to the FlakyBot.
 # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
 # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
 if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
  cleanup() {
    chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh
    EXIT=$?

    # If this is a periodic build, send the test log to the FlakyBot.
    # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
    # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
    if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
      chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
      $KOKORO_GFILE_DIR/linux_amd64/flakybot
diff --git a/CHANGELOG.md b/CHANGELOG.md
 [1]: https://pypi.org/project/google-cloud-bigquery/#history


 ## [2.26.0](https://www.github.com/googleapis/python-bigquery/compare/v2.25.2...v2.26.0) (2021-09-01)


 ### Features

 * set the X-Server-Timeout header when timeout is set ([#927](https://www.github.com/googleapis/python-bigquery/issues/927)) ([ba02f24](https://www.github.com/googleapis/python-bigquery/commit/ba02f248ba9c449c34859579a4011f4bfd2f4a93))


 ### Bug Fixes

 * guard imports against unsupported pyarrow versions ([#934](https://www.github.com/googleapis/python-bigquery/issues/934)) ([b289076](https://www.github.com/googleapis/python-bigquery/commit/b28907693bbe889becc1b9c8963f0a7e1ee6c35a))

 ### [2.25.2](https://www.github.com/googleapis/python-bigquery/compare/v2.25.1...v2.25.2) (2021-08-31)


 ### Bug Fixes

 * error inserting DataFrame with REPEATED field ([#925](https://www.github.com/googleapis/python-bigquery/issues/925)) ([656d2fa](https://www.github.com/googleapis/python-bigquery/commit/656d2fa6f870573a21235c83463752a2d084caba))
 * underscores weren't allowed in struct field names when passing parameters to the DB API ([#930](https://www.github.com/googleapis/python-bigquery/issues/930)) ([fcb0bc6](https://www.github.com/googleapis/python-bigquery/commit/fcb0bc68c972c2c98bb8542f54e9228308177ecb))


 ### Documentation

 * update docstring for bigquery_create_routine sample ([#883](https://www.github.com/googleapis/python-bigquery/issues/883)) ([#917](https://www.github.com/googleapis/python-bigquery/issues/917)) ([e2d12b7](https://www.github.com/googleapis/python-bigquery/commit/e2d12b795ef2dc51b0ee36f1b3000edb1e64ce05))

 ### [2.25.1](https://www.github.com/googleapis/python-bigquery/compare/v2.25.0...v2.25.1) (2021-08-25)


 ### Bug Fixes

 * populate default `timeout` and retry after client-side timeout ([#896](https://www.github.com/googleapis/python-bigquery/issues/896)) ([b508809](https://www.github.com/googleapis/python-bigquery/commit/b508809c0f887575274309a463e763c56ddd017d))
 * use REST API in cell magic when requested ([#892](https://www.github.com/googleapis/python-bigquery/issues/892)) ([1cb3e55](https://www.github.com/googleapis/python-bigquery/commit/1cb3e55253e824e3a1da5201f6ec09065fb6b627))

 ## [2.25.0](https://www.github.com/googleapis/python-bigquery/compare/v2.24.1...v2.25.0) (2021-08-24)


 ### Features

 * Support using GeoPandas for GEOGRAPHY columns ([#848](https://www.github.com/googleapis/python-bigquery/issues/848)) ([16f65e6](https://www.github.com/googleapis/python-bigquery/commit/16f65e6ae15979217ceea6c6d398c9057a363a13))

 ### [2.24.1](https://www.github.com/googleapis/python-bigquery/compare/v2.24.0...v2.24.1) (2021-08-13)


 ### Bug Fixes

 * remove pytz dependency and require pyarrow>=3.0.0 ([#875](https://www.github.com/googleapis/python-bigquery/issues/875)) ([2cb3563](https://www.github.com/googleapis/python-bigquery/commit/2cb3563ee863edef7eaf5d04d739bcfe7bc6438e))

 ## [2.24.0](https://www.github.com/googleapis/python-bigquery/compare/v2.23.3...v2.24.0) (2021-08-11)


diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
   # Configure remotes such that you can pull changes from the googleapis/python-bigquery
   # repository into your local repository.
   $ git remote add upstream git@github.com:googleapis/python-bigquery.git
   # fetch and merge changes from upstream intomaster
   # fetch and merge changes from upstream intomain
   $ git fetch upstream
   $ git merge upstream/master
   $ git merge upstream/main

 Now your local repo is set up such that you will push changes to your GitHub
 repo, from which you can submit a pull request.
  variables::

   export GOOGLE_CLOUD_TESTING_REMOTE="upstream"
   export GOOGLE_CLOUD_TESTING_BRANCH="master"
   export GOOGLE_CLOUD_TESTING_BRANCH="main"

  By doing this, you are specifying the location of the most up-to-date
  version of ``python-bigquery``. The the suggested remote name ``upstream``
  should point to the official ``googleapis`` checkout and the
  the branch should be themain branch on that remote (``master``).
  version of ``python-bigquery``. The
 remote name ``upstream``should point to the official ``googleapis``
 checkout andthe branch should be thedefault branch on that remote (``main``).

 - This repository contains configuration for the
  `pre-commit <https://pre-commit.com/>`__ tool, which automates checking
 ``README``. Due to the reStructuredText (``rst``) parser used by
 PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst``
 instead of
 ``https://github.com/googleapis/python-bigquery/blob/master/CONTRIBUTING.rst``)
 ``https://github.com/googleapis/python-bigquery/blob/main/CONTRIBUTING.rst``)
 may cause problems creating links or rendering the description.

 .. _description on PyPI: https://pypi.org/project/google-cloud-bigquery

 Supported versions can be found in our ``noxfile.py`` `config`_.

 .. _config: https://github.com/googleapis/python-bigquery/blob/master/noxfile.py
 .. _config: https://github.com/googleapis/python-bigquery/blob/main/noxfile.py


 We also explicitly decided to support Python 3 beginning with version 3.6.
diff --git a/docs/conf.py b/docs/conf.py
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'

 # Themaster toctree document.
 master_doc = "index"
 # Theroot toctree document.
 root_doc = "index"

 # General information about the project.
 project = "google-cloud-bigquery"
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
    (
 master_doc,
 root_doc,
        "google-cloud-bigquery.tex",
        "google-cloud-bigquery Documentation",
        author,
 # (source start file, name, description, authors, manual section).
 man_pages = [
    (
 master_doc,
 root_doc,
        "google-cloud-bigquery",
        "google-cloud-bigquery Documentation",
        [author],
 #  dir menu entry, description, category)
 texinfo_documents = [
    (
 master_doc,
 root_doc,
        "google-cloud-bigquery",
        "google-cloud-bigquery Documentation",
        author,
    "grpc": ("https://grpc.github.io/grpc/python/", None),
    "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None),
    "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None),
    "pandas": ("http://pandas.pydata.org/pandas-docs/dev", None),
    "geopandas": ("https://geopandas.org/", None),
 }


diff --git a/docs/usage/pandas.rst b/docs/usage/pandas.rst
     - Int64
     -

 Retrieve BigQuery GEOGRAPHY data as a GeoPandas GeoDataFrame
 ------------------------------------------------------------

 `GeoPandas <https://geopandas.org/>`_ adds geospatial analytics
 capabilities to Pandas.  To retrieve query results containing
 GEOGRAPHY data as a :class:`geopandas.GeoDataFrame`:

 .. literalinclude:: ../samples/geography/to_geodataframe.py
   :language: python
   :dedent: 4
   :start-after: [START bigquery_query_results_geodataframe]
   :end-before: [END bigquery_query_results_geodataframe]


 Load a Pandas DataFrame to a BigQuery Table
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py
        return self.installed_version >= _BQ_STORAGE_OPTIONAL_READ_SESSION_VERSION


 class PyarrowVersions:
    """Version comparisons for pyarrow package."""

    def __init__(self):
        self._installed_version = None

    @property
    def installed_version(self) -> packaging.version.Version:
        """Return the parsed version of pyarrow."""
        if self._installed_version is None:
            import pyarrow

            self._installed_version = packaging.version.parse(
                # Use 0.0.0, since it is earlier than any released version.
                # Legacy versions also have the same property, but
                # creating a LegacyVersion has been deprecated.
                # https://github.com/pypa/packaging/issues/321
                getattr(pyarrow, "__version__", "0.0.0")
            )

        return self._installed_version


 BQ_STORAGE_VERSIONS = BQStorageVersions()
 PYARROW_VERSIONS = PyarrowVersions()


 def _not_null(value, field):
diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py
    import pandas
 except ImportError:  # pragma: NO COVER
    pandas = None
 else:
    import numpy

 import pyarrow
 import pyarrow.parquet

 try:
    # _BaseGeometry is used to detect shapely objevys in `bq_to_arrow_array`
    from shapely.geometry.base import BaseGeometry as _BaseGeometry
 except ImportError:  # pragma: NO COVER
    # No shapely, use NoneType for _BaseGeometry as a placeholder.
    _BaseGeometry = type(None)
 else:
    if pandas is not None:  # pragma: NO COVER

        def _to_wkb():
            # Create a closure that:
            # - Adds a not-null check. This allows the returned function to
            #   be used directly with apply, unlike `shapely.wkb.dumps`.
            # - Avoid extra work done by `shapely.wkb.dumps` that we don't need.
            # - Caches the WKBWriter (and write method lookup :) )
            # - Avoids adding WKBWriter, lgeos, and notnull to the module namespace.
            from shapely.geos import WKBWriter, lgeos

            write = WKBWriter(lgeos).write
            notnull = pandas.notnull

            def _to_wkb(v):
                return write(v) if notnull(v) else v

            return _to_wkb

        _to_wkb = _to_wkb()

 try:
    from google.cloud.bigquery_storage import ArrowSerializationOptions
 except ImportError:
    "uint8": "INTEGER",
    "uint16": "INTEGER",
    "uint32": "INTEGER",
    "geometry": "GEOGRAPHY",
 }


    return data_type_constructor()


 def bq_to_arrow_field(bq_field):
 def bq_to_arrow_field(bq_field, array_type=None):
    """Return the Arrow field, corresponding to a given BigQuery column.

    Returns:
        None: if the Arrow type cannot be determined.
    """
    arrow_type = bq_to_arrow_data_type(bq_field)
    if arrow_type:
    if arrow_type is not None:
        if array_type is not None:
            arrow_type = array_type  # For GEOGRAPHY, at least initially
        is_nullable = bq_field.mode.upper() == "NULLABLE"
        return pyarrow.field(bq_field.name, arrow_type, nullable=is_nullable)



 def bq_to_arrow_array(series, bq_field):
    arrow_type = bq_to_arrow_data_type(bq_field)
    if bq_field.field_type.upper() == "GEOGRAPHY":
        arrow_type = None
        first = _first_valid(series)
        if first is not None:
            if series.dtype.name == "geometry" or isinstance(first, _BaseGeometry):
                arrow_type = pyarrow.binary()
                # Convert shapey geometry to WKB binary format:
                series = series.apply(_to_wkb)
            elif isinstance(first, bytes):
                arrow_type = pyarrow.binary()
        elif series.dtype.name == "geometry":
            # We have a GeoSeries containing all nulls, convert it to a pandas series
            series = pandas.Series(numpy.array(series))

        if arrow_type is None:
            arrow_type = bq_to_arrow_data_type(bq_field)
    else:
        arrow_type = bq_to_arrow_data_type(bq_field)

    field_type_upper = bq_field.field_type.upper() if bq_field.field_type else ""

    return columns_and_indexes


 def _first_valid(series):
    first_valid_index = series.first_valid_index()
    if first_valid_index is not None:
        return series.at[first_valid_index]


 def dataframe_to_bq_schema(dataframe, bq_schema):
    """Convert a pandas DataFrame schema to a BigQuery schema.

        # Otherwise, try to automatically determine the type based on the
        # pandas dtype.
        bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name)
        if bq_type is None:
            sample_data = _first_valid(dataframe[column])
            if (
                isinstance(sample_data, _BaseGeometry)
                and sample_data is not None  # Paranoia
            ):
                bq_type = "GEOGRAPHY"
        bq_field = schema.SchemaField(column, bq_type)
        bq_schema_out.append(bq_field)

    arrow_names = []
    arrow_fields = []
    for bq_field in bq_schema:
        arrow_fields.append(bq_to_arrow_field(bq_field))
        arrow_names.append(bq_field.name)
        arrow_arrays.append(
            bq_to_arrow_array(get_column_or_index(dataframe, bq_field.name), bq_field)
        )
        arrow_fields.append(bq_to_arrow_field(bq_field, arrow_arrays[-1].type))

    if all((field is not None for field in arrow_fields)):
        return pyarrow.Table.from_arrays(
        output = {}
        for column, value in zip(dataframe.columns, row):
            # Omit NaN values.
            if pandas.isna(value):
            is_nan = pandas.isna(value)

            # isna() can also return an array-like of bools, but the latter's boolean
            # value is ambiguous, hence an extra check. An array-like value is *not*
            # considered a NaN, however.
            if isinstance(is_nan, bool) and is_nan:
                continue
            output[column] = value

        yield output
Original file line number	Diff line number	Diff line change
		@@ -1,3 +1,3 @@
		docker:
		image: gcr.io/repo-automation-bots/owlbot-python:latest
		digest: sha256:50e35228649c47b6ca82aa0be3ff9eb2afce51c82b66c4a03fe4afeb5ff6c0fc
		digest: sha256:0ffe3bdd6c7159692df5f7744da74e5ef19966288a6bf76023e8e04e0c424d7d
Original file line number	Diff line number	Diff line change
		@@ -1,9 +1,9 @@
		# https://github.com/googleapis/repo-automation-bots/tree/master/packages/sync-repo-settings
		# Rules formaster branch protection
		# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings
		# Rules formain branch protection
		branchProtectionRules:
		# Identifies the protection rule pattern. Name of the branch to be protected.
		# Defaults to `master`
		- pattern:master
		# Defaults to `main`
		- pattern:main
		requiresCodeOwnerReviews: true
		requiresStrictStatusChecks: true
		requiredStatusCheckContexts:
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -41,7 +41,7 @@ python3 -m pip install --upgrade --quiet nox
		python3 -m nox --version

		# If this is a continuous build, send the test log to the FlakyBot.
		# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
		# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
		if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = "continuous" ]]; then
		cleanup() {
		chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -80,7 +80,7 @@ for file in samples/**/requirements.txt; do
		EXIT=$?

		# If this is a periodic build, send the test log to the FlakyBot.
		# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
		# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
		if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = "periodic" ]]; then
		chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
		$KOKORO_GFILE_DIR/linux_amd64/flakybot
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -5,6 +5,53 @@
		[1]: https://pypi.org/project/google-cloud-bigquery/#history


		## [2.26.0](https://www.github.com/googleapis/python-bigquery/compare/v2.25.2...v2.26.0) (2021-09-01)


		### Features

		* set the X-Server-Timeout header when timeout is set ([#927](https://www.github.com/googleapis/python-bigquery/issues/927)) ([ba02f24](https://www.github.com/googleapis/python-bigquery/commit/ba02f248ba9c449c34859579a4011f4bfd2f4a93))


		### Bug Fixes

		* guard imports against unsupported pyarrow versions ([#934](https://www.github.com/googleapis/python-bigquery/issues/934)) ([b289076](https://www.github.com/googleapis/python-bigquery/commit/b28907693bbe889becc1b9c8963f0a7e1ee6c35a))

		### [2.25.2](https://www.github.com/googleapis/python-bigquery/compare/v2.25.1...v2.25.2) (2021-08-31)


		### Bug Fixes

		* error inserting DataFrame with REPEATED field ([#925](https://www.github.com/googleapis/python-bigquery/issues/925)) ([656d2fa](https://www.github.com/googleapis/python-bigquery/commit/656d2fa6f870573a21235c83463752a2d084caba))
		* underscores weren't allowed in struct field names when passing parameters to the DB API ([#930](https://www.github.com/googleapis/python-bigquery/issues/930)) ([fcb0bc6](https://www.github.com/googleapis/python-bigquery/commit/fcb0bc68c972c2c98bb8542f54e9228308177ecb))


		### Documentation

		* update docstring for bigquery_create_routine sample ([#883](https://www.github.com/googleapis/python-bigquery/issues/883)) ([#917](https://www.github.com/googleapis/python-bigquery/issues/917)) ([e2d12b7](https://www.github.com/googleapis/python-bigquery/commit/e2d12b795ef2dc51b0ee36f1b3000edb1e64ce05))

		### [2.25.1](https://www.github.com/googleapis/python-bigquery/compare/v2.25.0...v2.25.1) (2021-08-25)


		### Bug Fixes

		* populate default `timeout` and retry after client-side timeout ([#896](https://www.github.com/googleapis/python-bigquery/issues/896)) ([b508809](https://www.github.com/googleapis/python-bigquery/commit/b508809c0f887575274309a463e763c56ddd017d))
		* use REST API in cell magic when requested ([#892](https://www.github.com/googleapis/python-bigquery/issues/892)) ([1cb3e55](https://www.github.com/googleapis/python-bigquery/commit/1cb3e55253e824e3a1da5201f6ec09065fb6b627))

		## [2.25.0](https://www.github.com/googleapis/python-bigquery/compare/v2.24.1...v2.25.0) (2021-08-24)


		### Features

		* Support using GeoPandas for GEOGRAPHY columns ([#848](https://www.github.com/googleapis/python-bigquery/issues/848)) ([16f65e6](https://www.github.com/googleapis/python-bigquery/commit/16f65e6ae15979217ceea6c6d398c9057a363a13))

		### [2.24.1](https://www.github.com/googleapis/python-bigquery/compare/v2.24.0...v2.24.1) (2021-08-13)


		### Bug Fixes

		* remove pytz dependency and require pyarrow>=3.0.0 ([#875](https://www.github.com/googleapis/python-bigquery/issues/875)) ([2cb3563](https://www.github.com/googleapis/python-bigquery/commit/2cb3563ee863edef7eaf5d04d739bcfe7bc6438e))

		## [2.24.0](https://www.github.com/googleapis/python-bigquery/compare/v2.23.3...v2.24.0) (2021-08-11)


Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -50,9 +50,9 @@ You'll have to create a development environment using a Git checkout:
		# Configure remotes such that you can pull changes from the googleapis/python-bigquery
		# repository into your local repository.
		$ git remote add upstream git@github.com:googleapis/python-bigquery.git
		# fetch and merge changes from upstream intomaster
		# fetch and merge changes from upstream intomain
		$ git fetch upstream
		$ git merge upstream/master
		$ git merge upstream/main

		Now your local repo is set up such that you will push changes to your GitHub
		repo, from which you can submit a pull request.
Expand DownExpand Up		@@ -110,12 +110,12 @@ Coding Style
		variables::

		export GOOGLE_CLOUD_TESTING_REMOTE="upstream"
		export GOOGLE_CLOUD_TESTING_BRANCH="master"
		export GOOGLE_CLOUD_TESTING_BRANCH="main"

		By doing this, you are specifying the location of the most up-to-date
		version of ``python-bigquery``. The the suggested remote name ``upstream``
		should point to the official ``googleapis`` checkout and the
		the branch should be themain branch on that remote (``master``).
		version of ``python-bigquery``. The
		remote name ``upstream``should point to the official ``googleapis``
		checkout andthe branch should be thedefault branch on that remote (``main``).

		- This repository contains configuration for the
		`pre-commit <https://pre-commit.com/>`__ tool, which automates checking
Expand DownExpand Up		@@ -209,7 +209,7 @@ The `description on PyPI`_ for the project comes directly from the
		``README``. Due to the reStructuredText (``rst``) parser used by
		PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst``
		instead of
		``https://github.com/googleapis/python-bigquery/blob/master/CONTRIBUTING.rst``)
		``https://github.com/googleapis/python-bigquery/blob/main/CONTRIBUTING.rst``)
		may cause problems creating links or rendering the description.

		.. _description on PyPI: https://pypi.org/project/google-cloud-bigquery
Expand All		@@ -234,7 +234,7 @@ We support:

		Supported versions can be found in our ``noxfile.py`` `config`_.

		.. _config: https://github.com/googleapis/python-bigquery/blob/master/noxfile.py
		.. _config: https://github.com/googleapis/python-bigquery/blob/main/noxfile.py


		We also explicitly decided to support Python 3 beginning with version 3.6.
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -76,8 +76,8 @@
		# The encoding of source files.
		# source_encoding = 'utf-8-sig'

		# Themaster toctree document.
		master_doc = "index"
		# Theroot toctree document.
		root_doc = "index"

		# General information about the project.
		project = "google-cloud-bigquery"
Expand DownExpand Up		@@ -281,7 +281,7 @@
		# author, documentclass [howto, manual, or own class]).
		latex_documents = [
		(
		master_doc,
		root_doc,
		"google-cloud-bigquery.tex",
		"google-cloud-bigquery Documentation",
		author,
Expand DownExpand Up		@@ -316,7 +316,7 @@
		# (source start file, name, description, authors, manual section).
		man_pages = [
		(
		master_doc,
		root_doc,
		"google-cloud-bigquery",
		"google-cloud-bigquery Documentation",
		[author],
Expand All		@@ -335,7 +335,7 @@
		# dir menu entry, description, category)
		texinfo_documents = [
		(
		master_doc,
		root_doc,
		"google-cloud-bigquery",
		"google-cloud-bigquery Documentation",
		author,
Expand DownExpand Up		@@ -366,6 +366,8 @@
		"grpc": ("https://grpc.github.io/grpc/python/", None),
		"proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None),
		"protobuf": ("https://googleapis.dev/python/protobuf/latest/", None),
		"pandas": ("http://pandas.pydata.org/pandas-docs/dev", None),
		"geopandas": ("https://geopandas.org/", None),
		}


Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -58,6 +58,20 @@ The following data types are used when creating a pandas DataFrame.
		- Int64
		-

		Retrieve BigQuery GEOGRAPHY data as a GeoPandas GeoDataFrame
		------------------------------------------------------------

		`GeoPandas <https://geopandas.org/>`_ adds geospatial analytics
		capabilities to Pandas. To retrieve query results containing
		GEOGRAPHY data as a :class:`geopandas.GeoDataFrame`:

		.. literalinclude:: ../samples/geography/to_geodataframe.py
		:language: python
		:dedent: 4
		:start-after: [START bigquery_query_results_geodataframe]
		:end-before: [END bigquery_query_results_geodataframe]


		Load a Pandas DataFrame to a BigQuery Table
		^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -74,7 +74,31 @@ def is_read_session_optional(self) -> bool:
		return self.installed_version >= _BQ_STORAGE_OPTIONAL_READ_SESSION_VERSION


		class PyarrowVersions:
plamut marked this conversation as resolved. Show resolvedHide resolved
		"""Version comparisons for pyarrow package."""

		def __init__(self):
		self._installed_version = None

		@property
		def installed_version(self) -> packaging.version.Version:
		"""Return the parsed version of pyarrow."""
		if self._installed_version is None:
		import pyarrow

		self._installed_version = packaging.version.parse(
		# Use 0.0.0, since it is earlier than any released version.
		# Legacy versions also have the same property, but
		# creating a LegacyVersion has been deprecated.
		# https://github.com/pypa/packaging/issues/321
		getattr(pyarrow, "__version__", "0.0.0")
		)

		return self._installed_version


		BQ_STORAGE_VERSIONS = BQStorageVersions()
		PYARROW_VERSIONS = PyarrowVersions()


		def _not_null(value, field):
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -25,10 +25,40 @@
		import pandas
		except ImportError: # pragma: NO COVER
		pandas = None
		else:
		import numpy

		import pyarrow
		import pyarrow.parquet

		try:
		# _BaseGeometry is used to detect shapely objevys in `bq_to_arrow_array`
		from shapely.geometry.base import BaseGeometry as _BaseGeometry
		except ImportError: # pragma: NO COVER
		# No shapely, use NoneType for _BaseGeometry as a placeholder.
		_BaseGeometry = type(None)
		else:
		if pandas is not None: # pragma: NO COVER

		def _to_wkb():
		# Create a closure that:
		# - Adds a not-null check. This allows the returned function to
		# be used directly with apply, unlike `shapely.wkb.dumps`.
		# - Avoid extra work done by `shapely.wkb.dumps` that we don't need.
		# - Caches the WKBWriter (and write method lookup :) )
		# - Avoids adding WKBWriter, lgeos, and notnull to the module namespace.
		from shapely.geos import WKBWriter, lgeos

		write = WKBWriter(lgeos).write
		notnull = pandas.notnull

		def _to_wkb(v):
		return write(v) if notnull(v) else v

		return _to_wkb

		_to_wkb = _to_wkb()

		try:
		from google.cloud.bigquery_storage import ArrowSerializationOptions
		except ImportError:
Expand DownExpand Up		@@ -71,6 +101,7 @@
		"uint8": "INTEGER",
		"uint16": "INTEGER",
		"uint32": "INTEGER",
		"geometry": "GEOGRAPHY",
		}


Expand DownExpand Up		@@ -191,14 +222,16 @@ def bq_to_arrow_data_type(field):
		return data_type_constructor()


		def bq_to_arrow_field(bq_field):
		def bq_to_arrow_field(bq_field, array_type=None):
		"""Return the Arrow field, corresponding to a given BigQuery column.

		Returns:
		None: if the Arrow type cannot be determined.
		"""
		arrow_type = bq_to_arrow_data_type(bq_field)
		if arrow_type:
		if arrow_type is not None:
		if array_type is not None:
		arrow_type = array_type # For GEOGRAPHY, at least initially
		is_nullable = bq_field.mode.upper() == "NULLABLE"
		return pyarrow.field(bq_field.name, arrow_type, nullable=is_nullable)

Expand DownExpand Up		@@ -245,7 +278,24 @@ def bq_schema_to_nullsafe_pandas_dtypes(


		def bq_to_arrow_array(series, bq_field):
		arrow_type = bq_to_arrow_data_type(bq_field)
		if bq_field.field_type.upper() == "GEOGRAPHY":
		arrow_type = None
		first = _first_valid(series)
		if first is not None:
		if series.dtype.name == "geometry" or isinstance(first, _BaseGeometry):
		arrow_type = pyarrow.binary()
		# Convert shapey geometry to WKB binary format:
		series = series.apply(_to_wkb)
		elif isinstance(first, bytes):
		arrow_type = pyarrow.binary()
		elif series.dtype.name == "geometry":
		# We have a GeoSeries containing all nulls, convert it to a pandas series
		series = pandas.Series(numpy.array(series))

		if arrow_type is None:
		arrow_type = bq_to_arrow_data_type(bq_field)
		else:
		arrow_type = bq_to_arrow_data_type(bq_field)

		field_type_upper = bq_field.field_type.upper() if bq_field.field_type else ""

Expand DownExpand Up		@@ -299,6 +349,12 @@ def list_columns_and_indexes(dataframe):
		return columns_and_indexes


		def _first_valid(series):
		first_valid_index = series.first_valid_index()
		if first_valid_index is not None:
		return series.at[first_valid_index]


		def dataframe_to_bq_schema(dataframe, bq_schema):
		"""Convert a pandas DataFrame schema to a BigQuery schema.

Expand DownExpand Up		@@ -339,6 +395,13 @@ def dataframe_to_bq_schema(dataframe, bq_schema):
		# Otherwise, try to automatically determine the type based on the
		# pandas dtype.
		bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name)
		if bq_type is None:
		sample_data = _first_valid(dataframe[column])
		if (
		isinstance(sample_data, _BaseGeometry)
		and sample_data is not None # Paranoia
		):
		bq_type = "GEOGRAPHY"
		bq_field = schema.SchemaField(column, bq_type)
		bq_schema_out.append(bq_field)

Expand DownExpand Up		@@ -463,11 +526,11 @@ def dataframe_to_arrow(dataframe, bq_schema):
		arrow_names = []
		arrow_fields = []
		for bq_field in bq_schema:
		arrow_fields.append(bq_to_arrow_field(bq_field))
		arrow_names.append(bq_field.name)
		arrow_arrays.append(
		bq_to_arrow_array(get_column_or_index(dataframe, bq_field.name), bq_field)
		)
		arrow_fields.append(bq_to_arrow_field(bq_field, arrow_arrays[-1].type))

		if all((field is not None for field in arrow_fields)):
		return pyarrow.Table.from_arrays(
Expand DownExpand Up		@@ -791,7 +854,13 @@ def dataframe_to_json_generator(dataframe):
		output = {}
		for column, value in zip(dataframe.columns, row):
		# Omit NaN values.
		if pandas.isna(value):
		is_nan = pandas.isna(value)

		# isna() can also return an array-like of bools, but the latter's boolean
		# value is ambiguous, hence an extra check. An array-like value is not
		# considered a NaN, however.
		if isinstance(is_nan, bool) and is_nan:
		continue
		output[column] = value

		yield output