Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd64f5b6

Browse files
authored
docs(samples): add create external table with hive partitioning (#1033)
* docs(samples): add create table hive partitioning sample* refactor
1 parentd9dfc24 commitd64f5b6

File tree

2 files changed

+104
-0
lines changed

2 files changed

+104
-0
lines changed
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
defcreate_table_external_hive_partitioned(table_id:str):
17+
original_table_id=table_id
18+
# [START bigquery_create_table_external_hivepartitioned]
19+
# Demonstrates creating an external table with hive partitioning.
20+
21+
# TODO(developer): Set table_id to the ID of the table to create.
22+
table_id="your-project.your_dataset.your_table_name"
23+
24+
# TODO(developer): Set source uri.
25+
# Example file:
26+
# gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/dt=2020-11-15/file1.parquet
27+
uri="gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
28+
29+
# TODO(developer): Set source uri prefix.
30+
source_uri_prefix= (
31+
"gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
32+
)
33+
34+
# [END bigquery_create_table_external_hivepartitioned]
35+
table_id=original_table_id
36+
# [START bigquery_create_table_external_hivepartitioned]
37+
fromgoogle.cloudimportbigquery
38+
39+
# Construct a BigQuery client object.
40+
client=bigquery.Client()
41+
42+
# Configure the external data source.
43+
external_config=bigquery.ExternalConfig("PARQUET")
44+
external_config.source_uris= [uri]
45+
external_config.autodetect=True
46+
47+
# Configure partitioning options.
48+
hive_partitioning_opts=bigquery.external_config.HivePartitioningOptions()
49+
50+
# The layout of the files in here is compatible with the layout requirements for hive partitioning,
51+
# so we can add an optional Hive partitioning configuration to leverage the object paths for deriving
52+
# partitioning column information.
53+
54+
# For more information on how partitions are extracted, see:
55+
# https://cloud.google.com/bigquery/docs/hive-partitioned-queries-gcs
56+
57+
# We have a "/dt=YYYY-MM-DD/" path component in our example files as documented above.
58+
# Autolayout will expose this as a column named "dt" of type DATE.
59+
hive_partitioning_opts.mode="AUTO"
60+
hive_partitioning_opts.require_partition_filter=True
61+
hive_partitioning_opts.source_uri_prefix=source_uri_prefix
62+
63+
external_config.hive_partitioning=hive_partitioning_opts
64+
65+
table=bigquery.Table(table_id)
66+
table.external_data_configuration=external_config
67+
68+
table=client.create_table(table)# Make an API request.
69+
print(
70+
"Created table {}.{}.{}".format(table.project,table.dataset_id,table.table_id)
71+
)
72+
# [END bigquery_create_table_external_hivepartitioned]
73+
returntable
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
importcreate_table_external_hive_partitioned
16+
17+
18+
deftest_create_table_external_hive_partitioned(capsys,random_table_id):
19+
table=create_table_external_hive_partitioned.create_table_external_hive_partitioned(
20+
random_table_id
21+
)
22+
23+
out,_=capsys.readouterr()
24+
hive_partioning=table.external_data_configuration.hive_partitioning
25+
assert"Created table {}".format(random_table_id)inout
26+
assert (
27+
hive_partioning.source_uri_prefix
28+
=="gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
29+
)
30+
asserthive_partioning.require_partition_filterisTrue
31+
asserthive_partioning.mode=="AUTO"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp