Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit3994be6

Browse files
author
Praful Makani
authored
docs(samples): add create external table using hivepartitioningoptions (#969)
1 parent08249bc commit3994be6

File tree

2 files changed

+155
-0
lines changed

2 files changed

+155
-0
lines changed
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
packagecom.example.bigquery;
18+
19+
// [START bigquery_set_hivepartitioningoptions]
20+
importcom.google.cloud.bigquery.BigQuery;
21+
importcom.google.cloud.bigquery.BigQueryException;
22+
importcom.google.cloud.bigquery.BigQueryOptions;
23+
importcom.google.cloud.bigquery.ExternalTableDefinition;
24+
importcom.google.cloud.bigquery.FormatOptions;
25+
importcom.google.cloud.bigquery.HivePartitioningOptions;
26+
importcom.google.cloud.bigquery.TableId;
27+
importcom.google.cloud.bigquery.TableInfo;
28+
29+
// Sample to create external table using hive partitioning
30+
publicclassSetHivePartitioningOptions {
31+
32+
publicstaticvoidmain(String[]args) {
33+
// TODO(developer): Replace these variables before running the sample.
34+
StringdatasetName ="MY_DATASET_NAME";
35+
StringtableName ="MY_TABLE_NAME";
36+
StringsourceUri ="gs://cloud-samples-data/bigquery/hive-partitioning-samples/customlayout/*";
37+
StringsourceUriPrefix =
38+
"gs://cloud-samples-data/bigquery/hive-partitioning-samples/customlayout/{pkey:STRING}/";
39+
setHivePartitioningOptions(datasetName,tableName,sourceUriPrefix,sourceUri);
40+
}
41+
42+
publicstaticvoidsetHivePartitioningOptions(
43+
StringdatasetName,StringtableName,StringsourceUriPrefix,StringsourceUri) {
44+
try {
45+
// Initialize client that will be used to send requests. This client only needs to be created
46+
// once, and can be reused for multiple requests.
47+
BigQuerybigquery =BigQueryOptions.getDefaultInstance().getService();
48+
49+
// Configuring partitioning options
50+
HivePartitioningOptionshivePartitioningOptions =
51+
HivePartitioningOptions.newBuilder()
52+
.setMode("CUSTOM")
53+
.setRequirePartitionFilter(true)
54+
.setSourceUriPrefix(sourceUriPrefix)
55+
.build();
56+
57+
TableIdtableId =TableId.of(datasetName,tableName);
58+
ExternalTableDefinitioncustomTable =
59+
ExternalTableDefinition.newBuilder(sourceUri,FormatOptions.parquet())
60+
.setAutodetect(true)
61+
.setHivePartitioningOptions(hivePartitioningOptions)
62+
.build();
63+
bigquery.create(TableInfo.of(tableId,customTable));
64+
System.out.println("External table created using hivepartitioningoptions");
65+
}catch (BigQueryExceptione) {
66+
System.out.println("External table was not created" +e.toString());
67+
}
68+
}
69+
}
70+
// [END bigquery_set_hivepartitioningoptions]
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
packagecom.example.bigquery;
18+
19+
importstaticcom.google.common.truth.Truth.assertThat;
20+
importstaticjunit.framework.TestCase.assertNotNull;
21+
22+
importjava.io.ByteArrayOutputStream;
23+
importjava.io.PrintStream;
24+
importjava.util.UUID;
25+
importjava.util.logging.Level;
26+
importjava.util.logging.Logger;
27+
importorg.junit.After;
28+
importorg.junit.Before;
29+
importorg.junit.BeforeClass;
30+
importorg.junit.Test;
31+
32+
publicclassSetHivePartitioningOptionsIT {
33+
34+
privatestaticfinalStringID =UUID.randomUUID().toString().substring(0,8);
35+
privatefinalLoggerlog =Logger.getLogger(this.getClass().getName());
36+
privateStringtableName;
37+
privateByteArrayOutputStreambout;
38+
privatePrintStreamout;
39+
privatePrintStreamoriginalPrintStream;
40+
41+
privatestaticfinalStringBIGQUERY_DATASET_NAME =requireEnvVar("BIGQUERY_DATASET_NAME");
42+
43+
privatestaticStringrequireEnvVar(StringvarName) {
44+
Stringvalue =System.getenv(varName);
45+
assertNotNull(
46+
"Environment variable " +varName +" is required to perform these tests.",
47+
System.getenv(varName));
48+
returnvalue;
49+
}
50+
51+
@BeforeClass
52+
publicstaticvoidcheckRequirements() {
53+
requireEnvVar("BIGQUERY_DATASET_NAME");
54+
}
55+
56+
@Before
57+
publicvoidsetUp() {
58+
// Create a test table
59+
tableName ="SET_HIVEPARTITIONINGOPTIONS_FROM_GCS_TEST_" +ID;
60+
bout =newByteArrayOutputStream();
61+
out =newPrintStream(bout);
62+
originalPrintStream =System.out;
63+
System.setOut(out);
64+
}
65+
66+
@After
67+
publicvoidtearDown() {
68+
// Clean up
69+
DeleteTable.deleteTable(BIGQUERY_DATASET_NAME,tableName);
70+
// restores print statements in the original method
71+
System.out.flush();
72+
System.setOut(originalPrintStream);
73+
log.log(Level.INFO,bout.toString());
74+
}
75+
76+
@Test
77+
publicvoidtestSetHivePartitioningOptions() {
78+
StringsourceUri ="gs://cloud-samples-data/bigquery/hive-partitioning-samples/customlayout/*";
79+
StringsourceUriPrefix =
80+
"gs://cloud-samples-data/bigquery/hive-partitioning-samples/customlayout/{pkey:STRING}/";
81+
SetHivePartitioningOptions.setHivePartitioningOptions(
82+
BIGQUERY_DATASET_NAME,tableName,sourceUriPrefix,sourceUri);
83+
assertThat(bout.toString()).contains("External table created using hivepartitioningoptions");
84+
}
85+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp