Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit42c083a

Browse files
feat: add fast query path support when empty jobId object is passed (#2349)
* feat: add fast query path support with job location* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* chore: fix NPE* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* chore: respect user specified location* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* chore: fix test* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* chore add IT tests* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* chore: allow users to specify project ID with JobID* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* chore: fix conditions* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.mdCo-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parente4b6035 commit42c083a

File tree

4 files changed

+122
-4
lines changed

4 files changed

+122
-4
lines changed

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1300,7 +1300,7 @@ public TableResult query(QueryJobConfiguration configuration, JobOption... optio
13001300
// If all parameters passed in configuration are supported by the query() method on the backend,
13011301
// put on fast path
13021302
QueryRequestInforequestInfo =newQueryRequestInfo(configuration);
1303-
if (requestInfo.isFastQuerySupported()) {
1303+
if (requestInfo.isFastQuerySupported(null)) {
13041304
StringprojectId =getOptions().getProjectId();
13051305
QueryRequestcontent =requestInfo.toPb();
13061306
returnqueryRpc(projectId,content,options);
@@ -1385,6 +1385,27 @@ public com.google.api.services.bigquery.model.QueryResponse call() {
13851385
publicTableResultquery(QueryJobConfigurationconfiguration,JobIdjobId,JobOption...options)
13861386
throwsInterruptedException,JobException {
13871387
Job.checkNotDryRun(configuration,"query");
1388+
// If all parameters passed in configuration are supported by the query() method on the backend,
1389+
// put on fast path
1390+
QueryRequestInforequestInfo =newQueryRequestInfo(configuration);
1391+
if (requestInfo.isFastQuerySupported(jobId)) {
1392+
// Be careful when setting the projectID in JobId, if a projectID is specified in the JobId,
1393+
// the job created by the query method will use that project. This may cause the query to
1394+
// fail with "Access denied" if the project do not have enough permissions to run the job.
1395+
1396+
StringprojectId =
1397+
jobId.getProject() !=null ?jobId.getProject() :getOptions().getProjectId();
1398+
QueryRequestcontent =requestInfo.toPb();
1399+
// Be careful when setting the location in JobId, if a location is specified in the JobId,
1400+
// the job created by the query method will be in that location, even if the table to be
1401+
// queried is in a different location. This may cause the query to fail with
1402+
// "BigQueryException: Not found"
1403+
if (jobId.getLocation() !=null) {
1404+
content.setLocation(jobId.getLocation());
1405+
}
1406+
1407+
returnqueryRpc(projectId,content,options);
1408+
}
13881409
returncreate(JobInfo.of(jobId,configuration),options).getQueryResults();
13891410
}
13901411

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryRequestInfo.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,15 @@ final class QueryRequestInfo {
5757
this.useQueryCache =config.useQueryCache();
5858
}
5959

60-
booleanisFastQuerySupported() {
60+
booleanisFastQuerySupported(JobIdjobId) {
61+
// Fast query path is not possible if job is specified in the JobID object
62+
// Respect Job field value in JobId specified by user.
63+
// Specifying it will force the query to take the slower path.
64+
if (jobId !=null) {
65+
if (jobId.getJob() !=null) {
66+
returnfalse;
67+
}
68+
}
6169
returnconfig.getClustering() ==null
6270
&&config.getCreateDisposition() ==null
6371
&&config.getDestinationEncryptionConfiguration() ==null

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryRequestInfoTest.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,8 +150,12 @@ public class QueryRequestInfoTest {
150150

151151
@Test
152152
publicvoidtestIsFastQuerySupported() {
153-
assertEquals(false,REQUEST_INFO.isFastQuerySupported());
154-
assertEquals(true,REQUEST_INFO_SUPPORTED.isFastQuerySupported());
153+
JobIdjobIdSupported =JobId.newBuilder().build();
154+
JobIdjobIdNotSupported =JobId.newBuilder().setJob("random-job-id").build();
155+
assertEquals(false,REQUEST_INFO.isFastQuerySupported(jobIdSupported));
156+
assertEquals(true,REQUEST_INFO_SUPPORTED.isFastQuerySupported(jobIdSupported));
157+
assertEquals(false,REQUEST_INFO.isFastQuerySupported(jobIdNotSupported));
158+
assertEquals(false,REQUEST_INFO_SUPPORTED.isFastQuerySupported(jobIdNotSupported));
155159
}
156160

157161
@Test

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ public class ITBigQueryTest {
180180
privatestaticfinalLongEXPIRATION_MS =86400000L;
181181
privatestaticfinalLoggerLOG =Logger.getLogger(ITBigQueryTest.class.getName());
182182
privatestaticfinalStringDATASET =RemoteBigQueryHelper.generateDatasetName();
183+
privatestaticfinalStringUK_DATASET =RemoteBigQueryHelper.generateDatasetName();
183184
privatestaticfinalStringDESCRIPTION ="Test dataset";
184185
privatestaticfinalStringOTHER_DATASET =RemoteBigQueryHelper.generateDatasetName();
185186
privatestaticfinalStringMODEL_DATASET =RemoteBigQueryHelper.generateDatasetName();
@@ -533,6 +534,8 @@ public class ITBigQueryTest {
533534
privatestaticfinalTableIdTABLE_ID =TableId.of(DATASET,"testing_table");
534535
privatestaticfinalTableIdTABLE_ID_DDL =TableId.of(DATASET,"ddl_testing_table");
535536
privatestaticfinalTableIdTABLE_ID_FASTQUERY =TableId.of(DATASET,"fastquery_testing_table");
537+
privatestaticfinalTableIdTABLE_ID_FASTQUERY_UK =
538+
TableId.of(UK_DATASET,"fastquery_testing_table");
536539
privatestaticfinalTableIdTABLE_ID_LARGE =TableId.of(DATASET,"large_data_testing_table");
537540
privatestaticfinalTableIdTABLE_ID_FASTQUERY_BQ_RESULTSET =
538541
TableId.of(DATASET,"fastquery_testing_bq_resultset");
@@ -717,6 +720,7 @@ public static void beforeClass() throws InterruptedException, IOException {
717720
DatasetInfoinfo3 =
718721
DatasetInfo.newBuilder(ROUTINE_DATASET).setDescription("java routine lifecycle").build();
719722
bigquery.create(info3);
723+
720724
LoadJobConfigurationconfiguration =
721725
LoadJobConfiguration.newBuilder(
722726
TABLE_ID,"gs://" +BUCKET +"/" +JSON_LOAD_FILE,FormatOptions.json())
@@ -781,6 +785,7 @@ public static void beforeClass() throws InterruptedException, IOException {
781785
publicstaticvoidafterClass()throwsExecutionException,InterruptedException {
782786
if (bigquery !=null) {
783787
RemoteBigQueryHelper.forceDelete(bigquery,DATASET);
788+
RemoteBigQueryHelper.forceDelete(bigquery,UK_DATASET);
784789
RemoteBigQueryHelper.forceDelete(bigquery,MODEL_DATASET);
785790
RemoteBigQueryHelper.forceDelete(bigquery,ROUTINE_DATASET);
786791
}
@@ -3284,6 +3289,86 @@ public void testFastSQLQuery() throws InterruptedException {
32843289
}
32853290
}
32863291

3292+
@Test
3293+
publicvoidtestProjectIDFastSQLQueryWithJobId()throwsInterruptedException {
3294+
Stringrandom_project_id ="RANDOM_PROJECT_" +UUID.randomUUID().toString().replace('-','_');
3295+
System.out.println(random_project_id);
3296+
Stringquery =
3297+
"SELECT TimestampField, StringField, BooleanField FROM " +TABLE_ID_FASTQUERY.getTable();
3298+
// With incorrect projectID in jobid
3299+
// The job will be created with the specified(incorrect) projectID
3300+
// hence failing the operation
3301+
JobIdjobIdWithProjectId =JobId.newBuilder().setProject(random_project_id).build();
3302+
QueryJobConfigurationconfigSelect =
3303+
QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(DATASET)).build();
3304+
try {
3305+
bigquery.query(configSelect,jobIdWithProjectId);
3306+
}catch (Exceptionexception) {
3307+
// error message for non-existent project
3308+
assertTrue(exception.getMessage().contains("Cannot parse as CloudRegion"));
3309+
assertEquals(BigQueryException.class,exception.getClass());
3310+
}
3311+
}
3312+
3313+
@Test
3314+
publicvoidtestLocationFastSQLQueryWithJobId()throwsInterruptedException {
3315+
DatasetInfoinfoUK =
3316+
DatasetInfo.newBuilder(UK_DATASET)
3317+
.setDescription(DESCRIPTION)
3318+
.setLocation("europe-west1")
3319+
.setLabels(LABELS)
3320+
.build();
3321+
bigquery.create(infoUK);
3322+
3323+
TableDefinitiontableDefinition =StandardTableDefinition.of(SIMPLE_SCHEMA);
3324+
TableInfotableInfo =TableInfo.newBuilder(TABLE_ID_FASTQUERY_UK,tableDefinition).build();
3325+
bigquery.create(tableInfo);
3326+
3327+
Stringinsert =
3328+
"INSERT " +UK_DATASET +"." +TABLE_ID_FASTQUERY_UK.getTable() +" VALUES('Anna');";
3329+
3330+
QueryJobConfigurationconfig =
3331+
QueryJobConfiguration.newBuilder(insert)
3332+
.setDefaultDataset(DatasetId.of(UK_DATASET))
3333+
.build();
3334+
TableResultresult =bigquery.query(config);
3335+
assertEquals(SIMPLE_SCHEMA,result.getSchema());
3336+
assertEquals(1,result.getTotalRows());
3337+
assertNull(result.getNextPage());
3338+
assertNull(result.getNextPageToken());
3339+
assertFalse(result.hasNextPage());
3340+
// Verify correctness of table content
3341+
for (FieldValueListrow :result.getValues()) {
3342+
FieldValuestringCell =row.get(0);
3343+
assertEquals(stringCell,row.get("StringField"));
3344+
assertEquals("Anna",stringCell.getStringValue());
3345+
}
3346+
// With incorrect location in jobid
3347+
// The job will be created with the specified(incorrect) location
3348+
// hence failing the operation
3349+
Stringquery ="SELECT StringField FROM " +TABLE_ID_FASTQUERY_UK.getTable();
3350+
JobIdjobIdWithLocation =JobId.newBuilder().setLocation("us-west1").build();
3351+
QueryJobConfigurationconfigSelect =
3352+
QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(UK_DATASET)).build();
3353+
try {
3354+
bigquery.query(configSelect,jobIdWithLocation);
3355+
}catch (BigQueryExceptionexception) {
3356+
assertTrue(exception.getMessage().contains("Not found"));
3357+
assertEquals(BigQueryException.class,exception.getClass());
3358+
}
3359+
3360+
// Without location in jobID, the query job defaults to the location of the dataset
3361+
JobIdjobIdNoLocation =JobId.newBuilder().build();
3362+
QueryJobConfigurationconfigNoLocation =
3363+
QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(UK_DATASET)).build();
3364+
TableResultresultNoLocation =bigquery.query(configNoLocation,jobIdNoLocation);
3365+
for (FieldValueListrow :resultNoLocation.getValues()) {
3366+
FieldValuestringCell =row.get(0);
3367+
assertEquals(stringCell,row.get("StringField"));
3368+
assertEquals("Anna",stringCell.getStringValue());
3369+
}
3370+
}
3371+
32873372
/* TODO(prasmish): replicate the entire test case for executeSelect */
32883373
@Test
32893374
publicvoidtestFastSQLQueryMultiPage()throwsInterruptedException {

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp