Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit506f781

Browse files
feat: reconfigure tqdm progress bar in %%bigquery magic (#1355)
* feat: add bigquery job id to tqdm progress bar descriptionChange-Id: I2add62e3cdd5f25f88ace2d08f212796918158b6* write to sys.stdout instead of sys.stderrChange-Id: I6c4001608af1bd8c305c53c6089d64f99605bd8c* configure progress barChange-Id: I5788448d580b53898e75fba68ff5d5a9d12e33d6* tqdm.notebookChange-Id: I87e45085b7535083327a5fe2e51dba4b6411db00* 🦉 Updates from OwlBot post-processorSeehttps://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md* reinclude ipywidgetsChange-Id: Ibe0fc01db05fcfaacdbe0c074b841ead3a39afc9* reinclude ipywidgetsChange-Id: I56f8f98853e83ead0e0ca743c03407a521370233* change test assertions to tqdm_notebookChange-Id: I2d55e529142ad0024ef4a98c2f15d10a73535380* change test assertions in test_magicsChange-Id: I7961ff1c5e9c54930d077e67ef9e01d79e351c5f* remove ipywidgetsChange-Id: I183e277fc7be8797c85d6802f4f8c3947871d4cc* update assertions in testChange-Id: I3b4a1b9460227ca49bf344362efbcc2c895d804d* update method args in query.py and table.pyChange-Id: I9a2bf2b54579668ff36ed992e599f4c7fabe918c* string formatting* fix typo* fix incorrect import structure for tqdm notebook* change default decorator back to tqdm* modify system test* add ipywidgets package for tqdm.notebook feature, set tqdm.notebook as default decorator for bq magic* change test assertion in test_query_pandas* revert test changes* reformat import statement* reformat import statement* remove timeouterror side effect* add tqdm mock patch* Revert "reformat import statement"This reverts commit4114221.* Revert "add tqdm mock patch"This reverts commitef809a0.* add timeout side effect* fix assertion* fix import* change mock patch to tqdm* move assertion* move assertions* add timeout side effect* adjust import statement, mock.patch tqdm* create fixture* revert import change* add import from helper* fix linting* remove unused imort* set ipywidgets version to 7.7.1* set ipywidgets version to 7.7.1* set ipywidgets version to 7.7.1* bump sphinx version* bump sphinx versionCo-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent4fce1d9 commit506f781

File tree

12 files changed

+87
-72
lines changed

12 files changed

+87
-72
lines changed

‎google/cloud/bigquery/_tqdm_helpers.py‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,15 @@
1515
"""Shared helper functions for tqdm progress bar."""
1616

1717
importconcurrent.futures
18+
importsys
1819
importtime
1920
importtyping
2021
fromtypingimportOptional
2122
importwarnings
2223

2324
try:
2425
importtqdm# type: ignore
26+
importtqdm.notebookasnotebook# type: ignore
2527

2628
exceptImportError:# pragma: NO COVER
2729
tqdm=None
@@ -47,9 +49,22 @@ def get_progress_bar(progress_bar_type, description, total, unit):
4749

4850
try:
4951
ifprogress_bar_type=="tqdm":
50-
returntqdm.tqdm(desc=description,total=total,unit=unit)
52+
returntqdm.tqdm(
53+
bar_format="{l_bar}{bar}|",
54+
colour="green",
55+
desc=description,
56+
file=sys.stdout,
57+
total=total,
58+
unit=unit,
59+
)
5160
elifprogress_bar_type=="tqdm_notebook":
52-
returntqdm.notebook.tqdm(desc=description,total=total,unit=unit)
61+
returnnotebook.tqdm(
62+
bar_format="{l_bar}{bar}|",
63+
desc=description,
64+
file=sys.stdout,
65+
total=total,
66+
unit=unit,
67+
)
5368
elifprogress_bar_type=="tqdm_gui":
5469
returntqdm.tqdm_gui(desc=description,total=total,unit=unit)
5570
except (KeyError,TypeError):
@@ -80,7 +95,7 @@ def wait_for_query(
8095
"""
8196
default_total=1
8297
current_stage=None
83-
start_time=time.time()
98+
start_time=time.perf_counter()
8499

85100
progress_bar=get_progress_bar(
86101
progress_bar_type,"Query is running",default_total,"query"
@@ -95,19 +110,15 @@ def wait_for_query(
95110
current_stage=query_job.query_plan[i]
96111
progress_bar.total=len(query_job.query_plan)
97112
progress_bar.set_description(
98-
"Query executing stage {} and status {} : {:0.2f}s".format(
99-
current_stage.name,
100-
current_stage.status,
101-
time.time()-start_time,
102-
),
113+
f"Query executing stage{current_stage.name} and status{current_stage.status} :{time.perf_counter()-start_time:.2f}s"
103114
)
104115
try:
105116
query_result=query_job.result(
106117
timeout=_PROGRESS_BAR_UPDATE_INTERVAL,max_results=max_results
107118
)
108119
progress_bar.update(default_total)
109120
progress_bar.set_description(
110-
"Query complete after {:0.2f}s".format(time.time()-start_time),
121+
f"Job ID{query_job.job_id} successfully executed",
111122
)
112123
break
113124
exceptconcurrent.futures.TimeoutError:

‎google/cloud/bigquery/job/query.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1556,9 +1556,9 @@ def to_arrow(
15561556
No progress bar.
15571557
``'tqdm'``
15581558
Use the :func:`tqdm.tqdm` function to print a progress bar
1559-
to :data:`sys.stderr`.
1559+
to :data:`sys.stdout`.
15601560
``'tqdm_notebook'``
1561-
Use the :func:`tqdm.tqdm_notebook` function to display a
1561+
Use the :func:`tqdm.notebook.tqdm` function to display a
15621562
progress bar as a Jupyter notebook widget.
15631563
``'tqdm_gui'``
15641564
Use the :func:`tqdm.tqdm_gui` function to display a

‎google/cloud/bigquery/magics/magics.py‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def __init__(self):
125125
self._default_query_job_config=bigquery.QueryJobConfig()
126126
self._bigquery_client_options=client_options.ClientOptions()
127127
self._bqstorage_client_options=client_options.ClientOptions()
128-
self._progress_bar_type="tqdm"
128+
self._progress_bar_type="tqdm_notebook"
129129

130130
@property
131131
defcredentials(self):
@@ -269,7 +269,7 @@ def progress_bar_type(self):
269269
Manually setting the progress_bar_type:
270270
271271
>>> from google.cloud.bigquery import magics
272-
>>> magics.context.progress_bar_type = "tqdm"
272+
>>> magics.context.progress_bar_type = "tqdm_notebook"
273273
"""
274274
returnself._progress_bar_type
275275

@@ -286,7 +286,7 @@ def _handle_error(error, destination_var=None):
286286
287287
Args:
288288
error (Exception):
289-
An exception that ocurred during the queryexectution.
289+
An exception that ocurred during the queryexecution.
290290
destination_var (Optional[str]):
291291
The name of the IPython session variable to store the query job.
292292
"""
@@ -329,22 +329,25 @@ def _run_query(client, query, job_config=None):
329329
Query complete after 2.07s
330330
'bf633912-af2c-4780-b568-5d868058632b'
331331
"""
332-
start_time=time.time()
332+
start_time=time.perf_counter()
333333
query_job=client.query(query,job_config=job_config)
334334

335335
ifjob_configandjob_config.dry_run:
336336
returnquery_job
337337

338-
print("Executing query with job ID: {}".format(query_job.job_id))
338+
print(f"Executing query with job ID:{query_job.job_id}")
339339

340340
whileTrue:
341-
print("\rQuery executing: {:0.2f}s".format(time.time()-start_time),end="")
341+
print(
342+
f"\rQuery executing:{time.perf_counter()-start_time:.2f}s".format(),
343+
end="",
344+
)
342345
try:
343346
query_job.result(timeout=0.5)
344347
break
345348
exceptfutures.TimeoutError:
346349
continue
347-
print("\nQuery complete after {:0.2f}s".format(time.time()-start_time))
350+
print(f"\nJob ID{query_job.job_id} successfully executed")
348351
returnquery_job
349352

350353

@@ -365,7 +368,7 @@ def _create_dataset_if_necessary(client, dataset_id):
365368
pass
366369
dataset=bigquery.Dataset(dataset_reference)
367370
dataset.location=client.location
368-
print("Creating dataset: {}".format(dataset_id))
371+
print(f"Creating dataset:{dataset_id}")
369372
dataset=client.create_dataset(dataset)
370373

371374

@@ -500,7 +503,7 @@ def _create_dataset_if_necessary(client, dataset_id):
500503
default=None,
501504
help=(
502505
"Sets progress bar type to display a progress bar while executing the query."
503-
"Defaults to usetqdm. Install the ``tqdm`` package to use this feature."
506+
"Defaults to usetqdm_notebook. Install the ``tqdm`` package to use this feature."
504507
),
505508
)
506509
def_cell_magic(line,query):

‎google/cloud/bigquery/table.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1728,9 +1728,9 @@ def to_arrow(
17281728
No progress bar.
17291729
``'tqdm'``
17301730
Use the :func:`tqdm.tqdm` function to print a progress bar
1731-
to :data:`sys.stderr`.
1731+
to :data:`sys.stdout`.
17321732
``'tqdm_notebook'``
1733-
Use the :func:`tqdm.tqdm_notebook` function to display a
1733+
Use the :func:`tqdm.notebook.tqdm` function to display a
17341734
progress bar as a Jupyter notebook widget.
17351735
``'tqdm_gui'``
17361736
Use the :func:`tqdm.tqdm_gui` function to display a
@@ -1921,9 +1921,9 @@ def to_dataframe(
19211921
No progress bar.
19221922
``'tqdm'``
19231923
Use the :func:`tqdm.tqdm` function to print a progress bar
1924-
to :data:`sys.stderr`.
1924+
to :data:`sys.stdout`.
19251925
``'tqdm_notebook'``
1926-
Use the :func:`tqdm.tqdm_notebook` function to display a
1926+
Use the :func:`tqdm.notebook.tqdm` function to display a
19271927
progress bar as a Jupyter notebook widget.
19281928
``'tqdm_gui'``
19291929
Use the :func:`tqdm.tqdm_gui` function to display a
@@ -2075,9 +2075,9 @@ def to_geodataframe(
20752075
No progress bar.
20762076
``'tqdm'``
20772077
Use the :func:`tqdm.tqdm` function to print a progress bar
2078-
to :data:`sys.stderr`.
2078+
to :data:`sys.stdout`.
20792079
``'tqdm_notebook'``
2080-
Use the :func:`tqdm.tqdm_notebook` function to display a
2080+
Use the :func:`tqdm.notebook.tqdm` function to display a
20812081
progress bar as a Jupyter notebook widget.
20822082
``'tqdm_gui'``
20832083
Use the :func:`tqdm.tqdm_gui` function to display a

‎noxfile.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ def default(session, install_extras=True):
8181
)
8282

8383
ifinstall_extrasandsession.python=="3.10":
84-
install_target=".[bqstorage,pandas,tqdm,opentelemetry]"
84+
install_target=".[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
8585
elifinstall_extras:
8686
install_target=".[all]"
8787
else:
@@ -186,7 +186,7 @@ def system(session):
186186
session.install("google-cloud-datacatalog","-c",constraints_path)
187187

188188
ifsession.python=="3.10":
189-
extras="[bqstorage,pandas,tqdm,opentelemetry]"
189+
extras="[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
190190
else:
191191
extras="[all]"
192192
session.install("-e",f".{extras}","-c",constraints_path)
@@ -235,7 +235,7 @@ def snippets(session):
235235
session.install("grpcio","-c",constraints_path)
236236

237237
ifsession.python=="3.10":
238-
extras="[bqstorage,pandas,tqdm,opentelemetry]"
238+
extras="[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
239239
else:
240240
extras="[all]"
241241
session.install("-e",f".{extras}","-c",constraints_path)
@@ -387,7 +387,7 @@ def blacken(session):
387387
defdocs(session):
388388
"""Build the docs."""
389389

390-
session.install("recommonmark","sphinx==4.0.1","sphinx_rtd_theme")
390+
session.install("recommonmark","sphinx==4.0.2","sphinx_rtd_theme")
391391
session.install("google-cloud-storage")
392392
session.install("-e",".[all]")
393393

@@ -412,7 +412,7 @@ def docfx(session):
412412

413413
session.install("-e",".")
414414
session.install(
415-
"sphinx==4.0.1","alabaster","recommonmark","gcp-sphinx-docfx-yaml"
415+
"sphinx==4.0.2","alabaster","recommonmark","gcp-sphinx-docfx-yaml"
416416
)
417417

418418
shutil.rmtree(os.path.join("docs","_build"),ignore_errors=True)

‎samples/magics/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ db-dtypes==1.0.4
22
google-cloud-bigquery-storage==2.16.1
33
google-auth-oauthlib==0.5.3
44
grpcio==1.49.1
5+
ipywidgets==7.7.1
56
ipython===7.31.1;python_version=='3.7'
67
ipython===8.0.1;python_version=='3.8'
78
ipython==8.5.0;python_version>='3.9'

‎samples/snippets/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ google-cloud-bigquery==3.3.3
33
google-cloud-bigquery-storage==2.16.1
44
google-auth-oauthlib==0.5.3
55
grpcio==1.49.1
6+
ipywidgets==7.7.1
67
ipython===7.31.1;python_version=='3.7'
78
ipython===8.0.1;python_version=='3.8'
89
ipython==8.5.0;python_version>='3.9'

‎setup.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
# See: https://github.com/googleapis/python-bigquery/issues/757
5353
"bqstorage": [],
5454
"pandas": ["pandas>=1.0.0","db-dtypes>=0.3.0,<2.0.0dev"],
55+
"ipywidgets": ["ipywidgets==7.7.1"],
5556
"geopandas": ["geopandas>=0.9.0, <1.0dev","Shapely>=1.6.0, <2.0dev"],
5657
"ipython": ["ipython>=7.0.1,!=8.1.0"],
5758
"tqdm": ["tqdm >= 4.7.4, <5.0.0dev"],

‎testing/constraints-3.7.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ google-cloud-bigquery-storage==2.0.0
1212
google-cloud-core==1.4.1
1313
google-resumable-media==0.6.0
1414
grpcio==1.47.0
15+
ipywidgets==7.7.1
1516
ipython==7.0.1
1617
opentelemetry-api==1.1.0
1718
opentelemetry-instrumentation==0.20b0

‎tests/system/test_magics.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,7 @@ def test_bigquery_magic(ipython_interactive):
7171
# Removes blanks & terminal code (result of display clearing)
7272
updates=list(filter(lambdax:bool(x)andx!="\x1b[2K",lines))
7373
assertre.match("Executing query with job ID: .*",updates[0])
74-
assertall(re.match("Query executing: .*s",line)forlineinupdates[1:-1])
75-
assertre.match("Query complete after .*s",updates[-1])
74+
assert (re.match("Query executing: .*s",line)forlineinupdates[1:-1])
7675
assertisinstance(result,pandas.DataFrame)
7776
assertlen(result)==10# verify row count
7877
assertlist(result)== ["url","view_count"]# verify column names

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp