Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit04510a7

Browse files
fix: preserve timestamp microsecond precision with rows from REST API (#402)
* feat: add formatOption default tru for tablelist and query result* feat: remove float point serialize* fix: lint* feat: remove comments
1 parentf421058 commit04510a7

File tree

7 files changed

+69
-53
lines changed

7 files changed

+69
-53
lines changed

‎google/cloud/bigquery/_helpers.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,8 @@ def _bytes_from_json(value, field):
8181
def_timestamp_from_json(value,field):
8282
"""Coerce 'value' to a datetime, if set or not nullable."""
8383
if_not_null(value,field):
84-
# value will be afloat in seconds, to microsecond precision, in UTC.
85-
return_datetime_from_microseconds(1e6*float(value))
84+
# value will be ainteger in seconds, to microsecond precision, in UTC.
85+
return_datetime_from_microseconds(int(value))
8686

8787

8888
def_timestamp_query_param_from_json(value,field):

‎google/cloud/bigquery/client.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3157,6 +3157,7 @@ def list_rows(
31573157
ifstart_indexisnotNone:
31583158
params["startIndex"]=start_index
31593159

3160+
params["formatOptions.useInt64Timestamp"]=True
31603161
row_iterator=RowIterator(
31613162
client=self,
31623163
api_request=functools.partial(self._call_api,retry,timeout=timeout),
@@ -3237,6 +3238,7 @@ def _list_rows_from_query_results(
32373238
ifstart_indexisnotNone:
32383239
params["startIndex"]=start_index
32393240

3241+
params["formatOptions.useInt64Timestamp"]=True
32403242
row_iterator=RowIterator(
32413243
client=self,
32423244
api_request=functools.partial(self._call_api,retry,timeout=timeout),

‎tests/unit/job/test_query.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -839,6 +839,7 @@ def test_result(self):
839839
query_params={
840840
"fields":_LIST_ROWS_FROM_QUERY_RESULTS_FIELDS,
841841
"location":"EU",
842+
"formatOptions.useInt64Timestamp":True,
842843
},
843844
timeout=None,
844845
)
@@ -887,6 +888,7 @@ def test_result_with_done_job_calls_get_query_results(self):
887888
query_params={
888889
"fields":_LIST_ROWS_FROM_QUERY_RESULTS_FIELDS,
889890
"location":"EU",
891+
"formatOptions.useInt64Timestamp":True,
890892
},
891893
timeout=None,
892894
)
@@ -1118,6 +1120,7 @@ def test_result_w_page_size(self):
11181120
"maxResults":3,
11191121
"fields":_LIST_ROWS_FROM_QUERY_RESULTS_FIELDS,
11201122
"location":"US",
1123+
"formatOptions.useInt64Timestamp":True,
11211124
},
11221125
timeout=None,
11231126
)
@@ -1129,6 +1132,7 @@ def test_result_w_page_size(self):
11291132
"maxResults":3,
11301133
"fields":_LIST_ROWS_FROM_QUERY_RESULTS_FIELDS,
11311134
"location":"US",
1135+
"formatOptions.useInt64Timestamp":True,
11321136
},
11331137
timeout=None,
11341138
)

‎tests/unit/job/test_query_pandas.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -501,16 +501,16 @@ def test_to_dataframe_column_dtypes():
501501
}
502502
row_data= [
503503
[
504-
"1.4338368E9",
504+
"1433836800000000",
505505
"420",
506506
"1.1",
507507
"1.77",
508508
"Cto_dataframeash",
509509
"true",
510510
"1999-12-01",
511511
],
512-
["1.3878117E9","2580","17.7","28.5","Cash","false","1953-06-14"],
513-
["1.3855653E9","2280","4.4","7.1","Credit","true","1981-11-04"],
512+
["1387811700000000","2580","17.7","28.5","Cash","false","1953-06-14"],
513+
["1385565300000000","2280","4.4","7.1","Credit","true","1981-11-04"],
514514
]
515515
rows= [{"f": [{"v":field}forfieldinrow]}forrowinrow_data]
516516
query_resource["rows"]=rows

‎tests/unit/test__helpers.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -190,18 +190,18 @@ def test_w_none_required(self):
190190
withself.assertRaises(TypeError):
191191
self._call_fut(None,_Field("REQUIRED"))
192192

193-
deftest_w_string_value(self):
193+
deftest_w_string_int_value(self):
194194
fromgoogle.cloud._helpersimport_EPOCH
195195

196-
coerced=self._call_fut("1.234567",object())
196+
coerced=self._call_fut("1234567",object())
197197
self.assertEqual(
198198
coerced,_EPOCH+datetime.timedelta(seconds=1,microseconds=234567)
199199
)
200200

201-
deftest_w_float_value(self):
201+
deftest_w_int_value(self):
202202
fromgoogle.cloud._helpersimport_EPOCH
203203

204-
coerced=self._call_fut(1.234567,object())
204+
coerced=self._call_fut(1234567,object())
205205
self.assertEqual(
206206
coerced,_EPOCH+datetime.timedelta(seconds=1,microseconds=234567)
207207
)

‎tests/unit/test_client.py‎

Lines changed: 38 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -6739,42 +6739,21 @@ def test_list_rows(self):
67396739
self.DS_ID,
67406740
self.TABLE_ID,
67416741
)
6742-
WHEN_TS=1437767599.006
6743-
WHEN=datetime.datetime.utcfromtimestamp(WHEN_TS).replace(tzinfo=UTC)
6744-
WHEN_1=WHEN+datetime.timedelta(seconds=1)
6745-
WHEN_2=WHEN+datetime.timedelta(seconds=2)
6742+
WHEN_TS=1437767599006000
6743+
6744+
WHEN=datetime.datetime.utcfromtimestamp(WHEN_TS/1e6).replace(tzinfo=UTC)
6745+
WHEN_1=WHEN+datetime.timedelta(microseconds=1)
6746+
WHEN_2=WHEN+datetime.timedelta(microseconds=2)
67466747
ROWS=1234
67476748
TOKEN="TOKEN"
67486749

6749-
def_bigquery_timestamp_float_repr(ts_float):
6750-
# Preserve microsecond precision for E+09 timestamps
6751-
return"%0.15E"% (ts_float,)
6752-
67536750
DATA= {
67546751
"totalRows":str(ROWS),
67556752
"pageToken":TOKEN,
67566753
"rows": [
6757-
{
6758-
"f": [
6759-
{"v":"Phred Phlyntstone"},
6760-
{"v":"32"},
6761-
{"v":_bigquery_timestamp_float_repr(WHEN_TS)},
6762-
]
6763-
},
6764-
{
6765-
"f": [
6766-
{"v":"Bharney Rhubble"},
6767-
{"v":"33"},
6768-
{"v":_bigquery_timestamp_float_repr(WHEN_TS+1)},
6769-
]
6770-
},
6771-
{
6772-
"f": [
6773-
{"v":"Wylma Phlyntstone"},
6774-
{"v":"29"},
6775-
{"v":_bigquery_timestamp_float_repr(WHEN_TS+2)},
6776-
]
6777-
},
6754+
{"f": [{"v":"Phred Phlyntstone"}, {"v":"32"}, {"v":WHEN_TS}]},
6755+
{"f": [{"v":"Bharney Rhubble"}, {"v":"33"}, {"v":WHEN_TS+1}]},
6756+
{"f": [{"v":"Wylma Phlyntstone"}, {"v":"29"}, {"v":WHEN_TS+2}]},
67786757
{"f": [{"v":"Bhettye Rhubble"}, {"v":None}, {"v":None}]},
67796758
],
67806759
}
@@ -6807,7 +6786,10 @@ def _bigquery_timestamp_float_repr(ts_float):
68076786
self.assertEqual(iterator.next_page_token,TOKEN)
68086787

68096788
conn.api_request.assert_called_once_with(
6810-
method="GET",path="/%s"%PATH,query_params={},timeout=7.5
6789+
method="GET",
6790+
path="/%s"%PATH,
6791+
query_params={"formatOptions.useInt64Timestamp":True},
6792+
timeout=7.5,
68116793
)
68126794

68136795
deftest_list_rows_w_start_index_w_page_size(self):
@@ -6856,20 +6838,30 @@ def test_list_rows_w_start_index_w_page_size(self):
68566838
self.assertEqual(len(rows),2)
68576839
self.assertEqual(rows[0],Row(("Wylma Phlyntstone",),f2i))
68586840
self.assertEqual(rows[1],Row(("Bhettye Rhubble",),f2i))
6859-
self.assertEqual(extra_params, {"startIndex":1})
6841+
self.assertEqual(
6842+
extra_params, {"startIndex":1,"formatOptions.useInt64Timestamp":True}
6843+
)
68606844

68616845
conn.api_request.assert_has_calls(
68626846
[
68636847
mock.call(
68646848
method="GET",
68656849
path="/%s"%PATH,
6866-
query_params={"startIndex":1,"maxResults":2},
6850+
query_params={
6851+
"startIndex":1,
6852+
"maxResults":2,
6853+
"formatOptions.useInt64Timestamp":True,
6854+
},
68676855
timeout=None,
68686856
),
68696857
mock.call(
68706858
method="GET",
68716859
path="/%s"%PATH,
6872-
query_params={"pageToken":"some-page-token","maxResults":2},
6860+
query_params={
6861+
"pageToken":"some-page-token",
6862+
"maxResults":2,
6863+
"formatOptions.useInt64Timestamp":True,
6864+
},
68736865
timeout=None,
68746866
),
68756867
]
@@ -6920,6 +6912,7 @@ def test_list_rows_query_params(self):
69206912
iterator=client.list_rows(table,**test[0])
69216913
six.next(iterator.pages)
69226914
req=conn.api_request.call_args_list[i]
6915+
test[1]["formatOptions.useInt64Timestamp"]=True
69236916
self.assertEqual(req[1]["query_params"],test[1],"for kwargs %s"%test[0])
69246917

69256918
deftest_list_rows_repeated_fields(self):
@@ -6979,7 +6972,10 @@ def test_list_rows_repeated_fields(self):
69796972
conn.api_request.assert_called_once_with(
69806973
method="GET",
69816974
path="/%s"%PATH,
6982-
query_params={"selectedFields":"color,struct"},
6975+
query_params={
6976+
"selectedFields":"color,struct",
6977+
"formatOptions.useInt64Timestamp":True,
6978+
},
69836979
timeout=None,
69846980
)
69856981

@@ -7047,7 +7043,10 @@ def test_list_rows_w_record_schema(self):
70477043
self.assertEqual(page_token,TOKEN)
70487044

70497045
conn.api_request.assert_called_once_with(
7050-
method="GET",path="/%s"%PATH,query_params={},timeout=None
7046+
method="GET",
7047+
path="/%s"%PATH,
7048+
query_params={"formatOptions.useInt64Timestamp":True},
7049+
timeout=None,
70517050
)
70527051

70537052
deftest_list_rows_with_missing_schema(self):
@@ -7109,7 +7108,10 @@ def test_list_rows_with_missing_schema(self):
71097108

71107109
rows=list(row_iter)
71117110
conn.api_request.assert_called_once_with(
7112-
method="GET",path=tabledata_path,query_params={},timeout=None
7111+
method="GET",
7112+
path=tabledata_path,
7113+
query_params={"formatOptions.useInt64Timestamp":True},
7114+
timeout=None,
71137115
)
71147116
self.assertEqual(row_iter.total_rows,3,msg=repr(table))
71157117
self.assertEqual(rows[0].name,"Phred Phlyntstone",msg=repr(table))

‎tests/unit/test_table.py‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2451,8 +2451,8 @@ def test_to_dataframe_timestamp_out_of_pyarrow_bounds(self):
24512451

24522452
schema= [SchemaField("some_timestamp","TIMESTAMP")]
24532453
rows= [
2454-
{"f": [{"v":"81953424000.0"}]},# 4567-01-01 00:00:00 UTC
2455-
{"f": [{"v":"253402214400.0"}]},# 9999-12-31 00:00:00 UTC
2454+
{"f": [{"v":"81953424000000000"}]},# 4567-01-01 00:00:00 UTC
2455+
{"f": [{"v":"253402214400000000"}]},# 9999-12-31 00:00:00 UTC
24562456
]
24572457
path="/foo"
24582458
api_request=mock.Mock(return_value={"rows":rows})
@@ -2675,9 +2675,9 @@ def test_to_dataframe_w_various_types_nullable(self):
26752675
]
26762676
row_data= [
26772677
[None,None,None,None,None,None],
2678-
["1.4338368E9","420","1.1",u"Cash","true","1999-12-01"],
2679-
["1.3878117E9","2580","17.7",u"Cash","false","1953-06-14"],
2680-
["1.3855653E9","2280","4.4",u"Credit","true","1981-11-04"],
2678+
["1433836800000000","420","1.1",u"Cash","true","1999-12-01"],
2679+
["1387811700000000","2580","17.7",u"Cash","false","1953-06-14"],
2680+
["1385565300000000","2280","4.4",u"Credit","true","1981-11-04"],
26812681
]
26822682
rows= [{"f": [{"v":field}forfieldinrow]}forrowinrow_data]
26832683
path="/foo"
@@ -2715,9 +2715,17 @@ def test_to_dataframe_column_dtypes(self):
27152715
SchemaField("date","DATE"),
27162716
]
27172717
row_data= [
2718-
["1.4338368E9","420","1.1","1.77",u"Cash","true","1999-12-01"],
2719-
["1.3878117E9","2580","17.7","28.5",u"Cash","false","1953-06-14"],
2720-
["1.3855653E9","2280","4.4","7.1",u"Credit","true","1981-11-04"],
2718+
["1433836800000000","420","1.1","1.77",u"Cash","true","1999-12-01"],
2719+
[
2720+
"1387811700000000",
2721+
"2580",
2722+
"17.7",
2723+
"28.5",
2724+
u"Cash",
2725+
"false",
2726+
"1953-06-14",
2727+
],
2728+
["1385565300000000","2280","4.4","7.1",u"Credit","true","1981-11-04"],
27212729
]
27222730
rows= [{"f": [{"v":field}forfieldinrow]}forrowinrow_data]
27232731
path="/foo"

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp