Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitf0bcbd6

Browse files
authored
fix(bigquery): fix inserting missing repeated fields (#10196)
* fix(bigquery): do not insert missing fields as explicit None* Omit all None values from JSON request body* Add an extra test for all missing row values* Flatten a block of code a bit
1 parent77dd923 commitf0bcbd6

File tree

3 files changed

+44
-16
lines changed

3 files changed

+44
-16
lines changed

‎bigquery/google/cloud/bigquery/_helpers.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -424,11 +424,12 @@ def _record_field_to_json(fields, row_value):
424424

425425
forsubindex,subfieldinenumerate(fields):
426426
subname=subfield.name
427-
ifisdict:
428-
subvalue=row_value.get(subname)
429-
else:
430-
subvalue=row_value[subindex]
431-
record[subname]=_field_to_json(subfield,subvalue)
427+
subvalue=row_value.get(subname)ifisdictelserow_value[subindex]
428+
429+
# None values are unconditionally omitted
430+
ifsubvalueisnotNone:
431+
record[subname]=_field_to_json(subfield,subvalue)
432+
432433
returnrecord
433434

434435

‎bigquery/tests/unit/test__helpers.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -856,14 +856,39 @@ def test_w_non_empty_dict(self):
856856
converted=self._call_fut(fields,original)
857857
self.assertEqual(converted, {"one":"42","two":"two"})
858858

859-
deftest_w_missing_nullable(self):
859+
deftest_w_some_missing_nullables(self):
860860
fields= [
861861
_make_field("INT64",name="one",mode="NULLABLE"),
862862
_make_field("STRING",name="two",mode="NULLABLE"),
863863
]
864864
original= {"one":42}
865865
converted=self._call_fut(fields,original)
866-
self.assertEqual(converted, {"one":"42","two":None})
866+
867+
# missing fields should not be converted to an explicit None
868+
self.assertEqual(converted, {"one":"42"})
869+
870+
deftest_w_all_missing_nullables(self):
871+
fields= [
872+
_make_field("INT64",name="one",mode="NULLABLE"),
873+
_make_field("STRING",name="two",mode="NULLABLE"),
874+
]
875+
original= {}
876+
converted=self._call_fut(fields,original)
877+
878+
# we should get an empty dict, not None
879+
self.assertEqual(converted, {})
880+
881+
deftest_w_explicit_none_value(self):
882+
fields= [
883+
_make_field("INT64",name="one",mode="NULLABLE"),
884+
_make_field("STRING",name="two",mode="NULLABLE"),
885+
_make_field("BOOL",name="three",mode="REPEATED"),
886+
]
887+
original= {"three":None,"one":42,"two":None}
888+
converted=self._call_fut(fields,original)
889+
890+
# None values should be dropped regardless of the field type
891+
self.assertEqual(converted, {"one":"42"})
867892

868893

869894
classTest_field_to_json(unittest.TestCase):

‎bigquery/tests/unit/test_client.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4668,10 +4668,13 @@ def test_insert_rows_w_schema(self):
46684668
]
46694669

46704670
def_row_data(row):
4671+
result= {"full_name":row[0],"age":str(row[1])}
46714672
joined=row[2]
4672-
ifisinstance(row[2],datetime.datetime):
4673+
ifisinstance(joined,datetime.datetime):
46734674
joined=_microseconds_from_datetime(joined)*1e-6
4674-
return {"full_name":row[0],"age":str(row[1]),"joined":joined}
4675+
ifjoinedisnotNone:
4676+
result["joined"]=joined
4677+
returnresult
46754678

46764679
SENT= {
46774680
"rows": [
@@ -4740,7 +4743,10 @@ def test_insert_rows_w_list_of_dictionaries(self):
47404743

47414744
def_row_data(row):
47424745
joined=row["joined"]
4743-
ifisinstance(joined,datetime.datetime):
4746+
ifjoinedisNone:
4747+
row=copy.deepcopy(row)
4748+
delrow["joined"]
4749+
elifisinstance(joined,datetime.datetime):
47444750
row["joined"]=_microseconds_from_datetime(joined)*1e-6
47454751
row["age"]=str(row["age"])
47464752
returnrow
@@ -4959,9 +4965,8 @@ def test_insert_rows_w_repeated_fields(self):
49594965
},
49604966
{
49614967
"json": {
4962-
"color":None,
49634968
"items": [],
4964-
"structs": [{"score":None,"times": [],"distances": [3.5]}],
4969+
"structs": [{"times": [],"distances": [3.5]}],
49654970
},
49664971
"insertId":"1",
49674972
},
@@ -5028,10 +5033,7 @@ def test_insert_rows_w_record_schema(self):
50285033
},
50295034
"insertId":"1",
50305035
},
5031-
{
5032-
"json": {"full_name":"Wylma Phlyntstone","phone":None},
5033-
"insertId":"2",
5034-
},
5036+
{"json": {"full_name":"Wylma Phlyntstone"},"insertId":"2"},
50355037
]
50365038
}
50375039

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp