Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit8fe7254

Browse files
plamuttseaver
andauthored
fix: raise error if inserting rows with unknown fields (#163)
Co-authored-by: Tres Seaver <tseaver@palladion.com>
1 parentf59fc9a commit8fe7254

File tree

2 files changed

+70
-1
lines changed

2 files changed

+70
-1
lines changed

‎google/cloud/bigquery/_helpers.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
importdatetime
2020
importdecimal
2121
importre
22+
importsix
2223

2324
fromgoogle.cloud._helpersimportUTC
2425
fromgoogle.cloud._helpersimport_date_from_iso8601_date
@@ -419,9 +420,23 @@ def _record_field_to_json(fields, row_value):
419420
Returns:
420421
Mapping[str, Any]: A JSON-serializable dictionary.
421422
"""
422-
record= {}
423423
isdict=isinstance(row_value,dict)
424424

425+
# If row is passed as a tuple, make the length sanity check to avoid either
426+
# uninformative index errors a few lines below or silently omitting some of
427+
# the values from the result (we cannot know exactly which fields are missing
428+
# or redundant, since we don't have their names).
429+
ifnotisdictandlen(row_value)!=len(fields):
430+
msg="The number of row fields ({}) does not match schema length ({}).".format(
431+
len(row_value),len(fields)
432+
)
433+
raiseValueError(msg)
434+
435+
record= {}
436+
437+
ifisdict:
438+
processed_fields=set()
439+
425440
forsubindex,subfieldinenumerate(fields):
426441
subname=subfield.name
427442
subvalue=row_value.get(subname)ifisdictelserow_value[subindex]
@@ -430,6 +445,20 @@ def _record_field_to_json(fields, row_value):
430445
ifsubvalueisnotNone:
431446
record[subname]=_field_to_json(subfield,subvalue)
432447

448+
ifisdict:
449+
processed_fields.add(subname)
450+
451+
# Unknown fields should not be silently dropped, include them. Since there
452+
# is no schema information available for them, include them as strings
453+
# to make them JSON-serializable.
454+
ifisdict:
455+
not_processed=set(row_value.keys())-processed_fields
456+
457+
forfield_nameinnot_processed:
458+
value=row_value[field_name]
459+
ifvalueisnotNone:
460+
record[field_name]=six.text_type(value)
461+
433462
returnrecord
434463

435464

‎tests/unit/test__helpers.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
importunittest
1919

2020
importmock
21+
importsix
2122

2223

2324
classTest_not_null(unittest.TestCase):
@@ -847,6 +848,26 @@ def test_w_non_empty_list(self):
847848
converted=self._call_fut(fields,original)
848849
self.assertEqual(converted, {"one":"42","two":"two"})
849850

851+
deftest_w_list_missing_fields(self):
852+
fields= [
853+
_make_field("INT64",name="one",mode="NULLABLE"),
854+
_make_field("STRING",name="two",mode="NULLABLE"),
855+
]
856+
original= [42]
857+
858+
withsix.assertRaisesRegex(self,ValueError,r".*not match schema length.*"):
859+
self._call_fut(fields,original)
860+
861+
deftest_w_list_too_many_fields(self):
862+
fields= [
863+
_make_field("INT64",name="one",mode="NULLABLE"),
864+
_make_field("STRING",name="two",mode="NULLABLE"),
865+
]
866+
original= [42,"two","three"]
867+
868+
withsix.assertRaisesRegex(self,ValueError,r".*not match schema length.*"):
869+
self._call_fut(fields,original)
870+
850871
deftest_w_non_empty_dict(self):
851872
fields= [
852873
_make_field("INT64",name="one",mode="NULLABLE"),
@@ -890,6 +911,25 @@ def test_w_explicit_none_value(self):
890911
# None values should be dropped regardless of the field type
891912
self.assertEqual(converted, {"one":"42"})
892913

914+
deftest_w_dict_unknown_fields(self):
915+
fields= [
916+
_make_field("INT64",name="one",mode="NULLABLE"),
917+
_make_field("STRING",name="two",mode="NULLABLE"),
918+
]
919+
original= {
920+
"whoami":datetime.date(2020,7,20),
921+
"one":111,
922+
"two":"222",
923+
"void":None,
924+
}
925+
926+
converted=self._call_fut(fields,original)
927+
928+
# Unknown fields should be included (if not None), but converted as strings.
929+
self.assertEqual(
930+
converted, {"whoami":"2020-07-20","one":"111","two":"222"},
931+
)
932+
893933

894934
classTest_field_to_json(unittest.TestCase):
895935
def_call_fut(self,field,value):

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp