Movatterモバイル変換


[0]ホーム

URL:


Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Sign up
Appearance settings

Commitba36ebe

Browse files
safer comparision
Signed-off-by: varun-edachali-dbx <varun.edachali@databricks.com>
1 parenta9b9006 commitba36ebe

File tree

1 file changed

+35
-21
lines changed

1 file changed

+35
-21
lines changed

‎examples/experimental/comparator.py‎

Lines changed: 35 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -235,8 +235,8 @@ def compare_cursor_description(
235235

236236
def_safe_compare(self,val1,val2):
237237
"""
238-
Safely compare two values, handlinglists, dicts, andcomplex types.
239-
238+
Safely compare two values, handlingRow objects andPyArrow tables.
239+
240240
Returns True if values are equal, False otherwise.
241241
"""
242242
try:
@@ -245,28 +245,40 @@ def _safe_compare(self, val1, val2):
245245
returnTrue
246246
ifval1isNoneorval2isNone:
247247
returnFalse
248-
249-
# For lists, tuples, and other sequences (but not strings)
248+
249+
# For Row objects, convert to dictionaries
250+
ifhasattr(val1,"asDict")andhasattr(val2,"asDict"):
251+
returnself._safe_compare(
252+
val1.asDict(recursive=True),val2.asDict(recursive=True)
253+
)
254+
255+
# For PyArrow arrays/tables
256+
ifhasattr(val1,"to_pylist")andhasattr(val2,"to_pylist"):
257+
returnval1.to_pylist()==val2.to_pylist()
258+
259+
# For lists and tuples
250260
ifisinstance(val1, (list,tuple))andisinstance(val2, (list,tuple)):
251261
iflen(val1)!=len(val2):
252262
returnFalse
253263
returnall(self._safe_compare(v1,v2)forv1,v2inzip(val1,val2))
254-
264+
255265
# For dictionaries
256266
ifisinstance(val1,dict)andisinstance(val2,dict):
257267
ifset(val1.keys())!=set(val2.keys()):
258268
returnFalse
259269
returnall(self._safe_compare(val1[k],val2[k])forkinval1.keys())
260-
261-
# For Row objects (which are tuples with special properties)
262-
ifhasattr(val1,'asDict')andhasattr(val2,'asDict'):
263-
returnself._safe_compare(val1.asDict(recursive=True),val2.asDict(recursive=True))
264-
265-
# Default comparison
266-
returnval1==val2
267-
except (ValueError,TypeError)ase:
268-
# If comparison fails (e.g., numpy arrays), convert to string
269-
returnstr(val1)==str(val2)
270+
271+
# Default comparison - ensure we always return a boolean
272+
result=val1==val2
273+
# If result is not a simple boolean, use bool() to convert it
274+
returnbool(result)
275+
276+
except (ValueError,TypeError):
277+
# Fallback to string comparison for problematic types
278+
try:
279+
returnstr(val1)==str(val2)
280+
except:
281+
returnFalse
270282

271283
defcompare_rows(
272284
self,thrift_rows:List[Row],sea_rows:List[Row],result:ComparisonResult
@@ -302,15 +314,17 @@ def compare_rows(
302314
# Check if dictionaries are different by comparing all fields
303315
all_fields=set(thrift_dict.keys())|set(sea_dict.keys())
304316
dicts_differ=False
305-
317+
306318
forfieldinall_fields:
307319
iffieldnotinthrift_dictorfieldnotinsea_dict:
308320
dicts_differ=True
309321
break
310-
elifnotself._safe_compare(thrift_dict.get(field),sea_dict.get(field)):
322+
elifnotself._safe_compare(
323+
thrift_dict.get(field),sea_dict.get(field)
324+
):
311325
dicts_differ=True
312326
break
313-
327+
314328
ifdicts_differ:
315329

316330
forfieldinall_fields:
@@ -353,9 +367,9 @@ def compare_rows(
353367
thrift_values= [m[1]forminmismatches]
354368
sea_values= [m[2]forminmismatches]
355369

356-
ifall(self._safe_compare(v,thrift_values[0])forvinthrift_values)andall(
357-
self._safe_compare(v,sea_values[0])forvinsea_values
358-
):
370+
ifall(
371+
self._safe_compare(v,thrift_values[0])forvinthrift_values
372+
)andall(self._safe_compare(v,sea_values[0])forvinsea_values):
359373
result.add_difference(
360374
f"Field '{field}' value mismatch in all rows",
361375
thrift_values[0],

0 commit comments

Comments
 (0)

[8]ページ先頭

©2009-2025 Movatter.jp