importtokenizeimportunittestfromioimportBytesIO,StringIOfromunittestimportTestCase,mockdefstringify_tokens_from_source(token_generator,source_string):result= []fortype,token,start,end,lineintoken_generator:result.append(f"{type:10}{token!r:13}{start}{end}")classTokenizeTest(TestCase):defcheck_tokenize(self,s,expected):self.check_tokenize("1 + 1","""\    """)deftest_tabs(self):self.check_tokenize("def f():\n""\tif x\n"'\x00',"""\    """)defcheck_tokenize(self,s,expected):f=StringIO(s)result=stringify_tokens_from_source(tokenize.generate_tokens(f.readline),s)forencodingin ["utf-8","latin-1","utf-16"]:tokens=list(tokenize._generate_tokens_from_c_tokenizer())self.check_tokenize('0xff <= 255',"""\    """)self.check_tokenize('''\  await = 2''',"""\    """)if__name__=="__main__":unittest.main()

...File "/usr/lib/python3.12/tokenize.py", line 577, in _generate_tokens_from_c_tokenizer    yield TokenInfo._make(info)          ^^^^^^^^^^^^^^^^^^^^^  File "/usr/lib/python3.12/collections/__init__.py", line 449, in _make    result = tuple_new(cls, iterable)             ^^^^^^^^^^^^^^^^^^^^^^^^SystemError: <built-in method __new__ of type object at 0xa42c40> returned a result with an exception set