May 21, 2023 · Apr 19, 2023 · Apr 27, 2023 · Apr 29, 2023 · May 2, 2023 · May 2, 2023
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
                print('tokenize', testfile)
            with open(testfile, 'rb') as f:
                # with self.subTest(file=testfile):
 self.check_roundtrip(f)
                self.check_roundtrip(f)


    def roundtrip(self, code):
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
            encoding = "utf-8"
        yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
    yield from _tokenize(rl_gen, encoding)


 def _tokenize(rl_gen, encoding):
    source = b"".join(rl_gen).decode(encoding)
    token = None
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1727,7 +1727,7 @@ def test_random_files(self):
		print('tokenize', testfile)
		with open(testfile, 'rb') as f:
		# with self.subTest(file=testfile):
		self.check_roundtrip(f)
		self.check_roundtrip(f)


		def roundtrip(self, code):
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -444,7 +444,7 @@ def tokenize(readline):
		encoding = "utf-8"
		yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
		yield from _tokenize(rl_gen, encoding)


		def _tokenize(rl_gen, encoding):
		source = b"".join(rl_gen).decode(encoding)
		token = None
Expand Down