Commitedeff24

committed

gh-99891: Fix infinite recursion in the tokenizer when showing warnings

1 parent05dfc53 commitedeff24Copy full SHA for edeff24

File tree

+25

-0

lines changed

+25

-0

lines changed

Lines changed: 13 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -160,6 +160,19 @@ def test_file_parse_error_multiline(self):`
`160`	`160`	`finally:`
`161`	`161`	`os.unlink(TESTFN)`
`162`	`162`
	`163`	`+deftest_tokenizer_fstring_warning_in_first_line(self):`
	`164`	`+source="0b1and 2"`
	`165`	`+withopen(TESTFN,"w")asfd:`
	`166`	`+fd.write("{}".format(source))`
	`167`	`+try:`
	`168`	`+retcode,stdout,stderr=script_helper.assert_python_failure(TESTFN,PYTHONWARNINGS="error")`
	`169`	`+self.assertGreater(retcode,0)`
	`170`	`+self.assertIn(b"SyntaxError: invalid binary litera",stderr)`
	`171`	`+self.assertEqual(stderr.count(source.encode()),1)`
	`172`	`+finally:`
	`173`	`+os.unlink(TESTFN)`
	`174`	`+`
	`175`	`+`
`163`	`176`	`classAbstractSourceEncodingTest:`
`164`	`177`
`165`	`178`	`deftest_default_coding(self):`

Lines changed: 3 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+Fix a bug in the tokenizer that could cause infinite recursion when showing`
	`2`	`+syntax warnings that happen in the first line of the source. Patch by Pablo`
	`3`	`+Galindo`

Lines changed: 8 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -97,6 +97,7 @@ tok_new(void)`
`97`	`97`	`tok->async_def_nl=0;`
`98`	`98`	`tok->interactive_underflow=IUNDERFLOW_NORMAL;`
`99`	`99`	`tok->str=NULL;`
	`100`	`+tok->report_warnings=1;`
`100`	`101`	`#ifdefPy_DEBUG`
`101`	`102`	`tok->debug=_Py_GetConfig()->parser_debug;`
`102`	`103`	`#endif`
`@@ -1201,6 +1202,10 @@ indenterror(struct tok_state *tok)`
`1201`	`1202`	`staticint`
`1202`	`1203`	`parser_warn(structtok_statetok,PyObjectcategory,constchar*format, ...)`
`1203`	`1204`	`{`
	`1205`	`+if (!tok->report_warnings) {`
	`1206`	`+return0;`
	`1207`	`+ }`
	`1208`	`+`
`1204`	`1209`	`PyObject*errmsg;`
`1205`	`1210`	`va_listvargs;`
`1206`	`1211`	`va_start(vargs,format);`
`@@ -2239,6 +2244,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)`
`2239`	`2244`	`}`
`2240`	`2245`	`}`
`2241`	`2246`	`structtokentoken;`
	`2247`	`+// We don't want to report warnings here because it could cause infinite recursion`
	`2248`	`+// if fetching the encoding shows a warning.`
	`2249`	`+tok->report_warnings=0;`
`2242`	`2250`	`while (tok->lineno<2&&tok->done==E_OK) {`
`2243`	`2251`	`_PyTokenizer_Get(tok,&token);`
`2244`	`2252`	`}`

Lines changed: 1 addition & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -92,6 +92,7 @@ struct tok_state {`
`92`	`92`	`NEWLINE token after it. */`
`93`	`93`	`/* How to proceed when asked for a new token in interactive mode */`
`94`	`94`	`enuminteractive_underflow_tinteractive_underflow;`
	`95`	`+intreport_warnings;`
`95`	`96`	`#ifdefPy_DEBUG`
`96`	`97`	`intdebug;`
`97`	`98`	`#endif`

Comments

(0)