Commite13d1d9

authored

gh-99581: Fix a buffer overflow in the tokenizer when copying lines that fill the available buffer (#99605)

1 parentabf5b6f commite13d1d9Copy full SHA for e13d1d9

File tree

+25

-1

lines changed

+25

-1

lines changed

Lines changed: 16 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,8 @@`
`10`	`10`	`fromunittestimportTestCase,mock`
`11`	`11`	`fromtest.test_grammarimport (VALID_UNDERSCORE_LITERALS,`
`12`	`12`	`INVALID_UNDERSCORE_LITERALS)`
	`13`	`+fromtest.supportimportos_helper`
	`14`	`+fromtest.support.script_helperimportrun_test_script,make_script`
`13`	`15`	`importos`
`14`	`16`	`importtoken`
`15`	`17`
`@@ -2631,5 +2633,19 @@ def fib(n):`
`2631`	`2633`	`self.assertEqual(get_tokens(code),get_tokens(code_no_cont))`
`2632`	`2634`
`2633`	`2635`
	`2636`	`+classCTokenizerBufferTests(unittest.TestCase):`
	`2637`	`+deftest_newline_at_the_end_of_buffer(self):`
	`2638`	`+# See issue 99581: Make sure that if we need to add a new line at the`
	`2639`	`+# end of the buffer, we have enough space in the buffer, specially when`
	`2640`	`+# the current line is as long as the buffer space available.`
	`2641`	`+test_script=f"""\`
	`2642`	`+ #coding: latin-1`
	`2643`	`+ #{"a"*10000}`
	`2644`	`+ #{"a"*10002}"""`
	`2645`	`+withos_helper.temp_dir()astemp_dir:`
	`2646`	`+file_name=make_script(temp_dir,'foo',test_script)`
	`2647`	`+run_test_script(file_name)`
	`2648`	`+`
	`2649`	`+`
`2634`	`2650`	`if__name__=="__main__":`
`2635`	`2651`	`unittest.main()`

Lines changed: 3 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+Fixed a bug that was causing a buffer overflow if the tokenizer copies a`
	`2`	`+line missing the newline caracter from a file that is as long as the`
	`3`	`+available tokenizer buffer. Patch by Pablo galindo`

Lines changed: 6 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -413,7 +413,11 @@ tok_readline_recode(struct tok_state *tok) {`
`413`	`413`	`error_ret(tok);`
`414`	`414`	`gotoerror;`
`415`	`415`	`}`
`416`		`-if (!tok_reserve_buf(tok,buflen+1)) {`
	`416`	`+// Make room for the null terminator and potentially`
	`417`	`+// an extra newline character that we may need to artificially`
	`418`	`+// add.`
	`419`	`+size_tbuffer_size=buflen+2;`
	`420`	`+if (!tok_reserve_buf(tok,buffer_size)) {`
`417`	`421`	`gotoerror;`
`418`	`422`	`}`
`419`	`423`	`memcpy(tok->inp,buf,buflen);`
`@@ -1000,6 +1004,7 @@ tok_underflow_file(struct tok_state *tok) {`
`1000`	`1004`	`return0;`
`1001`	`1005`	`}`
`1002`	`1006`	`if (tok->inp[-1]!='\n') {`
	`1007`	`+assert(tok->inp+1<tok->end);`
`1003`	`1008`	`/* Last line does not end in \n, fake one */`
`1004`	`1009`	`*tok->inp++='\n';`
`1005`	`1010`	`*tok->inp='\0';`

Comments

(0)