NotificationsYou must be signed in to change notification settings
Fork4.9k
Star17.7k

Commit33c697e

committed

Make ts_locale.c's character-type functions cope with UTF-16.

On Windows, in UTF8 database encoding, what char2wchar() produces isUTF16 not UTF32, ie, characters above U+FFFF will be represented bysurrogate pairs. t_isdigit() and siblings did not account for thisand failed to provide a large enough result buffer. That in turnled to bogus "invalid multibyte character for locale" errors, becausecontrary to what you might think from char2wchar()'s documentation,its Windows code path doesn't cope sanely with buffer overflow.The solution for t_isdigit() and siblings is pretty clear: providea 3-wchar_t result buffer not 2.char2wchar() also needs some work to provide more consistent, and moreaccurately documented, buffer overrun behavior. But that's a bigger joband it doesn't actually have any immediate payoff, so leave it for later.Per bug #15476 from Kenji Uno, who deserves credit for identifying thecause of the problem. Back-patch to all active branches.Discussion:https://postgr.es/m/15476-4314f480acf0f114@postgresql.org

1 parent1aad3a7 commit33c697eCopy full SHA for 33c697e

File tree

1 file changed

+19

-8

lines changed

src/backend/tsearch
- ts_locale.c

1 file changed

+19

-8

lines changed

`‎src/backend/tsearch/ts_locale.c`

Lines changed: 19 additions & 8 deletions

Original file line number	Diff line number	Diff line change
`@@ -23,18 +23,29 @@ static void tsearch_readline_callback(void *arg);`
`23`	`23`
`24`	`24`	`#ifdefUSE_WIDE_UPPER_LOWER`
`25`	`25`
	`26`	`+/*`
	`27`	`+ * The reason these functions use a 3-wchar_t output buffer, not 2 as you`
	`28`	`+ * might expect, is that on Windows "wchar_t" is 16 bits and what we'll be`
	`29`	`+ * getting from char2wchar() is UTF16 not UTF32. A single input character`
	`30`	`+ * may therefore produce a surrogate pair rather than just one wchar_t;`
	`31`	`+ * we also need room for a trailing null. When we do get a surrogate pair,`
	`32`	`+ * we pass just the first code to iswdigit() etc, so that these functions will`
	`33`	`+ * always return false for characters outside the Basic Multilingual Plane.`
	`34`	`+ */`
	`35`	`+#defineWC_BUF_LEN 3`
	`36`	`+`
`26`	`37`	`int`
`27`	`38`	`t_isdigit(constchar*ptr)`
`28`	`39`	`{`
`29`	`40`	`intclen=pg_mblen(ptr);`
`30`		`-wchar_tcharacter[2];`
	`41`	`+wchar_tcharacter[WC_BUF_LEN];`
`31`	`42`	`Oidcollation=DEFAULT_COLLATION_OID;/* TODO */`
`32`	`43`	`pg_locale_tmylocale=0;/* TODO */`
`33`	`44`
`34`	`45`	`if (clen==1\|\|lc_ctype_is_c(collation))`
`35`	`46`	`returnisdigit(TOUCHAR(ptr));`
`36`	`47`
`37`		`-char2wchar(character,2,ptr,clen,mylocale);`
	`48`	`+char2wchar(character,WC_BUF_LEN,ptr,clen,mylocale);`
`38`	`49`
`39`	`50`	`returniswdigit((wint_t)character[0]);`
`40`	`51`	`}`
`@@ -43,14 +54,14 @@ int`
`43`	`54`	`t_isspace(constchar*ptr)`
`44`	`55`	`{`
`45`	`56`	`intclen=pg_mblen(ptr);`
`46`		`-wchar_tcharacter[2];`
	`57`	`+wchar_tcharacter[WC_BUF_LEN];`
`47`	`58`	`Oidcollation=DEFAULT_COLLATION_OID;/* TODO */`
`48`	`59`	`pg_locale_tmylocale=0;/* TODO */`
`49`	`60`
`50`	`61`	`if (clen==1\|\|lc_ctype_is_c(collation))`
`51`	`62`	`returnisspace(TOUCHAR(ptr));`
`52`	`63`
`53`		`-char2wchar(character,2,ptr,clen,mylocale);`
	`64`	`+char2wchar(character,WC_BUF_LEN,ptr,clen,mylocale);`
`54`	`65`
`55`	`66`	`returniswspace((wint_t)character[0]);`
`56`	`67`	`}`
`@@ -59,14 +70,14 @@ int`
`59`	`70`	`t_isalpha(constchar*ptr)`
`60`	`71`	`{`
`61`	`72`	`intclen=pg_mblen(ptr);`
`62`		`-wchar_tcharacter[2];`
	`73`	`+wchar_tcharacter[WC_BUF_LEN];`
`63`	`74`	`Oidcollation=DEFAULT_COLLATION_OID;/* TODO */`
`64`	`75`	`pg_locale_tmylocale=0;/* TODO */`
`65`	`76`
`66`	`77`	`if (clen==1\|\|lc_ctype_is_c(collation))`
`67`	`78`	`returnisalpha(TOUCHAR(ptr));`
`68`	`79`
`69`		`-char2wchar(character,2,ptr,clen,mylocale);`
	`80`	`+char2wchar(character,WC_BUF_LEN,ptr,clen,mylocale);`
`70`	`81`
`71`	`82`	`returniswalpha((wint_t)character[0]);`
`72`	`83`	`}`
`@@ -75,14 +86,14 @@ int`
`75`	`86`	`t_isprint(constchar*ptr)`
`76`	`87`	`{`
`77`	`88`	`intclen=pg_mblen(ptr);`
`78`		`-wchar_tcharacter[2];`
	`89`	`+wchar_tcharacter[WC_BUF_LEN];`
`79`	`90`	`Oidcollation=DEFAULT_COLLATION_OID;/* TODO */`
`80`	`91`	`pg_locale_tmylocale=0;/* TODO */`
`81`	`92`
`82`	`93`	`if (clen==1\|\|lc_ctype_is_c(collation))`
`83`	`94`	`returnisprint(TOUCHAR(ptr));`
`84`	`95`
`85`		`-char2wchar(character,2,ptr,clen,mylocale);`
	`96`	`+char2wchar(character,WC_BUF_LEN,ptr,clen,mylocale);`
`86`	`97`
`87`	`98`	`returniswprint((wint_t)character[0]);`
`88`	`99`	`}`

0 commit comments

Comments

(0)

Movatterモバイル変換

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Commit33c697e

File tree

1 file changed

1 file changed

`‎src/backend/tsearch/ts_locale.c`

0 commit comments