CompositionExclusions.txtDerivedNormalizationProps.txtEastAsianWidth.txtNormalizationTest.txtUnicodeData.txt:$(top_builddir)/src/Makefile.global

CompositionExclusions.txtDerivedCoreProperties.txtDerivedNormalizationProps.txtEastAsianWidth.txtNormalizationTest.txtPropList.txtUnicodeData.txt:$(top_builddir)/src/Makefile.global

$(DOWNLOAD) https://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/$(@F)

unicode_version.h: generate-unicode_version.pl

$(PERL)$< --version$(UNICODE_VERSION)

unicode_category_table.h: generate-unicode_category_table.pl UnicodeData.txt

unicode_category_table.h: generate-unicode_category_table.plDerivedCoreProperties.txt PropList.txtUnicodeData.txt

rm -f$(OBJS) category_test category_test.o norm_test norm_test.o

distclean: clean

rm -f CompositionExclusions.txt DerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt UnicodeData.txt norm_test_table.h unicode_category_table.h unicode_norm_table.h

rm -f CompositionExclusions.txtDerivedCoreProperties.txtDerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt PropList.txt UnicodeData.txt norm_test_table.h unicode_category_table.h unicode_norm_table.h

This directory contains tools to generate the tables in

src/include/common/unicode_norm.h, used for Unicode normalization. The

generated .h file is included in the source tree, so these are normally not

needed to build PostgreSQL, only if you need to re-generate the .h file

from the Unicode data files for some reason, e.g. to update to a new version

of Unicode.

This directory contains tools to download new Unicode data files and

generate static tables. These tables are used to normalize or

determine various properties of Unicode data.

Generating unicode_norm_table.h

-------------------------------

The generated header files are copied to src/include/common/, and

included in the source tree, so these tools are not normally required

to build PostgreSQL.

Run

Update Unicode Version

----------------------

Edit src/Makefile.global.in and src/common/unicode/meson.build

to update the UNICODE_VERSION.

Then, generate the new header files with:

make update-unicode

from the top level of the source tree and commit the result.

or if using meson:

ninja update-unicode

from the top level of the source tree. Examine the result to make sure

the changes look reasonable (that is, that the diff size and scope is

comparable to the Unicode changes since the last update), and then

commit it.

Tests

-----

Normalization tests:

The Unicode consortium publishes a comprehensive test suite for the

normalization algorithm, in a file called NormalizationTest.txt. This

directory also contains a perl script and some C code, to run our

make normalization-check

This is also run as part of the update-unicode target.

Category & Property tests:

The file category_test.c exhaustively compares the category and

properties of each code point as determined by the generated tables

with the category and properties as reported by ICU. For this test to

be effective, the version of the Unicode data files must be similar to

the version of Unicode on which ICU is based, so attempt to match the

versions as closely as possible. A mismatched Unicode will skip over

codepoints that are assigned in one version and not the other, and may

falsely report failures. This test is run as a part of the

update-unicode target.

staticintpg_unicode_version=0;

staticinticu_unicode_version=0;

parse_unicode_version(constchar*version)

{

returnmajor*100+minor;

}

main(intargc,char**argv)

icu_test()

{

intpg_unicode_version=parse_unicode_version(PG_UNICODE_VERSION);

inticu_unicode_version=parse_unicode_version(U_UNICODE_VERSION);

intsuccessful=0;

intpg_skipped_codepoints=0;

inticu_skipped_codepoints=0;

printf("category_test: Postgres Unicode version:\t%s\n",PG_UNICODE_VERSION);

printf("category_test: ICU Unicode version:\t\t%s\n",U_UNICODE_VERSION);

for (UChar32code=0;code <=0x10ffff;code++)

for (pg_wcharcode=0;code <=0x10ffff;code++)

{

uint8_tpg_category=unicode_category(code);

uint8_ticu_category=u_charType(code);

boolprop_alphabetic=pg_u_prop_alphabetic(code);

boolprop_lowercase=pg_u_prop_lowercase(code);

boolprop_uppercase=pg_u_prop_uppercase(code);

boolprop_cased=pg_u_prop_cased(code);

boolprop_case_ignorable=pg_u_prop_case_ignorable(code);

boolprop_white_space=pg_u_prop_white_space(code);

boolprop_hex_digit=pg_u_prop_hex_digit(code);

boolprop_join_control=pg_u_prop_join_control(code);

boolicu_prop_alphabetic=u_hasBinaryProperty(

code,UCHAR_ALPHABETIC);

boolicu_prop_lowercase=u_hasBinaryProperty(

code,UCHAR_LOWERCASE);

boolicu_prop_uppercase=u_hasBinaryProperty(

code,UCHAR_UPPERCASE);

boolicu_prop_cased=u_hasBinaryProperty(

code,UCHAR_CASED);

boolicu_prop_case_ignorable=u_hasBinaryProperty(

code,UCHAR_CASE_IGNORABLE);

boolicu_prop_white_space=u_hasBinaryProperty(

code,UCHAR_WHITE_SPACE);

boolicu_prop_hex_digit=u_hasBinaryProperty(

code,UCHAR_HEX_DIGIT);

boolicu_prop_join_control=u_hasBinaryProperty(

code,UCHAR_JOIN_CONTROL);

boolisalpha=pg_u_isalpha(code);

boolislower=pg_u_islower(code);

boolisupper=pg_u_isupper(code);

boolispunct=pg_u_ispunct(code, false);

boolisdigit=pg_u_isdigit(code, false);

boolisxdigit=pg_u_isxdigit(code, false);

boolisalnum=pg_u_isalnum(code, false);

boolisspace=pg_u_isspace(code);

boolisblank=pg_u_isblank(code);

booliscntrl=pg_u_iscntrl(code);

boolisgraph=pg_u_isgraph(code);

boolisprint=pg_u_isprint(code);

boolicu_isalpha=u_isUAlphabetic(code);

boolicu_islower=u_isULowercase(code);

boolicu_isupper=u_isUUppercase(code);

boolicu_ispunct=u_ispunct(code);

boolicu_isdigit=u_isdigit(code);

boolicu_isxdigit=u_hasBinaryProperty(code,

UCHAR_POSIX_XDIGIT);

boolicu_isalnum=u_hasBinaryProperty(code,

UCHAR_POSIX_ALNUM);

boolicu_isspace=u_isUWhiteSpace(code);

boolicu_isblank=u_isblank(code);

boolicu_iscntrl=icu_category==PG_U_CONTROL;

boolicu_isgraph=u_hasBinaryProperty(code,

UCHAR_POSIX_GRAPH);

boolicu_isprint=u_hasBinaryProperty(code,

UCHAR_POSIX_PRINT);

if (pg_category==PG_U_UNASSIGNED&&

pg_unicode_version<icu_unicode_version)

{

pg_skipped_codepoints++;

continue;

}

if (icu_category==PG_U_UNASSIGNED&&

icu_unicode_version<pg_unicode_version)

{

icu_skipped_codepoints++;

continue;

}

if (pg_category!=icu_category)

{

if (pg_category==PG_U_UNASSIGNED&&

pg_unicode_version<icu_unicode_version)

pg_skipped_codepoints++;

elseif (icu_category==PG_U_UNASSIGNED&&

icu_unicode_version<pg_unicode_version)

icu_skipped_codepoints++;

{

printf("category_test: FAILURE for codepoint 0x%06x\n",code);

printf("category_test: Postgres category:%02d %s %s\n",pg_category,

unicode_category_abbrev(pg_category),

unicode_category_string(pg_category));

printf("category_test: ICU category:%02d %s %s\n",icu_category,

unicode_category_abbrev(icu_category),

unicode_category_string(icu_category));

printf("\n");

exit(1);

}

printf("category_test: FAILURE for codepoint 0x%06x\n",code);

printf("category_test: Postgres category:%02d %s %s\n",pg_category,

unicode_category_abbrev(pg_category),

unicode_category_string(pg_category));

printf("category_test: ICU category:%02d %s %s\n",icu_category,

unicode_category_abbrev(icu_category),

unicode_category_string(icu_category));

printf("\n");

exit(1);

}

if (prop_alphabetic!=icu_prop_alphabetic||

prop_join_control!=icu_prop_join_control)

{

printf("category_test: FAILURE for codepoint 0x%06x\n",code);

printf("category_test: Postgrespropertyalphabetic/lowercase/uppercase/cased/case_ignorable/white_space/hex_digit/join_control: %d/%d/%d/%d/%d/%d/%d/%d\n",

prop_alphabetic,prop_lowercase,prop_uppercase,

prop_cased,prop_case_ignorable,

prop_white_space,prop_hex_digit,prop_join_control);

printf("category_test: ICUpropertyalphabetic/lowercase/uppercase/cased/case_ignorable/white_space/hex_digit/join_control: %d/%d/%d/%d/%d/%d/%d/%d\n",

icu_prop_alphabetic,icu_prop_lowercase,icu_prop_uppercase,

icu_prop_cased,icu_prop_case_ignorable,

icu_prop_white_space,icu_prop_hex_digit,icu_prop_join_control);

printf("\n");

exit(1);

}

if (isalpha!=icu_isalpha||

isprint!=icu_isprint)

{

printf("category_test: FAILURE for codepoint 0x%06x\n",code);

printf("category_test: Postgresclassalpha/lower/upper/punct/digit/xdigit/alnum/space/blank/cntrl/graph/print: %d/%d/%d/%d/%d/%d/%d/%d/%d/%d/%d/%d\n",

isalpha,islower,isupper,ispunct,isdigit,isxdigit,isalnum,isspace,isblank,iscntrl,isgraph,isprint);

printf("category_test: ICU classalpha/lower/upper/punct/digit/xdigit/alnum/space/blank/cntrl/graph/print: %d/%d/%d/%d/%d/%d/%d/%d/%d/%d/%d/%d\n",

icu_isalpha,icu_islower,icu_isupper,icu_ispunct,icu_isdigit,icu_isxdigit,icu_isalnum,icu_isspace,icu_isblank,icu_iscntrl,icu_isgraph,icu_isprint);

printf("\n");

exit(1);

}

if (pg_category!=PG_U_UNASSIGNED)

successful++;

}

if (pg_skipped_codepoints>0)

printf("category_test: skipped %d codepoints unassigned in ICU due to Unicode version mismatch\n",

icu_skipped_codepoints);

printf("category_test: success\n");

exit(0);

printf("category_test: ICU test: %d codepoints successful\n",successful);

}

main(intargc,char**argv)

{

pg_unicode_version=parse_unicode_version(PG_UNICODE_VERSION);

printf("category_test: Postgres Unicode version:\t%s\n",PG_UNICODE_VERSION);

icu_unicode_version=parse_unicode_version(U_UNICODE_VERSION);

printf("category_test: ICU Unicode version:\t\t%s\n",U_UNICODE_VERSION);

icu_test();

printf("category_test: ICU support required for test; skipping\n");

exit(0);

printf("category_test: ICU not available; skipping\n");

}