Commitfa10dd3

committed

Fetch and generate sentence tests, property table

1 parent6fc6815 commitfa10dd3Copy full SHA for fa10dd3

File tree

+1323

-0

lines changed

+1323

-0

lines changed

Lines changed: 7 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -351,3 +351,10 @@ def emit_break_module(f, break_table, break_cats, name):`
`351`	`351`	`word_table.extend([(x,y,cat)for (x,y)inword_cats[cat]])`
`352`	`352`	`word_table.sort(key=lambdaw:w[0])`
`353`	`353`	`emit_break_module(rf,word_table,word_cats.keys(),"word")`
	`354`	`+`
	`355`	`+sentence_cats=load_properties("auxiliary/SentenceBreakProperty.txt", [])`
	`356`	`+sentence_table= []`
	`357`	`+forcatinsentence_cats:`
	`358`	`+sentence_table.extend([(x,y,cat)for (x,y)insentence_cats[cat]])`
	`359`	`+sentence_table.sort(key=lambdaw:w[0])`
	`360`	`+emit_break_module(rf,sentence_table,sentence_cats.keys(),"sentence")`

Lines changed: 15 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -190,8 +190,23 @@ def create_words_data(f):`
`190`	`190`	`f.write(" // http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakTest.txt\n")`
`191`	`191`	`unicode.emit_table(f,"TEST_WORD",test,wtype,True,showfun,True)`
`192`	`192`
	`193`	`+defcreate_sentence_data(f):`
	`194`	`+d=load_test_data("auxiliary/SentenceBreakTest.txt")`
	`195`	`+`
	`196`	`+test= []`
	`197`	`+`
	`198`	`+for (c,i)ind:`
	`199`	`+allchars= [cnforsincforcnins]`
	`200`	`+test.append((allchars,c))`
	`201`	`+`
	`202`	`+wtype="&'static [(&'static str, &'static [&'static str])]"`
	`203`	`+f.write(" // official Unicode test data\n")`
	`204`	`+f.write(" // http://www.unicode.org/Public/UNIDATA/auxiliary/SentenceBreakTest.txt\n")`
	`205`	`+unicode.emit_table(f,"TEST_SENTENCE",test,wtype,True,showfun,True)`
	`206`	`+`
`193`	`207`	`if__name__=="__main__":`
`194`	`208`	`withopen("testdata.rs","w")asrf:`
`195`	`209`	`rf.write(unicode.preamble)`
`196`	`210`	`create_grapheme_data(rf)`
`197`	`211`	`create_words_data(rf)`
	`212`	`+create_sentence_data(rf)`

Comments

(0)