Commit4347629

committed

Don't handle noncharacters differently than other unassigned codepoints

1 parent0b13808 commit4347629Copy full SHA for 4347629

File tree

-31

lines changed

-31

lines changed

Lines changed: 1 addition & 9 deletions

Original file line number	Diff line number	Diff line change
`@@ -106,7 +106,7 @@ def _load_unicode_data(self):`
`106`	`106`
`107`	`107`	`# Characters that cannot be part of a combining character sequence:`
`108`	`108`	`# control characters, format characters other than ZWJ and ZWNJ,`
`109`		`-# the line and paragraph separators, and noncharacters.`
	`109`	`+#andthe line and paragraph separators.`
`110`	`110`	`self.not_in_ccs= []`
`111`	`111`
`112`	`112`	`assigned_start=0;`
`@@ -147,14 +147,6 @@ def _load_unicode_data(self):`
`147`	`147`
`148`	`148`	`self.general_category_public_assigned.append((assigned_start,prev_char_int))`
`149`	`149`
`150`		`-# Mark noncharacters as nongraphic`
`151`		`-foriinrange(0xFDD0,0xFDF0):`
`152`		`-self.not_in_ccs.append(i)`
`153`		`-forprefixinrange(0,0x11):`
`154`		`-shifted=prefix<<16`
`155`		`-self.not_in_ccs.append(shifted\|0xFFFE)`
`156`		`-self.not_in_ccs.append(shifted\|0xFFFF)`
`157`		`-`
`158`	`150`	`self.not_in_ccs.sort()`
`159`	`151`
`160`	`152`	`def_load_default_ignorable_marks(self):`

Lines changed: 1 addition & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -41,8 +41,7 @@ impl CcsKind {`
`41`	`41`	`/// [defective combining character sequences](https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#I6.1.36487)`
`42`	`42`	`/// by inserting U+00A0 NO-BREAK SPACE in front of them.`
`43`	`43`	`///`
`44`		`-/// For the purposes of this iterator, private use characters,`
`45`		`-/// as well as unassigned codepoints other than noncharacters,`
	`44`	`+/// For the purposes of this iterator, private use characters and unassigned codepoints`
`46`	`45`	`/// are considered valid base characters,`
`47`	`46`	`/// so combining character sequences that follow such will not be modified.`
`48`	`47`	`///`

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -139,7 +139,7 @@ pub trait UnicodeNormalization<I: Iterator<Item = char>> {`
`139`	`139`	`/// with the correct advance width,`
`140`	`140`	`/// in diverse contexts (for example, when printed to a terminal).`
`141`	`141`	`///`
`142`		`-/// Sequences following a private use character or an unassigned codepoint that is not a noncharacter`
	`142`	`+/// Sequences following a private use character or an unassigned codepoint`
`143`	`143`	`/// are not corrected. Additionally, combining character sequences consisting entirely of`
`144`	`144`	`/// [default-ignorable code points](https://www.unicode.org/versions/Unicode15.0.0/UnicodeStandard-15.0.pdf#I8.1.40715)`
`145`	`145`	`/// are also left untouched. Handling this last case may require the iterator`

Lines changed: 1 addition & 19 deletions

Comments

(0)