NotificationsYou must be signed in to change notification settings
Fork0
Star0

Commit9cc3c2a

committed

Fix for issue 143; deal with handling non-BMP codepoints in serialization to non-unicode encodings.

1 parent4fa9fda commit9cc3c2aCopy full SHA for 9cc3c2a

File tree

4 files changed

+42

-14

lines changed

src/html5lib
- inputstream.py
- serializer
  - htmlserializer.py
- utils.py
tests
- test_serializer.py

4 files changed

+42

-14

lines changed

`‎src/html5lib/inputstream.py‎`

Lines changed: 3 additions & 7 deletions

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`
`6`	`6`	`fromconstantsimportEOF,spaceCharacters,asciiLetters,asciiUppercase`
`7`	`7`	`fromconstantsimportencodings,ReparseException`
	`8`	`+importutils`
`8`	`9`
`9`	`10`	`#Non-unicode versions of constants for use in the pre-parser`
`10`	`11`	`spaceCharactersBytes=frozenset([str(item)foriteminspaceCharacters])`
`@@ -381,14 +382,9 @@ def characterErrorsUCS2(self, data):`
`381`	`382`	`codepoint=ord(match.group())`
`382`	`383`	`pos=match.start()`
`383`	`384`	`#Pretty sure there should be endianness issues here`
`384`		`-if (codepoint>=0xD800andcodepoint<=0xDBFFand`
`385`		`-pos<len(data)-1and`
`386`		`-ord(data[pos+1])>=0xDC00and`
`387`		`-ord(data[pos+1])<=0xDFFF):`
	`385`	`+ifutils.isSurrogatePair(data[pos:pos+2]):`
`388`	`386`	`#We have a surrogate pair!`
`389`		`-#From a perl manpage`
`390`		`-char_val= (0x10000+ (codepoint-0xD800)*0x400+`
`391`		`- (ord(data[pos+1])-0xDC00))`
	`387`	`+char_val=utils.surrogatePairToCodepoint(data[pos:pos+2])`
`392`	`388`	`ifchar_valinnon_bmp_invalid_codepoints:`
`393`	`389`	`self.errors.append("invalid-codepoint")`
`394`	`390`	`skip=True`

`‎src/html5lib/serializer/htmlserializer.py‎`

Lines changed: 18 additions & 5 deletions

Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@`
`9`	`9`
`10`	`10`	`fromhtml5lib.constantsimportvoidElements,booleanAttributes,spaceCharacters`
`11`	`11`	`fromhtml5lib.constantsimportrcdataElements,entities,xmlEntities`
`12`		`-`
	`12`	`+fromhtml5libimportutils`
`13`	`13`	`fromxml.sax.saxutilsimportescape`
`14`	`14`
`15`	`15`	`spaceCharacters=u"".join(spaceCharacters)`
`@@ -27,20 +27,33 @@`
`27`	`27`	`fork,vinentities.items():`
`28`	`28`	`ifv!="&"andencode_entity_map.get(v)!=k.lower():`
`29`	`29`	`# prefer < over &LT; and similarly for &, >, etc.`
`30`		`-encode_entity_map[v]=k`
	`30`	`+encode_entity_map[ord(v)]=k`
`31`	`31`
`32`	`32`	`defhtmlentityreplace_errors(exc):`
`33`	`33`	`ifisinstance(exc, (UnicodeEncodeError,UnicodeTranslateError)):`
`34`	`34`	`res= []`
`35`		`-forcinexc.object[exc.start:exc.end]:`
`36`		`-e=encode_entity_map.get(c)`
	`35`	`+codepoints= []`
	`36`	`+skip=False`
	`37`	`+fori,cinenumerate(exc.object[exc.start:exc.end]):`
	`38`	`+ifskip:`
	`39`	`+skip=False`
	`40`	`+continue`
	`41`	`+index=i+exc.start`
	`42`	`+ifutils.isSurrogatePair(exc.object[index:min([exc.end,index+2])]):`
	`43`	`+codepoint=utils.surrogatePairToCodepoint(exc.object[index:index+2])`
	`44`	`+skip=True`
	`45`	`+else:`
	`46`	`+codepoint=ord(c)`
	`47`	`+codepoints.append(codepoint)`
	`48`	`+forcpincodepoints:`
	`49`	`+e=encode_entity_map.get(cp)`
`37`	`50`	`ife:`
`38`	`51`	`res.append("&")`
`39`	`52`	`res.append(e)`
`40`	`53`	`ifnote.endswith(";"):`
`41`	`54`	`res.append(";")`
`42`	`55`	`else:`
`43`		`-res.append(c.encode(exc.encoding,"xmlcharrefreplace"))`
	`56`	`+res.append("&#x%s;"%(hex(cp)[2:]))`
`44`	`57`	`return (u"".join(res),exc.end)`
`45`	`58`	`else:`
`46`	`59`	`returnxmlcharrefreplace_errors(exc)`

`‎src/html5lib/utils.py‎`

Lines changed: 20 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -153,4 +153,23 @@ def __deepcopy__(self, memo={}):`
`153`	`153`	`result=self.__class__()`
`154`	`154`	`memo[id(self)]=result`
`155`	`155`	`result.__init__(deepcopy(tuple(self),memo))`
`156`		`-returnresult`
	`156`	`+returnresult`
	`157`	`+`
	`158`	`+#Some utility functions to dal with weirdness around UCS2 vs UCS4`
	`159`	`+#python builds`
	`160`	`+`
	`161`	`+defencodingType():`
	`162`	`+iflen()==2:`
	`163`	`+return"UCS2"`
	`164`	`+else:`
	`165`	`+return"UCS4"`
	`166`	`+`
	`167`	`+defisSurrogatePair(data):`
	`168`	`+return (len(data)==2and`
	`169`	`+ord(data[0])>=0xD800andord(data[0])<=0xDBFFand`
	`170`	`+ord(data[1])>=0xDC00andord(data[1])<=0xDFFF)`
	`171`	`+`
	`172`	`+defsurrogatePairToCodepoint(data):`
	`173`	`+char_val= (0x10000+ (ord(data[0])-0xD800)*0x400+`
	`174`	`+ (ord(data[1])-0xDC00))`
	`175`	`+returnchar_val`

`‎tests/test_serializer.py‎`

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -132,7 +132,7 @@ def buildTestSuite():`
`132`	`132`	`allTests.append(unittest.TestLoader().loadTestsFromTestCase(LxmlTestCase))`
`133`	`133`
`134`	`134`	`returnunittest.TestSuite(allTests)`
`135`		`-`
	`135`	`+`
`136`	`136`
`137`	`137`	`defmain():`
`138`	`138`	`buildTestSuite()`

0 commit comments

Comments

(0)

Movatterモバイル変換

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Commit9cc3c2a

File tree

4 files changed

4 files changed

`‎src/html5lib/inputstream.py‎`

`‎src/html5lib/serializer/htmlserializer.py‎`

`‎src/html5lib/utils.py‎`

`‎tests/test_serializer.py‎`

0 commit comments