Commit911cf45

committed

Merge pull request#241 from gsnedders/encoding_tests

Get encoding reparsing actually working; r=nobody!

2 parents9dc49f6 +b0ae0c2 commit911cf45Copy full SHA for 911cf45

File tree

+28

-7

lines changed

+28

-7

lines changed

Lines changed: 5 additions & 6 deletions

Original file line number	Diff line number	Diff line change
`@@ -89,12 +89,11 @@ def _parse(self, stream, innerHTML=False, container="div",`
`89`	`89`	`parser=self,**kwargs)`
`90`	`90`	`self.reset()`
`91`	`91`
`92`		`-whileTrue:`
`93`		`-try:`
`94`		`-self.mainLoop()`
`95`		`-break`
`96`		`-exceptReparseException:`
`97`		`-self.reset()`
	`92`	`+try:`
	`93`	`+self.mainLoop()`
	`94`	`+exceptReparseException:`
	`95`	`+self.reset()`
	`96`	`+self.mainLoop()`
`98`	`97`
`99`	`98`	`defreset(self):`
`100`	`99`	`self.tree.reset()`

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -509,8 +509,8 @@ def changeEncoding(self, newEncoding):`
`509`	`509`	`self.charEncoding= (self.charEncoding[0],"certain")`
`510`	`510`	`else:`
`511`	`511`	`self.rawStream.seek(0)`
`512`		`-self.reset()`
`513`	`512`	`self.charEncoding= (newEncoding,"certain")`
	`513`	`+self.reset()`
`514`	`514`	`raiseReparseException("Encoding changed from %s to %s"% (self.charEncoding[0],newEncoding))`
`515`	`515`
`516`	`516`	`defdetectBOM(self):`

Lines changed: 22 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,28 @@`
`12`	`12`	`fromhtml5libimportHTMLParser,inputstream`
`13`	`13`
`14`	`14`
	`15`	`+deftest_basic_prescan_length():`
	`16`	`+data="<title>Caf\u00E9</title><!--a--><meta charset='utf-8'>".encode('utf-8')`
	`17`	`+pad=1024-len(data)+1`
	`18`	`+data=data.replace(b"-a-",b"-"+ (b"a"*pad)+b"-")`
	`19`	`+assertlen(data)==1024# Sanity`
	`20`	`+stream=inputstream.HTMLBinaryInputStream(data,chardet=False)`
	`21`	`+assert'utf-8'==stream.charEncoding[0].name`
	`22`	`+`
	`23`	`+`
	`24`	`+deftest_parser_reparse():`
	`25`	`+data="<title>Caf\u00E9</title><!--a--><meta charset='utf-8'>".encode('utf-8')`
	`26`	`+pad=10240-len(data)+1`
	`27`	`+data=data.replace(b"-a-",b"-"+ (b"a"*pad)+b"-")`
	`28`	`+assertlen(data)==10240# Sanity`
	`29`	`+stream=inputstream.HTMLBinaryInputStream(data,chardet=False)`
	`30`	`+assert'windows-1252'==stream.charEncoding[0].name`
	`31`	`+p=HTMLParser(namespaceHTMLElements=False)`
	`32`	`+doc=p.parse(data,useChardet=False)`
	`33`	`+assert'utf-8'==p.documentEncoding`
	`34`	`+assertdoc.find(".//title").text=="Caf\u00E9"`
	`35`	`+`
	`36`	`+`
`15`	`37`	`defrunParserEncodingTest(data,encoding):`
`16`	`38`	`p=HTMLParser()`
`17`	`39`	`assertp.documentEncodingisNone`

Comments

(0)