Commit980e9ad

committed

Fixed error when \r\n(EOF) falls across chunk boundary

--HG--extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%401159

1 parent66d242f commit980e9adCopy full SHA for 980e9ad

File tree

+14

-4

lines changed

+14

-4

lines changed

Lines changed: 9 additions & 4 deletions

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,8 @@ class HTMLInputStream(object):`
`24`	`24`
`25`	`25`	`"""`
`26`	`26`
	`27`	`+_defaultChunkSize=10240`
	`28`	`+`
`27`	`29`	`def__init__(self,source,encoding=None,parseMeta=True,chardet=True):`
`28`	`30`	`"""Initialises the HTMLInputStream.`
`29`	`31`
`@@ -260,11 +262,12 @@ def char(self):`
`260`	`262`	`self.readChars.append(char)`
`261`	`263`	`returnchar`
`262`	`264`
`263`		`-defreadChunk(self,chunkSize=10240):`
	`265`	`+defreadChunk(self,chunkSize=_defaultChunkSize):`
	`266`	`+self.chunk=u""`
	`267`	`+self.chunkOffset=0`
	`268`	`+`
`264`	`269`	`data=self.dataStream.read(chunkSize)`
`265`	`270`	`ifnotdata:`
`266`		`-self.chunk=u""`
`267`		`-self.chunkOffset=0`
`268`	`271`	`returnFalse`
`269`	`272`	`#Replace null characters`
`270`	`273`	`foriinxrange(data.count(u"\u0000")):`
`@@ -276,13 +279,15 @@ def readChunk(self, chunkSize=10240):`
`276`	`279`	`#Check for CR LF broken across chunks`
`277`	`280`	`if (self._lastChunkEndsWithCRanddata[0]=="\n"):`
`278`	`281`	`data=data[1:]`
	`282`	`+# Stop if the chunk is now empty`
	`283`	`+ifnotdata:`
	`284`	`+returnFalse`
`279`	`285`	`self._lastChunkEndsWithCR=data[-1]=="\r"`
`280`	`286`	`data=data.replace("\r\n","\n")`
`281`	`287`	`data=data.replace("\r","\n")`
`282`	`288`
`283`	`289`	`data=unicode(data)`
`284`	`290`	`self.chunk=data`
`285`		`-self.chunkOffset=0`
`286`	`291`
`287`	`292`	`self.updatePosition()`
`288`	`293`	`returnTrue`

Lines changed: 5 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -46,6 +46,11 @@ def test_newlines(self):`
`46`	`46`	`self.assertEquals(stream.charsUntil('e'),u"x")`
`47`	`47`	`self.assertEquals(stream.position(), (4,5))`
`48`	`48`
	`49`	`+deftest_newlines2(self):`
	`50`	`+size=HTMLInputStream._defaultChunkSize`
	`51`	`+stream=HTMLInputStream("\r"*size+"\n")`
	`52`	`+self.assertEquals(stream.charsUntil('x'),"\n"*size)`
	`53`	`+`
`49`	`54`	`deftest_position(self):`
`50`	`55`	`stream=HTMLInputStream(codecs.BOM_UTF8+"a\nbb\nccc\nddd")`
`51`	`56`	`self.assertEquals(stream.position(), (1,0))`

Comments

(0)