Commit2a332b9

committed

Save some calls to len() in the input stream (improves tokeniser performance by maybe 1-2%)

--HG--extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%401234

1 parent43a2727 commit2a332b9Copy full SHA for 2a332b9

File tree

-2

lines changed

-2

lines changed

Lines changed: 5 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -70,6 +70,7 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):`
`70`	`70`	`'replace')`
`71`	`71`
`72`	`72`	`self.chunk=u""`
	`73`	`+self.chunkSize=0`
`73`	`74`	`self.chunkOffset=0`
`74`	`75`	`self.ungetBuffer= []# reversed list of chars from unget()`
`75`	`76`	`self.readChars= []`
`@@ -255,7 +256,7 @@ def char(self):`
`255`	`256`	`self.readChars.append(char)`
`256`	`257`	`returnchar`
`257`	`258`
`258`		`-ifself.chunkOffset>=len(self.chunk):`
	`259`	`+ifself.chunkOffset>=self.chunkSize:`
`259`	`260`	`ifnotself.readChunk():`
`260`	`261`	`returnEOF`
`261`	`262`
`@@ -267,6 +268,7 @@ def char(self):`
`267`	`268`
`268`	`269`	`defreadChunk(self,chunkSize=_defaultChunkSize):`
`269`	`270`	`self.chunk=u""`
	`271`	`+self.chunkSize=0`
`270`	`272`	`self.chunkOffset=0`
`271`	`273`
`272`	`274`	`data=self.dataStream.read(chunkSize)`
`@@ -291,6 +293,7 @@ def readChunk(self, chunkSize=_defaultChunkSize):`
`291`	`293`
`292`	`294`	`data=unicode(data)`
`293`	`295`	`self.chunk=data`
	`296`	`+self.chunkSize=len(data)`
`294`	`297`
`295`	`298`	`self.updatePosition()`
`296`	`299`	`returnTrue`
`@@ -329,7 +332,7 @@ def charsUntil(self, characters, opposite = False):`
`329`	`332`	`m=chars.match(self.chunk,self.chunkOffset)`
`330`	`333`	`# If not everything matched, return everything up to the part that didn't match`
`331`	`334`	`end=m.end()`
`332`		`-ifend!=len(self.chunk):`
	`335`	`+ifend!=self.chunkSize:`
`333`	`336`	`rv.append(self.chunk[self.chunkOffset:end])`
`334`	`337`	`self.chunkOffset=end`
`335`	`338`	`break`

Comments

(0)