Commit981cbd1

committed

Update tokenizer.

1 parentbd57c61 commit981cbd1Copy full SHA for 981cbd1

File tree

+492

-136

lines changed

+492

-136

lines changed

Lines changed: 5 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -398,7 +398,11 @@ def parseRCDataRawtext(self, token, contentType):`
`398`	`398`	`assertcontentTypein ("RAWTEXT","RCDATA")`
`399`	`399`
`400`	`400`	`element=self.tree.insertElement(token)`
`401`		`-self.tokenizer.contentModelFlag=contentModelFlags[contentType]`
	`401`	`+`
	`402`	`+ifcontentType=="RAWTEXT":`
	`403`	`+self.tokenizer.state=self.tokenizer.rawtextState`
	`404`	`+else:`
	`405`	`+self.tokenizer.state=self.tokenizer.rcdataState`
`402`	`406`
`403`	`407`	`self.originalPhase=self.phase`
`404`	`408`

Lines changed: 14 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -449,6 +449,20 @@ def charsUntil(self, characters, opposite = False):`
`449`	`449`	`r=u"".join(rv)`
`450`	`450`	`returnr`
`451`	`451`
	`452`	`+defcharsUntilEOF(self):`
	`453`	`+""" Returns a string of characters from the stream up to EOF."""`
	`454`	`+`
	`455`	`+rv= []`
	`456`	`+`
	`457`	`+whileTrue:`
	`458`	`+rv.append(self.chunk[self.chunkOffset:])`
	`459`	`+ifnotself.readChunk():`
	`460`	`+# Reached EOF`
	`461`	`+break`
	`462`	`+`
	`463`	`+r=u"".join(rv)`
	`464`	`+returnr`
	`465`	`+`
`452`	`466`	`defunget(self,char):`
`453`	`467`	`# Only one character is allowed to be ungotten at once - it must`
`454`	`468`	`# be consumed again before any further call to unget`

Comments

(0)