Dec 29, 2020 · Dec 29, 2020 · Dec 29, 2020 · Dec 30, 2020 · Dec 30, 2020 · Dec 29, 2020
diff --git a/html5lib/_tokenizer.py b/html5lib/_tokenizer.py
diff --git a/html5lib/constants.py b/html5lib/constants.py
    0x9F: "\u0178",
 }

 tokenTypes = {
    "Doctype": 0,
    "Characters": 1,
    "SpaceCharacters": 2,
    "StartTag": 3,
    "EndTag": 4,
    "EmptyTag": 5,
    "Comment": 6,
    "ParseError": 7
 }

 tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
                           tokenTypes["EmptyTag"]])


 prefixes = {v: k for k, v in namespaces.items()}
 prefixes["http://www.w3.org/1998/Math/MathML"] = "math"
diff --git a/html5lib/html5parser.py b/html5lib/html5parser.py
diff --git a/html5lib/tests/test_tokenizer2.py b/html5lib/tests/test_tokenizer2.py

 from six import unichr, text_type

 from html5lib._tokenizer import HTMLTokenizer
 from html5lib.constants import tokenTypes
 from html5lib._tokenizer import HTMLTokenizer, ParseError, StartTag


 def ignore_parse_errors(toks):
    for tok in toks:
        if tok['type'] != tokenTypes['ParseError']:
        ifnot isinstance(tok,ParseError):
            yield tok


    out = list(ignore_parse_errors(toks))

    assert len(out) == 1
    assert out[0]['type'] == tokenTypes['StartTag']
    assertisinstance(out[0],StartTag)

    attrs_tok = out[0]['data']
    attrs_tok = out[0].attributes
    assert len(attrs_tok) == len(attrs)

    for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):
    out = list(ignore_parse_errors(toks))

    assert len(out) == 1
    assert out[0]['type'] == tokenTypes['StartTag']
    assertisinstance(out[0],StartTag)

    attrs_tok = out[0]['data']
    attrs_tok = out[0].attributes
    assert len(attrs_tok) == 1
    assert list(attrs_tok.items()) == [('a', '1')]

    out = list(ignore_parse_errors(toks))

    assert len(out) == 1
    assert out[0]['type'] == tokenTypes['StartTag']
    assertisinstance(out[0],StartTag)

    attrs_tok = out[0]['data']
    attrs_tok = out[0].attributes
    assert len(attrs_tok) == len(attrs)

    for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):
diff --git a/html5lib/tests/tokenizer.py b/html5lib/tests/tokenizer.py
 import pytest
 from six import unichr

 from html5lib._tokenizer import HTMLTokenizer
 from html5lib._tokenizer import HTMLTokenizer, StartTag
 from html5lib import constants, _utils



        tokenizer.state = getattr(tokenizer, self._state)
        if self._lastStartTag is not None:
            tokenizer.currentToken = {"type": "startTag",
                                      "name": self._lastStartTag}
            tokenizer.currentToken = StartTag(name=self._lastStartTag)

        types = {v: k for k, v in constants.tokenTypes.items()}
        for token in tokenizer:
            getattr(self, 'process%s' %types[token["type"]])(token)
            getattr(self, 'process%s' % token.__class__.__name__)(token)

        return self.outputTokens

    def processDoctype(self, token):
        self.outputTokens.append(["DOCTYPE", token["name"], token["publicId"],
                                  token["systemId"], token["correct"]])
        self.outputTokens.append(["DOCTYPE", token.name, token.publicId,
                                  token.systemId, token.correct])

    def processStartTag(self, token):
        self.outputTokens.append(["StartTag", token["name"],
                                  token["data"], token["selfClosing"]])
        self.outputTokens.append(["StartTag", token.name,
                                  token.attributes, token.self_closing])

    def processEmptyTag(self, token):
        if token["name"] not in constants.voidElements:
        if token.name not in constants.voidElements:
            self.outputTokens.append("ParseError")
        self.outputTokens.append(["StartTag", token["name"], dict(token["data"][::-1])])
        self.outputTokens.append(["StartTag", token.name,token.attributes])

    def processEndTag(self, token):
        self.outputTokens.append(["EndTag", token["name"],
                                  token["selfClosing"]])
        self.outputTokens.append(["EndTag", token.name, token.self_closing])

    def processComment(self, token):
        self.outputTokens.append(["Comment", token["data"]])
        self.outputTokens.append(["Comment", token.data])

    def processSpaceCharacters(self, token):
        self.outputTokens.append(["Character", token["data"]])
        self.outputTokens.append(["Character", token.data])
        self.processSpaceCharacters = self.processCharacters

    def processCharacters(self, token):
        self.outputTokens.append(["Character", token["data"]])
        self.outputTokens.append(["Character", token.data])

    def processEOF(self, token):
        pass

    def processParseError(self, token):
        self.outputTokens.append(["ParseError", token["data"]])
        self.outputTokens.append(["ParseError", token.data])


 def concatenateCharacterTokens(tokens):
diff --git a/html5lib/treebuilders/base.py b/html5lib/treebuilders/base.py
 from six import text_type

 from ..constants import scopingElements, tableInsertModeElements, namespaces
 from .._tokenizer import StartTag

 # The scope markers are inserted when entering object elements,
 # marquees, table cells, and table captions, and are used to prevent formatting
            clone = entry.cloneNode()  # Mainly to get a new copy of the attributes

            # Step 9
            element = self.insertElement({"type": "StartTag",
                                          "name": clone.name,
                                          "namespace": clone.namespace,
                                          "data": clone.attributes})
            tag = StartTag(name=clone.name, data=clone.attributes)
            tag.namespace = clone.namespace
            element = self.insertElement(tag)

            # Step 10
            self.activeFormattingElements[i] = element
        self.document.appendChild(element)

    def insertDoctype(self, token):
        name = token["name"]
        publicId = token["publicId"]
        systemId = token["systemId"]
        name = token.name
        publicId = token.publicId
        systemId = token.systemId

        doctype = self.doctypeClass(name, publicId, systemId)
        self.document.appendChild(doctype)

    def insertComment(self, token, parent=None):
        if parent is None:
            parent = self.openElements[-1]
        parent.appendChild(self.commentClass(token["data"]))
        parent.appendChild(self.commentClass(token.data))

    def createElement(self, token):
        """Create an element but don't insert it anywhere"""
        name = token["name"]
        namespace = token.get("namespace",self.defaultNamespace)
        name = token.name
        namespace = token.namespace orself.defaultNamespace
        element = self.elementClass(name, namespace)
        element.attributes = token["data"]
        element.attributes = token.attributes
        return element

    def _getInsertFromTable(self):
    insertFromTable = property(_getInsertFromTable, _setInsertFromTable)

    def insertElementNormal(self, token):
        name = token["name"]
        name = token.name
        assert isinstance(name, text_type), "Element %s not unicode" % name
        namespace = token.get("namespace",self.defaultNamespace)
        namespace = token.namespace orself.defaultNamespace
        element = self.elementClass(name, namespace)
        element.attributes = token["data"]
        element.attributes = token.attributes
        self.openElements[-1].appendChild(element)
        self.openElements.append(element)
        return element
diff --git a/html5lib/treebuilders/dom.py b/html5lib/treebuilders/dom.py
            return weakref.proxy(self)

        def insertDoctype(self, token):
            name = token["name"]
            publicId = token["publicId"]
            systemId = token["systemId"]
            name = token.name
            publicId = token.publicId
            systemId = token.systemId

            domimpl = Dom.getDOMImplementation()
            doctype = domimpl.createDocumentType(name, publicId, systemId)
diff --git a/html5lib/treebuilders/etree_lxml.py b/html5lib/treebuilders/etree_lxml.py
        return fragment

    def insertDoctype(self, token):
        name = token["name"]
        publicId = token["publicId"]
        systemId = token["systemId"]
        name = token.name
        publicId = token.publicId
        systemId = token.systemId

        if not name:
            warnings.warn("lxml cannot represent empty doctype", DataLossWarning)
                else:
                    docStr += "''"
            docStr += ">"
            if self.doctype.name != token["name"]:
            if self.doctype.name != token.name:
                warnings.warn("lxml cannot represent doctype with a different name to the root element", DataLossWarning)
        docStr += "<THIS_SHOULD_NEVER_APPEAR_PUBLICLY/>"
        root = etree.fromstring(docStr)

        # Append the initial comments:
        for comment_token in self.initial_comments:
            comment = self.commentClass(comment_token["data"])
            comment = self.commentClass(comment_token.data)
            root.addprevious(comment._element)

        # Create the root document and add the ElementTree to it
        self.document = self.documentClass()
        self.document._elementTree = root.getroottree()

        # Give the root element the right name
        name = token["name"]
        namespace = token.get("namespace",self.defaultNamespace)
        name = token.name
        namespace = token.namespace orself.defaultNamespace
        if namespace is None:
            etree_tag = name
        else:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2918,20 +2918,6 @@
		0x9F: "\u0178",
		}

		tokenTypes = {
		"Doctype": 0,
		"Characters": 1,
		"SpaceCharacters": 2,
		"StartTag": 3,
		"EndTag": 4,
		"EmptyTag": 5,
		"Comment": 6,
		"ParseError": 7
		}

		tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
		tokenTypes["EmptyTag"]])


		prefixes = {v: k for k, v in namespaces.items()}
		prefixes["http://www.w3.org/1998/Math/MathML"] = "math"
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,13 +4,12 @@

		from six import unichr, text_type

		from html5lib._tokenizer import HTMLTokenizer
		from html5lib.constants import tokenTypes
		from html5lib._tokenizer import HTMLTokenizer, ParseError, StartTag


		def ignore_parse_errors(toks):
		for tok in toks:
		if tok['type'] != tokenTypes['ParseError']:
		ifnot isinstance(tok,ParseError):
		yield tok


Expand All		@@ -23,9 +22,9 @@ def test_maintain_attribute_order():
		out = list(ignore_parse_errors(toks))

		assert len(out) == 1
		assert out[0]['type'] == tokenTypes['StartTag']
		assertisinstance(out[0],StartTag)

		attrs_tok = out[0]['data']
		attrs_tok = out[0].attributes
		assert len(attrs_tok) == len(attrs)

		for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):
Expand All		@@ -40,9 +39,9 @@ def test_duplicate_attribute():
		out = list(ignore_parse_errors(toks))

		assert len(out) == 1
		assert out[0]['type'] == tokenTypes['StartTag']
		assertisinstance(out[0],StartTag)

		attrs_tok = out[0]['data']
		attrs_tok = out[0].attributes
		assert len(attrs_tok) == 1
		assert list(attrs_tok.items()) == [('a', '1')]

Expand All		@@ -56,9 +55,9 @@ def test_maintain_duplicate_attribute_order():
		out = list(ignore_parse_errors(toks))

		assert len(out) == 1
		assert out[0]['type'] == tokenTypes['StartTag']
		assertisinstance(out[0],StartTag)

		attrs_tok = out[0]['data']
		attrs_tok = out[0].attributes
		assert len(attrs_tok) == len(attrs)

		for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,7 +8,7 @@
		import pytest
		from six import unichr

		from html5lib._tokenizer import HTMLTokenizer
		from html5lib._tokenizer import HTMLTokenizer, StartTag
		from html5lib import constants, _utils


Expand All		@@ -25,47 +25,44 @@ def parse(self, stream, encoding=None, innerHTML=False):

		tokenizer.state = getattr(tokenizer, self._state)
		if self._lastStartTag is not None:
		tokenizer.currentToken = {"type": "startTag",
		"name": self._lastStartTag}
		tokenizer.currentToken = StartTag(name=self._lastStartTag)

		types = {v: k for k, v in constants.tokenTypes.items()}
		for token in tokenizer:
		getattr(self, 'process%s' %types[token["type"]])(token)
		getattr(self, 'process%s' % token.__class__.__name__)(token)

		return self.outputTokens

		def processDoctype(self, token):
		self.outputTokens.append(["DOCTYPE", token["name"], token["publicId"],
		token["systemId"], token["correct"]])
		self.outputTokens.append(["DOCTYPE", token.name, token.publicId,
		token.systemId, token.correct])

		def processStartTag(self, token):
		self.outputTokens.append(["StartTag", token["name"],
		token["data"], token["selfClosing"]])
		self.outputTokens.append(["StartTag", token.name,
		token.attributes, token.self_closing])

		def processEmptyTag(self, token):
		if token["name"] not in constants.voidElements:
		if token.name not in constants.voidElements:
		self.outputTokens.append("ParseError")
		self.outputTokens.append(["StartTag", token["name"], dict(token["data"][::-1])])
		self.outputTokens.append(["StartTag", token.name,token.attributes])

		def processEndTag(self, token):
		self.outputTokens.append(["EndTag", token["name"],
		token["selfClosing"]])
		self.outputTokens.append(["EndTag", token.name, token.self_closing])

		def processComment(self, token):
		self.outputTokens.append(["Comment", token["data"]])
		self.outputTokens.append(["Comment", token.data])

		def processSpaceCharacters(self, token):
		self.outputTokens.append(["Character", token["data"]])
		self.outputTokens.append(["Character", token.data])
		self.processSpaceCharacters = self.processCharacters

		def processCharacters(self, token):
		self.outputTokens.append(["Character", token["data"]])
		self.outputTokens.append(["Character", token.data])

		def processEOF(self, token):
		pass

		def processParseError(self, token):
		self.outputTokens.append(["ParseError", token["data"]])
		self.outputTokens.append(["ParseError", token.data])


		def concatenateCharacterTokens(tokens):
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,6 +2,7 @@
		from six import text_type

		from ..constants import scopingElements, tableInsertModeElements, namespaces
		from .._tokenizer import StartTag

		# The scope markers are inserted when entering object elements,
		# marquees, table cells, and table captions, and are used to prevent formatting
Expand DownExpand Up		@@ -249,10 +250,9 @@ def reconstructActiveFormattingElements(self):
		clone = entry.cloneNode() # Mainly to get a new copy of the attributes

		# Step 9
		element = self.insertElement({"type": "StartTag",
		"name": clone.name,
		"namespace": clone.namespace,
		"data": clone.attributes})
		tag = StartTag(name=clone.name, data=clone.attributes)
		tag.namespace = clone.namespace
		element = self.insertElement(tag)

		# Step 10
		self.activeFormattingElements[i] = element
Expand DownExpand Up		@@ -286,24 +286,24 @@ def insertRoot(self, token):
		self.document.appendChild(element)

		def insertDoctype(self, token):
		name = token["name"]
		publicId = token["publicId"]
		systemId = token["systemId"]
		name = token.name
		publicId = token.publicId
		systemId = token.systemId

		doctype = self.doctypeClass(name, publicId, systemId)
		self.document.appendChild(doctype)

		def insertComment(self, token, parent=None):
		if parent is None:
		parent = self.openElements[-1]
		parent.appendChild(self.commentClass(token["data"]))
		parent.appendChild(self.commentClass(token.data))

		def createElement(self, token):
		"""Create an element but don't insert it anywhere"""
		name = token["name"]
		namespace = token.get("namespace",self.defaultNamespace)
		name = token.name
		namespace = token.namespace orself.defaultNamespace
		element = self.elementClass(name, namespace)
		element.attributes = token["data"]
		element.attributes = token.attributes
		return element

		def _getInsertFromTable(self):
Expand All		@@ -321,11 +321,11 @@ def _setInsertFromTable(self, value):
		insertFromTable = property(_getInsertFromTable, _setInsertFromTable)

		def insertElementNormal(self, token):
		name = token["name"]
		name = token.name
		assert isinstance(name, text_type), "Element %s not unicode" % name
		namespace = token.get("namespace",self.defaultNamespace)
		namespace = token.namespace orself.defaultNamespace
		element = self.elementClass(name, namespace)
		element.attributes = token["data"]
		element.attributes = token.attributes
		self.openElements[-1].appendChild(element)
		self.openElements.append(element)
		return element
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -126,9 +126,9 @@ def documentClass(self):
		return weakref.proxy(self)

		def insertDoctype(self, token):
		name = token["name"]
		publicId = token["publicId"]
		systemId = token["systemId"]
		name = token.name
		publicId = token.publicId
		systemId = token.systemId

		domimpl = Dom.getDOMImplementation()
		doctype = domimpl.createDocumentType(name, publicId, systemId)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -308,9 +308,9 @@ def getFragment(self):
		return fragment

		def insertDoctype(self, token):
		name = token["name"]
		publicId = token["publicId"]
		systemId = token["systemId"]
		name = token.name
		publicId = token.publicId
		systemId = token.systemId

		if not name:
		warnings.warn("lxml cannot represent empty doctype", DataLossWarning)
Expand DownExpand Up		@@ -359,23 +359,23 @@ def insertRoot(self, token):
		else:
		docStr += "''"
		docStr += ">"
		if self.doctype.name != token["name"]:
		if self.doctype.name != token.name:
		warnings.warn("lxml cannot represent doctype with a different name to the root element", DataLossWarning)
		docStr += "<THIS_SHOULD_NEVER_APPEAR_PUBLICLY/>"
		root = etree.fromstring(docStr)

		# Append the initial comments:
		for comment_token in self.initial_comments:
		comment = self.commentClass(comment_token["data"])
		comment = self.commentClass(comment_token.data)
		root.addprevious(comment._element)

		# Create the root document and add the ElementTree to it
		self.document = self.documentClass()
		self.document._elementTree = root.getroottree()

		# Give the root element the right name
		name = token["name"]
		namespace = token.get("namespace",self.defaultNamespace)
		name = token.name
		namespace = token.namespace orself.defaultNamespace
		if namespace is None:
		etree_tag = name
		else:
Expand Down