Oct 26, 2016
diff --git a/.travis.yml b/.travis.yml
 language: python
 python:
  - "2.6"
  - "2.7"
  - "3.3"
  - "3.4"
diff --git a/README.rst b/README.rst
 Installation
 ------------

 html5lib works on CPython 2.6+, CPython 3.3+ and PyPy.  To install it,
 html5lib works on CPython 2.7, CPython 3.3+ and PyPy.  To install it,
 use:

 .. code-block:: bash
 -----

 Unit tests require the ``pytest`` and ``mock`` libraries and can be
 run using the ``py.test`` command in the root directory;
 ``ordereddict`` is required under Python 2.6. All shouldpass.
 run using the ``py.test`` command in the root directory. All should
 pass.

 Test data are contained in a separate `html5lib-tests
 <https://github.com/html5lib/html5lib-tests>`_ repository and included
diff --git a/html5lib/_inputstream.py b/html5lib/_inputstream.py
 else:
    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)

 non_bmp_invalid_codepoints =set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
 0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE,
 0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
 0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF,
 0x10FFFE, 0x10FFFF])
 non_bmp_invalid_codepoints ={0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                              0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
                              0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE,
                              0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
                              0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
                              0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF,
                              0x10FFFE, 0x10FFFF}

 ascii_punctuation_re = re.compile("[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]")

diff --git a/html5lib/_trie/_base.py b/html5lib/_trie/_base.py
        if prefix is None:
            return set(keys)

        # Python 2.6: no set comprehensions
        return set([x for x in keys if x.startswith(prefix)])
        return {x for x in keys if x.startswith(prefix)}

    def has_keys_with_prefix(self, prefix):
        for key in self.keys():
diff --git a/html5lib/_utils.py b/html5lib/_utils.py
 from __future__ import absolute_import, division, unicode_literals

 import sys
 from types import ModuleType

 from six import text_type

 __all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
           "surrogatePairToCodepoint", "moduleFactoryFactory",
           "supports_lone_surrogates", "PY27"]
           "supports_lone_surrogates"]


 PY27 = sys.version_info[0] == 2 and sys.version_info[1] >= 7

 # Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
 # caught by the below test. In general this would be any platform
 # using UTF-16 as its encoding of unicode strings, such as
diff --git a/html5lib/constants.py b/html5lib/constants.py
    "xmlns:xlink": ("xmlns", "xlink", namespaces["xmlns"])
 }

 unadjustForeignAttributes =dict([((ns, local), qname) for qname, (prefix, local, ns) in
 adjustForeignAttributes.items()])
 unadjustForeignAttributes ={(ns, local): qname for qname, (prefix, local, ns) in
                             adjustForeignAttributes.items()}

 spaceCharacters = frozenset([
    "\t",
 digits = frozenset(string.digits)
 hexDigits = frozenset(string.hexdigits)

 asciiUpper2Lower =dict([(ord(c), ord(c.lower()))
 for c in string.ascii_uppercase])
 asciiUpper2Lower ={ord(c): ord(c.lower())
                    for c in string.ascii_uppercase}

 # Heading elements need to be ordered
 headingElements = (
                           tokenTypes["EmptyTag"]])


 prefixes =dict([(v, k) for k, v in namespaces.items()])
 prefixes ={v: k for k, v in namespaces.items()}
 prefixes["http://www.w3.org/1998/Math/MathML"] = "math"


diff --git a/html5lib/filters/alphabeticalattributes.py b/html5lib/filters/alphabeticalattributes.py

 from . import base

 try:
    from collections import OrderedDict
 except ImportError:
    from ordereddict import OrderedDict
 from collections import OrderedDict


 class Filter(base.Filter):
diff --git a/html5lib/html5parser.py b/html5lib/html5parser.py
 from __future__ import absolute_import, division, unicode_literals
 from six import with_metaclass, viewkeys, PY3
 from six import with_metaclass, viewkeys

 import types

 try:
    from collections import OrderedDict
 except ImportError:
    from ordereddict import OrderedDict
 from collections import OrderedDict

 from . import _inputstream
 from . import _tokenizer
        self.tree = tree(namespaceHTMLElements)
        self.errors = []

        self.phases =dict([(name, cls(self, self.tree)) for name, cls in
 getPhases(debug).items()])
        self.phases ={name: cls(self, self.tree)
 for name, cls ingetPhases(debug).items()}

    def _parse(self, stream, innerHTML=False, container="div", scripting=False, **kwargs):

            currentNode = self.tree.openElements[-1]
            if (token["name"] in self.breakoutElements or
                (token["name"] == "font" and
                 set(token["data"].keys()) &set(["color", "face", "size"]))):
                 set(token["data"].keys()) &{"color", "face", "size"})):
                self.parser.parseError("unexpected-html-element-in-foreign-content",
                                       {"name": token["name"]})
                while (self.tree.openElements[-1].namespace !=


 def adjust_attributes(token, replacements):
    if PY3 or _utils.PY27:
        needs_adjustment = viewkeys(token['data']) & viewkeys(replacements)
    else:
        needs_adjustment = frozenset(token['data']) & frozenset(replacements)
    needs_adjustment = viewkeys(token['data']) & viewkeys(replacements)
    if needs_adjustment:
        token['data'] = OrderedDict((replacements.get(k, k), v)
                                    for k, v in token['data'].items())
diff --git a/html5lib/tests/test_serializer.py b/html5lib/tests/test_serializer.py


 defserialize_html(input,options):
 options=dict([(str(k),v)fork,vinoptions.items()])
 options={str(k):vfork,vinoptions.items()}
 encoding=options.get("encoding",None)
 if"encoding"inoptions:
 deloptions["encoding"]
diff --git a/html5lib/treebuilders/base.py b/html5lib/treebuilders/base.py

 listElementsMap = {
    None: (frozenset(scopingElements), False),
    "button": (frozenset(scopingElements |set([(namespaces["html"], "button")])), False),
    "list": (frozenset(scopingElements |set([(namespaces["html"], "ol"),
 (namespaces["html"], "ul")])), False),
    "button": (frozenset(scopingElements |{(namespaces["html"], "button")}), False),
    "list": (frozenset(scopingElements |{(namespaces["html"], "ol"),
                                          (namespaces["html"], "ul")}), False),
    "table": (frozenset([(namespaces["html"], "html"),
                         (namespaces["html"], "table")]), False),
    "select": (frozenset([(namespaces["html"], "optgroup"),
diff --git a/html5lib/treewalkers/etree.py b/html5lib/treewalkers/etree.py
 from __future__ import absolute_import, division, unicode_literals

 try:
    from collections import OrderedDict
 except ImportError:
    try:
        from ordereddict import OrderedDict
    except ImportError:
        OrderedDict = dict
 from collections import OrderedDict

 import re

diff --git a/requirements.txt b/requirements.txt
 six>=1.9
 webencodings
 ordereddict ; python_version < '2.7'
diff --git a/setup.py b/setup.py
    'Operating System :: OS Independent',
    'Programming Language :: Python',
    'Programming Language :: Python :: 2',
    'Programming Language :: Python :: 2.6',
    'Programming Language :: Python :: 2.7',
    'Programming Language :: Python :: 3',
    'Programming Language :: Python :: 3.3',
          'webencodings',
      ],
      extras_require={
          # A empty extra that only has a conditional marker will be
          # unconditonally installed when the condition matches.
          ":python_version == '2.6'": ["ordereddict"],

          # A conditional extra will only install these items when the extra is
          # requested and the condition matches.
          "datrie:platform_python_implementation == 'CPython'": ["datrie"],
diff --git a/tox.ini b/tox.ini
 [tox]
 envlist = {py26,py27,py33,py34,py35,pypy}-{base,optional}
 envlist = {py27,py33,py34,py35,pypy}-{base,optional}

 [testenv]
 deps =
  mock
  base: six
  base: webencodings
  py26-base: ordereddict
  optional: -r{toxinidir}/requirements-optional.txt

 commands =
diff --git a/utils/spider.py b/utils/spider.py

        # Remove links not of content-type html or pages not found
        # XXX - need to deal with other status codes?
        toVisit =set([url for url in urls if url in responseHeaders and
 "html" in responseHeaders[url]['content-type'] and
 responseHeaders[url]['status'] == "200"])
        toVisit ={url for url in urls if url in responseHeaders and
                   "html" in responseHeaders[url]['content-type'] and
                   responseHeaders[url]['status'] == "200"}

        # Now check we are allowed to spider the page
        for url in toVisit:
Original file line number	Diff line number	Diff line change
		@@ -1,6 +1,5 @@
		language: python
		python:
		- "2.6"
		- "2.7"
		- "3.3"
		- "3.4"
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -90,7 +90,7 @@ More documentation is available at https://html5lib.readthedocs.io/.
		Installation
		------------

		html5lib works on CPython 2.6+, CPython 3.3+ and PyPy. To install it,
		html5lib works on CPython 2.7, CPython 3.3+ and PyPy. To install it,
		use:

		.. code-block:: bash
Expand DownExpand Up		@@ -128,8 +128,8 @@ Tests
		-----

		Unit tests require the ``pytest`` and ``mock`` libraries and can be
		run using the ``py.test`` command in the root directory;
		``ordereddict`` is required under Python 2.6. All shouldpass.
		run using the ``py.test`` command in the root directory. All should
		pass.

		Test data are contained in a separate `html5lib-tests
		<https://github.com/html5lib/html5lib-tests>`_ repository and included
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -40,13 +40,13 @@
		else:
		invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)

		non_bmp_invalid_codepoints =set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
		0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
		0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE,
		0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
		0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
		0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF,
		0x10FFFE, 0x10FFFF])
		non_bmp_invalid_codepoints ={0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
		0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
		0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE,
		0x8FFFF, 0x9FFFE, 0x9FFFF, 0xAFFFE, 0xAFFFF,
		0xBFFFE, 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
		0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF,
		0x10FFFE, 0x10FFFF}

		ascii_punctuation_re = re.compile("[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]")

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -13,8 +13,7 @@ def keys(self, prefix=None):
		if prefix is None:
		return set(keys)

		# Python 2.6: no set comprehensions
		return set([x for x in keys if x.startswith(prefix)])
		return {x for x in keys if x.startswith(prefix)}

		def has_keys_with_prefix(self, prefix):
		for key in self.keys():
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,6 +1,5 @@
		from __future__ import absolute_import, division, unicode_literals

		import sys
		from types import ModuleType

		from six import text_type
Expand All		@@ -13,11 +12,9 @@

		__all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
		"surrogatePairToCodepoint", "moduleFactoryFactory",
		"supports_lone_surrogates", "PY27"]
		"supports_lone_surrogates"]


		PY27 = sys.version_info[0] == 2 and sys.version_info[1] >= 7

		# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
		# caught by the below test. In general this would be any platform
		# using UTF-16 as its encoding of unicode strings, such as
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -519,8 +519,8 @@
		"xmlns:xlink": ("xmlns", "xlink", namespaces["xmlns"])
		}

		unadjustForeignAttributes =dict([((ns, local), qname) for qname, (prefix, local, ns) in
		adjustForeignAttributes.items()])
		unadjustForeignAttributes ={(ns, local): qname for qname, (prefix, local, ns) in
		adjustForeignAttributes.items()}

		spaceCharacters = frozenset([
		"\t",
Expand All		@@ -544,8 +544,8 @@
		digits = frozenset(string.digits)
		hexDigits = frozenset(string.hexdigits)

		asciiUpper2Lower =dict([(ord(c), ord(c.lower()))
		for c in string.ascii_uppercase])
		asciiUpper2Lower ={ord(c): ord(c.lower())
		for c in string.ascii_uppercase}

		# Heading elements need to be ordered
		headingElements = (
Expand DownExpand Up		@@ -2933,7 +2933,7 @@
		tokenTypes["EmptyTag"]])


		prefixes =dict([(v, k) for k, v in namespaces.items()])
		prefixes ={v: k for k, v in namespaces.items()}
		prefixes["http://www.w3.org/1998/Math/MathML"] = "math"


Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,10 +2,7 @@

		from . import base

		try:
		from collections import OrderedDict
		except ImportError:
		from ordereddict import OrderedDict
		from collections import OrderedDict


		class Filter(base.Filter):
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,12 +1,9 @@
		from __future__ import absolute_import, division, unicode_literals
		from six import with_metaclass, viewkeys, PY3
		from six import with_metaclass, viewkeys

		import types

		try:
		from collections import OrderedDict
		except ImportError:
		from ordereddict import OrderedDict
		from collections import OrderedDict

		from . import _inputstream
		from . import _tokenizer
Expand DownExpand Up		@@ -74,8 +71,8 @@ def __init__(self, tree=None, strict=False, namespaceHTMLElements=True, debug=Fa
		self.tree = tree(namespaceHTMLElements)
		self.errors = []

		self.phases =dict([(name, cls(self, self.tree)) for name, cls in
		getPhases(debug).items()])
		self.phases ={name: cls(self, self.tree)
		for name, cls ingetPhases(debug).items()}

		def _parse(self, stream, innerHTML=False, container="div", scripting=False, **kwargs):

Expand DownExpand Up		@@ -2417,7 +2414,7 @@ def processStartTag(self, token):
		currentNode = self.tree.openElements[-1]
		if (token["name"] in self.breakoutElements or
		(token["name"] == "font" and
		set(token["data"].keys()) &set(["color", "face", "size"]))):
		set(token["data"].keys()) &{"color", "face", "size"})):
		self.parser.parseError("unexpected-html-element-in-foreign-content",
		{"name": token["name"]})
		while (self.tree.openElements[-1].namespace !=
Expand DownExpand Up		@@ -2711,10 +2708,7 @@ def processEndTag(self, token):


		def adjust_attributes(token, replacements):
		if PY3 or _utils.PY27:
		needs_adjustment = viewkeys(token['data']) & viewkeys(replacements)
		else:
		needs_adjustment = frozenset(token['data']) & frozenset(replacements)
		needs_adjustment = viewkeys(token['data']) & viewkeys(replacements)
		if needs_adjustment:
		token['data'] = OrderedDict((replacements.get(k, k), v)
		for k, v in token['data'].items())
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -80,7 +80,7 @@ def _convertAttrib(self, attribs):


		defserialize_html(input,options):
		options=dict([(str(k),v)fork,vinoptions.items()])
		options={str(k):vfork,vinoptions.items()}
		encoding=options.get("encoding",None)
		if"encoding"inoptions:
		deloptions["encoding"]
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,9 +10,9 @@

		listElementsMap = {
		None: (frozenset(scopingElements), False),
		"button": (frozenset(scopingElements \|set([(namespaces["html"], "button")])), False),
		"list": (frozenset(scopingElements \|set([(namespaces["html"], "ol"),
		(namespaces["html"], "ul")])), False),
		"button": (frozenset(scopingElements \|{(namespaces["html"], "button")}), False),
		"list": (frozenset(scopingElements \|{(namespaces["html"], "ol"),
		(namespaces["html"], "ul")}), False),
		"table": (frozenset([(namespaces["html"], "html"),
		(namespaces["html"], "table")]), False),
		"select": (frozenset([(namespaces["html"], "optgroup"),
Expand Down