Jan 12, 2016 · Nov 24, 2014
diff --git a/.pytest.expect b/.pytest.expect
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
            # Such platforms will have already checked for such
            # surrogate errors, so no need to do this checking.
            self.reportCharacterErrors = None
            self.replaceCharactersRegexp = None
        elif len("\U0010FFFF") == 1:
            self.reportCharacterErrors = self.characterErrorsUCS4
            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
        else:
            self.reportCharacterErrors = self.characterErrorsUCS2
            self.replaceCharactersRegexp = re.compile(
                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))

        # List of where new lines occur
        self.newLines = [0]
        if self.reportCharacterErrors:
            self.reportCharacterErrors(data)

            # Replace invalid characters
            # Note U+0000 is dealt with in the tokenizer
            data = self.replaceCharactersRegexp.sub("\ufffd", data)

        # Replace invalid characters
        data = data.replace("\r\n", "\n")
        data = data.replace("\r", "\n")
Original file line number	Diff line number	Diff line change
Expand Up		@@ -185,14 +185,10 @@ def __init__(self, source):
		# Such platforms will have already checked for such
		# surrogate errors, so no need to do this checking.
		self.reportCharacterErrors = None
		self.replaceCharactersRegexp = None
		elif len("\U0010FFFF") == 1:
		self.reportCharacterErrors = self.characterErrorsUCS4
		self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
		else:
		self.reportCharacterErrors = self.characterErrorsUCS2
		self.replaceCharactersRegexp = re.compile(
		eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])\|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))

		# List of where new lines occur
		self.newLines = [0]
Expand DownExpand Up		@@ -290,10 +286,7 @@ def readChunk(self, chunkSize=None):
		if self.reportCharacterErrors:
		self.reportCharacterErrors(data)

		# Replace invalid characters
		# Note U+0000 is dealt with in the tokenizer
		data = self.replaceCharactersRegexp.sub("\ufffd", data)

		# Replace invalid characters
		data = data.replace("\r\n", "\n")
		data = data.replace("\r", "\n")

Expand Down