Get rid of obsolete replacement of unpaired surrogates with U+FFFD.

gsnedders · gsnedders · commit 30ea6e4a8c5d · 2014-11-24T02:22:27.000Z
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
@@ -167,10 +167,8 @@ def __init__(self, source):
         # Craziness
         if len("\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]")
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
 
         # List of where new lines occur
         self.newLines = [0]
@@ -268,9 +266,6 @@ def readChunk(self, chunkSize=None):
         self.reportCharacterErrors(data)
 
         # Replace invalid characters
-        # Note U+0000 is dealt with in the tokenizer
-        data = self.replaceCharactersRegexp.sub("\ufffd", data)
-
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")
 
