From f27af7000897cd2e589d07bf0ef0308054cb6024 Mon Sep 17 00:00:00 2001
From: Geoffrey Sneddon <me@gsnedders.com>
Date: Mon, 24 Nov 2014 02:21:01 +0000
Subject: [PATCH] Get rid of obsolete replacement of unpaired surrogates with
 U+FFFD.

---
 .pytest.expect          | Bin 55230 -> 55002 bytes
 html5lib/inputstream.py |   9 +--------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/.pytest.expect b/.pytest.expect
index 5f3b61942c093094876a3c29dfee8f19f63e6013..e818b51683df6a369de1cb6f44e86dbd9bcc9a27 100644
GIT binary patch
delta 124
zcmV-?0E7R&uLIhx1CS~K+miucIk9B<DU-WgHnYqs0|t}go-30uL=BVah!c}uhcc7E
zXA6@i$PSZmq(`#`GQB*L`?5l_K5yLtli+7UlMyi&laMJnv$1j0FO#s7LX!}o1e0KK
e7?ZqKLbLy&q%)KF#VnJrf)bN|7Y4H+)}8@z7B`vz

delta 185
zcmcb$mU-WL<_TJi_a{H}vD}#QQ*(0WEE85UGYf;*$qPkwCM#I-ZJw_6pLO!)Y1)$m
z(!?iEED)c3##CqW1u?G8rwvkVC+Az3umIH?PYQCLY&TzUvO@v?<O{pBCnqQfZ@!RM
z&p27(yyIkt1uT<)Es&TLYO(o2!4-X&CZoxP!n%_@#3!fm2u<!*)tPL-%RjlS5NOCT
ir_Em%G8=<j!wxjvF!pA^WI<b-$@SuFn{BSnV*~);-$%Is

diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
index 20f6c95a..5694efe3 100644
--- a/html5lib/inputstream.py
+++ b/html5lib/inputstream.py
@@ -185,14 +185,10 @@ def __init__(self, source):
             # Such platforms will have already checked for such
             # surrogate errors, so no need to do this checking.
             self.reportCharacterErrors = None
-            self.replaceCharactersRegexp = None
         elif len("\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile(
-                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))
 
         # List of where new lines occur
         self.newLines = [0]
@@ -290,10 +286,7 @@ def readChunk(self, chunkSize=None):
         if self.reportCharacterErrors:
             self.reportCharacterErrors(data)
 
-            # Replace invalid characters
-            # Note U+0000 is dealt with in the tokenizer
-            data = self.replaceCharactersRegexp.sub("\ufffd", data)
-
+        # Replace invalid characters
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")