Replace invalid characters with U+FFFD (fixes html5lib#96)

Leif Arne Storset · Leif Arne Storset · commit 6a6b74a0ebf1 · 2014-06-06T15:04:55.000+02:00
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
@@ -270,6 +270,7 @@ def readChunk(self, chunkSize=None):
         # Replace invalid characters
         # Note U+0000 is dealt with in the tokenizer
         data = self.replaceCharactersRegexp.sub("\ufffd", data)
+        data = invalid_unicode_re.sub("\ufffd", data)
 
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")
