Fix for issue 143; deal with handling non-BMP codepoints in serialization to non-unicode encodings.

jgraham · jgraham · commit 9cc3c2aa5739 · 2010-04-02T12:02:39.000-05:00
diff --git a/src/html5lib/inputstream.py b/src/html5lib/inputstream.py
@@ -5,6 +5,7 @@
 
 from constants import EOF, spaceCharacters, asciiLetters, asciiUppercase
 from constants import encodings, ReparseException
+import utils
 
 #Non-unicode versions of constants for use in the pre-parser
 spaceCharactersBytes = frozenset([str(item) for item in spaceCharacters])
@@ -381,14 +382,9 @@ def characterErrorsUCS2(self, data):
             codepoint = ord(match.group())
             pos = match.start()
             #Pretty sure there should be endianness issues here
-            if (codepoint >= 0xD800 and codepoint <= 0xDBFF and
-                pos < len(data) - 1 and
-                ord(data[pos + 1]) >= 0xDC00 and
-                ord(data[pos + 1]) <= 0xDFFF):
+            if utils.isSurrogatePair(data[pos:pos+2]):
                 #We have a surrogate pair!
-                #From a perl manpage
-                char_val = (0x10000 + (codepoint - 0xD800) * 0x400 + 
-                            (ord(data[pos + 1]) - 0xDC00))
+                char_val = utils.surrogatePairToCodepoint(data[pos:pos+2])
                 if char_val in non_bmp_invalid_codepoints:
                     self.errors.append("invalid-codepoint")
                 skip = True
diff --git a/src/html5lib/serializer/htmlserializer.py b/src/html5lib/serializer/htmlserializer.py
@@ -9,7 +9,7 @@
 
 from html5lib.constants import voidElements, booleanAttributes, spaceCharacters
 from html5lib.constants import rcdataElements, entities, xmlEntities
-
+from html5lib import utils
 from xml.sax.saxutils import escape
 
 spaceCharacters = u"".join(spaceCharacters)
@@ -27,20 +27,33 @@
     for k, v in entities.items():
         if v != "&" and encode_entity_map.get(v) != k.lower():
             # prefer &lt; over &LT; and similarly for &amp;, &gt;, etc.
-            encode_entity_map[v] = k
+            encode_entity_map[ord(v)] = k
 
     def htmlentityreplace_errors(exc):
         if isinstance(exc, (UnicodeEncodeError, UnicodeTranslateError)):
             res = []
-            for c in exc.object[exc.start:exc.end]:
-                e = encode_entity_map.get(c)
+            codepoints = []
+            skip = False
+            for i, c in enumerate(exc.object[exc.start:exc.end]):
+                if skip:
+                    skip = False
+                    continue
+                index = i + exc.start
+                if utils.isSurrogatePair(exc.object[index:min([exc.end, index+2])]):
+                    codepoint = utils.surrogatePairToCodepoint(exc.object[index:index+2])
+                    skip = True
+                else:
+                    codepoint = ord(c)
+                codepoints.append(codepoint)
+            for cp in codepoints:
+                e = encode_entity_map.get(cp)
                 if e:
                     res.append("&")
                     res.append(e)
                     if not e.endswith(";"):
                         res.append(";")
                 else:
-                    res.append(c.encode(exc.encoding, "xmlcharrefreplace"))
+                    res.append("&#x%s;"%(hex(cp)[2:]))
             return (u"".join(res), exc.end)
         else:
             return xmlcharrefreplace_errors(exc)
diff --git a/src/html5lib/utils.py b/src/html5lib/utils.py
@@ -153,4 +153,23 @@ def __deepcopy__(self, memo={}):
         result = self.__class__()
         memo[id(self)] = result
         result.__init__(deepcopy(tuple(self), memo))
-        return result
+        return result
+
+#Some utility functions to dal with weirdness around UCS2 vs UCS4
+#python builds
+
+def encodingType():
+    if len() == 2:
+        return "UCS2"
+    else:
+        return "UCS4"
+
+def isSurrogatePair(data):   
+    return (len(data) == 2 and
+            ord(data[0]) >= 0xD800 and ord(data[0]) <= 0xDBFF and
+            ord(data[1]) >= 0xDC00 and ord(data[1]) <= 0xDFFF)
+
+def surrogatePairToCodepoint(data):
+    char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + 
+                (ord(data[1]) - 0xDC00))
+    return char_val
diff --git a/tests/test_serializer.py b/tests/test_serializer.py
@@ -132,7 +132,7 @@ def buildTestSuite():
         allTests.append(unittest.TestLoader().loadTestsFromTestCase(LxmlTestCase))
 
     return unittest.TestSuite(allTests)
-                        
+
 
 def main():
     buildTestSuite()
