Work with lxml entities (based on patch by fantasai)

jgraham · jgraham · commit e834288d96ce · 2010-03-13T21:48:00.000+01:00
diff --git a/src/html5lib/constants.py b/src/html5lib/constants.py
@@ -508,6 +508,8 @@
     376    # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
 )
 
+xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))
+
 entities = {
     "AElig;": u"\u00C6",
     "AElig": u"\u00C6",
diff --git a/src/html5lib/serializer/htmlserializer.py b/src/html5lib/serializer/htmlserializer.py
@@ -8,7 +8,7 @@
 _ = gettext.gettext
 
 from html5lib.constants import voidElements, booleanAttributes, spaceCharacters
-from html5lib.constants import rcdataElements
+from html5lib.constants import rcdataElements, entities, xmlEntities
 
 from xml.sax.saxutils import escape
 
@@ -54,26 +54,32 @@ def encode(text, encoding):
 
 class HTMLSerializer(object):
 
+    # attribute quoting options
     quote_attr_values = False
     quote_char = '"'
     use_best_quote_char = True
-    minimize_boolean_attributes = True
 
+    # tag syntax options
+    omit_optional_tags = True
+    minimize_boolean_attributes = True
     use_trailing_solidus = False
     space_before_trailing_solidus = True
+
+    # escaping options
     escape_lt_in_attrs = False
     escape_rcdata = False
+    resolve_entities = True
 
+    # miscellaneous options
     inject_meta_charset = True
     strip_whitespace = False
     sanitize = False
-    omit_optional_tags = True
 
     options = ("quote_attr_values", "quote_char", "use_best_quote_char",
           "minimize_boolean_attributes", "use_trailing_solidus",
           "space_before_trailing_solidus", "omit_optional_tags",
           "strip_whitespace", "inject_meta_charset", "escape_lt_in_attrs",
-          "escape_rcdata", 'use_trailing_solidus', "sanitize")
+          "escape_rcdata", "resolve_entities", "sanitize")
 
     def __init__(self, **kwargs):
         if kwargs.has_key('quote_char'):
@@ -214,6 +220,19 @@ def serialize(self, treewalker, encoding=None):
                     comment = comment.encode(encoding, unicode_encode_errors)
                 yield comment
 
+            elif type == "Entity":
+                name = token["name"]
+                key = name + ";"
+                if not key in entities:
+                    self.serializeError(_("Entity %s not recognized" % name))
+                if self.resolve_entities and key not in xmlEntities:
+                    data = entities[key]
+                else:
+                    data = u"&%s;" % name
+                if encoding:
+                    data = data.encode(encoding, unicode_encode_errors)
+                yield data
+
             else:
                 self.serializeError(token["data"])
 
diff --git a/src/html5lib/treewalkers/_base.py b/src/html5lib/treewalkers/_base.py
@@ -64,6 +64,9 @@ def doctype(self, name, publicId=None, systemId=None, correct=True):
                 "systemId": systemId,
                 "correct": correct}
 
+    def entity(self, name):
+        return {"type": "Entity", "name": unicode(name)}
+
     def unknown(self, nodeType):
         return self.error(_("Unknown node type: ") + nodeType)
 
@@ -89,6 +92,7 @@ def element(self, node, namespace, name, attrs, hasChildren):
 TEXT = Node.TEXT_NODE
 ELEMENT = Node.ELEMENT_NODE
 COMMENT = Node.COMMENT_NODE
+ENTITY = Node.ENTITY_NODE
 UNKNOWN = "<#UNKNOWN#>"
 
 class NonRecursiveTreeWalker(TreeWalker):
@@ -133,6 +137,9 @@ def __iter__(self):
             elif type == COMMENT:
                 yield self.comment(details[0])
 
+            elif type == ENTITY:
+                yield self.entity(details[0])
+
             elif type == DOCUMENT:
                 hasChildren = True
 
diff --git a/src/html5lib/treewalkers/lxmletree.py b/src/html5lib/treewalkers/lxmletree.py
@@ -126,6 +126,9 @@ def getNodeDetails(self, node):
         elif node.tag == etree.Comment:
             return _base.COMMENT, node.text
 
+        elif node.tag == etree.Entity:
+            return _base.ENTITY, node.text[1:-1] # strip &;
+
         else:
             #This is assumed to be an ordinary element
             match = tag_regexp.match(node.tag)
diff --git a/tests/test_serializer.py b/tests/test_serializer.py
@@ -2,9 +2,18 @@
 import unittest
 from support import simplejson, html5lib_test_files
 
+import html5lib
 from html5lib import html5parser, serializer, constants
 from html5lib.treewalkers._base import TreeWalker
 
+optionals_loaded = []
+
+try:
+    from lxml import etree
+    optionals_loaded.append("lxml")
+except ImportError:
+    pass
+
 default_namespace = constants.namespaces["html"]
 
 class JsonWalker(TreeWalker):
@@ -80,7 +89,32 @@ def serialize_xhtml(self, input, options):
         return u''.join(serializer.XHTMLSerializer(**options).
                 serialize(JsonWalker(input),options.get("encoding",None)))
 
-def buildTestSuite():
+class LxmlTestCase(unittest.TestCase):
+    def setUp(self):
+        self.parser = etree.XMLParser(resolve_entities=False)
+        self.treewalker = html5lib.getTreeWalker("lxml")
+        self.serializer = serializer.HTMLSerializer()
+
+    def testEntityReplacement(self):
+        doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&beta;</html>"""
+        tree = etree.fromstring(doc, parser = self.parser).getroottree()
+        result = serializer.serialize(tree, tree="lxml", omit_optional_tags=False)
+        self.assertEquals(u"""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>\u03B2</html>""", result)
+
+    def testEntityXML(self):
+        doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&gt;</html>"""
+        tree = etree.fromstring(doc, parser = self.parser).getroottree()
+        result = serializer.serialize(tree, tree="lxml", omit_optional_tags=False)
+        self.assertEquals(u"""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&gt;</html>""", result)
+
+    def testEntityNoResolve(self):
+        doc = """<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&beta;</html>"""
+        tree = etree.fromstring(doc, parser = self.parser).getroottree()
+        result = serializer.serialize(tree, tree="lxml", omit_optional_tags=False,
+                                      resolve_entities=False)
+        self.assertEquals(u"""<!DOCTYPE html SYSTEM "about:legacy-compat"><html>&beta;</html>""", result)
+
+def buildBasicTestSuite():
     for filename in html5lib_test_files('serializer', '*.test'):
         test_name = os.path.basename(filename).replace('.test','')
         tests = simplejson.load(file(filename))
@@ -92,6 +126,14 @@ def buildTestSuite():
                 test.get("options", {}))
     return unittest.TestLoader().loadTestsFromTestCase(TestCase)
 
+def buildTestSuite():
+    allTests = [buildBasicTestSuite()]
+    if "lxml" in optionals_loaded:
+        allTests.append(unittest.TestLoader().loadTestsFromTestCase(LxmlTestCase))
+
+    return unittest.TestSuite(allTests)
+                        
+
 def main():
     buildTestSuite()
     unittest.main()

Original file line number	Diff line number	Diff line change
`@@ -508,6 +508,8 @@`
`508`	`508`	`376 # 0x9F 0x0178 LATIN CAPITAL LETTER Y WITH DIAERESIS`
`509`	`509`	`)`
`510`	`510`
	`511`	`+xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))`
	`512`	`+`
`511`	`513`	`entities = {`
`512`	`514`	`"AElig;": u"\u00C6",`
`513`	`515`	`"AElig": u"\u00C6",`