Make sure entities that represent spaces become space tokens

James Graham · James Graham · commit 97ed427b1cf5 · 2010-09-22T12:20:17.000+02:00
--HG--
extra : transplant_source : 3%95%88%AE%86%01%FA%8E%23%F3%0B%DB%40X%06%E9%D7%F6%8B%7E
diff --git a/html5lib/tokenizer.py b/html5lib/tokenizer.py
@@ -223,7 +223,11 @@ def entitiesStartingWith(name):
         if fromAttribute:
             self.currentToken["data"][-1][1] += output
         else:
-            self.tokenQueue.append({"type": tokenTypes["Characters"], "data": output})
+            if output in spaceCharacters:
+                tokenType = "SpaceCharacters"
+            else:
+                tokenType = "Characters"
+            self.tokenQueue.append({"type": tokenTypes[tokenType], "data": output})
 
     def processEntityInAttribute(self, allowedChar):
         """This method replaces the need for "entityInAttributeValueState".
