Move entities to using DATrie if possible; if not, use a new custom Trie-like interface.

gsnedders · gsnedders · commit 8205aae5c2ec · 2013-02-01T14:16:08.000Z
This provides a significant performance gain, both parse-time and memory-usage,
over the previous entitiesStartingWith magic, regardless of which impl is used.
diff --git a/html5lib/tokenizer.py b/html5lib/tokenizer.py
@@ -18,10 +18,9 @@
 
 from .inputstream import HTMLInputStream
 
-# Group entities by their first character, for faster lookups
-entitiesByFirstChar = {}
-for e in entities:
-    entitiesByFirstChar.setdefault(e[0], []).append(e)
+from .trie import Trie
+
+entitiesTrie = Trie(entities)
 
 class HTMLTokenizer(object):
     """ This class takes care of tokenizing HTML.
@@ -179,28 +178,20 @@ def consumeEntity(self, allowedChar=None, fromAttribute=False):
             #
             # Consume characters and compare to these to a substring of the
             # entity names in the list until the substring no longer matches.
-            filteredEntityList = entitiesByFirstChar.get(charStack[0], [])
-
-            def entitiesStartingWith(name):
-                return [e for e in filteredEntityList if e.startswith(name)]
-
             while (charStack[-1] is not EOF):
-                filteredEntityList = entitiesStartingWith("".join(charStack))
-                if not filteredEntityList:
+                if not entitiesTrie.has_keys_with_prefix("".join(charStack)):
                     break
                 charStack.append(self.stream.char())
 
             # At this point we have a string that starts with some characters
             # that may match an entity
-            entityName = None
-
             # Try to find the longest entity the string will match to take care
             # of &noti for instance.
-            for entityLength in range(len(charStack)-1, 1, -1):
-                possibleEntityName = "".join(charStack[:entityLength])
-                if possibleEntityName in entities:
-                    entityName = possibleEntityName
-                    break
+            try:
+                entityName = entitiesTrie.longest_prefix("".join(charStack[:-1]))
+                entityLength = len(entityName)
+            except KeyError:
+                entityName = None
 
             if entityName is not None:
                 if entityName[-1] != ";":
diff --git a/html5lib/trie/__init__.py b/html5lib/trie/__init__.py
@@ -0,0 +1,10 @@
+from .py import Trie as PyTrie
+
+Trie = PyTrie
+
+try:
+    from .datrie import Trie as DATrie
+except ImportError:
+    pass
+else:
+    Trie = DATrie
diff --git a/html5lib/trie/_base.py b/html5lib/trie/_base.py
@@ -0,0 +1,33 @@
+from collections import Mapping
+
+class Trie(Mapping):
+    """Abstract base class for tries"""
+
+    def keys(self, prefix=None):
+        keys = super().keys()
+
+        if prefix is None:
+            return set(keys)
+
+        return {x for x in keys if x.startswith(prefix)}
+
+    def has_keys_with_prefix(self, prefix):
+        for key in self.keys():
+            if key.startswith(prefix):
+                return True
+
+        return False
+
+    def longest_prefix(self, prefix):
+        if prefix in self:
+            return prefix
+
+        for i in range(1, len(prefix) + 1):
+            if prefix[:-i] in self:
+                return prefix[:-i]
+
+        raise KeyError(prefix)
+
+    def longest_prefix_item(self, prefix):
+        lprefix = self.longest_prefix(prefix)
+        return (lprefix, self[lprefix])
diff --git a/html5lib/trie/datrie.py b/html5lib/trie/datrie.py
@@ -0,0 +1,42 @@
+from itertools import chain
+
+from datrie import Trie as DATrie
+
+from ._base import Trie as ABCTrie
+
+class Trie(ABCTrie):
+    def __init__(self, data):
+        chars = set()
+        for key in data.keys():
+            if not isinstance(key, str):
+                raise TypeError("All keys must be strings")
+            for char in key:
+                chars.add(char)
+
+        self._data = DATrie("".join(chars))
+        for key, value in data.items():
+            self._data[key] = value
+
+    def __contains__(self, key):
+        return key in self._data
+
+    def __len__(self):
+        return len(self._data)
+
+    def __iter__(self):
+        raise NotImplementedError()
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+    def keys(self, prefix=None):
+        return self._data.keys(prefix)
+
+    def has_keys_with_prefix(self, prefix):
+        return self._data.has_keys_with_prefix(prefix)
+
+    def longest_prefix(self, prefix):
+        return self._data.longest_prefix(prefix)
+
+    def longest_prefix_item(self, prefix):
+        return self._data.longest_prefix_item(prefix)
diff --git a/html5lib/trie/py.py b/html5lib/trie/py.py
@@ -0,0 +1,63 @@
+from bisect import bisect_left
+
+from ._base import Trie as ABCTrie
+
+class Trie(ABCTrie):
+    def __init__(self, data):
+        if not all(isinstance(x, str) for x in data.keys()):
+            raise TypeError("All keys must be strings")
+
+        self._data = data
+        self._keys = sorted(data.keys())
+        self._cachestr = ""
+        self._cachepoints = (0, len(data))
+
+    def __contains__(self, key):
+        return key in self._data
+
+    def __len__(self):
+        return len(self._data)
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+    def keys(self, prefix=None):
+        if prefix is None or prefix == "" or not self._keys:
+            return set(self._keys)
+        
+        if prefix.startswith(self._cachestr):
+            lo, hi = self._cachepoints
+            start = i = bisect_left(self._keys, prefix, lo, hi)
+        else:
+            start = i = bisect_left(self._keys, prefix)
+
+        keys = set()
+        if start == len(self._keys):
+            return keys
+
+        while self._keys[i].startswith(prefix):
+            keys.add(self._keys[i])
+            i += 1
+
+        self._cachestr = prefix
+        self._cachepoints = (start, i)
+
+        return keys
+
+    def has_keys_with_prefix(self, prefix):
+        if prefix in self._data:
+            return True
+
+        if prefix.startswith(self._cachestr):
+            lo, hi = self._cachepoints
+            i = bisect_left(self._keys, prefix, lo, hi)
+        else:
+            i = bisect_left(self._keys, prefix)
+
+        if i == len(self._keys):
+            return False
+        
+        return self._keys[i].startswith(prefix)
