Save some calls to len() in the input stream (improves tokeniser performance by maybe 1-2%)

philiptaylor · philiptaylor · commit 2a332b9755a9 · 2008-12-16T23:54:37.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%401234
diff --git a/src/html5lib/inputstream.py b/src/html5lib/inputstream.py
@@ -70,6 +70,7 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
                                                                  'replace')
 
         self.chunk = u""
+        self.chunkSize = 0
         self.chunkOffset = 0
         self.ungetBuffer = [] # reversed list of chars from unget()
         self.readChars = []
@@ -255,7 +256,7 @@ def char(self):
             self.readChars.append(char)
             return char
 
-        if self.chunkOffset >= len(self.chunk):
+        if self.chunkOffset >= self.chunkSize:
             if not self.readChunk():
                 return EOF
 
@@ -267,6 +268,7 @@ def char(self):
 
     def readChunk(self, chunkSize=_defaultChunkSize):
         self.chunk = u""
+        self.chunkSize = 0
         self.chunkOffset = 0
 
         data = self.dataStream.read(chunkSize)
@@ -291,6 +293,7 @@ def readChunk(self, chunkSize=_defaultChunkSize):
 
         data = unicode(data)
         self.chunk = data
+        self.chunkSize = len(data)
 
         self.updatePosition()
         return True
@@ -329,7 +332,7 @@ def charsUntil(self, characters, opposite = False):
             m = chars.match(self.chunk, self.chunkOffset)
             # If not everything matched, return everything up to the part that didn't match
             end = m.end()
-            if end != len(self.chunk):
+            if end != self.chunkSize:
                 rv.append(self.chunk[self.chunkOffset:end])
                 self.chunkOffset = end
                 break
