Fixed error when \r\n(EOF) falls across chunk boundary

philiptaylor · philiptaylor · commit 980e9ad88e27 · 2008-05-29T14:50:45.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%401159
diff --git a/src/html5lib/inputstream.py b/src/html5lib/inputstream.py
@@ -24,6 +24,8 @@ class HTMLInputStream(object):
 
     """
 
+    _defaultChunkSize = 10240
+
     def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         """Initialises the HTMLInputStream.
 
@@ -260,11 +262,12 @@ def char(self):
         self.readChars.append(char)
         return char
 
-    def readChunk(self, chunkSize=10240):
+    def readChunk(self, chunkSize=_defaultChunkSize):
+        self.chunk = u""
+        self.chunkOffset = 0
+
         data = self.dataStream.read(chunkSize)
         if not data:
-            self.chunk = u""
-            self.chunkOffset = 0
             return False
         #Replace null characters
         for i in xrange(data.count(u"\u0000")):
@@ -276,13 +279,15 @@ def readChunk(self, chunkSize=10240):
         #Check for CR LF broken across chunks
         if (self._lastChunkEndsWithCR and data[0] == "\n"):
             data = data[1:]
+            # Stop if the chunk is now empty
+            if not data:
+                return False
         self._lastChunkEndsWithCR = data[-1] == "\r"
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")
 
         data = unicode(data)
         self.chunk = data
-        self.chunkOffset = 0
 
         self.updatePosition()
         return True
diff --git a/tests/test_stream.py b/tests/test_stream.py
@@ -46,6 +46,11 @@ def test_newlines(self):
         self.assertEquals(stream.charsUntil('e'),u"x")
         self.assertEquals(stream.position(), (4,5))
 
+    def test_newlines2(self):
+        size = HTMLInputStream._defaultChunkSize
+        stream = HTMLInputStream("\r" * size + "\n")
+        self.assertEquals(stream.charsUntil('x'), "\n" * size)
+
     def test_position(self):
         stream = HTMLInputStream(codecs.BOM_UTF8 + "a\nbb\nccc\nddd")
         self.assertEquals(stream.position(), (1, 0))
