Get Python tokenizer up to the version of the spec of the end of September.

gsnedders · gsnedders · commit bd57c619bc40 · 2009-10-23T14:01:00.000+02:00
diff --git a/src/html5lib/tokenizer.py b/src/html5lib/tokenizer.py
@@ -955,7 +955,7 @@ def afterDoctypeNameState(self):
                         matched = False
                         break
                 if matched:
-                    self.state = self.beforeDoctypePublicIdentifierState
+                    self.state = self.afterDoctypePublicKeywordState
                     return True
             elif data in (u"s", u"S"):
                 matched = True
@@ -966,7 +966,7 @@ def afterDoctypeNameState(self):
                         matched = False
                         break
                 if matched:
-                    self.state = self.beforeDoctypeSystemIdentifierState
+                    self.state = self.afterDoctypeSystemKeywordState
                     return True
 
             # All the characters read before the current 'data' will be
@@ -981,6 +981,26 @@ def afterDoctypeNameState(self):
             self.state = self.bogusDoctypeState
 
         return True
+    
+    def afterDoctypePublicKeywordState(self):
+        data = self.stream.char()
+        if data in spaceCharacters:
+            self.state = self.beforeDoctypePublicIdentifierState
+        elif data in ("'", '"'):
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "unexpected-char-in-doctype"})
+            self.stream.unget(data)
+            self.state = self.beforeDoctypePublicIdentifierState
+        elif data is EOF:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "eof-in-doctype"})
+            self.currentToken["correct"] = False
+            self.tokenQueue.append(self.currentToken)
+            self.state = self.dataState
+        else:
+            self.stream.unget(data)
+            self.state = self.beforeDoctypePublicIdentifierState
+        return True
 
     def beforeDoctypePublicIdentifierState(self):
         data = self.stream.char()
@@ -1054,17 +1074,47 @@ def doctypePublicIdentifierSingleQuotedState(self):
     def afterDoctypePublicIdentifierState(self):
         data = self.stream.char()
         if data in spaceCharacters:
-            pass
-        elif data == "\"":
+            self.state = self.betweenDoctypePublicAndSystemIdentifiersState
+        elif data == ">":
+            self.tokenQueue.append(self.currentToken)
+            self.state = self.dataState
+        elif data == '"':
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "unexpected-char-in-doctype"})
             self.currentToken["systemId"] = u""
             self.state = self.doctypeSystemIdentifierDoubleQuotedState
         elif data == "'":
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "unexpected-char-in-doctype"})
             self.currentToken["systemId"] = u""
             self.state = self.doctypeSystemIdentifierSingleQuotedState
+        elif data is EOF:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "eof-in-doctype"})
+            self.currentToken["correct"] = False
+            self.tokenQueue.append(self.currentToken)
+            self.state = self.dataState
+        else:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "unexpected-char-in-doctype"})
+            self.currentToken["correct"] = False
+            self.state = self.bogusDoctypeState
+        return True
+    
+    def betweenDoctypePublicAndSystemIdentifiersState(self):
+        data = self.stream.char()
+        if data in spaceCharacters:
+            pass
         elif data == ">":
             self.tokenQueue.append(self.currentToken)
             self.state = self.dataState
-        elif data is EOF:
+        elif data == '"':
+            self.currentToken["systemId"] = u""
+            self.state = self.doctypeSystemIdentifierDoubleQuotedState
+        elif data == "'":
+            self.currentToken["systemId"] = u""
+            self.state = self.doctypeSystemIdentifierSingleQuotedState
+        elif data == EOF:
             self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
               "eof-in-doctype"})
             self.currentToken["correct"] = False
@@ -1077,6 +1127,26 @@ def afterDoctypePublicIdentifierState(self):
             self.state = self.bogusDoctypeState
         return True
     
+    def afterDoctypeSystemKeywordState(self):
+        data = self.stream.char()
+        if data in spaceCharacters:
+            self.state = self.beforeDoctypeSystemIdentifierState
+        elif data in ("'", '"'):
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "unexpected-char-in-doctype"})
+            self.stream.unget(data)
+            self.state = self.beforeDoctypeSystemIdentifierState
+        elif data is EOF:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+              "eof-in-doctype"})
+            self.currentToken["correct"] = False
+            self.tokenQueue.append(self.currentToken)
+            self.state = self.dataState
+        else:
+            self.stream.unget(data)
+            self.state = self.beforeDoctypeSystemIdentifierState
+        return True
+    
     def beforeDoctypeSystemIdentifierState(self):
         data = self.stream.char()
         if data in spaceCharacters:
