move away from emitCurrentToken; part 1 / ...

annevk · annevk · commit 0da2aa1dc072 · 2006-12-30T19:51:33.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40381
diff --git a/src/tokenizer.py b/src/tokenizer.py
@@ -274,7 +274,6 @@ def emitCurrentToken(self):
 
         # Add token to the queue to be yielded
         self.tokenQueue.append(token)
-
         self.state = self.states["data"]
 
     def emitCurrentTokenWithParseError(self, data=None):
@@ -596,13 +595,14 @@ def attributeValueUnQuotedState(self):
     def bogusCommentState(self):
         # Make a new comment token and give it as value all the characters
         # until the first > or EOF (charsUntil checks for EOF automatically)
-        self.currentToken =\
-          {"type": "Comment", "data": self.stream.charsUntil((u">"))}
-        self.emitCurrentToken()
+        # and emit it.
+        self.tokenQueue.append(
+          {"type": "Comment", "data": self.stream.charsUntil((u">"))})
 
         # Eat the character directly after the bogus comment which is either a
         # ">" or an EOF.
         self.stream.char()
+        self.state = self.states["data"]
         return True
 
     def markupDeclarationOpenState(self):
@@ -631,6 +631,7 @@ def commentState(self):
         if data == u"-":
             self.state = self.states["commentDash"]
         elif data == EOF:
+            # XXX EMIT
             self.emitCurrentTokenWithParseError()
         else:
             self.currentToken["data"] += data + self.stream.charsUntil(u"-")
@@ -641,6 +642,7 @@ def commentDashState(self):
         if data == u"-":
             self.state = self.states["commentEnd"]
         elif data == EOF:
+            # XXX EMIT
             self.emitCurrentTokenWithParseError()
         else:
             self.currentToken["data"] += u"-" + data +\
@@ -654,12 +656,14 @@ def commentDashState(self):
     def commentEndState(self):
         data = self.stream.char()
         if data == u">":
+            # XXX EMIT
             self.emitCurrentToken()
         elif data == u"-":
             self.tokenQueue.append({"type": "ParseError", "data":
               _("Unexpected '--' sequence found in comment.")})
             self.currentToken["data"] += data
         elif data == EOF:
+            # XXX EMIT
             self.emitCurrentTokenWithParseError()
         else:
             # XXX
@@ -690,9 +694,11 @@ def beforeDoctypeNameState(self):
         elif data == u">":
             # Character needs to be consumed per the specification so don't
             # invoke emitCurrentTokenWithParseError with "data" as argument.
+            # XXX EMIT
             self.emitCurrentTokenWithParseError()
         elif data == EOF:
-            self.emitCurrentTokenWithParseError(data)
+            # XXX EMIT
+            self.emitCurrentTokenWithParseError()
         else:
             self.currentToken["name"] = data
             self.state = self.states["doctypeName"]
@@ -705,9 +711,11 @@ def doctypeNameState(self):
             self.state = self.states["afterDoctypeName"]
             needsDoctypeCheck = True
         elif data == u">":
-            self.emitCurrentToken()
+            self.tokenQueue.append(self.currentToken)
+            self.state = self.states["data"]
         elif data == EOF:
-            self.emitCurrentTokenWithParseError(data)
+            # XXX EMIT
+            self.emitCurrentTokenWithParseError()
         else:
             # We can't just uppercase everything that arrives here. For
             # instance, non-ASCII characters.
@@ -727,9 +735,11 @@ def afterDoctypeNameState(self):
         if data in spaceCharacters:
             pass
         elif data == u">":
-            self.emitCurrentToken()
+            self.tokenQueue.append(self.currentToken)
+            self.state = self.states["data"]
         elif data == EOF:
             self.currentToken["data"] = True
+            # XXX EMIT
             self.emitCurrentTokenWithParseError(data)
         else:
             self.tokenQueue.append({"type": "ParseError", "data":
@@ -741,8 +751,10 @@ def afterDoctypeNameState(self):
     def bogusDoctypeState(self):
         data = self.stream.char()
         if data == u">":
-            self.emitCurrentToken()
+            self.tokenQueue.append(self.currentToken)
+            self.state = self.states["data"]
         elif data == EOF:
+            # XXX EMIT
             self.emitCurrentTokenWithParseError(data)
         else:
             pass
