Move where we concatenate expected tokens to handle ignoreErrorOrder

gsnedders · gsnedders · commit d0f3b7b81a3a · 2014-11-24T01:47:24.000Z
Also update the expected failures now we no longer fail a test.
diff --git a/html5lib/tests/expected-failures/tokenizer.dat b/html5lib/tests/expected-failures/tokenizer.dat
@@ -36,8 +36,5 @@
 #data
 <!DOCTYPE
 
-#data
-I'm &no
-
 #data
 <!DOCTYPE 	
diff --git a/html5lib/tests/test_tokenizer.py b/html5lib/tests/test_tokenizer.py
@@ -108,6 +108,7 @@ def tokensMatch(expectedTokens, receivedTokens, ignoreErrorOrder,
                 token.pop()
 
     if not ignoreErrorOrder and not ignoreErrors:
+        expectedTokens = concatenateCharacterTokens(expectedTokens)
         return expectedTokens == receivedTokens
     else:
         # Sort the tokens into two groups; non-parse errors and parse errors
@@ -120,6 +121,7 @@ def tokensMatch(expectedTokens, receivedTokens, ignoreErrorOrder,
                 else:
                     if not ignoreErrors:
                         tokens[tokenType][1].append(token)
+            tokens[tokenType][0] = concatenateCharacterTokens(tokens[tokenType][0])
         return tokens["expected"] == tokens["received"]
 
 
@@ -144,7 +146,7 @@ def runTokenizerTest(test):
     warnings.resetwarnings()
     warnings.simplefilter("error")
 
-    expected = concatenateCharacterTokens(test['output'])
+    expected = test['output']
     if 'lastStartTag' not in test:
         test['lastStartTag'] = None
     parser = TokenizerTestParser(test['initialState'],
