Add expected failures for the tokenizer (except lone surrogates!).

gsnedders · gsnedders · commit 0081e81acaa4 · 2014-11-24T00:47:39.000Z
We can't currently add the lone-surrogates tests because the
expected failures file is UTF-8 and we can't have lone-surrogates
there. Le sigh…
diff --git a/html5lib/tests/expected-failures/tokenizer.dat b/html5lib/tests/expected-failures/tokenizer.dat
@@ -0,0 +1,43 @@
+#data
+<!DOCTYPE>
+
+#data
+<!DOCTYPE >
+
+#data
+<!DOCTYPE	
+
+#data
+<!DOCTYPE
+
+
+#data
+<!DOCTYPE
+
+#data
+<!DOCTYPE
+
+#data
+<!DOCTYPE 
+
+
+#data
+<!DOCTYPE 
+
+#data
+<!DOCTYPE 
+
+#data
+<!DOCTYPE  
+
+#data
+<!DOCTYPE 
+
+#data
+<!DOCTYPE
+
+#data
+I'm &no
+
+#data
+<!DOCTYPE 	
diff --git a/html5lib/tests/test_tokenizer.py b/html5lib/tests/test_tokenizer.py
@@ -3,8 +3,9 @@
 import json
 import warnings
 import re
+import os
 
-from .support import get_data_files
+from .support import get_data_files, TestData, xfail
 
 from html5lib.tokenizer import HTMLTokenizer
 from html5lib import constants
@@ -161,6 +162,11 @@ def runTokenizerTest(test):
     assert tokensMatch(expected, received, ignoreErrorOrder, True), errorMsg
 
 
+@xfail
+def xfailRunTokenizerTest(*args, **kwargs):
+    return runTokenizerTest(*args, **kwargs)
+
+
 def _doCapitalize(match):
     return match.group(1).upper()
 
@@ -174,6 +180,14 @@ def capitalize(s):
 
 
 def testTokenizer():
+    # Get xfails
+    filename = os.path.join(os.path.split(__file__)[0],
+                            "expected-failures",
+                            "tokenizer.dat")
+    xfails = TestData(filename, "data")
+    xfails = frozenset([x["data"] for x in xfails])
+
+    # Get tests
     for filename in get_data_files('tokenizer', '*.test'):
         with open(filename) as fp:
             tests = json.load(fp)
@@ -185,4 +199,8 @@ def testTokenizer():
                         test = unescape(test)
                     for initialState in test["initialStates"]:
                         test["initialState"] = capitalize(initialState)
-                        yield runTokenizerTest, test
+                        if test['input'] in xfails:
+                            testFunc = xfailRunTokenizerTest
+                        else:
+                            testFunc = runTokenizerTest
+                        yield testFunc, test
