use charade instead of chardet for Python 2.6 - 3.3 compatibility

ambv · gsnedders · commit d442ec320e55 · 2013-05-04T20:56:41.000+01:00
diff --git a/.travis.yml b/.travis.yml
@@ -28,8 +28,6 @@ before_install:
 install:
   - pip install -r requirements.txt -r requirements-test.txt --use-mirrors
   - if [[ $USE_OPTIONAL == "true" ]]; then pip install -r requirements-optional.txt --use-mirrors; fi
-  - if [[ $TRAVIS_PYTHON_VERSION != 3.* && $USE_OPTIONAL == "true" ]]; then pip install -r requirements-optional-2.txt --use-mirrors; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3.* && $USE_OPTIONAL == "true" ]]; then pip install -r requirements-optional-3.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION != "pypy" && $USE_OPTIONAL == "true" ]]; then pip install -r requirements-optional-cpython.txt --use-mirrors; fi
   - if [[ $FLAKE == "true" ]]; then pip install --use-mirrors flake8; fi
 
diff --git a/README.rst b/README.rst
@@ -30,10 +30,9 @@ Optionally:
 
 - ``genshi`` has a treewalker (but not builder); and
 
-- ``chardet`` can be used as a fallback when character encoding cannot
-  be determined (note currently this is only packaged on PyPI for
-  Python 2, though several package managers include unofficial ports
-  to Python 3).
+- ``charade`` can be used as a fallback when character encoding cannot
+  be determined; ``chardet``, from which it was forked, can also be used
+  on Python 2.
 
 
 Installation
diff --git a/debug-info.py b/debug-info.py
@@ -12,7 +12,7 @@
     "maxsize": sys.maxsize
 }
 
-search_modules = ["chardet", "datrie", "genshi", "html5lib", "lxml", "six"]
+search_modules = ["charade", "chardet", "datrie", "genshi", "html5lib", "lxml", "six"]
 found_modules = []
 
 for m in search_modules:
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
@@ -457,7 +457,10 @@ def detectEncoding(self, parseMeta=True, chardet=True):
         if encoding is None and chardet:
             confidence = "tentative"
             try:
-                from chardet.universaldetector import UniversalDetector
+                try:
+                    from charade.universaldetector import UniversalDetector
+                except ImportError:
+                    from chardet.universaldetector import UniversalDetector
                 buffers = []
                 detector = UniversalDetector()
                 while not detector.done:
diff --git a/html5lib/tests/test_encoding.py b/html5lib/tests/test_encoding.py
@@ -53,9 +53,12 @@ def test_encoding():
             yield (runPreScanEncodingTest, test[b'data'], test[b'encoding'])
 
 try:
-    import chardet  # flake8: noqa
+    try:
+        import charade  # flake8: noqa
+    except ImportError:
+        import chardet  # flake8: noqa
 except ImportError:
-    print("chardet not found, skipping chardet tests")
+    print("charade/chardet not found, skipping chardet tests")
 else:
     def test_chardet():
         with open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt"), "rb") as fp:
diff --git a/requirements-optional-2.txt b/requirements-optional-2.txt
diff --git a/requirements-optional-3.txt b/requirements-optional-3.txt
diff --git a/requirements-optional.txt b/requirements-optional.txt
@@ -5,3 +5,7 @@ genshi
 # DATrie can be used in place of our Python trie implementation for
 # slightly better parsing performance.
 datrie
+
+# charade can be used as a fallback in case we are unable to determine
+# the encoding of a document.
+charade

Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@`
`12`	`12`	`"maxsize": sys.maxsize`
`13`	`13`	`}`
`14`	`14`
`15`		`-search_modules = ["chardet", "datrie", "genshi", "html5lib", "lxml", "six"]`
	`15`	`+search_modules = ["charade", "chardet", "datrie", "genshi", "html5lib", "lxml", "six"]`
`16`	`16`	`found_modules = []`
`17`	`17`
`18`	`18`	`for m in search_modules:`