More encoding bugfixes

jgraham · jgraham · commit 9c136b08d374 · 2007-02-23T20:18:27.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40507
diff --git a/src/constants.py b/src/constants.py
@@ -112,7 +112,8 @@
     u"\n",
     u"\u000B",
     u"\u000C",
-    u" "
+    u" ",
+    u"\r"
 ))
 
 tableInsertModeElements = frozenset((
@@ -455,3 +456,203 @@
     "zwj": u"\u200D",
     "zwnj": u"\u200C"
 }
+
+encodings = [
+    "ascii",
+    "646",
+    "us-ascii",
+    "big5",
+    "big5-tw",
+    "csbig5",
+    "big5hkscs",
+    "big5-hkscs",
+    "hkscs",
+    "cp874",
+    "cp875",
+    "cp932",
+    "932",
+    "ms932",
+    "mskanji",
+    "ms-kanji",
+    "cp949",
+    "949",
+    "ms949",
+    "uhc",
+    "cp950",
+    "950",
+    "ms950",
+    "cp1006",
+    "cp1250",
+    "windows-1250",
+    "cp1251",
+    "windows-1251",
+    "cp1252",
+    "windows-1252",
+    "cp1253",
+    "windows-1253",
+    "cp1254",
+    "windows-1254",
+    "cp1255",
+    "windows-1255",
+    "cp1256",
+    "windows1256",
+    "cp1257",
+    "windows-1257",
+    "cp1258",
+    "windows-1258",
+    "euc_jp",
+    "eucjp",
+    "ujis",
+    "u-jis",
+    "euc_jis_2004",
+    "jisx0213",
+    "eucjis2004",
+    "euc_jisx0213",
+    "eucjisx0213",
+    "euc_kr",
+    "euckr",
+    "korean",
+    "ksc5601",
+    "ks_c-5601",
+    "ks_c-5601-1987",
+    "ksx1001",
+    "ks_x-1001",
+    "gb2312",
+    "chinese",
+    "csiso58gb231280",
+    "euc-cn",
+    "euccn",
+    "eucgb2312-cn",
+    "gb2312-1980",
+    "gb2312-80",
+    "iso-ir-58",
+    "gbk",
+    "936",
+    "cp936",
+    "ms936",
+    "gb18030",
+    "gb18030-2000",
+    "hz",
+    "hzgb",
+    "hz-gb",
+    "hz-gb-2312",
+    "iso2022_jp",
+    "csiso2022jp",
+    "iso2022jp",
+    "iso-2022-jp",
+    "iso2022_jp_1",
+    "iso2022jp-1",
+    "iso-2022-jp-1",
+    "iso2022_jp_2",
+    "iso2022jp-2",
+    "iso-2022-jp-2",
+    "iso2022_jp_2004",
+    "iso2022jp-2004",
+    "iso-2022-jp-2004",
+    "iso2022_jp_3",
+    "iso2022jp-3",
+    "iso-2022-jp-3",
+    "iso2022_jp_ext",
+    "iso2022jp-ext",
+    "iso-2022-jp-ext",
+    "iso2022_kr",
+    "csiso2022kr",
+    "iso2022kr",
+    "iso-2022-kr",
+    "latin_1",
+    "iso-8859-1",
+    "iso8859-1",
+    "8859",
+    "cp819",
+    "latin",
+    "latin1",
+    "L1",
+    "iso8859_2",
+    "iso-8859-2",
+    "latin2",
+    "L2",
+    "iso8859_3",
+    "iso-8859-3",
+    "latin3",
+    "L3",
+    "iso8859_4",
+    "iso-8859-4",
+    "latin4",
+    "L4",
+    "iso8859_5",
+    "iso-8859-5",
+    "cyrillic",
+    "iso8859_6",
+    "iso-8859-6",
+    "arabic",
+    "iso8859_7",
+    "iso-8859-7",
+    "greek",
+    "greek8",
+    "iso8859_8",
+    "iso-8859-8",
+    "hebrew",
+    "iso8859_9",
+    "iso-8859-9",
+    "latin5",
+    "L5",
+    "iso8859_10",
+    "iso-8859-10",
+    "latin6",
+    "L6",
+    "iso8859_13",
+    "iso-8859-13",
+    "iso8859_14",
+    "iso-8859-14",
+    "latin8",
+    "L8",
+    "iso8859_15",
+    "iso-8859-15",
+    "johab",
+    "cp1361",
+    "ms1361",
+    "koi8_r",
+    "koi8_u",
+    "mac_cyrillic",
+    "maccyrillic",
+    "mac_greek",
+    "macgreek",
+    "mac_iceland",
+    "maciceland",
+    "mac_latin2",
+    "maclatin2",
+    "maccentraleurope",
+    "mac_roman",
+    "macroman",
+    "mac_turkish",
+    "macturkish",
+    "ptcp154",
+    "csptcp154",
+    "pt154",
+    "cp154",
+    "cyrillic-asian",
+    "shift_jis",
+    "csshiftjis",
+    "shiftjis",
+    "sjis",
+    "s_jis",
+    "shift_jis_2004",
+    "shiftjis2004",
+    "sjis_2004",
+    "sjis2004",
+    "shift_jisx0213",
+    "shiftjisx0213",
+    "sjisx0213",
+    "s_jisx0213",
+    "utf_16",
+    "U16",
+    "utf16",
+    "utf_16_be",
+    "UTF-16BE",
+    "utf_16_le",
+    "UTF-16LE",
+    "utf_8",
+    "U8",
+    "UTF",
+    "utf8",
+    "utf_8_sig"]
diff --git a/src/inputstream.py b/src/inputstream.py
@@ -2,6 +2,7 @@
 import re
 
 from constants import EOF, spaceCharacters, asciiLetters, asciiUppercase
+from constants import encodings
 from utils import MethodDispatcher
 
 class HTMLInputStream(object):
@@ -356,14 +357,8 @@ def getAttribute(self):
         return attr
 
     def isValidEncoding(self, encoding):
-        """Determine if encoding is a valid encoding and, if it is, set it 
-        as the encoding on the inputstream"""
-        try:
-            codecs.lookup(encoding)
-            rv = True
-        except LookupError:
-            rv = False
-        return rv
+        """Determine if a string is a supported encoding"""
+        return encoding is not None and encoding.lower() in encodings
 
 class FragmentParser(object):
     """Helper object for parsing document fragments e.g. attributes and content
diff --git a/tests/test_encoding.py b/tests/test_encoding.py
@@ -4,13 +4,14 @@
 import StringIO
 import unittest
 import new
+import codecs
 
 # XXX Allow us to import the sibling module
 os.chdir(os.path.split(os.path.abspath(__file__))[0])
 sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, "src")))
 
 import inputstream
-import codecs
+
 
 def parseTestcase(testString):
     testString = testString.split("\n")
