GlobalCompleter returns unicode matches

thomasballinger · thomasballinger · commit b4cee8210ccf · 2015-02-17T19:17:20.000-05:00
diff --git a/bpython/_py3compat.py b/bpython/_py3compat.py
@@ -49,3 +49,13 @@ def prepare_for_exec(arg, encoding=None):
 else:
     def prepare_for_exec(arg, encoding=None):
         return arg.encode(encoding)
+
+
+def try_decode(s, encoding):
+    """Try to decode s which is str names. Return None if not decodable"""
+    if not py3 and not isinstance(s, unicode):
+        try:
+            return s.decode(encoding)
+        except UnicodeDecodeError:
+            return None
+    return s
diff --git a/bpython/autocomplete.py b/bpython/autocomplete.py
@@ -34,7 +34,7 @@
 from bpython import inspection
 from bpython import importcompletion
 from bpython import line as lineparts
-from bpython._py3compat import py3
+from bpython._py3compat import py3, try_decode
 from bpython.lazyre import LazyReCompile
 
 
@@ -286,11 +286,17 @@ def matches(self, cursor_offset, line, **kwargs):
         n = len(text)
         for word in keyword.kwlist:
             if method_match(word, n, text):
-                matches.add(word)
+                word = try_decode(word, 'ascii')  # py2 keywords are all ascii
+                if word is not None:
+                    matches.add(word)
         for nspace in [builtins.__dict__, locals_]:
             for word, val in nspace.items():
                 if (method_match(word, len(text), text) and
                         word != "__builtins__"):
+                    word = try_decode(word, 'ascii')
+                    # if identifier isn't ascii, don't complete (syntax error)
+                    if word is None:
+                        continue
                     matches.add(_callable_postfix(val, word))
         return matches
 
diff --git a/bpython/importcompletion.py b/bpython/importcompletion.py
@@ -20,7 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-from bpython._py3compat import py3
+from bpython._py3compat import py3, try_decode
 from bpython.line import current_word, current_import, \
     current_from_import_from, current_from_import_import
 
@@ -42,17 +42,6 @@
 fully_loaded = False
 
 
-def try_decode_module(module, encoding):
-    """Try to decode module names."""
-    if not py3 and not isinstance(module, unicode):
-        try:
-            return module.decode(encoding)
-        except UnicodeDecodeError:
-            # Not importable anyway, ignore it
-            return None
-    return module
-
-
 def module_matches(cw, prefix=''):
     """Modules names to replace cw with"""
     full = '%s.%s' % (prefix, cw) if prefix else cw
@@ -83,7 +72,7 @@ def attr_matches(cw, prefix='', only_modules=False):
     if module_part:
         matches = ('%s.%s' % (module_part, m) for m in matches)
 
-    generator = (try_decode_module(match, 'ascii') for match in matches)
+    generator = (try_decode(match, 'ascii') for match in matches)
     return set(filter(lambda x: x is not None, generator))
 
 
@@ -184,7 +173,7 @@ def find_all_modules(path=None):
         if not p:
             p = os.curdir
         for module in find_modules(p):
-            module = try_decode_module(module, sys.getfilesystemencoding())
+            module = try_decode(module, sys.getfilesystemencoding())
             if module is None:
                 continue
             modules.add(module)
diff --git a/bpython/test/test_autocomplete.py b/bpython/test/test_autocomplete.py
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 from collections import namedtuple
 import inspect
 from bpython._py3compat import py3
@@ -273,6 +275,15 @@ def function():
                                           locals_={'function': function}),
                          set(('function(', )))
 
+    def test_completions_are_unicode(self):
+        for m in self.com.matches(1, 'a', locals_={'abc': 10}):
+            self.assertIsInstance(m, type(u''))
+
+    @unittest.skipIf(py3, "in Python 3 invalid identifiers are passed through")
+    def test_ignores_nonascii_encodable(self):
+        self.assertSetEqual(self.com.matches(1, 'abc', locals_={'abcß': 10}),
+                            set())
+
 
 class TestParameterNameCompletion(unittest.TestCase):
     def test_set_of_params_returns_when_matches_found(self):
