micropython · dpgeorge · Jun 28, 2014 · Jun 12, 2014 · Jun 3, 2014 · Jun 12, 2014
diff --git a/py/builtin.c b/py/builtin.c
@@ -172,13 +172,40 @@ STATIC mp_obj_t mp_builtin_callable(mp_obj_t o_in) {
 MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_callable_obj, mp_builtin_callable);
 
 STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) {
-    int ord = mp_obj_get_int(o_in);
+    #if MICROPY_PY_BUILTINS_STR_UNICODE
+    machine_int_t c = mp_obj_get_int(o_in);
+    char str[4];
+    int len = 0;
+    if (c < 0x80) {
+        *str = c; len = 1;
+    } else if (c < 0x800) {
+        str[0] = (c >> 6) | 0xC0;
+        str[1] = (c & 0x3F) | 0x80;
+        len = 2;
+    } else if (c < 0x10000) {
+        str[0] = (c >> 12) | 0xE0;
+        str[1] = ((c >> 6) & 0x3F) | 0x80;
+        str[2] = (c & 0x3F) | 0x80;
+        len = 3;
+    } else if (c < 0x110000) {
+        str[0] = (c >> 18) | 0xF0;
+        str[1] = ((c >> 12) & 0x3F) | 0x80;
+        str[2] = ((c >> 6) & 0x3F) | 0x80;
+        str[3] = (c & 0x3F) | 0x80;
+        len = 4;
+    } else {
+        nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "chr() arg not in range(0x110000)"));
+    }
+    return mp_obj_new_str(str, len, true);
+    #else
+    machine_int_t ord = mp_obj_get_int(o_in);
     if (0 <= ord && ord <= 0x10ffff) {
         char str[1] = {ord};
         return mp_obj_new_str(str, 1, true);
     } else {
         nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "chr() arg not in range(0x110000)"));
     }
+    #endif
 }
 
 MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_chr_obj, mp_builtin_chr);
@@ -344,13 +371,32 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_oct_obj, mp_builtin_oct);
 STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) {
     uint len;
     const char *str = mp_obj_str_get_data(o_in, &len);
+    #if MICROPY_PY_BUILTINS_STR_UNICODE
+    uint charlen = unichar_charlen(str, len);
+    if (charlen == 1) {
+        if (MP_OBJ_IS_STR(o_in) && UTF8_IS_NONASCII(*str)) {
+            machine_int_t ord = *str++ & 0x7F;
+            for (machine_int_t mask = 0x40; ord & mask; mask >>= 1) {
+                ord &= ~mask;
+            }
+            while (UTF8_IS_CONT(*str)) {
+                ord = (ord << 6) | (*str++ & 0x3F);
+            }
+            return mp_obj_new_int(ord);
+        } else {
+            return mp_obj_new_int(((const byte*)str)[0]);
+        }
+    } else {
+        nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "ord() expected a character, but string of length %d found", charlen));
+    }
+    #else
     if (len == 1) {
         // don't sign extend when converting to ord
-        // TODO unicode
         return mp_obj_new_int(((const byte*)str)[0]);
     } else {
         nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "ord() expected a character, but string of length %d found", len));
     }
+    #endif
 }
 
 MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_ord_obj, mp_builtin_ord);

diff --git a/py/lexer.c b/py/lexer.c
@@ -502,19 +502,32 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
                             case 'v': c = 0x0b; break;
                             case 'f': c = 0x0c; break;
                             case 'r': c = 0x0d; break;
+                            case 'u':
+                            case 'U':
+                                if (is_bytes) {
+                                    // b'\u1234' == b'\\u1234'
+                                    vstr_add_char(&lex->vstr, '\\');
+                                    break;
+                                }
+                                // Otherwise fall through.
                             case 'x':
                             {
                                 uint num = 0;
-                                if (!get_hex(lex, 2, &num)) {
+                                if (!get_hex(lex, (c == 'x' ? 2 : c == 'u' ? 4 : 8), &num)) {
                                     // TODO error message
                                     assert(0);
                                 }
                                 c = num;
                                 break;
                             }
-                            case 'N': break; // TODO \N{name} only in strings
-                            case 'u': break; // TODO \uxxxx only in strings
-                            case 'U': break; // TODO \Uxxxxxxxx only in strings
+                            case 'N':
+                                // Supporting '\N{LATIN SMALL LETTER A}' == 'a' would require keeping the
+                                // entire Unicode name table in the core. As of Unicode 6.3.0, that's nearly
+                                // 3MB of text; even gzip-compressed and with minimal structure, it'll take
+                                // roughly half a meg of storage. This form of Unicode escape may be added
+                                // later on, but it's definitely not a priority right now. -- CJA 20140607
+                                assert(!"Unicode name escapes not supported");
+                                break;
                             default:
                                 if (c >= '0' && c <= '7') {
                                     // Octal sequence, 1-3 chars
@@ -533,7 +546,13 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
                         }
                     }
                     if (c != MP_LEXER_CHAR_EOF) {
-                        vstr_add_char(&lex->vstr, c);
+                        if (c < 0x110000 && !is_bytes) {
+                            vstr_add_char(&lex->vstr, c);
+                        } else if (c < 0x100 && is_bytes) {
+                            vstr_add_byte(&lex->vstr, c);
+                        } else {
+                            assert(!"TODO: Throw an error, invalid escape code probably");
+                        }
                     }
                 } else {
                     vstr_add_char(&lex->vstr, CUR_CHAR(lex));

diff --git a/py/misc.h b/py/misc.h
@@ -100,7 +100,9 @@ bool unichar_isupper(unichar c);
 bool unichar_islower(unichar c);
 unichar unichar_tolower(unichar c);
 unichar unichar_toupper(unichar c);
-#define unichar_charlen(s, bytelen) (bytelen)
+uint unichar_charlen(const char *str, uint len); // TODO this should return machine_uint_t
+#define UTF8_IS_NONASCII(ch) ((ch) & 0x80)
+#define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80)
 
 /** variable string *********************************************/
 
@@ -164,4 +166,18 @@ int DEBUG_printf(const char *fmt, ...);
 
 extern uint mp_verbose_flag;
 
+// This is useful for unicode handling. Some CPU archs has
+// special instructions for efficient implentation of this
+// function (e.g. CLZ on ARM).
+// NOTE: this function is unused at the moment
+#ifndef count_lead_ones
+static inline uint count_lead_ones(byte val) {
+    uint c = 0;
+    for (byte mask = 0x80; val & mask; mask >>= 1) {
+        c++;
+    }
+    return c;
+}
+#endif
+
 #endif // _INCLUDED_MINILIB_H
diff --git a/py/mpconfig.h b/py/mpconfig.h
@@ -249,6 +249,11 @@ typedef double mp_float_t;
 /*****************************************************************************/
 /* Fine control over Python builtins, classes, modules, etc                  */
 
+// Whether str object is proper unicode
+#ifndef MICROPY_PY_BUILTINS_STR_UNICODE
+#define MICROPY_PY_BUILTINS_STR_UNICODE (0)
+#endif
+
 // Whether to support bytearray object
 #ifndef MICROPY_PY_BUILTINS_BYTEARRAY
 #define MICROPY_PY_BUILTINS_BYTEARRAY (1)

diff --git a/py/obj.c b/py/obj.c
@@ -357,7 +357,12 @@ uint mp_get_index(const mp_obj_type_t *type, machine_uint_t len, mp_obj_t index,
 
 // may return MP_OBJ_NULL
 mp_obj_t mp_obj_len_maybe(mp_obj_t o_in) {
-    if (MP_OBJ_IS_STR(o_in) || MP_OBJ_IS_TYPE(o_in, &mp_type_bytes)) {
+    if (
+#if !MICROPY_PY_BUILTINS_STR_UNICODE
+        // It's simple - unicode is slow, non-unicode is fast
+        MP_OBJ_IS_STR(o_in) ||
+#endif
+        MP_OBJ_IS_TYPE(o_in, &mp_type_bytes)) {
         return MP_OBJ_NEW_SMALL_INT((machine_int_t)mp_obj_str_get_len(o_in));
     } else {
         mp_obj_type_t *type = mp_obj_get_type(o_in);