Skip to content

Commit a95f7e2

Browse files
amitabamitab
committed
WL10659: Support utf8mb4 as default charset
This patch updates the list of supported charsets as of MySQL 8.0.1. Unittests can now be run with MySQL 8.0.1. Tests have been updated.
1 parent 6482770 commit a95f7e2

File tree

8 files changed

+141
-66
lines changed

8 files changed

+141
-66
lines changed

‎lib/mysqlx/charsets.py

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# -*- coding: utf-8 -*-
22

33
# MySQL Connector/Python - MySQL driver written in Python.
4-
# Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
4+
# Copyright (c) 2013, 2017, Oracle and/or its affiliates. All rights reserved.
55

66
# MySQL Connector/Python is licensed under the terms of the GPLv2
77
# <http://www.gnu.org/licenses/old-licenses/gpl-2.0.html>, like most
@@ -24,8 +24,8 @@
2424
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
2525

2626
# This file was auto-generated.
27-
_GENERATED_ON = '2016-06-06'
28-
_MYSQL_VERSION = (5, 7, 12)
27+
_GENERATED_ON = '2017-04-27'
28+
_MYSQL_VERSION = (8, 0, 1)
2929

3030
"""This module contains the MySQL Server Character Sets"""
3131

@@ -76,7 +76,7 @@
7676
("latin7", "latin7_general_cs", False), # 42
7777
("macce", "macce_bin", False), # 43
7878
("cp1250", "cp1250_croatian_ci", False), # 44
79-
("utf8mb4", "utf8mb4_general_ci", True), # 45
79+
("utf8mb4", "utf8mb4_general_ci", False), # 45
8080
("utf8mb4", "utf8mb4_bin", False), # 46
8181
("latin1", "latin1_bin", False), # 47
8282
("latin1", "latin1_general_ci", False), # 48
@@ -107,7 +107,7 @@
107107
("keybcs2", "keybcs2_bin", False), # 73
108108
("koi8r", "koi8r_bin", False), # 74
109109
("koi8u", "koi8u_bin", False), # 75
110-
None,
110+
("utf8", "utf8_tolower_ci", False), # 76
111111
("latin2", "latin2_bin", False), # 77
112112
("latin5", "latin5_bin", False), # 78
113113
("latin7", "latin7_bin", False), # 79
@@ -282,5 +282,58 @@
282282
("gb18030", "gb18030_chinese_ci", True), # 248
283283
("gb18030", "gb18030_bin", False), # 249
284284
("gb18030", "gb18030_unicode_520_ci", False), # 250
285+
None,
286+
None,
287+
None,
288+
None,
289+
("utf8mb4", "utf8mb4_0900_ai_ci", True), # 255
290+
("utf8mb4", "utf8mb4_de_pb_0900_ai_ci", False), # 256
291+
("utf8mb4", "utf8mb4_is_0900_ai_ci", False), # 257
292+
("utf8mb4", "utf8mb4_lv_0900_ai_ci", False), # 258
293+
("utf8mb4", "utf8mb4_ro_0900_ai_ci", False), # 259
294+
("utf8mb4", "utf8mb4_sl_0900_ai_ci", False), # 260
295+
("utf8mb4", "utf8mb4_pl_0900_ai_ci", False), # 261
296+
("utf8mb4", "utf8mb4_et_0900_ai_ci", False), # 262
297+
("utf8mb4", "utf8mb4_es_0900_ai_ci", False), # 263
298+
("utf8mb4", "utf8mb4_sv_0900_ai_ci", False), # 264
299+
("utf8mb4", "utf8mb4_tr_0900_ai_ci", False), # 265
300+
("utf8mb4", "utf8mb4_cs_0900_ai_ci", False), # 266
301+
("utf8mb4", "utf8mb4_da_0900_ai_ci", False), # 267
302+
("utf8mb4", "utf8mb4_lt_0900_ai_ci", False), # 268
303+
("utf8mb4", "utf8mb4_sk_0900_ai_ci", False), # 269
304+
("utf8mb4", "utf8mb4_es_trad_0900_ai_ci", False), # 270
305+
("utf8mb4", "utf8mb4_la_0900_ai_ci", False), # 271
306+
None,
307+
("utf8mb4", "utf8mb4_eo_0900_ai_ci", False), # 273
308+
("utf8mb4", "utf8mb4_hu_0900_ai_ci", False), # 274
309+
("utf8mb4", "utf8mb4_hr_0900_ai_ci", False), # 275
310+
None,
311+
("utf8mb4", "utf8mb4_vi_0900_ai_ci", False), # 277
312+
("utf8mb4", "utf8mb4_0900_as_cs", False), # 278
313+
("utf8mb4", "utf8mb4_de_pb_0900_as_cs", False), # 279
314+
("utf8mb4", "utf8mb4_is_0900_as_cs", False), # 280
315+
("utf8mb4", "utf8mb4_lv_0900_as_cs", False), # 281
316+
("utf8mb4", "utf8mb4_ro_0900_as_cs", False), # 282
317+
("utf8mb4", "utf8mb4_sl_0900_as_cs", False), # 283
318+
("utf8mb4", "utf8mb4_pl_0900_as_cs", False), # 284
319+
("utf8mb4", "utf8mb4_et_0900_as_cs", False), # 285
320+
("utf8mb4", "utf8mb4_es_0900_as_cs", False), # 286
321+
("utf8mb4", "utf8mb4_sv_0900_as_cs", False), # 287
322+
("utf8mb4", "utf8mb4_tr_0900_as_cs", False), # 288
323+
("utf8mb4", "utf8mb4_cs_0900_as_cs", False), # 289
324+
("utf8mb4", "utf8mb4_da_0900_as_cs", False), # 290
325+
("utf8mb4", "utf8mb4_lt_0900_as_cs", False), # 291
326+
("utf8mb4", "utf8mb4_sk_0900_as_cs", False), # 292
327+
("utf8mb4", "utf8mb4_es_trad_0900_as_cs", False), # 293
328+
("utf8mb4", "utf8mb4_la_0900_as_cs", False), # 294
329+
None,
330+
("utf8mb4", "utf8mb4_eo_0900_as_cs", False), # 296
331+
("utf8mb4", "utf8mb4_hu_0900_as_cs", False), # 297
332+
("utf8mb4", "utf8mb4_hr_0900_as_cs", False), # 298
333+
None,
334+
("utf8mb4", "utf8mb4_vi_0900_as_cs", False), # 300
335+
None,
336+
None,
337+
("utf8mb4", "utf8mb4_ja_0900_as_cs", False), # 303
285338
]
286339

‎src/mysql_capi.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,8 +1042,13 @@ MySQL_connect(MySQL *self, PyObject *args, PyObject *kwds)
10421042
#if MYSQL_VERSION_ID >= 50711
10431043
unsigned int ssl_mode;
10441044
#endif
1045-
my_bool abool;
1046-
my_bool ssl_enabled= 0;
1045+
#if MYSQL_VERSION_ID >= 80001
1046+
bool abool;
1047+
bool ssl_enabled= 0;
1048+
#else
1049+
my_bool abool;
1050+
my_bool ssl_enabled= 0;
1051+
#endif
10471052
MYSQL *res;
10481053

10491054
static char *kwlist[]=

‎tests/cext/test_cext_api.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -301,10 +301,10 @@ def test_get_server_version(self):
301301
version = cmy.get_server_version()
302302
self.assertIsInstance(version, tuple)
303303
self.assertEqual(3, len(version))
304-
self.assertTrue(all([isinstance(v, int) and v > 0 for v in version]))
304+
self.assertTrue(all([isinstance(v, int) for v in version]))
305305

306-
self.assertTrue(3 < version[0] < 7)
307-
self.assertTrue(0 < version[1] < 20)
306+
self.assertTrue(3 < version[0] < 9)
307+
self.assertTrue(0 <= version[1] < 20)
308308
self.assertTrue(0 < version[2] < 99)
309309

310310
def test_thread_id(self):

‎tests/mysqld.py

Lines changed: 60 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ def __init__(self, basedir, topdir, cnf, bind_address, port, mysqlx_port,
376376
super(MySQLServer, self).__init__(self._basedir,
377377
self._option_file,
378378
sharedir=self._sharedir)
379+
self._init_sql = os.path.join(self._topdir, 'init.sql')
379380

380381
def _create_directories(self):
381382
"""Create directory structure for bootstrapping
@@ -389,9 +390,12 @@ def _create_directories(self):
389390
dirs = [
390391
self._topdir,
391392
os.path.join(self._topdir, 'tmp'),
392-
self._datadir,
393-
os.path.join(self._datadir, 'mysql')
394393
]
394+
395+
if self._version[0:3] < (8, 0, 1):
396+
dirs.append(self._datadir)
397+
dirs.append(os.path.join(self._datadir, 'mysql'))
398+
395399
for adir in dirs:
396400
LOGGER.debug("Creating directory %s", adir)
397401
os.mkdir(adir)
@@ -407,7 +411,6 @@ def _get_bootstrap_cmd(self):
407411
cmd = [
408412
os.path.join(self._sbindir, EXEC_MYSQLD),
409413
'--no-defaults',
410-
'--bootstrap',
411414
'--basedir=%s' % self._basedir,
412415
'--datadir=%s' % self._datadir,
413416
'--log-warnings=0',
@@ -417,6 +420,13 @@ def _get_bootstrap_cmd(self):
417420
'--tmpdir=%s' % self._tmpdir,
418421
'--innodb_log_file_size=1Gb',
419422
]
423+
424+
if self._version[0:2] >= (8, 0):
425+
cmd.append("--initialize-insecure")
426+
cmd.append("--init-file={0}".format(self._init_sql))
427+
else:
428+
cmd.append("--bootstrap")
429+
420430
if self._version[0:2] < (5, 5):
421431
cmd.append('--language={0}/english'.format(self._lc_messages_dir))
422432
else:
@@ -453,68 +463,62 @@ def bootstrap(self):
453463
extra_sql = [
454464
"CREATE DATABASE myconnpy;"
455465
]
456-
insert = (
457-
"INSERT INTO mysql.user VALUES ('localhost','root'{0},"
458-
"'Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y',"
459-
"'Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y',"
460-
"'Y','Y','Y','Y','Y','','','','',0,0,0,0,"
461-
"@@default_authentication_plugin,'','N',"
462-
"CURRENT_TIMESTAMP,NULL{1}), ('::1','root'{0},"
463-
"'Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y',"
464-
"'Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y',"
465-
"'Y','Y','Y','Y','Y','','','','',0,0,0,0,"
466-
"@@default_authentication_plugin,'','N',"
467-
"CURRENT_TIMESTAMP,NULL{1});"
468-
)
469-
# MySQL 5.7.5+ creates no user while bootstrapping
470-
if self._version[0:3] >= (5, 7, 6):
471-
# MySQL 5.7.6+ have extra account_locked col and no password col
472-
extra_sql.append(insert.format("", ",'N'"))
466+
defaults = ("'root'{0}, "
467+
"'Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y',"
468+
"'Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y',"
469+
"'Y','Y','Y','Y','Y','','','','',0,0,0,0,"
470+
"@@default_authentication_plugin,'','N',"
471+
"CURRENT_TIMESTAMP,NULL{1}")
472+
473+
hosts = ["::1", "127.0.0.1"]
474+
if self._version[0:3] < (8, 0, 1):
475+
# because we use --initialize-insecure for 8.0 above
476+
# which already creates root @ localhost
477+
hosts.append("localhost")
478+
479+
insert = "INSERT INTO mysql.user VALUES {0};".format(
480+
", ".join("('{0}', {{0}})".format(host) for host in hosts))
481+
482+
if self._version[0:3] >= (8, 0, 1):
483+
# No password column, has account_locked, Create_role_priv and
484+
# Drop_role_priv columns
485+
defaults = defaults.format("", ", 'N', 'Y', 'Y'")
486+
elif self._version[0:3] >= (5, 7, 6):
487+
# No password column, has account_locked column
488+
defaults = defaults.format("", ", 'N'")
473489
elif self._version[0:3] >= (5, 7, 5):
474-
extra_sql.append(insert.format(",''", ""))
475-
476-
insert_localhost = (
477-
"INSERT INTO mysql.user SELECT '127.0.0.1', `User`{0},"
478-
" `Select_priv`, `Insert_priv`, `Update_priv`, `Delete_priv`,"
479-
" `Create_priv`, `Drop_priv`, `Reload_priv`, `Shutdown_priv`,"
480-
" `Process_priv`, `File_priv`, `Grant_priv`, `References_priv`,"
481-
" `Index_priv`, `Alter_priv`, `Show_db_priv`, `Super_priv`,"
482-
" `Create_tmp_table_priv`, `Lock_tables_priv`, `Execute_priv`,"
483-
" `Repl_slave_priv`, `Repl_client_priv`, `Create_view_priv`,"
484-
" `Show_view_priv`, `Create_routine_priv`, "
485-
"`Alter_routine_priv`,"
486-
" `Create_user_priv`, `Event_priv`, `Trigger_priv`, "
487-
"`Create_tablespace_priv`, `ssl_type`, `ssl_cipher`,"
488-
"`x509_issuer`, `x509_subject`, `max_questions`, `max_updates`,"
489-
"`max_connections`, `max_user_connections`, `plugin`,"
490-
"`authentication_string`, `password_expired`,"
491-
"`password_last_changed`, `password_lifetime`{1} FROM mysql.user "
492-
"WHERE `user` = 'root' and `host` = 'localhost';"
493-
)
494-
# MySQL 5.7.4+ only creates root@localhost
495-
if self._version[0:3] >= (5, 7, 6):
496-
extra_sql.append(insert_localhost.format("", ",`account_locked`"))
497-
elif self._version[0:3] >= (5, 7, 4):
498-
extra_sql.append(insert_localhost.format(",`Password`", ""))
490+
# The password column
491+
defaults = defaults.format(", ''", "")
499492

493+
extra_sql.append(insert.format(defaults))
500494
bootstrap_log = os.path.join(self._topdir, 'bootstrap.log')
501495
try:
502496
self._create_directories()
503497
cmd = self._get_bootstrap_cmd()
504498
sql = ["USE mysql;"]
505-
for filename in script_files:
506-
full_path = os.path.join(self._scriptdir, filename)
507-
LOGGER.debug("Reading SQL from '%s'", full_path)
508-
with open(full_path, 'r') as fp:
509-
sql.extend([line.strip() for line in fp.readlines()])
510-
sql.extend(extra_sql)
499+
500+
if self._version[0:2] >= (8, 0):
501+
test_sql = open(self._init_sql, "w")
502+
test_sql.write("\n".join(extra_sql))
503+
test_sql.close()
504+
else:
505+
for filename in script_files:
506+
full_path = os.path.join(self._scriptdir, filename)
507+
LOGGER.debug("Reading SQL from '%s'", full_path)
508+
with open(full_path, 'r') as fp:
509+
sql.extend([line.strip() for line in fp.readlines()])
510+
511511
fp_log = open(bootstrap_log, 'w')
512-
prc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
513-
stderr=subprocess.STDOUT, stdout=fp_log)
514-
if sys.version_info[0] == 2:
515-
prc.communicate('\n'.join(sql))
512+
if self._version[0:2] < (8, 0):
513+
sql.extend(extra_sql)
514+
prc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
515+
stderr=subprocess.STDOUT,
516+
stdout=fp_log)
517+
prc.communicate('\n'.join(sql) if sys.version_info[0] == 2
518+
else bytearray('\n'.join(sql), 'utf8'))
516519
else:
517-
prc.communicate(bytearray('\n'.join(sql), 'utf8'))
520+
prc = subprocess.call(cmd, stderr=subprocess.STDOUT,
521+
stdout=fp_log)
518522
fp_log.close()
519523
except OSError as err:
520524
raise MySQLBootstrapError(

‎tests/test_abstracts.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,8 @@ def test_cmd_quit(self):
207207
self.cnx.cmd_quit()
208208
self.assertFalse(self.cnx.is_connected())
209209

210+
@unittest.skipIf(tests.MYSQL_VERSION >= (8, 0, 1),
211+
"As of MySQL 8.0.1, CMD_SHUTDOWN is not recognized.")
210212
@foreach_cnx()
211213
def test_cmd_shutdown(self):
212214
server = tests.MYSQL_SERVERS[0]

‎tests/test_bugs.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1821,6 +1821,8 @@ def test_load_csv(self):
18211821
self.assertEqual(self.exp_rows, cur.fetchone()[0])
18221822

18231823

1824+
@unittest.skipIf(tests.MYSQL_VERSION >= (8, 0, 1),
1825+
"BugOra17422299 not tested with MySQL version >= 8.0.1")
18241826
class BugOra17422299(tests.MySQLConnectorTests):
18251827
"""BUG#17422299: cmd_shutdown fails with malformed connection packet
18261828
"""

‎tests/test_mysqlx_crud.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1422,6 +1422,9 @@ def test_column_metadata(self):
14221422
self.assertEqual("name", col.get_column_name())
14231423
self.assertEqual("test", col.get_table_name())
14241424
self.assertEqual(mysqlx.ColumnType.STRING, col.get_type())
1425+
if tests.MYSQL_VERSION >= (8, 0, 1):
1426+
self.assertEqual("utf8mb4_0900_ai_ci", col.get_collation_name())
1427+
self.assertEqual("utf8mb4", col.get_character_set_name())
14251428

14261429
col = result.columns[2]
14271430
self.assertEqual("pic", col.get_column_name())

‎unittests.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,12 @@
9999
# MySQL option file template. Platform specifics dynamically added later.
100100
MY_CNF = """
101101
# MySQL option file for MySQL Connector/Python tests
102+
[mysqld-8.0]
103+
information-schema-stats=LATEST
104+
plugin-load={mysqlx_plugin}
105+
loose_mysqlx_port={mysqlx_port}
106+
{mysqlx_bind_address}
107+
102108
[mysqld-5.7]
103109
plugin-load={mysqlx_plugin}
104110
loose_mysqlx_port={mysqlx_port}

0 commit comments

Comments
 (0)