From c3becee5e94223fac50d3fcfe8a3b157d76bbea7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 5 Jul 2018 11:47:08 +0900 Subject: [PATCH 001/227] Update documents --- README.rst | 2 +- docs/source/user/installation.rst | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 1c7fba54c..163c4f2a4 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ Package is uploaded on `PyPI `_. You can install it with pip:: - $ pip3 install PyMySQL + $ python3 -m pip install PyMySQL Documentation diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index e3bfe84d0..8a81fddb6 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -6,24 +6,22 @@ Installation The last stable release is available on PyPI and can be installed with ``pip``:: - $ pip install PyMySQL + $ python3 -m pip install PyMySQL Requirements ------------- * Python -- one of the following: - - CPython_ >= 2.6 or >= 3.3 - - PyPy_ >= 4.0 - - IronPython_ 2.7 + - CPython_ >= 2.7 or >= 3.4 + - Latest PyPy_ * MySQL Server -- one of the following: - - MySQL_ >= 4.1 (tested with only 5.5~) - - MariaDB_ >= 5.1 + - MySQL_ >= 5.5 + - MariaDB_ >= 5.5 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ -.. _IronPython: http://ironpython.net/ .. _MySQL: http://www.mysql.com/ .. _MariaDB: https://mariadb.org/ From 7ea89711cf4ade1970a8cf359dc074b19c272184 Mon Sep 17 00:00:00 2001 From: aaron jheng Date: Mon, 16 Jul 2018 19:42:04 +0800 Subject: [PATCH 002/227] Update Pipfile (#712) --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 0e142ba35..a18fa51af 100644 --- a/Pipfile +++ b/Pipfile @@ -1,5 +1,5 @@ [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = true name = "pypi" From 3571e904f4f598a9010456ba8e56542846a68b60 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Mon, 16 Jul 2018 21:56:17 -0700 Subject: [PATCH 003/227] Return "mysql_native_password" in auth response (#709) --- pymysql/connections.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 1e580d21d..e9dd4c99c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -804,7 +804,11 @@ def _request_authentication(self): authresp = b'' plugin_name = None - if self._auth_plugin_name in ('', 'mysql_native_password'): + if self._auth_plugin_name == '': + plugin_name = b'' + authresp = _auth.scramble_native_password(self.password, self.salt) + elif self._auth_plugin_name == 'mysql_native_password': + plugin_name = b'mysql_native_password' authresp = _auth.scramble_native_password(self.password, self.salt) elif self._auth_plugin_name == 'caching_sha2_password': plugin_name = b'caching_sha2_password' From 1c8ee8fa71757f1a8be2b0c041a9174ae35963f6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 25 Jul 2018 11:56:34 +0900 Subject: [PATCH 004/227] Update CHANGES --- CHANGELOG | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index b4372fed6..d73ddd79d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,13 @@ # Changes +## 0.9.2 + +Release date: 2018-07-04 + +* Disalbled unintentinally enabled debug log +* Removed unintentionally installed tests + + ## 0.9.1 Release date: 2018-07-03 From 55e195cdf2f19c3f36007b72c1a9093dd9b0ebdb Mon Sep 17 00:00:00 2001 From: pick2510 Date: Sun, 12 Aug 2018 05:10:49 +0200 Subject: [PATCH 005/227] Fix old password support (#713) --- pymysql/_auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index bbb742d3a..7a7377bf7 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -4,6 +4,8 @@ from ._compat import text_type, PY2 from .constants import CLIENT from .err import OperationalError +from .util import byte2int, int2byte + from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization, hashes @@ -11,6 +13,7 @@ from functools import partial import hashlib +import io import struct From 104e84f5c9c84e5e75bfcad3d23306871c838a8b Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 10 Sep 2018 16:56:32 +0100 Subject: [PATCH 006/227] Cleanup after Python 2.6 support removed (#725) --- pymysql/connections.py | 4 ++-- pymysql/cursors.py | 6 +++--- tox.ini | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index e9dd4c99c..f1ae621dc 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -302,8 +302,8 @@ def _config(key, arg): conv = converters.conversions # Need for MySQLdb compatibility. - self.encoders = dict([(k, v) for (k, v) in conv.items() if type(k) is not int]) - self.decoders = dict([(k, v) for (k, v) in conv.items() if type(k) is int]) + self.encoders = {k: v for (k, v) in conv.items() if type(k) is not int} + self.decoders = {k: v for (k, v) in conv.items() if type(k) is int} self.sql_mode = sql_mode self.init_command = init_command self.max_allowed_packet = max_allowed_packet diff --git a/pymysql/cursors.py b/pymysql/cursors.py index cc169987b..a6d645d41 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -122,9 +122,9 @@ def _escape_args(self, args, conn): return tuple(conn.literal(arg) for arg in args) elif isinstance(args, dict): if PY2: - args = dict((ensure_bytes(key), ensure_bytes(val)) for - (key, val) in args.items()) - return dict((key, conn.literal(val)) for (key, val) in args.items()) + args = {ensure_bytes(key): ensure_bytes(val) for + (key, val) in args.items()} + return {key: conn.literal(val) for (key, val) in args.items()} else: # If it's not a dictionary let's try escaping it anyways. # Worst case it will throw a Value error diff --git a/tox.ini b/tox.ini index a50364c9c..e2f2917cd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34,pypy,pypy3 +envlist = py27,py34,py35,py36,py37,pypy,pypy3 [testenv] commands = coverage run ./runtests.py From 09040abb0a8153b68b76113539632e7eaa53ce69 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Sep 2018 12:45:31 +1000 Subject: [PATCH 007/227] Fix docstring (#727) capath and cipher are supported. --- pymysql/connections.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index f1ae621dc..4dae75d36 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -152,7 +152,6 @@ class Connection(object): (default: 10, min: 1, max: 31536000) :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. - For now the capath and cipher arguments are not supported. :param read_default_group: Group to read from in the configuration file. :param compress: Not supported :param named_pipe: Not supported From 3ab3b275e3d60be733f2c3f1bf6cfd644863466c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 29 Oct 2018 16:41:14 +0900 Subject: [PATCH 008/227] sys.argv is not always available (#739) fixes #736 --- pymysql/connections.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 4dae75d36..92ba3861c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -315,10 +315,12 @@ def _config(key, arg): '_pid': str(os.getpid()), '_client_version': VERSION_STRING, } + if program_name is None: + argv = getattr(sys, "argv") + if argv: + program_name = argv[0] if program_name: self._connect_attrs["program_name"] = program_name - elif sys.argv: - self._connect_attrs["program_name"] = sys.argv[0] if defer_connect: self._sock = None From 27546ef14fc33f058b1003492d4ba72b7a1b58da Mon Sep 17 00:00:00 2001 From: IWAMOTO Toshihiro Date: Wed, 28 Nov 2018 10:38:25 +0900 Subject: [PATCH 009/227] Hide Connection.autocommit_mode docstring (#746) It is an internal variable and the rendered document can be misread as if pymysql defaults to server default autocommit mode. --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 92ba3861c..32a071698 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -294,7 +294,7 @@ def _config(key, arg): self._affected_rows = 0 self.host_info = "Not connected" - #: specified autocommit mode. None means use server default. + # specified autocommit mode. None means use server default. self.autocommit_mode = autocommit if conv is None: From c7154e6c6b091611fa105b124cc6975e07f37441 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 6 Dec 2018 20:56:24 +0900 Subject: [PATCH 010/227] Support non-ascii program_name on Python 2 (#748) Fixes #747 --- pymysql/connections.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 32a071698..c01d9993c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -319,6 +319,9 @@ def _config(key, arg): argv = getattr(sys, "argv") if argv: program_name = argv[0] + if PY2: + program_name = program_name.decode('utf-8', 'replace') + if program_name: self._connect_attrs["program_name"] = program_name @@ -847,9 +850,9 @@ def _request_authentication(self): if self.server_capabilities & CLIENT.CONNECT_ATTRS: connect_attrs = b'' for k, v in self._connect_attrs.items(): - k = k.encode('utf8') + k = k.encode('utf-8') connect_attrs += struct.pack('B', len(k)) + k - v = v.encode('utf8') + v = v.encode('utf-8') connect_attrs += struct.pack('B', len(v)) + v data += struct.pack('B', len(connect_attrs)) + connect_attrs From 9fa5daf8a8157fb80725e6bca78e29cb80833618 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 15 Dec 2018 11:10:20 +0900 Subject: [PATCH 011/227] Remove auto program_name from sys.argv (#755) --- pymysql/connections.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index c01d9993c..7f5646e81 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -315,12 +315,6 @@ def _config(key, arg): '_pid': str(os.getpid()), '_client_version': VERSION_STRING, } - if program_name is None: - argv = getattr(sys, "argv") - if argv: - program_name = argv[0] - if PY2: - program_name = program_name.decode('utf-8', 'replace') if program_name: self._connect_attrs["program_name"] = program_name From c42ddff9412740ad785c633e31967174eec931e0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 16 Dec 2018 13:45:44 +0900 Subject: [PATCH 012/227] Warn when old password is used (#756) --- pymysql/_auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 7a7377bf7..e0a48f743 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -15,6 +15,7 @@ import hashlib import io import struct +import warnings DEBUG = False @@ -72,6 +73,8 @@ def my_rnd(self): def scramble_old_password(password, message): """Scramble for old_password""" + warnings.warn("old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" + "old password support will be removed in future PyMySQL version") hash_pass = _hash_password_323(password) hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) hash_pass_n = struct.unpack(">LL", hash_pass) From 7b18bb6588903bce502308099d6f007fd165e8fc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Dec 2018 21:19:36 +0900 Subject: [PATCH 013/227] test: self.connections[0] -> self.connect() (#758) --- pymysql/tests/test_DictCursor.py | 2 +- pymysql/tests/test_basic.py | 40 ++++++------ pymysql/tests/test_connection.py | 108 ++++++++++++++++--------------- pymysql/tests/test_cursor.py | 2 +- pymysql/tests/test_issues.py | 32 ++++----- pymysql/tests/test_load_local.py | 8 +-- 6 files changed, 98 insertions(+), 94 deletions(-) diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index 9a0d638b2..122882e65 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -14,7 +14,7 @@ class TestDictCursor(base.PyMySQLTestCase): def setUp(self): super(TestDictCursor, self).setUp() - self.conn = conn = self.connections[0] + self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) # create a table ane some data to query diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index a53373224..940661f75 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -18,7 +18,7 @@ class TestConversion(base.PyMySQLTestCase): def test_datatypes(self): """ test every data type """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)") try: @@ -57,7 +57,7 @@ def test_datatypes(self): def test_dict(self): """ test dict escaping """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a integer, b integer, c integer)") try: @@ -68,7 +68,7 @@ def test_dict(self): c.execute("drop table test_dict") def test_string(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a text)") test_value = "I am a test string" @@ -80,7 +80,7 @@ def test_string(self): c.execute("drop table test_dict") def test_integer(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a integer)") test_value = 12345 @@ -94,7 +94,7 @@ def test_integer(self): def test_binary(self): """test binary data""" data = bytes(bytearray(range(255))) - conn = self.connections[0] + conn = self.connect() self.safe_create_table( conn, "test_binary", "create table test_binary (b binary(255))") @@ -106,7 +106,7 @@ def test_binary(self): def test_blob(self): """test blob data""" data = bytes(bytearray(range(256)) * 4) - conn = self.connections[0] + conn = self.connect() self.safe_create_table( conn, "test_blob", "create table test_blob (b blob)") @@ -117,7 +117,7 @@ def test_blob(self): def test_untyped(self): """ test conversion of null, empty string """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("select null,''") self.assertEqual((None,u''), c.fetchone()) @@ -126,7 +126,7 @@ def test_untyped(self): def test_timedelta(self): """ test timedelta conversion """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')") self.assertEqual((datetime.timedelta(0, 45000), @@ -141,7 +141,7 @@ def test_timedelta(self): def test_datetime_microseconds(self): """ test datetime conversion w microseconds""" - conn = self.connections[0] + conn = self.connect() if not self.mysql_server_is(conn, (5, 6, 4)): raise SkipTest("target backend does not support microseconds") c = conn.cursor() @@ -206,7 +206,7 @@ class TestCursor(base.PyMySQLTestCase): # ('max_updates', 3, 1, 11, 11, 0, 0), # ('max_connections', 3, 1, 11, 11, 0, 0), # ('max_user_connections', 3, 1, 11, 11, 0, 0)) - # conn = self.connections[0] + # conn = self.connect() # c = conn.cursor() # c.execute("select * from mysql.user") # @@ -214,7 +214,7 @@ class TestCursor(base.PyMySQLTestCase): def test_fetch_no_result(self): """ test a fetchone() with no rows """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_nr (b varchar(32))") try: @@ -226,7 +226,7 @@ def test_fetch_no_result(self): def test_aggregates(self): """ test aggregate functions """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() try: c.execute('create table test_aggregates (i integer)') @@ -240,7 +240,7 @@ def test_aggregates(self): def test_single_tuple(self): """ test a single tuple """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() self.safe_create_table( conn, 'mystuff', @@ -283,7 +283,7 @@ class TestBulkInserts(base.PyMySQLTestCase): def setUp(self): super(TestBulkInserts, self).setUp() - self.conn = conn = self.connections[0] + self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) # create a table ane some data to query @@ -299,14 +299,14 @@ def setUp(self): """) def _verify_records(self, data): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() cursor.execute("SELECT id, name, age, height from bulkinsert") result = cursor.fetchall() self.assertEqual(sorted(data), sorted(result)) def test_bulk_insert(self): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] @@ -320,7 +320,7 @@ def test_bulk_insert(self): self._verify_records(data) def test_bulk_insert_multiline_statement(self): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] cursor.executemany("""insert @@ -344,7 +344,7 @@ def test_bulk_insert_multiline_statement(self): self._verify_records(data) def test_bulk_insert_single_record(self): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123)] cursor.executemany("insert into bulkinsert (id, name, age, height) " @@ -354,7 +354,7 @@ def test_bulk_insert_single_record(self): def test_issue_288(self): """executemany should work with "insert ... on update" """ - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] cursor.executemany("""insert @@ -380,7 +380,7 @@ def test_issue_288(self): self._verify_records(data) def test_warnings(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 5e95b1c8c..3f1627803 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -96,18 +96,19 @@ class TestAuthentication(base.PyMySQLTestCase): # print("plugin: %r" % r[0]) def test_plugin(self): - if not self.mysql_server_is(self.connections[0], (5, 5, 0)): + conn = self.connect() + if not self.mysql_server_is(conn, (5, 5, 0)): raise unittest2.SkipTest("MySQL-5.5 required for plugins") - cur = self.connections[0].cursor() + cur = conn.cursor() cur.execute("select plugin from mysql.user where concat(user, '@', host)=current_user()") for r in cur: - self.assertIn(self.connections[0]._auth_plugin_name, (r[0], 'mysql_native_password')) + self.assertIn(conn._auth_plugin_name, (r[0], 'mysql_native_password')) @unittest2.skipUnless(socket_auth, "connection to unix_socket required") @unittest2.skipIf(socket_found, "socket plugin already installed") def testSocketAuthInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin auth_socket soname 'auth_socket.so'") TestAuthentication.socket_found = True @@ -132,7 +133,7 @@ def testSocketAuth(self): self.realtestSocketAuth() def realtestSocketAuth(self): - with TempUser(self.connections[0].cursor(), TestAuthentication.osuser + '@localhost', + with TempUser(self.connect().cursor(), TestAuthentication.osuser + '@localhost', self.databases[0]['db'], self.socket_plugin_name) as u: c = pymysql.connect(user=TestAuthentication.osuser, **self.db) @@ -180,7 +181,7 @@ def __init__(self, con): @unittest2.skipIf(two_questions_found, "two_questions plugin already installed") def testDialogAuthTwoQuestionsInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin two_questions soname 'dialog_examples.so'") TestAuthentication.two_questions_found = True @@ -200,7 +201,7 @@ def realTestDialogAuthTwoQuestions(self): TestAuthentication.Dialog.fail=False TestAuthentication.Dialog.m = {b'Password, please:': b'notverysecret', b'Are you sure ?': b'yes, of course'} - with TempUser(self.connections[0].cursor(), 'pymysql_2q@localhost', + with TempUser(self.connect().cursor(), 'pymysql_2q@localhost', self.databases[0]['db'], 'two_questions', 'notverysecret') as u: with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user='pymysql_2q', **self.db) @@ -210,7 +211,7 @@ def realTestDialogAuthTwoQuestions(self): @unittest2.skipIf(three_attempts_found, "three_attempts plugin already installed") def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin three_attempts soname 'dialog_examples.so'") TestAuthentication.three_attempts_found = True @@ -229,7 +230,7 @@ def testDialogAuthThreeAttempts(self): def realTestDialogAuthThreeAttempts(self): TestAuthentication.Dialog.m = {b'Password, please:': b'stillnotverysecret'} TestAuthentication.Dialog.fail=True # fail just once. We've got three attempts after all - with TempUser(self.connections[0].cursor(), 'pymysql_3a@localhost', + with TempUser(self.connect().cursor(), 'pymysql_3a@localhost', self.databases[0]['db'], 'three_attempts', 'stillnotverysecret') as u: pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.DialogHandler}, **self.db) @@ -253,7 +254,7 @@ def realTestDialogAuthThreeAttempts(self): @unittest2.skipIf(os.environ.get('PAMSERVICE') is None, "PAMSERVICE env var required") def testPamAuthInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin pam soname 'auth_pam.so'") TestAuthentication.pam_found = True @@ -276,7 +277,7 @@ def realTestPamAuth(self): db = self.db.copy() import os db['password'] = os.environ.get('PASSWORD') - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute('show grants for ' + TestAuthentication.osuser + '@localhost') grants = cur.fetchone()[0] @@ -311,51 +312,54 @@ def realTestPamAuth(self): @unittest2.skipUnless(socket_auth, "connection to unix_socket required") @unittest2.skipUnless(mysql_old_password_found, "no mysql_old_password plugin") def testMySQLOldPasswordAuth(self): - if self.mysql_server_is(self.connections[0], (5, 7, 0)): + conn = self.connect() + if self.mysql_server_is(conn, (5, 7, 0)): raise unittest2.SkipTest('Old passwords aren\'t supported in 5.7') # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") # from login in MySQL-5.6 - if self.mysql_server_is(self.connections[0], (5, 6, 0)): + if self.mysql_server_is(conn, (5, 6, 0)): raise unittest2.SkipTest('Old passwords don\'t authenticate in 5.6') db = self.db.copy() db['password'] = "crummy p\tassword" - with self.connections[0] as c: - # deprecated in 5.6 - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(self.connections[0], (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) - else: + c = conn.cursor() + + # deprecated in 5.6 + if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + with self.assertWarns(pymysql.err.Warning) as cm: c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) - v = c.fetchone()[0] - self.assertEqual(v, '2a01785203b08770') - # only works in MariaDB and MySQL-5.6 - can't separate out by version - #if self.mysql_server_is(self.connections[0], (5, 5, 0)): - # with TempUser(c, 'old_pass_user@localhost', - # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: - # cur = pymysql.connect(user='old_pass_user', **db).cursor() - # cur.execute("SELECT VERSION()") - c.execute("SELECT @@secure_auth") - secure_auth_setting = c.fetchone()[0] - c.execute('set old_passwords=1') - # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(self.connections[0], (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute('set global secure_auth=0') - else: + else: + c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) + v = c.fetchone()[0] + self.assertEqual(v, '2a01785203b08770') + # only works in MariaDB and MySQL-5.6 - can't separate out by version + #if self.mysql_server_is(self.connect(), (5, 5, 0)): + # with TempUser(c, 'old_pass_user@localhost', + # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: + # cur = pymysql.connect(user='old_pass_user', **db).cursor() + # cur.execute("SELECT VERSION()") + c.execute("SELECT @@secure_auth") + secure_auth_setting = c.fetchone()[0] + c.execute('set old_passwords=1') + # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead + if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + with self.assertWarns(pymysql.err.Warning) as cm: c.execute('set global secure_auth=0') - with TempUser(c, 'old_pass_user@localhost', - self.databases[0]['db'], password=db['password']) as u: - cur = pymysql.connect(user='old_pass_user', **db).cursor() - cur.execute("SELECT VERSION()") - c.execute('set global secure_auth=%r' % secure_auth_setting) + else: + c.execute('set global secure_auth=0') + with TempUser(c, 'old_pass_user@localhost', + self.databases[0]['db'], password=db['password']) as u: + cur = pymysql.connect(user='old_pass_user', **db).cursor() + cur.execute("SELECT VERSION()") + c.execute('set global secure_auth=%r' % secure_auth_setting) @unittest2.skipUnless(socket_auth, "connection to unix_socket required") @unittest2.skipUnless(sha256_password_found, "no sha256 password authentication plugin found") def testAuthSHA256(self): - c = self.connections[0].cursor() + conn = self.connect() + c = conn.cursor() with TempUser(c, 'pymysql_sha256@localhost', self.databases[0]['db'], 'sha256_password') as u: - if self.mysql_server_is(self.connections[0], (5, 7, 0)): + if self.mysql_server_is(conn, (5, 7, 0)): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") else: c.execute('SET old_passwords = 2') @@ -377,7 +381,7 @@ def test_utf8mb4(self): def test_largedata(self): """Large query and response (>=16MB)""" - cur = self.connections[0].cursor() + cur = self.connect().cursor() cur.execute("SELECT @@max_allowed_packet") if cur.fetchone()[0] < 16*1024*1024 + 10: print("Set max_allowed_packet to bigger than 17MB") @@ -387,7 +391,7 @@ def test_largedata(self): assert cur.fetchone()[0] == t def test_autocommit(self): - con = self.connections[0] + con = self.connect() self.assertFalse(con.get_autocommit()) cur = con.cursor() @@ -400,7 +404,7 @@ def test_autocommit(self): self.assertEqual(cur.fetchone()[0], 0) def test_select_db(self): - con = self.connections[0] + con = self.connect() current_db = self.databases[0]['db'] other_db = self.databases[1]['db'] @@ -503,7 +507,7 @@ def escape_foo(x, d): class TestEscape(base.PyMySQLTestCase): def test_escape_string(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() self.assertEqual(con.escape("foo'bar"), "'foo\\'bar'") @@ -516,21 +520,21 @@ def test_escape_string(self): self.assertEqual(con.escape("foo'bar"), "'foo''bar'") def test_escape_builtin_encoders(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() val = datetime.datetime(2012, 3, 4, 5, 6) self.assertEqual(con.escape(val, con.encoders), "'2012-03-04 05:06:00'") def test_escape_custom_object(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() mapping = {Foo: escape_foo} self.assertEqual(con.escape(Foo(), mapping), "bar") def test_escape_fallback_encoder(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() class Custom(str): @@ -540,13 +544,13 @@ class Custom(str): self.assertEqual(con.escape(Custom('foobar'), mapping), "'foobar'") def test_escape_no_default(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() self.assertRaises(TypeError, con.escape, 42, {}) def test_escape_dict_value(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() mapping = con.encoders.copy() @@ -554,7 +558,7 @@ def test_escape_dict_value(self): self.assertEqual(con.escape({'foo': Foo()}, mapping), {'foo': "bar"}) def test_escape_list_item(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() mapping = con.encoders.copy() diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index add047550..fb3e8bedc 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -7,7 +7,7 @@ class CursorTest(base.PyMySQLTestCase): def setUp(self): super(CursorTest, self).setUp() - conn = self.connections[0] + conn = self.connect() self.safe_create_table( conn, "test", "create table test (data varchar(10))", diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index cedd09258..8dca31b73 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -21,7 +21,7 @@ class TestOldIssues(base.PyMySQLTestCase): def test_issue_3(self): """ undefined methods datetime_or_None, date_or_None """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -42,7 +42,7 @@ def test_issue_3(self): def test_issue_4(self): """ can't retrieve TIMESTAMP fields """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -57,7 +57,7 @@ def test_issue_4(self): def test_issue_5(self): """ query on information_schema.tables fails """ - con = self.connections[0] + con = self.connect() cur = con.cursor() cur.execute("select * from information_schema.tables") @@ -73,7 +73,7 @@ def test_issue_6(self): def test_issue_8(self): """ Primary Key and Index error when selecting data """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -98,7 +98,7 @@ def test_issue_9(self): def test_issue_13(self): """ can't handle large result fields """ - conn = self.connections[0] + conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -117,7 +117,7 @@ def test_issue_13(self): def test_issue_15(self): """ query should be expanded before perform character encoding """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -132,7 +132,7 @@ def test_issue_15(self): def test_issue_16(self): """ Patch for string and tuple escaping """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -148,7 +148,7 @@ def test_issue_16(self): @unittest2.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") def test_issue_17(self): """could not connect mysql use passwod""" - conn = self.connections[0] + conn = self.connect() host = self.databases[0]["host"] db = self.databases[0]["db"] c = conn.cursor() @@ -191,7 +191,7 @@ def test_issue_33(self): @unittest2.skip("This test requires manual intervention") def test_issue_35(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() print("sudo killall -9 mysqld within the next 10 seconds") try: @@ -237,7 +237,7 @@ def test_issue_36(self): del self.connections[1] def test_issue_37(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() self.assertEqual(1, c.execute("SELECT @foo")) self.assertEqual((None,), c.fetchone()) @@ -245,7 +245,7 @@ def test_issue_37(self): c.execute("set @foo = 'bar'") def test_issue_38(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() datum = "a" * 1024 * 1023 # reduced size for most default mysql installs @@ -259,7 +259,7 @@ def test_issue_38(self): c.execute("drop table issue38") def disabled_test_issue_54(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -278,7 +278,7 @@ def disabled_test_issue_54(self): class TestGitHubIssues(base.PyMySQLTestCase): def test_issue_66(self): """ 'Connection' object has no attribute 'insert_id' """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() self.assertEqual(0, conn.insert_id()) try: @@ -294,7 +294,7 @@ def test_issue_66(self): def test_issue_79(self): """ Duplicate field overwrites the previous one in the result of DictCursor """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor(pymysql.cursors.DictCursor) with warnings.catch_warnings(): @@ -321,7 +321,7 @@ def test_issue_79(self): def test_issue_95(self): """ Leftover trailing OK packet for "CALL my_sp" queries """ - conn = self.connections[0] + conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -366,7 +366,7 @@ def test_issue_114(self): def test_issue_175(self): """ The number of fields returned by server is read in wrong way """ - conn = self.connections[0] + conn = self.connect() cur = conn.cursor() for length in (200, 300): columns = ', '.join('c{0} integer'.format(i) for i in range(length)) diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index 85fd94ea5..eafa6e19e 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -10,7 +10,7 @@ class TestLoadLocal(base.PyMySQLTestCase): def test_no_file(self): """Test load local infile when the file does not exist""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") try: @@ -26,7 +26,7 @@ def test_no_file(self): def test_load_file(self): """Test load local infile with a valid file""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -44,7 +44,7 @@ def test_load_file(self): def test_unbuffered_load_file(self): """Test unbuffered load local infile with a valid file""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor(cursors.SSCursor) c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -66,7 +66,7 @@ def test_unbuffered_load_file(self): def test_load_warnings(self): """Test load local infile produces the appropriate warnings""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), From 15eaee5f0a40125275de2c53671fff888b8c7439 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Dec 2018 21:27:40 +0900 Subject: [PATCH 014/227] Close connection on unknown error (#759) Fixes #275 --- pymysql/connections.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pymysql/connections.py b/pymysql/connections.py index 7f5646e81..642461149 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -694,6 +694,10 @@ def _read_bytes(self, num_bytes): raise err.OperationalError( CR.CR_SERVER_LOST, "Lost connection to MySQL server during query (%s)" % (e,)) + except BaseException: + # Don't convert unknown exception to MySQLError. + self._force_close() + raise if len(data) < num_bytes: self._force_close() raise err.OperationalError( From 1ef6c587337bd6ff3272c2c4771948676fd2a9e6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Dec 2018 21:58:01 +0900 Subject: [PATCH 015/227] Deprecate context manager interface of Connection (#742) --- pymysql/connections.py | 3 +++ pymysql/tests/test_connection.py | 30 +++++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 642461149..c8ed12a34 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -492,6 +492,9 @@ def cursor(self, cursor=None): def __enter__(self): """Context manager that returns a Cursor""" + warnings.warn( + "Context manager API of Connection object is deprecated; Use conn.begin()", + DeprecationWarning) return self.cursor() def __exit__(self, exc, value, traceback): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 3f1627803..7f31f6c21 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -7,6 +7,8 @@ from pymysql._compat import text_type from pymysql.constants import CLIENT +import pytest + class TempUser: def __init__(self, c, user, db, auth=None, authdata=None, password=None): @@ -451,21 +453,23 @@ def test_read_default_group(self): def test_context(self): with self.assertRaises(ValueError): c = self.connect() + with pytest.warns(DeprecationWarning): + with c as cur: + cur.execute('create table test ( a int ) ENGINE=InnoDB') + c.begin() + cur.execute('insert into test values ((1))') + raise ValueError('pseudo abort') + c = self.connect() + with pytest.warns(DeprecationWarning): with c as cur: - cur.execute('create table test ( a int ) ENGINE=InnoDB') - c.begin() + cur.execute('select count(*) from test') + self.assertEqual(0, cur.fetchone()[0]) cur.execute('insert into test values ((1))') - raise ValueError('pseudo abort') - c.commit() - c = self.connect() - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(0, cur.fetchone()[0]) - cur.execute('insert into test values ((1))') - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(1,cur.fetchone()[0]) - cur.execute('drop table test') + with pytest.warns(DeprecationWarning): + with c as cur: + cur.execute('select count(*) from test') + self.assertEqual(1,cur.fetchone()[0]) + cur.execute('drop table test') def test_set_charset(self): c = self.connect() From 4e6d5f3d1db5241c47fe8742fe951c49819fd44c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 18 Dec 2018 17:17:22 +0900 Subject: [PATCH 016/227] Update URLs in README --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 163c4f2a4..cee5053d7 100644 --- a/README.rst +++ b/README.rst @@ -127,15 +127,15 @@ This example will print: Resources --------- -* DB-API 2.0: http://www.python.org/dev/peps/pep-0249 +* DB-API 2.0: https://www.python.org/dev/peps/pep-0249/ -* MySQL Reference Manuals: http://dev.mysql.com/doc/ +* MySQL Reference Manuals: https://dev.mysql.com/doc/ * MySQL client/server protocol: - http://dev.mysql.com/doc/internals/en/client-server-protocol.html + https://dev.mysql.com/doc/internals/en/client-server-protocol.html * "Connector" channel in MySQL Community Slack: - http://lefred.be/mysql-community-on-slack/ + https://lefred.be/mysql-community-on-slack/ * PyMySQL mailing list: https://groups.google.com/forum/#!forum/pymysql-users From 91954b1ca84b454ec995c65a2c4b08fc67934067 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 18 Dec 2018 18:32:49 +0900 Subject: [PATCH 017/227] Update charsets based on MySQL 8.0.12 (#733) --- pymysql/charset.py | 90 ++++++++---------------------------------- pymysql/connections.py | 2 +- pymysql/converters.py | 15 ------- 3 files changed, 17 insertions(+), 90 deletions(-) diff --git a/pymysql/charset.py b/pymysql/charset.py index 968376cfa..07d80638f 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -18,7 +18,7 @@ def __repr__(self): @property def encoding(self): name = self.name - if name == 'utf8mb4': + if name in ('utf8mb4', 'utf8mb3'): return 'utf8' return name @@ -30,18 +30,18 @@ def is_binary(self): class Charsets: def __init__(self): self._by_id = {} + self._by_name = {} def add(self, c): self._by_id[c.id] = c + if c.is_default: + self._by_name[c.name] = c def by_id(self, id): return self._by_id[id] def by_name(self, name): - name = name.lower() - for c in self._by_id.values(): - if c.name == name and c.is_default: - return c + return self._by_name.get(name.lower()) _charsets = Charsets() """ @@ -89,7 +89,6 @@ def by_name(self, name): _charsets.add(Charset(32, 'armscii8', 'armscii8_general_ci', 'Yes')) _charsets.add(Charset(33, 'utf8', 'utf8_general_ci', 'Yes')) _charsets.add(Charset(34, 'cp1250', 'cp1250_czech_cs', '')) -_charsets.add(Charset(35, 'ucs2', 'ucs2_general_ci', 'Yes')) _charsets.add(Charset(36, 'cp866', 'cp866_general_ci', 'Yes')) _charsets.add(Charset(37, 'keybcs2', 'keybcs2_general_ci', 'Yes')) _charsets.add(Charset(38, 'macce', 'macce_general_ci', 'Yes')) @@ -108,13 +107,9 @@ def by_name(self, name): _charsets.add(Charset(51, 'cp1251', 'cp1251_general_ci', 'Yes')) _charsets.add(Charset(52, 'cp1251', 'cp1251_general_cs', '')) _charsets.add(Charset(53, 'macroman', 'macroman_bin', '')) -_charsets.add(Charset(54, 'utf16', 'utf16_general_ci', 'Yes')) -_charsets.add(Charset(55, 'utf16', 'utf16_bin', '')) _charsets.add(Charset(57, 'cp1256', 'cp1256_general_ci', 'Yes')) _charsets.add(Charset(58, 'cp1257', 'cp1257_bin', '')) _charsets.add(Charset(59, 'cp1257', 'cp1257_general_ci', 'Yes')) -_charsets.add(Charset(60, 'utf32', 'utf32_general_ci', 'Yes')) -_charsets.add(Charset(61, 'utf32', 'utf32_bin', '')) _charsets.add(Charset(63, 'binary', 'binary', 'Yes')) _charsets.add(Charset(64, 'armscii8', 'armscii8_bin', '')) _charsets.add(Charset(65, 'ascii', 'ascii_bin', '')) @@ -128,6 +123,7 @@ def by_name(self, name): _charsets.add(Charset(73, 'keybcs2', 'keybcs2_bin', '')) _charsets.add(Charset(74, 'koi8r', 'koi8r_bin', '')) _charsets.add(Charset(75, 'koi8u', 'koi8u_bin', '')) +_charsets.add(Charset(76, 'utf8', 'utf8_tolower_ci', '')) _charsets.add(Charset(77, 'latin2', 'latin2_bin', '')) _charsets.add(Charset(78, 'latin5', 'latin5_bin', '')) _charsets.add(Charset(79, 'latin7', 'latin7_bin', '')) @@ -141,7 +137,6 @@ def by_name(self, name): _charsets.add(Charset(87, 'gbk', 'gbk_bin', '')) _charsets.add(Charset(88, 'sjis', 'sjis_bin', '')) _charsets.add(Charset(89, 'tis620', 'tis620_bin', '')) -_charsets.add(Charset(90, 'ucs2', 'ucs2_bin', '')) _charsets.add(Charset(91, 'ujis', 'ujis_bin', '')) _charsets.add(Charset(92, 'geostd8', 'geostd8_general_ci', 'Yes')) _charsets.add(Charset(93, 'geostd8', 'geostd8_bin', '')) @@ -151,67 +146,6 @@ def by_name(self, name): _charsets.add(Charset(97, 'eucjpms', 'eucjpms_japanese_ci', 'Yes')) _charsets.add(Charset(98, 'eucjpms', 'eucjpms_bin', '')) _charsets.add(Charset(99, 'cp1250', 'cp1250_polish_ci', '')) -_charsets.add(Charset(101, 'utf16', 'utf16_unicode_ci', '')) -_charsets.add(Charset(102, 'utf16', 'utf16_icelandic_ci', '')) -_charsets.add(Charset(103, 'utf16', 'utf16_latvian_ci', '')) -_charsets.add(Charset(104, 'utf16', 'utf16_romanian_ci', '')) -_charsets.add(Charset(105, 'utf16', 'utf16_slovenian_ci', '')) -_charsets.add(Charset(106, 'utf16', 'utf16_polish_ci', '')) -_charsets.add(Charset(107, 'utf16', 'utf16_estonian_ci', '')) -_charsets.add(Charset(108, 'utf16', 'utf16_spanish_ci', '')) -_charsets.add(Charset(109, 'utf16', 'utf16_swedish_ci', '')) -_charsets.add(Charset(110, 'utf16', 'utf16_turkish_ci', '')) -_charsets.add(Charset(111, 'utf16', 'utf16_czech_ci', '')) -_charsets.add(Charset(112, 'utf16', 'utf16_danish_ci', '')) -_charsets.add(Charset(113, 'utf16', 'utf16_lithuanian_ci', '')) -_charsets.add(Charset(114, 'utf16', 'utf16_slovak_ci', '')) -_charsets.add(Charset(115, 'utf16', 'utf16_spanish2_ci', '')) -_charsets.add(Charset(116, 'utf16', 'utf16_roman_ci', '')) -_charsets.add(Charset(117, 'utf16', 'utf16_persian_ci', '')) -_charsets.add(Charset(118, 'utf16', 'utf16_esperanto_ci', '')) -_charsets.add(Charset(119, 'utf16', 'utf16_hungarian_ci', '')) -_charsets.add(Charset(120, 'utf16', 'utf16_sinhala_ci', '')) -_charsets.add(Charset(128, 'ucs2', 'ucs2_unicode_ci', '')) -_charsets.add(Charset(129, 'ucs2', 'ucs2_icelandic_ci', '')) -_charsets.add(Charset(130, 'ucs2', 'ucs2_latvian_ci', '')) -_charsets.add(Charset(131, 'ucs2', 'ucs2_romanian_ci', '')) -_charsets.add(Charset(132, 'ucs2', 'ucs2_slovenian_ci', '')) -_charsets.add(Charset(133, 'ucs2', 'ucs2_polish_ci', '')) -_charsets.add(Charset(134, 'ucs2', 'ucs2_estonian_ci', '')) -_charsets.add(Charset(135, 'ucs2', 'ucs2_spanish_ci', '')) -_charsets.add(Charset(136, 'ucs2', 'ucs2_swedish_ci', '')) -_charsets.add(Charset(137, 'ucs2', 'ucs2_turkish_ci', '')) -_charsets.add(Charset(138, 'ucs2', 'ucs2_czech_ci', '')) -_charsets.add(Charset(139, 'ucs2', 'ucs2_danish_ci', '')) -_charsets.add(Charset(140, 'ucs2', 'ucs2_lithuanian_ci', '')) -_charsets.add(Charset(141, 'ucs2', 'ucs2_slovak_ci', '')) -_charsets.add(Charset(142, 'ucs2', 'ucs2_spanish2_ci', '')) -_charsets.add(Charset(143, 'ucs2', 'ucs2_roman_ci', '')) -_charsets.add(Charset(144, 'ucs2', 'ucs2_persian_ci', '')) -_charsets.add(Charset(145, 'ucs2', 'ucs2_esperanto_ci', '')) -_charsets.add(Charset(146, 'ucs2', 'ucs2_hungarian_ci', '')) -_charsets.add(Charset(147, 'ucs2', 'ucs2_sinhala_ci', '')) -_charsets.add(Charset(159, 'ucs2', 'ucs2_general_mysql500_ci', '')) -_charsets.add(Charset(160, 'utf32', 'utf32_unicode_ci', '')) -_charsets.add(Charset(161, 'utf32', 'utf32_icelandic_ci', '')) -_charsets.add(Charset(162, 'utf32', 'utf32_latvian_ci', '')) -_charsets.add(Charset(163, 'utf32', 'utf32_romanian_ci', '')) -_charsets.add(Charset(164, 'utf32', 'utf32_slovenian_ci', '')) -_charsets.add(Charset(165, 'utf32', 'utf32_polish_ci', '')) -_charsets.add(Charset(166, 'utf32', 'utf32_estonian_ci', '')) -_charsets.add(Charset(167, 'utf32', 'utf32_spanish_ci', '')) -_charsets.add(Charset(168, 'utf32', 'utf32_swedish_ci', '')) -_charsets.add(Charset(169, 'utf32', 'utf32_turkish_ci', '')) -_charsets.add(Charset(170, 'utf32', 'utf32_czech_ci', '')) -_charsets.add(Charset(171, 'utf32', 'utf32_danish_ci', '')) -_charsets.add(Charset(172, 'utf32', 'utf32_lithuanian_ci', '')) -_charsets.add(Charset(173, 'utf32', 'utf32_slovak_ci', '')) -_charsets.add(Charset(174, 'utf32', 'utf32_spanish2_ci', '')) -_charsets.add(Charset(175, 'utf32', 'utf32_roman_ci', '')) -_charsets.add(Charset(176, 'utf32', 'utf32_persian_ci', '')) -_charsets.add(Charset(177, 'utf32', 'utf32_esperanto_ci', '')) -_charsets.add(Charset(178, 'utf32', 'utf32_hungarian_ci', '')) -_charsets.add(Charset(179, 'utf32', 'utf32_sinhala_ci', '')) _charsets.add(Charset(192, 'utf8', 'utf8_unicode_ci', '')) _charsets.add(Charset(193, 'utf8', 'utf8_icelandic_ci', '')) _charsets.add(Charset(194, 'utf8', 'utf8_latvian_ci', '')) @@ -232,6 +166,10 @@ def by_name(self, name): _charsets.add(Charset(209, 'utf8', 'utf8_esperanto_ci', '')) _charsets.add(Charset(210, 'utf8', 'utf8_hungarian_ci', '')) _charsets.add(Charset(211, 'utf8', 'utf8_sinhala_ci', '')) +_charsets.add(Charset(212, 'utf8', 'utf8_german2_ci', '')) +_charsets.add(Charset(213, 'utf8', 'utf8_croatian_ci', '')) +_charsets.add(Charset(214, 'utf8', 'utf8_unicode_520_ci', '')) +_charsets.add(Charset(215, 'utf8', 'utf8_vietnamese_ci', '')) _charsets.add(Charset(223, 'utf8', 'utf8_general_mysql500_ci', '')) _charsets.add(Charset(224, 'utf8mb4', 'utf8mb4_unicode_ci', '')) _charsets.add(Charset(225, 'utf8mb4', 'utf8mb4_icelandic_ci', '')) @@ -257,14 +195,18 @@ def by_name(self, name): _charsets.add(Charset(245, 'utf8mb4', 'utf8mb4_croatian_ci', '')) _charsets.add(Charset(246, 'utf8mb4', 'utf8mb4_unicode_520_ci', '')) _charsets.add(Charset(247, 'utf8mb4', 'utf8mb4_vietnamese_ci', '')) - +_charsets.add(Charset(248, 'gb18030', 'gb18030_chinese_ci', 'Yes')) +_charsets.add(Charset(249, 'gb18030', 'gb18030_bin', '')) +_charsets.add(Charset(250, 'gb18030', 'gb18030_unicode_520_ci', '')) +_charsets.add(Charset(255, 'utf8mb4', 'utf8mb4_0900_ai_ci', '')) charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id +#TODO: remove this def charset_to_encoding(name): """Convert MySQL's charset name to Python's codec name""" - if name == 'utf8mb4': + if name in ('utf8mb4', 'utf8mb3'): return 'utf8' return name diff --git a/pymysql/connections.py b/pymysql/connections.py index c8ed12a34..2e4122b48 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -88,7 +88,7 @@ def _makefile(sock, mode): } -DEFAULT_CHARSET = 'utf8mb4' # TODO: change to utf8mb4 +DEFAULT_CHARSET = 'utf8mb4' MAX_PACKET_LEN = 2**24-1 diff --git a/pymysql/converters.py b/pymysql/converters.py index bf1db9d77..ce2be062d 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -354,21 +354,6 @@ def through(x): convert_bit = through -def convert_characters(connection, field, data): - field_charset = charset_by_id(field.charsetnr).name - encoding = charset_to_encoding(field_charset) - if field.flags & FLAG.SET: - return convert_set(data.decode(encoding)) - if field.flags & FLAG.BINARY: - return data - - if connection.use_unicode: - data = data.decode(encoding) - elif connection.charset != field_charset: - data = data.decode(encoding) - data = data.encode(connection.encoding) - return data - encoders = { bool: escape_bool, int: escape_int, From 4bf04205359251ca35015a01359d6687d0b4bbcf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 18 Dec 2018 19:25:12 +0900 Subject: [PATCH 018/227] Optional cryptography (#760) --- Pipfile | 1 + README.rst | 5 +++++ docs/source/user/installation.rst | 5 +++++ pymysql/_auth.py | 12 +++++++++--- pymysql/util.py | 9 --------- setup.py | 6 +++--- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Pipfile b/Pipfile index a18fa51af..07939550b 100644 --- a/Pipfile +++ b/Pipfile @@ -10,3 +10,4 @@ cryptography = "*" pytest = "*" unittest2 = "*" twine = "*" +flake8 = "*" diff --git a/README.rst b/README.rst index cee5053d7..cd1e3bd99 100644 --- a/README.rst +++ b/README.rst @@ -61,6 +61,11 @@ You can install it with pip:: $ python3 -m pip install PyMySQL +To use "sha256_password" or "caching_sha2_password" for authenticate, +you need to install additional dependency:: + + $ python3 -m pip install PyMySQL[rsa] + Documentation ------------- diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index 8a81fddb6..656e3c7a4 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -8,6 +8,11 @@ The last stable release is available on PyPI and can be installed with ``pip``:: $ python3 -m pip install PyMySQL +To use "sha256_password" or "caching_sha2_password" for authenticate, +you need to install additional dependency:: + + $ python3 -m pip install PyMySQL[rsa] + Requirements ------------- diff --git a/pymysql/_auth.py b/pymysql/_auth.py index e0a48f743..199f36c7f 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -7,9 +7,13 @@ from .util import byte2int, int2byte -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization, hashes -from cryptography.hazmat.primitives.asymmetric import padding +try: + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import serialization, hashes + from cryptography.hazmat.primitives.asymmetric import padding + _have_cryptography = True +except ImportError: + _have_cryptography = False from functools import partial import hashlib @@ -134,6 +138,8 @@ def sha2_rsa_encrypt(password, salt, public_key): Used for sha256_password and caching_sha2_password. """ + if not _have_cryptography: + raise RuntimeError("cryptography is required for sha256_password or caching_sha2_password") message = _xor_password(password + b'\0', salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) return rsa_key.encrypt( diff --git a/pymysql/util.py b/pymysql/util.py index 3e82ac7b5..04683f83c 100644 --- a/pymysql/util.py +++ b/pymysql/util.py @@ -11,12 +11,3 @@ def byte2int(b): def int2byte(i): return struct.pack("!B", i) - -def join_bytes(bs): - if len(bs) == 0: - return "" - else: - rv = bs[0] - for b in bs[1:]: - rv += b - return rv diff --git a/setup.py b/setup.py index 14650d1c7..71bc09b8b 100755 --- a/setup.py +++ b/setup.py @@ -17,9 +17,9 @@ description='Pure Python MySQL Driver', long_description=readme, packages=find_packages(exclude=['tests*', 'pymysql.tests*']), - install_requires=[ - "cryptography", - ], + extras_require={ + "rsa": ["cryptography"], + }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python :: 2', From 8eee26692726e7c923186322a68e15e4d98c138e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 18 Dec 2018 20:35:06 +0900 Subject: [PATCH 019/227] v0.9.3 --- CHANGELOG | 11 +++++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d73ddd79d..9ddb8f0bc 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,16 @@ # Changes +## 0.9.3 + +Release date: 2018-12-18 + +* cryptography dependency is optional now. +* Fix old_password (used before MySQL 4.1) support. +* Deprecate old_password. +* Stop sending ``sys.argv[0]`` for connection attribute "program_name". +* Close connection when unknown error is happened. +* Deprecate context manager API of Connection object. + ## 0.9.2 Release date: 2018-07-04 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index b79b4b83e..0cb5006cc 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 9, 2, None) +VERSION = (0, 9, 3, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 71bc09b8b..6157243af 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = "0.9.2" +version = "0.9.3" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() From fe0cd60e6f9e3bc0fb81f76f7e4fa30a1e1b34cc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 18 Dec 2018 21:41:06 +0900 Subject: [PATCH 020/227] Drop Python 3.4 support (#762) --- .travis.yml | 10 +++---- README.rst | 2 +- docs/source/user/installation.rst | 2 +- pymysql/converters.py | 43 +------------------------------ pymysql/err.py | 7 +---- pymysql/tests/test_err.py | 6 ----- setup.py | 1 - 7 files changed, 9 insertions(+), 62 deletions(-) diff --git a/.travis.yml b/.travis.yml index f4a7cc74e..32e0e4b58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,12 @@ # vim: sw=2 ts=2 sts=2 expandtab -sudo: required +dist: xenial language: python +cache: pip + services: - docker -cache: pip - matrix: include: - env: @@ -17,7 +17,7 @@ matrix: python: "3.6" - env: - DB=mariadb:10.1 - python: "pypy" + python: "pypy3.5" - env: - DB=mariadb:10.2 python: "2.7" @@ -32,7 +32,7 @@ matrix: python: "3.6" - env: - DB=mysql:5.7 - python: "3.4" + python: "3.7" - env: - DB=mysql:8.0 - TEST_AUTH=yes diff --git a/README.rst b/README.rst index cd1e3bd99..175bf43ed 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ Requirements * Python -- one of the following: - - CPython_ : 2.7 and >= 3.4 + - CPython_ : 2.7 and >= 3.5 - PyPy_ : Latest version * MySQL Server -- one of the following: diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index 656e3c7a4..d95961c66 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -18,7 +18,7 @@ Requirements * Python -- one of the following: - - CPython_ >= 2.7 or >= 3.4 + - CPython_ >= 2.7 or >= 3.5 - Latest PyPy_ * MySQL Server -- one of the following: diff --git a/pymysql/converters.py b/pymysql/converters.py index ce2be062d..be2e697c6 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -301,46 +301,6 @@ def convert_date(obj): return obj -def convert_mysql_timestamp(timestamp): - """Convert a MySQL TIMESTAMP to a Timestamp object. - - MySQL >= 4.1 returns TIMESTAMP in the same format as DATETIME: - - >>> mysql_timestamp_converter('2007-02-25 22:32:17') - datetime.datetime(2007, 2, 25, 22, 32, 17) - - MySQL < 4.1 uses a big string of numbers: - - >>> mysql_timestamp_converter('20070225223217') - datetime.datetime(2007, 2, 25, 22, 32, 17) - - Illegal values are returned as None: - - >>> mysql_timestamp_converter('2007-02-31 22:32:17') is None - True - >>> mysql_timestamp_converter('00000000000000') is None - True - - """ - if not PY2 and isinstance(timestamp, (bytes, bytearray)): - timestamp = timestamp.decode('ascii') - if timestamp[4] == '-': - return convert_datetime(timestamp) - timestamp += "0"*(14-len(timestamp)) # padding - year, month, day, hour, minute, second = \ - int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]), \ - int(timestamp[8:10]), int(timestamp[10:12]), int(timestamp[12:14]) - try: - return datetime.datetime(year, month, day, hour, minute, second) - except ValueError: - return timestamp - -def convert_set(s): - if isinstance(s, (bytes, bytearray)): - return set(s.split(b",")) - return set(s.split(",")) - - def through(x): return x @@ -388,11 +348,10 @@ def through(x): FIELD_TYPE.LONGLONG: int, FIELD_TYPE.INT24: int, FIELD_TYPE.YEAR: int, - FIELD_TYPE.TIMESTAMP: convert_mysql_timestamp, + FIELD_TYPE.TIMESTAMP: convert_datetime, FIELD_TYPE.DATETIME: convert_datetime, FIELD_TYPE.TIME: convert_timedelta, FIELD_TYPE.DATE: convert_date, - FIELD_TYPE.SET: convert_set, FIELD_TYPE.BLOB: through, FIELD_TYPE.TINY_BLOB: through, FIELD_TYPE.MEDIUM_BLOB: through, diff --git a/pymysql/err.py b/pymysql/err.py index fbc60558e..e93ba9bea 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -99,11 +99,6 @@ def _map_error(exc, *errors): def raise_mysql_exception(data): errno = struct.unpack(' Date: Wed, 19 Dec 2018 17:45:43 +0900 Subject: [PATCH 021/227] Remove context manager (#763) --- pymysql/connections.py | 14 -------------- pymysql/tests/test_connection.py | 21 --------------------- 2 files changed, 35 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 2e4122b48..af074e21d 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -490,20 +490,6 @@ def cursor(self, cursor=None): return cursor(self) return self.cursorclass(self) - def __enter__(self): - """Context manager that returns a Cursor""" - warnings.warn( - "Context manager API of Connection object is deprecated; Use conn.begin()", - DeprecationWarning) - return self.cursor() - - def __exit__(self, exc, value, traceback): - """On successful exit, commit. On exception, rollback""" - if exc: - self.rollback() - else: - self.commit() - # The following methods are INTERNAL USE ONLY (called from Cursor) def query(self, sql, unbuffered=False): # if DEBUG: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 7f31f6c21..7c258df89 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -450,27 +450,6 @@ def test_read_default_group(self): ) self.assertTrue(conn.open) - def test_context(self): - with self.assertRaises(ValueError): - c = self.connect() - with pytest.warns(DeprecationWarning): - with c as cur: - cur.execute('create table test ( a int ) ENGINE=InnoDB') - c.begin() - cur.execute('insert into test values ((1))') - raise ValueError('pseudo abort') - c = self.connect() - with pytest.warns(DeprecationWarning): - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(0, cur.fetchone()[0]) - cur.execute('insert into test values ((1))') - with pytest.warns(DeprecationWarning): - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(1,cur.fetchone()[0]) - cur.execute('drop table test') - def test_set_charset(self): c = self.connect() c.set_charset('utf8mb4') From a500fcd64d4500417540a2a2ff7b16a88d1872ad Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 19 Dec 2018 21:39:58 +0900 Subject: [PATCH 022/227] Remove unittest2 dependency (#764) --- .travis.yml | 2 +- Pipfile | 13 ---- pymysql/tests/__init__.py | 4 +- pymysql/tests/base.py | 26 ++++++-- pymysql/tests/test_basic.py | 6 +- pymysql/tests/test_connection.py | 64 +++++++++---------- pymysql/tests/test_err.py | 4 +- pymysql/tests/test_issues.py | 7 +- pymysql/tests/test_nextset.py | 4 +- pymysql/tests/thirdparty/__init__.py | 5 +- .../thirdparty/test_MySQLdb/capabilities.py | 5 +- .../tests/thirdparty/test_MySQLdb/dbapi20.py | 6 +- .../test_MySQLdb/test_MySQLdb_capabilities.py | 5 +- .../test_MySQLdb/test_MySQLdb_dbapi20.py | 5 +- .../test_MySQLdb/test_MySQLdb_nonstandard.py | 5 +- runtests.py | 31 --------- tox.ini | 7 +- 17 files changed, 77 insertions(+), 122 deletions(-) delete mode 100644 Pipfile delete mode 100755 runtests.py diff --git a/.travis.yml b/.travis.yml index 32e0e4b58..b8b07b901 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ before_script: - export COVERALLS_PARALLEL=true script: - - coverage run ./runtests.py + - pytest -v --cov-config .coveragerc pymysql - if [ "${TEST_AUTH}" = "yes" ]; then pytest -v --cov-config .coveragerc tests; fi diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 07939550b..000000000 --- a/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -cryptography = "*" - -[dev-packages] -pytest = "*" -unittest2 = "*" -twine = "*" -flake8 = "*" diff --git a/pymysql/tests/__init__.py b/pymysql/tests/__init__.py index a9f5a4bfe..91ad57631 100644 --- a/pymysql/tests/__init__.py +++ b/pymysql/tests/__init__.py @@ -14,5 +14,5 @@ from pymysql.tests.thirdparty import * if __name__ == "__main__": - import unittest2 - unittest2.main() + import unittest + unittest.main() diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index 091cccfa6..22bed9d8d 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -3,14 +3,33 @@ import os import re import warnings - -import unittest2 +import unittest import pymysql from .._compat import CPYTHON -class PyMySQLTestCase(unittest2.TestCase): +if CPYTHON: + import atexit + gc.set_debug(gc.DEBUG_UNCOLLECTABLE) + + @atexit.register + def report_uncollectable(): + import gc + if not gc.garbage: + print("No garbages!") + return + print('uncollectable objects') + for obj in gc.garbage: + print(obj) + if hasattr(obj, '__dict__'): + print(obj.__dict__) + for ref in gc.get_referrers(obj): + print("referrer:", ref) + print('---') + + +class PyMySQLTestCase(unittest.TestCase): # You can specify your test environment creating a file named # "databases.json" or editing the `databases` variable below. fname = os.path.join(os.path.dirname(__file__), "databases.json") @@ -97,7 +116,6 @@ def safe_gc_collect(self): """Ensure cycles are collected via gc. Runs additional times on non-CPython platforms. - """ gc.collect() if not CPYTHON: diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 940661f75..c2d539047 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -4,7 +4,7 @@ import time import warnings -from unittest2 import SkipTest +import pytest from pymysql import util import pymysql.cursors @@ -143,7 +143,7 @@ def test_datetime_microseconds(self): conn = self.connect() if not self.mysql_server_is(conn, (5, 6, 4)): - raise SkipTest("target backend does not support microseconds") + pytest.skip("target backend does not support microseconds") c = conn.cursor() dt = datetime.datetime(2013, 11, 12, 9, 9, 9, 123450) c.execute("create table test_datetime (id int, ts datetime(6))") @@ -256,7 +256,7 @@ def test_json(self): args["charset"] = "utf8mb4" conn = pymysql.connect(**args) if not self.mysql_server_is(conn, (5, 7, 0)): - raise SkipTest("JSON type is not supported on MySQL <= 5.6") + pytest.skip("JSON type is not supported on MySQL <= 5.6") self.safe_create_table(conn, "test_json", """\ create table test_json ( diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 7c258df89..e4d24c445 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,7 +1,7 @@ import datetime import sys import time -import unittest2 +import pytest import pymysql from pymysql.tests import base from pymysql._compat import text_type @@ -100,14 +100,14 @@ class TestAuthentication(base.PyMySQLTestCase): def test_plugin(self): conn = self.connect() if not self.mysql_server_is(conn, (5, 5, 0)): - raise unittest2.SkipTest("MySQL-5.5 required for plugins") + pytest.skip("MySQL-5.5 required for plugins") cur = conn.cursor() cur.execute("select plugin from mysql.user where concat(user, '@', host)=current_user()") for r in cur: self.assertIn(conn._auth_plugin_name, (r[0], 'mysql_native_password')) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(socket_found, "socket plugin already installed") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(socket_found, reason="socket plugin already installed") def testSocketAuthInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -124,13 +124,13 @@ def testSocketAuthInstallPlugin(self): self.realtestSocketAuth() except pymysql.err.InternalError: TestAuthentication.socket_found = False - raise unittest2.SkipTest('we couldn\'t install the socket plugin') + pytest.skip('we couldn\'t install the socket plugin') finally: if TestAuthentication.socket_found: cur.execute("uninstall plugin %s" % self.socket_plugin_name) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(socket_found, "no socket plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not socket_found, reason="no socket plugin") def testSocketAuth(self): self.realtestSocketAuth() @@ -179,8 +179,8 @@ def __init__(self, con): self.con=con - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(two_questions_found, "two_questions plugin already installed") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(two_questions_found, reason="two_questions plugin already installed") def testDialogAuthTwoQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -189,13 +189,13 @@ def testDialogAuthTwoQuestionsInstallPlugin(self): TestAuthentication.two_questions_found = True self.realTestDialogAuthTwoQuestions() except pymysql.err.InternalError: - raise unittest2.SkipTest('we couldn\'t install the two_questions plugin') + pytest.skip('we couldn\'t install the two_questions plugin') finally: if TestAuthentication.two_questions_found: cur.execute("uninstall plugin two_questions") - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(two_questions_found, "no two questions auth plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not two_questions_found, reason="no two questions auth plugin") def testDialogAuthTwoQuestions(self): self.realTestDialogAuthTwoQuestions() @@ -209,8 +209,8 @@ def realTestDialogAuthTwoQuestions(self): pymysql.connect(user='pymysql_2q', **self.db) pymysql.connect(user='pymysql_2q', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(three_attempts_found, "three_attempts plugin already installed") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(three_attempts_found, reason="three_attempts plugin already installed") def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -219,13 +219,13 @@ def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): TestAuthentication.three_attempts_found = True self.realTestDialogAuthThreeAttempts() except pymysql.err.InternalError: - raise unittest2.SkipTest('we couldn\'t install the three_attempts plugin') + pytest.skip('we couldn\'t install the three_attempts plugin') finally: if TestAuthentication.three_attempts_found: cur.execute("uninstall plugin three_attempts") - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(three_attempts_found, "no three attempts plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not three_attempts_found, reason="no three attempts plugin") def testDialogAuthThreeAttempts(self): self.realTestDialogAuthThreeAttempts() @@ -250,10 +250,10 @@ def realTestDialogAuthThreeAttempts(self): with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(pam_found, "pam plugin already installed") - @unittest2.skipIf(os.environ.get('PASSWORD') is None, "PASSWORD env var required") - @unittest2.skipIf(os.environ.get('PAMSERVICE') is None, "PAMSERVICE env var required") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(pam_found, reason="pam plugin already installed") + @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") + @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") def testPamAuthInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -262,16 +262,16 @@ def testPamAuthInstallPlugin(self): TestAuthentication.pam_found = True self.realTestPamAuth() except pymysql.err.InternalError: - raise unittest2.SkipTest('we couldn\'t install the auth_pam plugin') + pytest.skip('we couldn\'t install the auth_pam plugin') finally: if TestAuthentication.pam_found: cur.execute("uninstall plugin pam") - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(pam_found, "no pam plugin") - @unittest2.skipIf(os.environ.get('PASSWORD') is None, "PASSWORD env var required") - @unittest2.skipIf(os.environ.get('PAMSERVICE') is None, "PAMSERVICE env var required") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not pam_found, reason="no pam plugin") + @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") + @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") def testPamAuth(self): self.realTestPamAuth() @@ -311,16 +311,16 @@ def realTestPamAuth(self): # select old_password("crummy p\tassword"); #| old_password("crummy p\tassword") | #| 2a01785203b08770 | - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(mysql_old_password_found, "no mysql_old_password plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not mysql_old_password_found, reason="no mysql_old_password plugin") def testMySQLOldPasswordAuth(self): conn = self.connect() if self.mysql_server_is(conn, (5, 7, 0)): - raise unittest2.SkipTest('Old passwords aren\'t supported in 5.7') + pytest.skip('Old passwords aren\'t supported in 5.7') # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") # from login in MySQL-5.6 if self.mysql_server_is(conn, (5, 6, 0)): - raise unittest2.SkipTest('Old passwords don\'t authenticate in 5.6') + pytest.skip('Old passwords don\'t authenticate in 5.6') db = self.db.copy() db['password'] = "crummy p\tassword" c = conn.cursor() @@ -354,8 +354,8 @@ def testMySQLOldPasswordAuth(self): cur.execute("SELECT VERSION()") c.execute('set global secure_auth=%r' % secure_auth_setting) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(sha256_password_found, "no sha256 password authentication plugin found") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not sha256_password_found, reason="no sha256 password authentication plugin found") def testAuthSHA256(self): conn = self.connect() c = conn.cursor() diff --git a/pymysql/tests/test_err.py b/pymysql/tests/test_err.py index 895c2afba..bb6a5c499 100644 --- a/pymysql/tests/test_err.py +++ b/pymysql/tests/test_err.py @@ -1,4 +1,4 @@ -import unittest2 +import unittest from pymysql import err @@ -6,7 +6,7 @@ __all__ = ["TestRaiseException"] -class TestRaiseException(unittest2.TestCase): +class TestRaiseException(unittest.TestCase): def test_raise_mysql_exception(self): data = b"\xff\x15\x04#28000Access denied" diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 8dca31b73..05ecf2863 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -3,11 +3,12 @@ import warnings import sys +import pytest + import pymysql from pymysql import cursors from pymysql._compat import text_type from pymysql.tests import base -import unittest2 try: import imp @@ -145,7 +146,7 @@ def test_issue_16(self): finally: c.execute("drop table issue16") - @unittest2.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") + @pytest.mark.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") def test_issue_17(self): """could not connect mysql use passwod""" conn = self.connect() @@ -189,7 +190,7 @@ def test_issue_33(self): c.execute(u"select name from hei\xdfe") self.assertEqual(u"Pi\xdfata", c.fetchone()[0]) - @unittest2.skip("This test requires manual intervention") + @pytest.mark.skip("This test requires manual intervention") def test_issue_35(self): conn = self.connect() c = conn.cursor() diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index 998441072..d5467b11b 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -1,4 +1,4 @@ -import unittest2 +import pytest import pymysql from pymysql import util @@ -50,7 +50,7 @@ def test_ok_and_next(self): self.assertEqual([(2,)], list(cur)) self.assertFalse(bool(cur.nextset())) - @unittest2.expectedFailure + @pytest.mark.xfail def test_multi_cursor(self): con = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) cur1 = con.cursor() diff --git a/pymysql/tests/thirdparty/__init__.py b/pymysql/tests/thirdparty/__init__.py index 6d59e1127..7a613478d 100644 --- a/pymysql/tests/thirdparty/__init__.py +++ b/pymysql/tests/thirdparty/__init__.py @@ -1,8 +1,5 @@ from .test_MySQLdb import * if __name__ == "__main__": - try: - import unittest2 as unittest - except ImportError: - import unittest + import unittest unittest.main() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index bcf9eecbf..6be9d1ba4 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -6,10 +6,7 @@ """ import sys from time import time -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest PY2 = sys.version_info[0] == 2 diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 3cbf2263d..1cc202e23 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -14,12 +14,8 @@ __version__ = '$Revision$'[11:-2] __author__ = 'Stuart Bishop ' -try: - import unittest2 as unittest -except ImportError: - import unittest - import time +import unittest # $Log$ # Revision 1.1.2.1 2006/02/25 03:44:32 adustman diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 0fc5e8316..13b43d3f5 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -1,8 +1,5 @@ from . import capabilities -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import pymysql from pymysql.tests import base import warnings diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index a26691626..2c9a06005 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -2,10 +2,7 @@ import pymysql from pymysql.tests import base -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest class test_MySQLdb(dbapi20.DatabaseAPI20Test): diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py index 17fc2cde5..5c739a420 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py @@ -1,8 +1,5 @@ import sys -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import pymysql _mysql = pymysql diff --git a/runtests.py b/runtests.py deleted file mode 100755 index ea3d9e8dd..000000000 --- a/runtests.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -import unittest2 - -from pymysql._compat import PYPY, JYTHON, IRONPYTHON - -#import pymysql -#pymysql.connections.DEBUG = True -#pymysql._auth.DEBUG = True - -if not (PYPY or JYTHON or IRONPYTHON): - import atexit - import gc - gc.set_debug(gc.DEBUG_UNCOLLECTABLE) - - @atexit.register - def report_uncollectable(): - import gc - if not gc.garbage: - print("No garbages!") - return - print('uncollectable objects') - for obj in gc.garbage: - print(obj) - if hasattr(obj, '__dict__'): - print(obj.__dict__) - for ref in gc.get_referrers(obj): - print("referrer:", ref) - print('---') - -import pymysql.tests -unittest2.main(pymysql.tests, verbosity=2) diff --git a/tox.ini b/tox.ini index e2f2917cd..d13e49f4f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py27,py34,py35,py36,py37,pypy,pypy3 +envlist = py27,py35,py36,py37,pypy,pypy3 [testenv] -commands = coverage run ./runtests.py -deps = unittest2 - coverage +commands = pytest -v pymysql/tests/ +deps = coverage pytest passenv = USER PASSWORD PAMSERVICE From a4d6630d8c9f5b098d4bcb0f2dce3eec39190aad Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 19 Dec 2018 22:09:59 +0900 Subject: [PATCH 023/227] travis: Remove unittest2 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b8b07b901..b2f91aab5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ matrix: # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version # really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't install: - - pip install -U coveralls unittest2 coverage cryptography pytest pytest-cov + - pip install -U coveralls coverage cryptography pytest pytest-cov before_script: - ./.travis/initializedb.sh @@ -55,9 +55,9 @@ before_script: - export COVERALLS_PARALLEL=true script: - - pytest -v --cov-config .coveragerc pymysql + - pytest -v --cov --cov-config .coveragerc pymysql - if [ "${TEST_AUTH}" = "yes" ]; - then pytest -v --cov-config .coveragerc tests; + then pytest -v --cov --cov-config .coveragerc tests; fi - if [ ! -z "${DB}" ]; then docker logs mysqld; From 6d85d0ad1133e419b53b6517cf87b1ccdebf7ab1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 13:42:45 +0900 Subject: [PATCH 024/227] requirements->requirements-dev --- requirements.txt => requirements-dev.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename requirements.txt => requirements-dev.txt (100%) diff --git a/requirements.txt b/requirements-dev.txt similarity index 100% rename from requirements.txt rename to requirements-dev.txt From 0b91c36715c2755cb593d2f0fc616e38c22d97c4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 13:43:04 +0900 Subject: [PATCH 025/227] Add pytest to requirements-dev --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 70f051613..5e85e5226 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ cryptography - +pytest From 3b51a620ce825c826bb217e9326d52a849298798 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 13:59:02 +0900 Subject: [PATCH 026/227] Use set -exv in initializedb.sh --- .travis/initializedb.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index d9897e49c..251c1a71e 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -1,9 +1,7 @@ #!/bin/bash -#debug -set -x -#verbose -set -v +#error,debug,verbose, +set -exv if [ ! -z "${DB}" ]; then # disable existing database server in case of accidential connection From 60b6dfe9a3c844aa666e54d9486f2c9af72c6362 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 14:21:27 +0900 Subject: [PATCH 027/227] fix travis --- .travis/initializedb.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 251c1a71e..3f71a5494 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -4,12 +4,7 @@ set -exv if [ ! -z "${DB}" ]; then - # disable existing database server in case of accidential connection - sudo service mysql stop - - docker pull ${DB} docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} - sleep 10 mysql() { docker exec mysqld mysql "${@}" @@ -17,16 +12,11 @@ if [ ! -z "${DB}" ]; then while : do sleep 5 - mysql -e 'select version()' - if [ $? = 0 ]; then - break - fi + mysql -e 'select version()' && break echo "server logs" docker logs --tail 5 mysqld done - mysql -e 'select VERSION()' - if [ $DB == 'mysql:8.0' ]; then WITH_PLUGIN='with mysql_native_password' mysql -e 'SET GLOBAL local_infile=on' From a3c06319791009436315eb9387ca3fa00bfd4b17 Mon Sep 17 00:00:00 2001 From: Scott Cole Date: Mon, 7 Jan 2019 00:23:43 -0800 Subject: [PATCH 028/227] Update error message for cryptography package (#769) --- pymysql/_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 199f36c7f..aa082dfe5 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -139,7 +139,7 @@ def sha2_rsa_encrypt(password, salt, public_key): Used for sha256_password and caching_sha2_password. """ if not _have_cryptography: - raise RuntimeError("cryptography is required for sha256_password or caching_sha2_password") + raise RuntimeError("'cryptography' package is required for sha256_password or caching_sha2_password auth methods") message = _xor_password(password + b'\0', salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) return rsa_key.encrypt( From 501abf0d33b23e95f1a2ec3b40d4b33b513c1c33 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 16 Jan 2019 21:37:10 +0900 Subject: [PATCH 029/227] travis: Sleep more (#772) --- .travis/initializedb.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 3f71a5494..9ec35d312 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -4,7 +4,9 @@ set -exv if [ ! -z "${DB}" ]; then + docker pull ${DB} docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} + sleep 15 mysql() { docker exec mysqld mysql "${@}" From 34adc28316e2e92de7c46e81fa24435c5a562914 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 17 Jan 2019 20:08:18 +0900 Subject: [PATCH 030/227] travis: faster DB startup wait (#773) --- .travis/initializedb.sh | 96 +++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 9ec35d312..17d06100d 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -1,62 +1,46 @@ #!/bin/bash -#error,debug,verbose, -set -exv - -if [ ! -z "${DB}" ]; then - docker pull ${DB} - docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} - sleep 15 - - mysql() { - docker exec mysqld mysql "${@}" - } - while : - do - sleep 5 - mysql -e 'select version()' && break - echo "server logs" - docker logs --tail 5 mysqld - done - - if [ $DB == 'mysql:8.0' ]; then - WITH_PLUGIN='with mysql_native_password' - mysql -e 'SET GLOBAL local_infile=on' - docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - - # Test user for auth test - mysql -e ' - CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", - nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", - nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" - PASSWORD EXPIRE NEVER;' - mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' - else - WITH_PLUGIN='' - fi - - mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' - mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - - mysql -u root -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" - mysql -u root -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - - cp .travis/docker.json pymysql/tests/databases.json +set -ex + +docker pull ${DB} +docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} + +mysql() { + docker exec mysqld mysql "${@}" +} +while : +do + sleep 3 + mysql --protocol=tcp -e 'select version()' && break +done +docker logs mysqld + +if [ $DB == 'mysql:8.0' ]; then + WITH_PLUGIN='with mysql_native_password' + mysql -e 'SET GLOBAL local_infile=on' + docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" + + # Test user for auth test + mysql -e ' + CREATE USER + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", + nopass_sha256 IDENTIFIED WITH "sha256_password", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER;' + mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' else - cat ~/.my.cnf + WITH_PLUGIN='' +fi - mysql -e 'select VERSION()' - mysql -e 'create database test1 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' - mysql -e 'create database test2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' +mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' +mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - mysql -u root -e "create user test2 identified by 'some password'; grant all on test2.* to test2;" - mysql -u root -e "create user test2@localhost identified by 'some password'; grant all on test2.* to test2@localhost;" +mysql -u root -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" +mysql -u root -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - cp .travis/database.json pymysql/tests/databases.json -fi +cp .travis/docker.json pymysql/tests/databases.json From d063f68e890739f0e582f1b8049e1af170c94d77 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 17 Jan 2019 21:39:18 +0900 Subject: [PATCH 031/227] Remove auto show warnings (#774) --- pymysql/cursors.py | 31 ------------------------------- pymysql/tests/test_basic.py | 12 ------------ pymysql/tests/test_issues.py | 25 ------------------------- pymysql/tests/test_load_local.py | 24 ------------------------ 4 files changed, 92 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index a6d645d41..b3a690e6b 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -2,7 +2,6 @@ from __future__ import print_function, absolute_import from functools import partial import re -import warnings from ._compat import range_type, text_type, PY2 from . import err @@ -35,8 +34,6 @@ class Cursor(object): #: Default value of max_allowed_packet is 1048576. max_stmt_length = 1024000 - _defer_warnings = False - def __init__(self, connection): self.connection = connection self.description = None @@ -46,7 +43,6 @@ def __init__(self, connection): self._executed = None self._result = None self._rows = None - self._warnings_handled = False def close(self): """ @@ -90,9 +86,6 @@ def _nextset(self, unbuffered=False): """Get the next query set""" conn = self._get_db() current_result = self._result - # for unbuffered queries warnings are only available once whole result has been read - if unbuffered: - self._show_warnings() if current_result is None or current_result is not conn._result: return None if not current_result.has_next: @@ -347,26 +340,6 @@ def _do_get_result(self): self.description = result.description self.lastrowid = result.insert_id self._rows = result.rows - self._warnings_handled = False - - if not self._defer_warnings: - self._show_warnings() - - def _show_warnings(self): - if self._warnings_handled: - return - self._warnings_handled = True - if self._result and (self._result.has_next or not self._result.warning_count): - return - ws = self._get_db().show_warnings() - if ws is None: - return - for w in ws: - msg = w[-1] - if PY2: - if isinstance(msg, unicode): - msg = msg.encode('utf-8', 'replace') - warnings.warn(err.Warning(*w[1:3]), stacklevel=4) def __iter__(self): return iter(self.fetchone, None) @@ -427,8 +400,6 @@ class SSCursor(Cursor): possible to scroll backwards, as only the current row is held in memory. """ - _defer_warnings = True - def _conv_row(self, row): return row @@ -468,7 +439,6 @@ def fetchone(self): self._check_executed() row = self.read_next() if row is None: - self._show_warnings() return None self.rownumber += 1 return row @@ -502,7 +472,6 @@ def fetchmany(self, size=None): for i in range_type(size): row = self.read_next() if row is None: - self._show_warnings() break rows.append(row) self.rownumber += 1 diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index c2d539047..38c8cb649 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -2,7 +2,6 @@ import datetime import json import time -import warnings import pytest @@ -378,14 +377,3 @@ def test_issue_288(self): age = values(age)""")) cursor.execute('commit') self._verify_records(data) - - def test_warnings(self): - con = self.connect() - cur = con.cursor() - with warnings.catch_warnings(record=True) as ws: - warnings.simplefilter("always") - cur.execute("drop table if exists no_exists_table") - self.assertEqual(len(ws), 1) - self.assertEqual(ws[0].category, pymysql.Warning) - if u"no_exists_table" not in str(ws[0].message): - self.fail("'no_exists_table' not in %s" % (str(ws[0].message),)) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 05ecf2863..3775f314a 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -485,28 +485,3 @@ def test_issue_363(self): # don't assert the exact internal binary value, as it could # vary across implementations self.assertTrue(isinstance(row[0], bytes)) - - def test_issue_491(self): - """ Test warning propagation """ - conn = pymysql.connect(charset="utf8", **self.databases[0]) - - with warnings.catch_warnings(): - # Ignore all warnings other than pymysql generated ones - warnings.simplefilter("ignore") - warnings.simplefilter("error", category=pymysql.Warning) - - # verify for both buffered and unbuffered cursor types - for cursor_class in (cursors.Cursor, cursors.SSCursor): - c = conn.cursor(cursor_class) - try: - c.execute("SELECT CAST('124b' AS SIGNED)") - c.fetchall() - except pymysql.Warning as e: - # Warnings should have errorcode and string message, just like exceptions - self.assertEqual(len(e.args), 2) - self.assertEqual(e.args[0], 1292) - self.assertTrue(isinstance(e.args[1], text_type)) - else: - self.fail("Should raise Warning") - finally: - c.close() diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index eafa6e19e..30186e3a5 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -2,7 +2,6 @@ from pymysql.tests import base import os -import warnings __all__ = ["TestLoadLocal"] @@ -64,29 +63,6 @@ def test_unbuffered_load_file(self): c = conn.cursor() c.execute("DROP TABLE test_load_local") - def test_load_warnings(self): - """Test load local infile produces the appropriate warnings""" - conn = self.connect() - c = conn.cursor() - c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") - filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'data', - 'load_local_warn_data.txt') - try: - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - c.execute( - ("LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + - "test_load_local FIELDS TERMINATED BY ','").format(filename) - ) - self.assertEqual(w[0].category, Warning) - expected_message = "Incorrect integer value" - if expected_message not in str(w[-1].message): - self.fail("%r not in %r" % (expected_message, w[-1].message)) - finally: - c.execute("DROP TABLE test_load_local") - c.close() - if __name__ == "__main__": import unittest From 3539f87ed59b443916f195967ece4ba5b32cd8e3 Mon Sep 17 00:00:00 2001 From: Deneby67 Date: Tue, 5 Feb 2019 10:55:38 +0300 Subject: [PATCH 032/227] Optimize reading huge packet (#779) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index af074e21d..7cf3fef67 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -638,7 +638,7 @@ def _read_packet(self, packet_type=MysqlPacket): :raise OperationalError: If the connection to the MySQL server is lost. :raise InternalError: If the packet sequence number is wrong. """ - buff = b'' + buff = bytearray() while True: packet_header = self._read_bytes(4) #if DEBUG: dump_packet(packet_header) @@ -666,7 +666,7 @@ def _read_packet(self, packet_type=MysqlPacket): if bytes_to_read < MAX_PACKET_LEN: break - packet = packet_type(buff, self.encoding) + packet = packet_type(bytes(buff), self.encoding) packet.check_error() return packet From 3f04fcaa8fc9d96f717b76e58553e87f890c6ba4 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 1 Mar 2019 16:03:51 +0800 Subject: [PATCH 033/227] Fix typo in CHANGELOG (#783) --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 9ddb8f0bc..a7272aa93 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -51,7 +51,7 @@ Release date: 2018-05-07 * Many test suite improvements, especially adding MySQL 8.0 and using Docker. Thanks to Daniel Black. -* Droppped support for old Python and MySQL whih is not tested long time. +* Droppped support for old Python and MySQL which is not tested long time. ## 0.8 From 188efc56ef0e3c5e6745e52907e8677bf86f7487 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Mon, 4 Mar 2019 13:56:15 +0800 Subject: [PATCH 034/227] Fix minor typos in comments (#784) --- pymysql/connections.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 7cf3fef67..f54344ed9 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -140,7 +140,7 @@ class Connection(object): Specifies my.cnf file to read these parameters from under the [client] section. :param conv: Conversion dictionary to use instead of the default one. - This is used to provide custom marshalling and unmarshaling of types. + This is used to provide custom marshalling and unmarshalling of types. See converters. :param use_unicode: Whether or not to default to unicode strings. @@ -159,14 +159,14 @@ class Connection(object): :param local_infile: Boolean to enable the use of LOAD DATA LOCAL command. (default: False) :param max_allowed_packet: Max size of packet sent to server in bytes. (default: 16MB) Only used to limit size of "LOAD LOCAL INFILE" data packet smaller than default (16KB). - :param defer_connect: Don't explicitly connect on contruction - wait for connect call. + :param defer_connect: Don't explicitly connect on construction - wait for connect call. (default: False) :param auth_plugin_map: A dict of plugin names to a class that processes that plugin. The class will take the Connection object as the argument to the constructor. The class needs an authenticate method taking an authentication packet as an argument. For the dialog plugin, a prompt(echo, prompt) method can be used (if no authenticate method) for returning a string from the user. (experimental) - :param server_public_key: SHA256 authenticaiton plugin public key value. (default: None) + :param server_public_key: SHA256 authentication plugin public key value. (default: None) :param db: Alias for database. (for compatibility to MySQLdb) :param passwd: Alias for password. (for compatibility to MySQLdb) :param binary_prefix: Add _binary prefix on bytes and bytearray. (default: False) @@ -622,9 +622,9 @@ def connect(self, sock=None): def write_packet(self, payload): """Writes an entire "mysql packet" in its entirety to the network - addings its length and sequence number. + adding its length and sequence number. """ - # Internal note: when you build packet manualy and calls _write_bytes() + # Internal note: when you build packet manually and calls _write_bytes() # directly, you should set self._next_seq_id properly. data = pack_int24(len(payload)) + int2byte(self._next_seq_id) + payload if DEBUG: dump_packet(data) From 3674bc6fd064bf88524e839c07690e8c35223709 Mon Sep 17 00:00:00 2001 From: Ludger Heide Date: Wed, 13 Mar 2019 18:05:11 +0100 Subject: [PATCH 035/227] Setting SO_KEEPALIVE only for TCP (#785) --- pymysql/connections.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index f54344ed9..d9ade9a2e 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -575,8 +575,9 @@ def connect(self, sock=None): self.host_info = "socket %s:%d" % (self.host, self.port) if DEBUG: print('connected using socket') sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.settimeout(None) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + self._sock = sock self._rfile = _makefile(sock, 'rb') self._next_seq_id = 0 From 383c0438fe74464ad65b9850bd13f310de7a878e Mon Sep 17 00:00:00 2001 From: parthgandhi Date: Fri, 30 Aug 2019 12:00:19 +0530 Subject: [PATCH 036/227] fix spelling mistakes in changelog (#808) --- CHANGELOG | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a7272aa93..503b043ad 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -15,7 +15,7 @@ Release date: 2018-12-18 Release date: 2018-07-04 -* Disalbled unintentinally enabled debug log +* Disabled unintentinally enabled debug log * Removed unintentionally installed tests @@ -51,7 +51,7 @@ Release date: 2018-05-07 * Many test suite improvements, especially adding MySQL 8.0 and using Docker. Thanks to Daniel Black. -* Droppped support for old Python and MySQL which is not tested long time. +* Dropped support for old Python and MySQL which is not tested long time. ## 0.8 From f8c31d40c5abda9e03de5df34ea692b428fb6677 Mon Sep 17 00:00:00 2001 From: ppd0705 Date: Fri, 13 Sep 2019 13:16:40 +0800 Subject: [PATCH 037/227] Fix error packet handling for SSCursor (#810) --- pymysql/connections.py | 5 ++++- pymysql/protocol.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index d9ade9a2e..93efd9be6 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -668,7 +668,10 @@ def _read_packet(self, packet_type=MysqlPacket): break packet = packet_type(bytes(buff), self.encoding) - packet.check_error() + if packet.is_error_packet(): + if self._result is not None and self._result.unbuffered_active is True: + self._result.unbuffered_active = False + packet.raise_for_error() return packet def _read_bytes(self, num_bytes): diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 8ccf7c4d7..e302edab3 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -213,11 +213,14 @@ def is_error_packet(self): def check_error(self): if self.is_error_packet(): - self.rewind() - self.advance(1) # field_count == error (we already know that) - errno = self.read_uint16() - if DEBUG: print("errno =", errno) - err.raise_mysql_exception(self._data) + self.raise_for_error() + + def raise_for_error(self): + self.rewind() + self.advance(1) # field_count == error (we already know that) + errno = self.read_uint16() + if DEBUG: print("errno =", errno) + err.raise_mysql_exception(self._data) def dump(self): dump_packet(self._data) From 18b0bcb9bf0561fa2d191ff946e97d99a244b211 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 21 Sep 2019 18:16:35 +0900 Subject: [PATCH 038/227] use better format for float (#806) --- pymysql/converters.py | 7 ++++++- .../thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index be2e697c6..889cd7a20 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -54,7 +54,12 @@ def escape_int(value, mapping=None): return str(value) def escape_float(value, mapping=None): - return ('%.15g' % value) + s = repr(value) + if s in ('inf', 'nan'): + raise ProgrammingError("%s can not be used with MySQL" % s) + if 'e' not in s: + s += 'e0' + return s _escape_table = [unichr(x) for x in range(128)] _escape_table[0] = u'\\0' diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 13b43d3f5..8c1dd535a 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -90,7 +90,7 @@ def test_literal_int(self): self.assertTrue("2" == self.connection.literal(2)) def test_literal_float(self): - self.assertTrue("3.1415" == self.connection.literal(3.1415)) + self.assertEqual("3.1415e0", self.connection.literal(3.1415)) def test_literal_string(self): self.assertTrue("'foo'" == self.connection.literal("foo")) From ec8306b2331881bedc3aa19c13ec1400aa939ec3 Mon Sep 17 00:00:00 2001 From: brettl-sprint <57368682+brettl-sprint@users.noreply.github.com> Date: Thu, 7 Nov 2019 00:33:14 -0500 Subject: [PATCH 039/227] Updates link to error handling documentation (#821) --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 93efd9be6..227386064 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1,7 +1,7 @@ # Python implementation of the MySQL client-server protocol # http://dev.mysql.com/doc/internals/en/client-server-protocol.html # Error codes: -# http://dev.mysql.com/doc/refman/5.5/en/error-messages-client.html +# https://dev.mysql.com/doc/refman/5.5/en/error-handling.html from __future__ import print_function from ._compat import PY2, range_type, text_type, str_type, JYTHON, IRONPYTHON From 9dcefe9814bb053b1718a4407bb06790cb5de955 Mon Sep 17 00:00:00 2001 From: Bastien Vallet Date: Thu, 7 Nov 2019 16:55:03 +0100 Subject: [PATCH 040/227] Add Python 3.8 support (#822) --- .travis.yml | 6 +++--- setup.py | 1 + tox.ini | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b2f91aab5..69ca5317c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,13 +17,13 @@ matrix: python: "3.6" - env: - DB=mariadb:10.1 - python: "pypy3.5" + python: "pypy3" - env: - DB=mariadb:10.2 python: "2.7" - env: - DB=mariadb:10.3 - python: "3.7-dev" + python: "3.7" - env: - DB=mysql:5.5 python: "3.5" @@ -36,7 +36,7 @@ matrix: - env: - DB=mysql:8.0 - TEST_AUTH=yes - python: "3.7-dev" + python: "3.8" - env: - DB=mysql:8.0 - TEST_AUTH=yes diff --git a/setup.py b/setup.py index b888c01fd..6a9b2d807 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', diff --git a/tox.ini b/tox.ini index d13e49f4f..95430ae82 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py35,py36,py37,pypy,pypy3 +envlist = py{27,35,36,37,38,py,py3} [testenv] commands = pytest -v pymysql/tests/ From c3e5a63514c57d1f4c9d5e7bf4b7e10b0608b0e1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 13 Nov 2019 14:14:58 +0900 Subject: [PATCH 041/227] Use OperationalError for unknown error with code>1000. (#823) Fixes #816. --- pymysql/err.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymysql/err.py b/pymysql/err.py index e93ba9bea..8ca23655c 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -100,5 +100,7 @@ def _map_error(exc, *errors): def raise_mysql_exception(data): errno = struct.unpack(' Date: Thu, 21 Nov 2019 15:51:19 +0900 Subject: [PATCH 042/227] Use cp1252 encoding for latin1 charset (#824) --- pymysql/charset.py | 14 ++++++-------- pymysql/converters.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pymysql/charset.py b/pymysql/charset.py index 07d80638f..d3ced67ca 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -20,6 +20,12 @@ def encoding(self): name = self.name if name in ('utf8mb4', 'utf8mb3'): return 'utf8' + if name == 'latin1': + return 'cp1252' + if name == 'koi8r': + return 'koi8_r' + if name == 'koi8u': + return 'koi8_u' return name @property @@ -202,11 +208,3 @@ def by_name(self, name): charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id - - -#TODO: remove this -def charset_to_encoding(name): - """Convert MySQL's charset name to Python's codec name""" - if name in ('utf8mb4', 'utf8mb3'): - return 'utf8' - return name diff --git a/pymysql/converters.py b/pymysql/converters.py index 889cd7a20..2793a2ae2 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -6,7 +6,7 @@ import time from .constants import FIELD_TYPE, FLAG -from .charset import charset_by_id, charset_to_encoding +from .charset import charset_by_id def escape_item(val, charset, mapping=None): From 2330bf798894b35f3fcc796e9c5df5bac44105ab Mon Sep 17 00:00:00 2001 From: brettl-sprint <57368682+brettl-sprint@users.noreply.github.com> Date: Thu, 21 Nov 2019 11:33:03 -0500 Subject: [PATCH 043/227] Raise more graceful error when port is not int (#820) --- pymysql/connections.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymysql/connections.py b/pymysql/connections.py index 227386064..d74af4fad 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -254,6 +254,8 @@ def _config(key, arg): self.host = host or "localhost" self.port = port or 3306 + if type(self.port) is not int: + raise ValueError("port should be of type int") self.user = user or DEFAULT_USER self.password = password or b"" if isinstance(self.password, text_type): From 0f4d45e5a20b47959ba7d16f130cbc0c7ce8506c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 26 Nov 2019 20:56:41 +0900 Subject: [PATCH 044/227] Fix decimal literal. (#828) Fixes #818. --- pymysql/converters.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 2793a2ae2..efb0e4d44 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -159,6 +159,11 @@ def escape_date(obj, mapping=None): def escape_struct_time(obj, mapping=None): return escape_datetime(datetime.datetime(*obj[:6])) + +def Decimal2Literal(o, d): + return format(o, "f") + + def _convert_second_fraction(s): if not s: return 0 @@ -337,7 +342,7 @@ def through(x): datetime.timedelta: escape_timedelta, datetime.time: escape_time, time.struct_time: escape_struct_time, - Decimal: escape_object, + Decimal: Decimal2Literal, } if not PY2 or JYTHON or IRONPYTHON: From c3c87a7e773dbb09def0b081c70dd55fe83b9633 Mon Sep 17 00:00:00 2001 From: Sebastien Volle Date: Wed, 4 Dec 2019 11:31:11 +0100 Subject: [PATCH 045/227] Fix connection timeout error messages (#830) Fix inconsistency between connection read/write timeout error messages and actual value checks. --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index d74af4fad..a1cd8c25e 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -267,10 +267,10 @@ def _config(key, arg): raise ValueError("connect_timeout should be >0 and <=31536000") self.connect_timeout = connect_timeout or None if read_timeout is not None and read_timeout <= 0: - raise ValueError("read_timeout should be >= 0") + raise ValueError("read_timeout should be > 0") self._read_timeout = read_timeout if write_timeout is not None and write_timeout <= 0: - raise ValueError("write_timeout should be >= 0") + raise ValueError("write_timeout should be > 0") self._write_timeout = write_timeout if charset: self.charset = charset From 577276a952499fdc4c6786e164dfb3f12dad7272 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 8 Dec 2019 01:06:34 +1100 Subject: [PATCH 046/227] Fix typo. (#833) Closes #832 --- pymysql/cursors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index b3a690e6b..033b5e7ff 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -8,7 +8,7 @@ #: Regular expression for :meth:`Cursor.executemany`. -#: executemany only suports simple bulk insert. +#: executemany only supports simple bulk insert. #: You can use it to load large dataset. RE_INSERT_VALUES = re.compile( r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" + From 6faa8b679df6ca97a83f3028228eaa2803278171 Mon Sep 17 00:00:00 2001 From: Vilhelm Prytz Date: Wed, 11 Dec 2019 23:24:33 +0100 Subject: [PATCH 047/227] Remove unused imports (#835) --- pymysql/_auth.py | 3 +-- pymysql/converters.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index aa082dfe5..a7fdaa48e 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -1,8 +1,7 @@ """ Implements auth methods """ -from ._compat import text_type, PY2 -from .constants import CLIENT +from ._compat import PY2 from .err import OperationalError from .util import byte2int, int2byte diff --git a/pymysql/converters.py b/pymysql/converters.py index efb0e4d44..b084ed2f3 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -5,8 +5,7 @@ import re import time -from .constants import FIELD_TYPE, FLAG -from .charset import charset_by_id +from .constants import FIELD_TYPE def escape_item(val, charset, mapping=None): From 9f1b8569032ec7eaff36fe9ef5e40f82c47260b2 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 14 Feb 2020 10:18:40 +0000 Subject: [PATCH 048/227] Fix test suite compatibility with MySQL 8 (#840) MySQL 8 deprecates the use of display format for int columns: https://dev.mysql.com/doc/refman/8.0/en/numeric-type-syntax.html This results in warnings being generated during test suite execution which results in test failures. Drop use of display widths - they don't materially change the tests so this should be safe across all MySQL versions and variants. --- pymysql/tests/test_basic.py | 2 +- pymysql/tests/test_issues.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 38c8cb649..aa23e065d 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -289,7 +289,7 @@ def setUp(self): self.safe_create_table(conn, 'bulkinsert', """\ CREATE TABLE bulkinsert ( -id int(11), +id int, name char(20), age int, height int, diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 3775f314a..604aeaffe 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -79,8 +79,8 @@ def test_issue_8(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists test") - c.execute("""CREATE TABLE `test` (`station` int(10) NOT NULL DEFAULT '0', `dh` -datetime NOT NULL DEFAULT '2015-01-01 00:00:00', `echeance` int(1) NOT NULL + c.execute("""CREATE TABLE `test` (`station` int NOT NULL DEFAULT '0', `dh` +datetime NOT NULL DEFAULT '2015-01-01 00:00:00', `echeance` int NOT NULL DEFAULT '0', `me` double DEFAULT NULL, `mo` double DEFAULT NULL, PRIMARY KEY (`station`,`dh`,`echeance`)) ENGINE=MyISAM DEFAULT CHARSET=latin1;""") try: From 8f9060042f0987656039d0588a54b6df30d3ba57 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 Mar 2020 18:15:10 +0900 Subject: [PATCH 049/227] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..f2bd4d300 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,39 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Complete steps to reproduce the behavior: + +Schema: + +``` +CREATE DATABASE ... +CREATE TABLE ... +``` + +Code: + +```py +import pymysql +con = pymysql.connect(...) +``` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment** + - OS: [e.g. Windows, Linux] + - Server and version: [e.g. MySQL 8.0.19, MariaDB] + - PyMySQL version: + +**Additional context** +Add any other context about the problem here. From 33bb6b6640bd7004054b105de3da62f489f0df03 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 Mar 2020 18:19:49 +0900 Subject: [PATCH 050/227] Remove old ISSUE_TEMPLATE --- .github/ISSUE_TEMPLATE.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 3e0fbe826..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,11 +0,0 @@ -This project is maintained one busy person with a frail wife and an infant daughter. -My time and energy is a very limited resource. I'm not a teacher or free tech support. -Don't ask a question here. Don't file an issue until you believe it's a not a problem with your code. -Search for friendly volunteers who can teach you or review your code on ML or Q&A sites. - -See also: https://medium.com/@methane/why-you-must-not-ask-questions-on-github-issues-51d741d83fde - - -If you're sure it's PyMySQL's issue, report the complete steps to reproduce, from creating database. - -I don't have time to investigate your issue from an incomplete code snippet. From d895719372d00378b17a42d60109d10b0d1a10ed Mon Sep 17 00:00:00 2001 From: Uri Date: Wed, 13 May 2020 07:45:43 +0300 Subject: [PATCH 051/227] updated doctored version info for MySQLdb compatibility (#858) Fixes #790 --- pymysql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 0cb5006cc..6ffb2ae60 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -108,7 +108,7 @@ def get_client_info(): # for MySQLdb compatibility connect = Connection = Connect # we include a doctored version_info here for MySQLdb compatibility -version_info = (1, 3, 12, "final", 0) +version_info = (1, 3, 13, "final", 0) NULL = "NULL" From 466ecfe61eab666658b6f2141b0dfb457c4c72a5 Mon Sep 17 00:00:00 2001 From: Justin Chang Date: Mon, 13 Jul 2020 00:10:36 -0400 Subject: [PATCH 052/227] Fix InterfaceError response when connection lost (#872) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index a1cd8c25e..fe7a2abdc 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -739,7 +739,7 @@ def _execute_command(self, command, sql): :raise ValueError: If no username was specified. """ if not self._sock: - raise err.InterfaceError("(0, '')") + raise err.InterfaceError(0, '') # If the last query was unbuffered, make sure it finishes before # sending new commands @@ -1253,7 +1253,7 @@ def __init__(self, filename, connection): def send_data(self): """Send data packets from the local file to the server""" if not self.connection._sock: - raise err.InterfaceError("(0, '')") + raise err.InterfaceError(0, '') conn = self.connection try: From 221d411cb2acfae34d95908aa841f7bb5a1d6e74 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 09:16:05 +0900 Subject: [PATCH 053/227] travis: Use Python 3.9-dev --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 69ca5317c..bff6a0ee1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ matrix: python: "3.5" - env: - DB=mysql:5.6 - python: "3.6" + python: "3.9-dev" - env: - DB=mysql:5.7 python: "3.7" From f75c0024c6bd89a165b559b2bacd7afdb8858cce Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 10:03:05 +0900 Subject: [PATCH 054/227] Update error mapping (#873) --- pymysql/constants/ER.py | 1 - pymysql/err.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/constants/ER.py b/pymysql/constants/ER.py index 79b88afbe..ddcc4e907 100644 --- a/pymysql/constants/ER.py +++ b/pymysql/constants/ER.py @@ -1,4 +1,3 @@ - ERROR_FIRST = 1000 HASHCHK = 1000 NISAMCHK = 1001 diff --git a/pymysql/err.py b/pymysql/err.py index 8ca23655c..94100cfea 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -83,7 +83,8 @@ def _map_error(exc, *errors): ) _map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL, ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL, - ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW) + ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW, ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, + ER.ILLEGAL_VALUE_FOR_TYPE) _map_error(IntegrityError, ER.DUP_ENTRY, ER.NO_REFERENCED_ROW, ER.NO_REFERENCED_ROW_2, ER.ROW_IS_REFERENCED, ER.ROW_IS_REFERENCED_2, ER.CANNOT_ADD_FOREIGN, ER.BAD_NULL_ERROR) From 73f977029e2c076719a7ea8d0c3df84cb44ebe7c Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Fri, 17 Jul 2020 03:06:23 +0200 Subject: [PATCH 055/227] Support for MariaDB's auth_ed25519 authentication plugin (#786) (#791) --- .travis.yml | 8 +++-- .travis/initializedb.sh | 10 ++++++- README.rst | 5 ++++ pymysql/_auth.py | 60 ++++++++++++++++++++++++++++++++++++++ pymysql/connections.py | 2 ++ requirements-dev.txt | 1 + setup.py | 1 + tests/test_mariadb_auth.py | 23 +++++++++++++++ 8 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 tests/test_mariadb_auth.py diff --git a/.travis.yml b/.travis.yml index bff6a0ee1..553d9cd1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ matrix: python: "2.7" - env: - DB=mariadb:10.3 + - TEST_MARIADB_AUTH=yes python: "3.7" - env: - DB=mysql:5.5 @@ -46,7 +47,7 @@ matrix: # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version # really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't install: - - pip install -U coveralls coverage cryptography pytest pytest-cov + - pip install -U coveralls coverage cryptography PyNaCl pytest pytest-cov before_script: - ./.travis/initializedb.sh @@ -57,7 +58,10 @@ before_script: script: - pytest -v --cov --cov-config .coveragerc pymysql - if [ "${TEST_AUTH}" = "yes" ]; - then pytest -v --cov --cov-config .coveragerc tests; + then pytest -v --cov --cov-config .coveragerc tests/test_auth.py; + fi + - if [ "${TEST_MARIADB_AUTH}" = "yes" ]; + then pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py; fi - if [ ! -z "${DB}" ]; then docker logs mysqld; diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 17d06100d..98c1cd3b4 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -6,7 +6,7 @@ docker pull ${DB} docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} mysql() { - docker exec mysqld mysql "${@}" + docker exec -i mysqld mysql "${@}" } while : do @@ -33,6 +33,14 @@ if [ $DB == 'mysql:8.0' ]; then nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" PASSWORD EXPIRE NEVER;' mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' +elif [[ $DB == mariadb:10.* ]] && [ ${DB#mariadb:10.} -ge 3 ]; then + mysql -e ' + INSTALL SONAME "auth_ed25519"; + CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' + # we need to pass the hashed password manually until 10.4, so hide it here + mysql -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql + mysql -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql + WITH_PLUGIN='' else WITH_PLUGIN='' fi diff --git a/README.rst b/README.rst index 175bf43ed..7bed7f7e9 100644 --- a/README.rst +++ b/README.rst @@ -66,6 +66,11 @@ you need to install additional dependency:: $ python3 -m pip install PyMySQL[rsa] +To use MariaDB's "ed25519" authentication method, you need to install +additional dependency:: + + $ python3 -m pip install PyMySQL[ed25519] + Documentation ------------- diff --git a/pymysql/_auth.py b/pymysql/_auth.py index a7fdaa48e..72e9579bb 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -113,6 +113,66 @@ def _hash_password_323(password): return struct.pack(">LL", r1, r2) +# MariaDB's client_ed25519-plugin +# https://mariadb.com/kb/en/library/connection/#client_ed25519-plugin + +_nacl_bindings = False + + +def _init_nacl(): + global _nacl_bindings + try: + from nacl import bindings + _nacl_bindings = bindings + except ImportError: + raise RuntimeError("'pynacl' package is required for ed25519_password auth method") + + +def _scalar_clamp(s32): + ba = bytearray(s32) + ba0 = bytes(bytearray([ba[0] & 248])) + ba31 = bytes(bytearray([(ba[31] & 127) | 64])) + return ba0 + bytes(s32[1:31]) + ba31 + + +def ed25519_password(password, scramble): + """Sign a random scramble with elliptic curve Ed25519. + + Secret and public key are derived from password. + """ + # variable names based on rfc8032 section-5.1.6 + # + if not _nacl_bindings: + _init_nacl() + + # h = SHA512(password) + h = hashlib.sha512(password).digest() + + # s = prune(first_half(h)) + s = _scalar_clamp(h[:32]) + + # r = SHA512(second_half(h) || M) + r = hashlib.sha512(h[32:] + scramble).digest() + + # R = encoded point [r]B + r = _nacl_bindings.crypto_core_ed25519_scalar_reduce(r) + R = _nacl_bindings.crypto_scalarmult_ed25519_base_noclamp(r) + + # A = encoded point [s]B + A = _nacl_bindings.crypto_scalarmult_ed25519_base_noclamp(s) + + # k = SHA512(R || A || M) + k = hashlib.sha512(R + A + scramble).digest() + + # S = (k * s + r) mod L + k = _nacl_bindings.crypto_core_ed25519_scalar_reduce(k) + ks = _nacl_bindings.crypto_core_ed25519_scalar_mul(k, s) + S = _nacl_bindings.crypto_core_ed25519_scalar_add(ks, r) + + # signature = R || S + return R + S + + # sha256_password diff --git a/pymysql/connections.py b/pymysql/connections.py index fe7a2abdc..75e07f34d 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -894,6 +894,8 @@ def _process_auth(self, plugin_name, auth_packet): return _auth.sha256_password_auth(self, auth_packet) elif plugin_name == b"mysql_native_password": data = _auth.scramble_native_password(self.password, auth_packet.read_all()) + elif plugin_name == b'client_ed25519': + data = _auth.ed25519_password(self.password, auth_packet.read_all()) elif plugin_name == b"mysql_old_password": data = _auth.scramble_old_password(self.password, auth_packet.read_all()) + b'\0' elif plugin_name == b"mysql_clear_password": diff --git a/requirements-dev.txt b/requirements-dev.txt index 5e85e5226..d65512fbb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ cryptography +PyNaCl>=1.4.0 pytest diff --git a/setup.py b/setup.py index 6a9b2d807..3dbdca2d1 100755 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ packages=find_packages(exclude=['tests*', 'pymysql.tests*']), extras_require={ "rsa": ["cryptography"], + "ed25519": ["PyNaCl>=1.4.0"], }, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/test_mariadb_auth.py b/tests/test_mariadb_auth.py new file mode 100644 index 000000000..2f336feca --- /dev/null +++ b/tests/test_mariadb_auth.py @@ -0,0 +1,23 @@ +"""Test for auth methods supported by MariaDB 10.3+""" + +import pymysql + +# pymysql.connections.DEBUG = True +# pymysql._auth.DEBUG = True + +host = "127.0.0.1" +port = 3306 + + +def test_ed25519_no_password(): + con = pymysql.connect(user="nopass_ed25519", host=host, port=port, ssl=None) + con.close() + + +def test_ed25519_password(): # nosec + con = pymysql.connect(user="user_ed25519", password="pass_ed25519", + host=host, port=port, ssl=None) + con.close() + + +# default mariadb docker images aren't configured with SSL From e929f94b2e26bd71eeb8253d16ab5e537f27ae91 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 11:34:04 +0900 Subject: [PATCH 056/227] Update changelog --- CHANGELOG | 18 ++++++++++++++++++ setup.cfg | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 503b043ad..8f13b9ff0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,23 @@ # Changes +## 0.10 + +Release date: 2020-07-17 + +* MariaDB ed25519 auth is supported. +* Python 3.4 support is dropped. +* Context manager interface is removed from `Connection`. It will be added + with different meaning. +* MySQL warnings are not shown by default because many user report issue to + PyMySQL issue tracker when they see warning. You need to call "SHOW WARNINGS" + explicitly when you want to see warnings. +* Formatting of float object is changed from "3.14" to "3.14e0". +* Use cp1252 codec for latin1 charset. +* Fix decimal literal. +* TRUNCATED_WRONG_VALUE_FOR_FIELD, and ILLEGAL_VALUE_FOR_TYPE are now + DataError instead of InternalError. + + ## 0.9.3 Release date: 2018-12-18 diff --git a/setup.cfg b/setup.cfg index a26a846b9..ca7a9ae3f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,5 +13,5 @@ license_file = LICENSE author=yutaka.matsubara author_email=yutaka.matsubara@gmail.com -maintainer=INADA Naoki +maintainer=Inada Naoki maintainer_email=songofacandy@gmail.com From d78581ec246a22758fc397242b74ccaebf07cb62 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 17:34:31 +0900 Subject: [PATCH 057/227] v0.10.0 --- CHANGELOG | 2 +- pymysql/__init__.py | 2 +- setup.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 8f13b9ff0..d2e3bd86b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,6 @@ # Changes -## 0.10 +## v0.10.0 Release date: 2020-07-17 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 6ffb2ae60..9c4e8f576 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 9, 3, None) +VERSION = (0, 10, 0, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 3dbdca2d1..8c72060f4 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = "0.9.3" +version = "0.10.0" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() @@ -30,6 +30,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', From a262df2d5f0bf0f39864521f9efcc37dbee5005d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 Jul 2020 16:25:02 +0900 Subject: [PATCH 058/227] Update Changelog --- CHANGELOG | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index d2e3bd86b..186c75eaa 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,7 +2,9 @@ ## v0.10.0 -Release date: 2020-07-17 +Release date: 2020-07-18 + +This version is the last version supporting Python 2.7. * MariaDB ed25519 auth is supported. * Python 3.4 support is dropped. From 95e313acdbd522827fb2eaea5520c3b280b08195 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 Jul 2020 16:25:58 +0900 Subject: [PATCH 059/227] CHANGELOG -> CHANGELOG.md --- CHANGELOG => CHANGELOG.md | 0 MANIFEST.in | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename CHANGELOG => CHANGELOG.md (100%) diff --git a/CHANGELOG b/CHANGELOG.md similarity index 100% rename from CHANGELOG rename to CHANGELOG.md diff --git a/MANIFEST.in b/MANIFEST.in index 0a5207928..e9e1eebcb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst LICENSE CHANGELOG +include README.rst LICENSE CHANGELOG.md From a27cbcb9be99b5b0038855eb6313083fe7feed3b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 Jul 2020 16:30:58 +0900 Subject: [PATCH 060/227] fix warning --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ca7a9ae3f..db1af5452 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ universal = 1 [metadata] license = "MIT" -license_file = LICENSE +license_files = LICENSE author=yutaka.matsubara author_email=yutaka.matsubara@gmail.com From 3e71dd32e8ce868b090c282759eebdeabc960f58 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 28 Jul 2020 13:06:07 +0900 Subject: [PATCH 061/227] Add missing import (#879) Fixes #878 --- pymysql/converters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymysql/converters.py b/pymysql/converters.py index b084ed2f3..1b5829040 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -5,6 +5,7 @@ import re import time +from .err import ProgrammingError from .constants import FIELD_TYPE From 2f6bb5d720286ef4efb84749877980c3157f15d5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Sep 2020 18:03:05 +0900 Subject: [PATCH 062/227] Fix sha256 and caching_sha2 auth (#892) --- .travis/initializedb.sh | 4 ++-- pymysql/_auth.py | 3 +++ tests/test_auth.py | 17 ++++++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 98c1cd3b4..6991cfe60 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -27,9 +27,9 @@ if [ $DB == 'mysql:8.0' ]; then # Test user for auth test mysql -e ' CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" PASSWORD EXPIRE NEVER;' mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 72e9579bb..57f9abb1a 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -184,6 +184,9 @@ def _roundtrip(conn, send_data): def _xor_password(password, salt): + # Trailing NUL character will be added in Auth Switch Request. + # See https://github.com/mysql/mysql-server/blob/7d10c82196c8e45554f27c00681474a9fb86d137/sql/auth/sha2_password.cc#L939-L945 + salt = salt[:SCRAMBLE_LENGTH] password_bytes = bytearray(password) salt = bytearray(salt) # for PY2 compat. salt_len = len(salt) diff --git a/tests/test_auth.py b/tests/test_auth.py index 7d8573442..61957655e 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -12,6 +12,9 @@ ca = os.path.expanduser("~/ca.pem") ssl = {'ca': ca, 'check_hostname': False} +pass_sha256 = "pass_sha256_01234567890123456789" +pass_caching_sha2 = "pass_caching_sha2_01234567890123456789" + def test_sha256_no_password(): con = pymysql.connect(user="nopass_sha256", host=host, port=port, ssl=None) @@ -24,12 +27,12 @@ def test_sha256_no_passowrd_ssl(): def test_sha256_password(): - con = pymysql.connect(user="user_sha256", password="pass_sha256", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=None) con.close() def test_sha256_password_ssl(): - con = pymysql.connect(user="user_sha256", password="pass_sha256", host=host, port=port, ssl=ssl) + con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=ssl) con.close() @@ -38,26 +41,26 @@ def test_caching_sha2_no_password(): con.close() -def test_caching_sha2_no_password(): +def test_caching_sha2_no_password_ssl(): con = pymysql.connect(user="nopass_caching_sha2", host=host, port=port, ssl=ssl) con.close() def test_caching_sha2_password(): - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) con.query("FLUSH PRIVILEGES") con.close() def test_caching_sha2_password_ssl(): - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=ssl) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=ssl) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) con.query("FLUSH PRIVILEGES") con.close() From 37fd1e1b0126d75d80eef59c053f80634b09bd75 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 10 Sep 2020 16:29:31 +0900 Subject: [PATCH 063/227] v0.10.1 --- CHANGELOG.md | 8 ++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 186c75eaa..0d1313aa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changes +## v0.10.1 + +Release date: 2020-09-10 + +* Fix missing import of ProgrammingError. (#878) +* Fix auth switch request handling. (#890) + + ## v0.10.0 Release date: 2020-07-18 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 9c4e8f576..5148fa77b 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 10, 0, None) +VERSION = (0, 10, 1, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 8c72060f4..e35e7b290 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = "0.10.0" +version = "0.10.1" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() From 99b703cccb8011692c398caf0c0fbd97b1355e90 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 10 Dec 2020 13:51:06 +1100 Subject: [PATCH 064/227] Fix test unix_socket for MariaDB-10.4 (#907) --- .travis.yml | 12 +++++++----- pymysql/tests/test_connection.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 553d9cd1b..e1398170f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # vim: sw=2 ts=2 sts=2 expandtab -dist: xenial +dist: bionic language: python cache: pip @@ -13,16 +13,18 @@ matrix: - DB=mariadb:5.5 python: "3.5" - env: - - DB=mariadb:10.0 + - DB=mariadb:10.2 python: "3.6" - env: - - DB=mariadb:10.1 + - DB=mariadb:10.3 + - TEST_MARIADB_AUTH=yes python: "pypy3" - env: - - DB=mariadb:10.2 + - DB=mariadb:10.4 + - TEST_MARIADB_AUTH=yes python: "2.7" - env: - - DB=mariadb:10.3 + - DB=mariadb:10.5 - TEST_MARIADB_AUTH=yes python: "3.7" - env: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index e4d24c445..51b9f3a55 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -70,7 +70,7 @@ class TestAuthentication(base.PyMySQLTestCase): for r in cur: if (r[1], r[2]) != (u'ACTIVE', u'AUTHENTICATION'): continue - if r[3] == u'auth_socket.so': + if r[3] == u'auth_socket.so' or r[0] == u'unix_socket': socket_plugin_name = r[0] socket_found = True elif r[3] == u'dialog_examples.so': From 907b45374ec8d09f1b83f4afca00b291d09e5d16 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 16:16:13 +0900 Subject: [PATCH 065/227] travis: Remove Python 2.7, 3.5, MySQL 5.5, MariaDB 5.5. (#913) --- .travis.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1398170f..aa1f0f341 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,6 @@ services: matrix: include: - - env: - - DB=mariadb:5.5 - python: "3.5" - env: - DB=mariadb:10.2 python: "3.6" @@ -19,20 +16,13 @@ matrix: - DB=mariadb:10.3 - TEST_MARIADB_AUTH=yes python: "pypy3" - - env: - - DB=mariadb:10.4 - - TEST_MARIADB_AUTH=yes - python: "2.7" - env: - DB=mariadb:10.5 - TEST_MARIADB_AUTH=yes python: "3.7" - - env: - - DB=mysql:5.5 - python: "3.5" - env: - DB=mysql:5.6 - python: "3.9-dev" + python: "3.9" - env: - DB=mysql:5.7 python: "3.7" @@ -40,10 +30,6 @@ matrix: - DB=mysql:8.0 - TEST_AUTH=yes python: "3.8" - - env: - - DB=mysql:8.0 - - TEST_AUTH=yes - python: "2.7" # different py version from 5.6 and 5.7 as cache seems to be based on py version # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version From 4e481fa52262e35498cd7ee187ebe4903f9a1771 Mon Sep 17 00:00:00 2001 From: CJ Mauro <57578688+cmauro1@users.noreply.github.com> Date: Sat, 2 Jan 2021 02:18:18 -0500 Subject: [PATCH 066/227] Add context manager support to Connection (#886) --- pymysql/connections.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 75e07f34d..9e87e0b02 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -325,7 +325,14 @@ def _config(key, arg): self._sock = None else: self.connect() - + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + del exc_info + self.close() + def _create_ssl_ctx(self, sslp): if isinstance(sslp, ssl.SSLContext): return sslp From b2e580f6edfe4198efe03bff07847580599df649 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 16:18:50 +0900 Subject: [PATCH 067/227] Create FUNDING.yml (#914) --- .github/FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..89fc5cf8f --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: [methane] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 2d440dfcbeadb26d13c1779c02872f840ec455f5 Mon Sep 17 00:00:00 2001 From: Uri Date: Sat, 2 Jan 2021 09:33:07 +0200 Subject: [PATCH 068/227] Updated mysqlclient version to 1.4.0 (#885) --- pymysql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 5148fa77b..29e6b87c1 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -108,7 +108,7 @@ def get_client_info(): # for MySQLdb compatibility connect = Connection = Connect # we include a doctored version_info here for MySQLdb compatibility -version_info = (1, 3, 13, "final", 0) +version_info = (1, 4, 0, "final", 0) NULL = "NULL" From 1489819a47cdeae830002435ac2fc4d43c6c949d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 16:35:24 +0900 Subject: [PATCH 069/227] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7bed7f7e9..0a09f8923 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ .. image:: https://badge.fury.io/py/PyMySQL.svg :target: https://badge.fury.io/py/PyMySQL -.. image:: https://travis-ci.org/PyMySQL/PyMySQL.svg?branch=master - :target: https://travis-ci.org/PyMySQL/PyMySQL +.. image:: https://travis-ci.com/PyMySQL/PyMySQL.svg?branch=master + :target: https://travis-ci.com/PyMySQL/PyMySQL .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master From aefbdbe1dc6dc022f2b02d2f4c4564d4ec929175 Mon Sep 17 00:00:00 2001 From: Moriyoshi Koizumi Date: Sat, 2 Jan 2021 17:11:19 +0900 Subject: [PATCH 070/227] Add MySQL Connector/Python compatible SSL options. (#903) Add connector-python compatible options. Also fixes #842. https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html --- pymysql/connections.py | 50 ++++++++-- pymysql/tests/test_connection.py | 160 ++++++++++++++++++++++++++++++- requirements-dev.txt | 1 + 3 files changed, 201 insertions(+), 10 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 9e87e0b02..7ecfb616c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -152,6 +152,12 @@ class Connection(object): (default: 10, min: 1, max: 31536000) :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. + :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate + :param ssl_cert: Path to the file that contains a PEM-formatted client certificate + :param ssl_disabled: A boolean value that disables usage of TLS + :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate + :param ssl_verify_cert: Set to true to check the validity of server certificates + :param ssl_verify_identity: Set to true to check the server's identity :param read_default_group: Group to read from in the configuration file. :param compress: Not supported :param named_pipe: Not supported @@ -191,7 +197,9 @@ def __init__(self, host=None, user=None, password="", max_allowed_packet=16*1024*1024, defer_connect=False, auth_plugin_map=None, read_timeout=None, write_timeout=None, bind_address=None, binary_prefix=False, program_name=None, - server_public_key=None): + server_public_key=None, ssl_ca=None, ssl_cert=None, + ssl_disabled=None, ssl_key=None, ssl_verify_cert=None, + ssl_verify_identity=None): if use_unicode is None and sys.version_info[0] > 2: use_unicode = True @@ -245,12 +253,23 @@ def _config(key, arg): ssl[key] = value self.ssl = False - if ssl: - if not SSL_ENABLED: - raise NotImplementedError("ssl module not found") - self.ssl = True - client_flag |= CLIENT.SSL - self.ctx = self._create_ssl_ctx(ssl) + if not ssl_disabled: + if ssl_ca or ssl_cert or ssl_key or ssl_verify_cert or ssl_verify_identity: + ssl = { + "ca": ssl_ca, + "check_hostname": bool(ssl_verify_identity), + "verify_mode": ssl_verify_cert if ssl_verify_cert is not None else False, + } + if ssl_cert is not None: + ssl["cert"] = ssl_cert + if ssl_key is not None: + ssl["key" ] = ssl_key + if ssl: + if not SSL_ENABLED: + raise NotImplementedError("ssl module not found") + self.ssl = True + client_flag |= CLIENT.SSL + self.ctx = self._create_ssl_ctx(ssl) self.host = host or "localhost" self.port = port or 3306 @@ -341,7 +360,22 @@ def _create_ssl_ctx(self, sslp): hasnoca = ca is None and capath is None ctx = ssl.create_default_context(cafile=ca, capath=capath) ctx.check_hostname = not hasnoca and sslp.get('check_hostname', True) - ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED + verify_mode_value = sslp.get('verify_mode') + if verify_mode_value is None: + ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED + elif isinstance(verify_mode_value, bool): + ctx.verify_mode = ssl.CERT_REQUIRED if verify_mode_value else ssl.CERT_NONE + else: + if isinstance(verify_mode_value, (text_type, str_type)): + verify_mode_value = verify_mode_value.lower() + if verify_mode_value in ("none", "0", "false", "no"): + ctx.verify_mode = ssl.CERT_NONE + elif verify_mode_value == "optional": + ctx.verify_mode = ssl.CERT_OPTIONAL + elif verify_mode_value in ("required", "1", "true", "yes"): + ctx.verify_mode = ssl.CERT_REQUIRED + else: + ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED if 'cert' in sslp: ctx.load_cert_chain(sslp['cert'], keyfile=sslp.get('key')) if 'cipher' in sslp: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 51b9f3a55..d04cdd488 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,14 +1,14 @@ import datetime +import ssl import sys import time +import mock import pytest import pymysql from pymysql.tests import base from pymysql._compat import text_type from pymysql.constants import CLIENT -import pytest - class TempUser: def __init__(self, c, user, db, auth=None, authdata=None, password=None): @@ -478,6 +478,162 @@ def test_defer_connect(self): c.close() sock.close() + def test_ssl_connect(self): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + "cipher": "cipher", + }, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_called_with("cipher") + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + }, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_not_called + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + for ssl_verify_cert in (True, "1", "yes", "true"): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_cert="cert", + ssl_key="key", + ssl_verify_cert=ssl_verify_cert, + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + for ssl_verify_cert in (None, False, "0", "no", "false"): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_cert="cert", + ssl_key="key", + ssl_verify_cert=ssl_verify_cert, + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + for ssl_ca in ("ca", None): + for ssl_verify_cert in ("foo", "bar", ""): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca=ssl_ca, + ssl_cert="cert", + ssl_key="key", + ssl_verify_cert=ssl_verify_cert, + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == (ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE), (ssl_ca, ssl_verify_cert) + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ssl_verify_identity=True, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_disabled=True, + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + }, + ) + assert not create_default_context.called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_disabled=True, + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ) + assert not create_default_context.called + # A custom type and function to escape it class Foo(object): diff --git a/requirements-dev.txt b/requirements-dev.txt index d65512fbb..69d3f68ae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ cryptography PyNaCl>=1.4.0 pytest +mock From 66947bf8ccba9986a8503d4a7d5b77b1b21be54e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 18:22:37 +0900 Subject: [PATCH 071/227] Remove Python 2.7 and 3.5 support. (#915) --- pymysql/__init__.py | 6 +- pymysql/_auth.py | 9 +-- pymysql/_compat.py | 21 ------ pymysql/charset.py | 2 +- pymysql/connections.py | 60 +++++----------- pymysql/converters.py | 68 ++++--------------- pymysql/cursors.py | 49 ++++--------- pymysql/optionfile.py | 7 +- pymysql/protocol.py | 26 +++---- pymysql/tests/base.py | 30 -------- pymysql/tests/test_SSCursor.py | 1 + pymysql/tests/test_basic.py | 1 - pymysql/tests/test_connection.py | 11 ++- pymysql/tests/test_converters.py | 9 --- pymysql/tests/test_issues.py | 15 ---- pymysql/tests/test_optionfile.py | 14 +--- .../test_MySQLdb/test_MySQLdb_nonstandard.py | 22 ++---- 17 files changed, 71 insertions(+), 280 deletions(-) delete mode 100644 pymysql/_compat.py diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 29e6b87c1..1e126dcdc 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -23,7 +23,6 @@ """ import sys -from ._compat import PY2 from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string from .err import ( @@ -79,10 +78,7 @@ def __hash__(self): def Binary(x): """Return x as a binary type.""" - if PY2: - return bytearray(x) - else: - return bytes(x) + return bytes(x) def Connect(*args, **kwargs): diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 57f9abb1a..77caeafd8 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -1,7 +1,6 @@ """ Implements auth methods """ -from ._compat import PY2 from .err import OperationalError from .util import byte2int, int2byte @@ -46,8 +45,6 @@ def scramble_native_password(password, message): def _my_crypt(message1, message2): result = bytearray(message1) - if PY2: - message2 = bytearray(message2) for i in range(len(result)): result[i] ^= message2[i] @@ -61,7 +58,7 @@ def _my_crypt(message1, message2): SCRAMBLE_LENGTH_323 = 8 -class RandStruct_323(object): +class RandStruct_323: def __init__(self, seed1, seed2): self.max_value = 0x3FFFFFFF @@ -188,7 +185,7 @@ def _xor_password(password, salt): # See https://github.com/mysql/mysql-server/blob/7d10c82196c8e45554f27c00681474a9fb86d137/sql/auth/sha2_password.cc#L939-L945 salt = salt[:SCRAMBLE_LENGTH] password_bytes = bytearray(password) - salt = bytearray(salt) # for PY2 compat. + #salt = bytearray(salt) # for PY2 compat. salt_len = len(salt) for i in range(len(password_bytes)): password_bytes[i] ^= salt[i % salt_len] @@ -259,8 +256,6 @@ def scramble_caching_sha2(password, nonce): p3 = hashlib.sha256(p2 + nonce).digest() res = bytearray(p1) - if PY2: - p3 = bytearray(p3) for i in range(len(p3)): res[i] ^= p3[i] diff --git a/pymysql/_compat.py b/pymysql/_compat.py deleted file mode 100644 index 252789ec4..000000000 --- a/pymysql/_compat.py +++ /dev/null @@ -1,21 +0,0 @@ -import sys - -PY2 = sys.version_info[0] == 2 -PYPY = hasattr(sys, 'pypy_translation_info') -JYTHON = sys.platform.startswith('java') -IRONPYTHON = sys.platform == 'cli' -CPYTHON = not PYPY and not JYTHON and not IRONPYTHON - -if PY2: - import __builtin__ - range_type = xrange - text_type = unicode - long_type = long - str_type = basestring - unichr = __builtin__.unichr -else: - range_type = range - text_type = str - long_type = int - str_type = str - unichr = chr diff --git a/pymysql/charset.py b/pymysql/charset.py index d3ced67ca..3ef3ea461 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -6,7 +6,7 @@ } -class Charset(object): +class Charset: def __init__(self, id, name, collation, is_default): self.id, self.name, self.collation = id, name, collation self.is_default = is_default == 'Yes' diff --git a/pymysql/connections.py b/pymysql/connections.py index 7ecfb616c..e426d151c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -2,11 +2,7 @@ # http://dev.mysql.com/doc/internals/en/client-server-protocol.html # Error codes: # https://dev.mysql.com/doc/refman/5.5/en/error-handling.html -from __future__ import print_function -from ._compat import PY2, range_type, text_type, str_type, JYTHON, IRONPYTHON - import errno -import io import os import socket import struct @@ -47,32 +43,11 @@ _py_version = sys.version_info[:2] -if PY2: - pass -elif _py_version < (3, 6): - # See http://bugs.python.org/issue24870 - _surrogateescape_table = [chr(i) if i < 0x80 else chr(i + 0xdc00) for i in range(256)] - - def _fast_surrogateescape(s): - return s.decode('latin1').translate(_surrogateescape_table) -else: - def _fast_surrogateescape(s): - return s.decode('ascii', 'surrogateescape') - -# socket.makefile() in Python 2 is not usable because very inefficient and -# bad behavior about timeout. -# XXX: ._socketio doesn't work under IronPython. -if PY2 and not IRONPYTHON: - # read method of file-like returned by sock.makefile() is very slow. - # So we copy io-based one from Python 3. - from ._socketio import SocketIO - - def _makefile(sock, mode): - return io.BufferedReader(SocketIO(sock, mode)) -else: - # socket.makefile in Python 3 is nice. - def _makefile(sock, mode): - return sock.makefile(mode) +def _fast_surrogateescape(s): + return s.decode('ascii', 'surrogateescape') + +def _makefile(sock, mode): + return sock.makefile(mode) TEXT_TYPES = { @@ -113,7 +88,7 @@ def lenenc_int(i): raise ValueError("Encoding %x is larger than %x - no representation in LengthEncodedInteger" % (i, (1 << 64))) -class Connection(object): +class Connection: """ Representation of a socket with a mysql server. @@ -277,7 +252,7 @@ def _config(key, arg): raise ValueError("port should be of type int") self.user = user or DEFAULT_USER self.password = password or b"" - if isinstance(self.password, text_type): + if isinstance(self.password, str): self.password = self.password.encode('latin1') self.db = database self.unix_socket = unix_socket @@ -493,7 +468,7 @@ def escape(self, obj, mapping=None): Non-standard, for internal use; do not use this in your applications. """ - if isinstance(obj, str_type): + if isinstance(obj, str): return "'" + self.escape_string(obj) + "'" if isinstance(obj, (bytes, bytearray)): ret = self._quote_bytes(obj) @@ -537,11 +512,8 @@ def cursor(self, cursor=None): def query(self, sql, unbuffered=False): # if DEBUG: # print("DEBUG: sending query:", sql) - if isinstance(sql, text_type) and not (JYTHON or IRONPYTHON): - if PY2: - sql = sql.encode(self.encoding) - else: - sql = sql.encode(self.encoding, 'surrogateescape') + if isinstance(sql, str): + sql = sql.encode(self.encoding, 'surrogateescape') self._execute_command(COMMAND.COM_QUERY, sql) self._affected_rows = self._read_query_result(unbuffered=unbuffered) return self._affected_rows @@ -792,7 +764,7 @@ def _execute_command(self, command, sql): self.next_result() self._result = None - if isinstance(sql, text_type): + if isinstance(sql, str): sql = sql.encode(self.encoding) packet_size = min(MAX_PACKET_LEN, len(sql) + 1) # +1 is for command @@ -825,7 +797,7 @@ def _request_authentication(self): raise ValueError("Did not specify a username") charset_id = charset_by_name(self.charset).id - if isinstance(self.user, text_type): + if isinstance(self.user, str): self.user = self.user.encode(self.encoding) data_init = struct.pack(' max_stmt_length: rows += self.execute(sql + postfix) sql = bytearray(prefix) @@ -265,7 +242,7 @@ def callproc(self, procname, args=()): q = "CALL %s(%s)" % (procname, ','.join(['@_%s_%d' % (procname, i) - for i in range_type(len(args))])) + for i in range(len(args))])) self._query(q) self._executed = q return args @@ -356,7 +333,7 @@ def __iter__(self): NotSupportedError = err.NotSupportedError -class DictCursorMixin(object): +class DictCursorMixin: # You can override this to use OrderedDict or other dict-like types. dict_type = dict @@ -469,7 +446,7 @@ def fetchmany(self, size=None): size = self.arraysize rows = [] - for i in range_type(size): + for i in range(size): row = self.read_next() if row is None: break @@ -485,7 +462,7 @@ def scroll(self, value, mode='relative'): raise err.NotSupportedError( "Backwards scrolling not supported by this cursor") - for _ in range_type(value): + for _ in range(value): self.read_next() self.rownumber += value elif mode == 'absolute': @@ -494,7 +471,7 @@ def scroll(self, value, mode='relative'): "Backwards scrolling not supported by this cursor") end = value - self.rownumber - for _ in range_type(end): + for _ in range(end): self.read_next() self.rownumber = value else: diff --git a/pymysql/optionfile.py b/pymysql/optionfile.py index 91e2dfe30..79810ef34 100644 --- a/pymysql/optionfile.py +++ b/pymysql/optionfile.py @@ -1,9 +1,4 @@ -from ._compat import PY2 - -if PY2: - import ConfigParser as configparser -else: - import configparser +import configparser class Parser(configparser.RawConfigParser): diff --git a/pymysql/protocol.py b/pymysql/protocol.py index e302edab3..541475ad6 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -1,9 +1,7 @@ # Python implementation of low level MySQL client-server protocol # http://dev.mysql.com/doc/internals/en/client-server-protocol.html -from __future__ import print_function from .charset import MBLENGTH -from ._compat import PY2, range_type from .constants import FIELD_TYPE, SERVER_STATUS from . import err from .util import byte2int @@ -37,7 +35,7 @@ def printable(data): print("-" * 66) except ValueError: pass - dump_data = [data[i:i+16] for i in range_type(0, min(len(data), 256), 16)] + dump_data = [data[i:i+16] for i in range(0, min(len(data), 256), 16)] for d in dump_data: print(' '.join("{:02X}".format(byte2int(x)) for x in d) + ' ' * (16 - len(d)) + ' ' * 2 + @@ -46,7 +44,7 @@ def printable(data): print() -class MysqlPacket(object): +class MysqlPacket: """Representation of a MySQL response packet. Provides an interface for reading/parsing the packet results. @@ -108,16 +106,10 @@ def get_bytes(self, position, length=1): """ return self._data[position:(position+length)] - if PY2: - def read_uint8(self): - result = ord(self._data[self._position]) - self._position += 1 - return result - else: - def read_uint8(self): - result = self._data[self._position] - self._position += 1 - return result + def read_uint8(self): + result = self._data[self._position] + self._position += 1 + return result def read_uint16(self): result = struct.unpack_from(' Date: Sun, 3 Jan 2021 10:11:53 +0900 Subject: [PATCH 072/227] Use GitHub Actions (#917) --- .github/workflows/test.yaml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 000000000..369b50676 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,54 @@ +name: Test + +on: + push: + pull_request: + +jobs: + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - db: "mariadb:10.2" + py: "3.9" + - db: "mariadb:10.3" + py: "3.8" + - db: "mariadb:10.5" + py: "3.7" + - db: "mysql:5.6" + py: "3.6" + - db: "mysql:5.7" + py: "pypy-3.6" + - db: "mysql:8.0" + py: "3.9" + + services: + mysql: + image: "${{ matrix.db }}" + ports: + - 3306:3306 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + options: "--name=mysqld" + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.py }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py }} + - name: Set up MySQL + run: | + sleep 10 + mysql -h 127.0.0.1 -uroot -e "select version()" + mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" + mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' + mysql -h 127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' + mysql -h 127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" + mysql -h 127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" + cp .travis/docker.json pymysql/tests/databases.json + - name: Run test + run: | + pip install -U cryptography PyNaCl pytest pytest-cov mock + pytest -v --cov --cov-config .coveragerc pymysql From 8d0c6c20f608f40726ee94d3b56be71481e55c59 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 10:16:53 +0900 Subject: [PATCH 073/227] Update README.rst --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 0a09f8923..269928b82 100644 --- a/README.rst +++ b/README.rst @@ -5,9 +5,6 @@ .. image:: https://badge.fury.io/py/PyMySQL.svg :target: https://badge.fury.io/py/PyMySQL -.. image:: https://travis-ci.com/PyMySQL/PyMySQL.svg?branch=master - :target: https://travis-ci.com/PyMySQL/PyMySQL - .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master From 6ec449aa068922405350813df1001f635871d437 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 11:32:08 +0900 Subject: [PATCH 074/227] Fix regression, enable coveralls (#918) --- .github/workflows/test.yaml | 23 +++++++++++++++++-- pymysql/connections.py | 2 +- pymysql/converters.py | 4 ++-- pymysql/cursors.py | 1 - pymysql/tests/test_cursor.py | 3 --- .../thirdparty/test_MySQLdb/capabilities.py | 18 +++++---------- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 369b50676..c68f7239b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,7 +5,7 @@ on: pull_request: jobs: - build: + test: runs-on: ubuntu-20.04 strategy: matrix: @@ -50,5 +50,24 @@ jobs: cp .travis/docker.json pymysql/tests/databases.json - name: Run test run: | - pip install -U cryptography PyNaCl pytest pytest-cov mock + pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls pytest -v --cov --cov-config .coveragerc pymysql + - name: Report coverage + run: coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: ${{ matrix.test-name }} + COVERALLS_PARALLEL: true + + coveralls: + name: Finish coveralls + runs-on: ubuntu-20.04 + needs: test + container: python:3-slim + steps: + - name: Finished + run: | + pip3 install --upgrade coveralls + coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pymysql/connections.py b/pymysql/connections.py index e426d151c..6fd15e132 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -341,7 +341,7 @@ def _create_ssl_ctx(self, sslp): elif isinstance(verify_mode_value, bool): ctx.verify_mode = ssl.CERT_REQUIRED if verify_mode_value else ssl.CERT_NONE else: - if isinstance(verify_mode_value, (text_type, str_type)): + if isinstance(verify_mode_value, str): verify_mode_value = verify_mode_value.lower() if verify_mode_value in ("none", "0", "false", "no"): ctx.verify_mode = ssl.CERT_NONE diff --git a/pymysql/converters.py b/pymysql/converters.py index 0e40eab7f..6d1fc9ee0 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -74,11 +74,11 @@ def escape_string(value, mapping=None): def escape_bytes_prefixed(value, mapping=None): - return "_binary'%s'" % value.decode('ascii', 'surrogateescape') + return "_binary'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) def escape_bytes(value, mapping=None): - return "'%s'" % value.decode('ascii', 'surrogateescape') + return "'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) def escape_str(value, mapping=None): diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 6f72ba35d..a8c528367 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -1,5 +1,4 @@ import re - from . import err diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index fb3e8bedc..4c9174f5b 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -30,7 +30,6 @@ def test_cleanup_rows_unbuffered(self): break del cursor - self.safe_gc_collect() c2 = conn.cursor() @@ -48,10 +47,8 @@ def test_cleanup_rows_buffered(self): break del cursor - self.safe_gc_collect() c2 = conn.cursor() - c2.execute("select 1") self.assertEqual( diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index 6be9d1ba4..e261a78eb 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -8,7 +8,6 @@ from time import time import unittest -PY2 = sys.version_info[0] == 2 class DatabaseTest(unittest.TestCase): @@ -24,10 +23,7 @@ def setUp(self): self.connection = db self.cursor = db.cursor() self.BLOBText = ''.join([chr(i) for i in range(256)] * 100); - if PY2: - self.BLOBUText = unicode().join(unichr(i) for i in range(16834)) - else: - self.BLOBUText = "".join(chr(i) for i in range(16834)) + self.BLOBUText = "".join(chr(i) for i in range(16834)) data = bytearray(range(256)) * 16 self.BLOBBinary = self.db_module.Binary(data) @@ -64,14 +60,12 @@ def new_table_name(self): i = i + 1 def create_table(self, columndefs): + """ + Create a table using a list of column definitions given in columndefs. - """ Create a table using a list of column definitions given in - columndefs. - - generator must be a function taking arguments (row_number, - col_number) returning a suitable data object for insertion - into the table. - + generator must be a function taking arguments (row_number, + col_number) returning a suitable data object for insertion + into the table. """ self.table = self.new_table_name() self.cursor.execute('CREATE TABLE %s (%s) %s' % From b93a87a25ea22c1563cbbcaf943799b3f7e40887 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 11:55:37 +0900 Subject: [PATCH 075/227] Actions: Run auth tests (#919) --- .github/workflows/test.yaml | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c68f7239b..71cc4e828 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,16 +12,24 @@ jobs: include: - db: "mariadb:10.2" py: "3.9" + - db: "mariadb:10.3" py: "3.8" + mariadb_auth: true + - db: "mariadb:10.5" py: "3.7" + mariadb_auth: true + - db: "mysql:5.6" py: "3.6" + - db: "mysql:5.7" py: "pypy-3.6" + - db: "mysql:8.0" py: "3.9" + mysql_auth: true services: mysql: @@ -48,10 +56,41 @@ jobs: mysql -h 127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" mysql -h 127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" cp .travis/docker.json pymysql/tests/databases.json + - name: Run test run: | pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls pytest -v --cov --cov-config .coveragerc pymysql + + - name: Run MySQL8 auth test + if: ${{ matrix.mysql_auth }} + run: | + docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" + mysql -uroot -h127.0.0.1 -e ' + CREATE USER + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", + nopass_sha256 IDENTIFIED WITH "sha256_password", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER; + GRANT RELOAD ON *.* TO user_caching_sha2;' + pytest -v --cov --cov-config .coveragerc tests/test_auth.py; + + - name: Run MariaDB auth test + if: ${{ matrix.mariadb_auth }} + run: | + mysql -uroot -h127.0.0.1 -e ' + INSTALL SONAME "auth_ed25519"; + CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' + # we need to pass the hashed password manually until 10.4, so hide it here + mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql -uroot -h127.0.0.1 + mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql -uroot -h127.0.0.1 + pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py + - name: Report coverage run: coveralls env: From f889038f1b6b134806fb158d34cfb59f31905da2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:05:46 +0900 Subject: [PATCH 076/227] Reformat with black (#920) --- pymysql/__init__.py | 125 ++- pymysql/_auth.py | 45 +- pymysql/_socketio.py | 16 +- pymysql/charset.py | 317 ++++--- pymysql/connections.py | 423 ++++++---- pymysql/constants/CLIENT.py | 13 +- pymysql/constants/COMMAND.py | 25 +- pymysql/constants/CR.py | 100 +-- pymysql/constants/FIELD_TYPE.py | 2 - pymysql/constants/SERVER_STATUS.py | 1 - pymysql/converters.py | 86 +- pymysql/cursors.py | 75 +- pymysql/err.py | 78 +- pymysql/optionfile.py | 4 +- pymysql/protocol.py | 111 ++- pymysql/tests/__init__.py | 1 + pymysql/tests/base.py | 18 +- pymysql/tests/test_DictCursor.py | 52 +- pymysql/tests/test_SSCursor.py | 102 ++- pymysql/tests/test_basic.py | 186 ++-- pymysql/tests/test_connection.py | 445 ++++++---- pymysql/tests/test_converters.py | 24 +- pymysql/tests/test_cursor.py | 74 +- pymysql/tests/test_err.py | 3 +- pymysql/tests/test_issues.py | 140 +-- pymysql/tests/test_load_local.py | 31 +- pymysql/tests/test_nextset.py | 12 +- pymysql/tests/test_optionfile.py | 7 +- pymysql/tests/thirdparty/__init__.py | 1 + .../tests/thirdparty/test_MySQLdb/__init__.py | 1 + .../thirdparty/test_MySQLdb/capabilities.py | 243 +++--- .../tests/thirdparty/test_MySQLdb/dbapi20.py | 794 +++++++++--------- .../test_MySQLdb/test_MySQLdb_capabilities.py | 73 +- .../test_MySQLdb/test_MySQLdb_dbapi20.py | 200 +++-- .../test_MySQLdb/test_MySQLdb_nonstandard.py | 46 +- pymysql/util.py | 1 - tests/test_auth.py | 42 +- tests/test_mariadb_auth.py | 5 +- 38 files changed, 2296 insertions(+), 1626 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 1e126dcdc..5b49262e9 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -26,12 +26,26 @@ from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string from .err import ( - Warning, Error, InterfaceError, DataError, - DatabaseError, OperationalError, IntegrityError, InternalError, - NotSupportedError, ProgrammingError, MySQLError) + Warning, + Error, + InterfaceError, + DataError, + DatabaseError, + OperationalError, + IntegrityError, + InternalError, + NotSupportedError, + ProgrammingError, + MySQLError, +) from .times import ( - Date, Time, Timestamp, - DateFromTicks, TimeFromTicks, TimestampFromTicks) + Date, + Time, + Timestamp, + DateFromTicks, + TimeFromTicks, + TimestampFromTicks, +) VERSION = (0, 10, 1, None) @@ -45,7 +59,6 @@ class DBAPISet(frozenset): - def __ne__(self, other): if isinstance(other, set): return frozenset.__ne__(self, other) @@ -62,18 +75,32 @@ def __hash__(self): return frozenset.__hash__(self) -STRING = DBAPISet([FIELD_TYPE.ENUM, FIELD_TYPE.STRING, - FIELD_TYPE.VAR_STRING]) -BINARY = DBAPISet([FIELD_TYPE.BLOB, FIELD_TYPE.LONG_BLOB, - FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.TINY_BLOB]) -NUMBER = DBAPISet([FIELD_TYPE.DECIMAL, FIELD_TYPE.DOUBLE, FIELD_TYPE.FLOAT, - FIELD_TYPE.INT24, FIELD_TYPE.LONG, FIELD_TYPE.LONGLONG, - FIELD_TYPE.TINY, FIELD_TYPE.YEAR]) -DATE = DBAPISet([FIELD_TYPE.DATE, FIELD_TYPE.NEWDATE]) -TIME = DBAPISet([FIELD_TYPE.TIME]) +STRING = DBAPISet([FIELD_TYPE.ENUM, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING]) +BINARY = DBAPISet( + [ + FIELD_TYPE.BLOB, + FIELD_TYPE.LONG_BLOB, + FIELD_TYPE.MEDIUM_BLOB, + FIELD_TYPE.TINY_BLOB, + ] +) +NUMBER = DBAPISet( + [ + FIELD_TYPE.DECIMAL, + FIELD_TYPE.DOUBLE, + FIELD_TYPE.FLOAT, + FIELD_TYPE.INT24, + FIELD_TYPE.LONG, + FIELD_TYPE.LONGLONG, + FIELD_TYPE.TINY, + FIELD_TYPE.YEAR, + ] +) +DATE = DBAPISet([FIELD_TYPE.DATE, FIELD_TYPE.NEWDATE]) +TIME = DBAPISet([FIELD_TYPE.TIME]) TIMESTAMP = DBAPISet([FIELD_TYPE.TIMESTAMP, FIELD_TYPE.DATETIME]) -DATETIME = TIMESTAMP -ROWID = DBAPISet() +DATETIME = TIMESTAMP +ROWID = DBAPISet() def Binary(x): @@ -87,9 +114,12 @@ def Connect(*args, **kwargs): more information. """ from .connections import Connection + return Connection(*args, **kwargs) + from . import connections as _orig_conn + if _orig_conn.Connection.__init__.__doc__ is not None: Connect.__doc__ = _orig_conn.Connection.__init__.__doc__ del _orig_conn @@ -99,7 +129,8 @@ def get_client_info(): # for MySQLdb compatibility version = VERSION if VERSION[3] is None: version = VERSION[:3] - return '.'.join(map(str, version)) + return ".".join(map(str, version)) + connect = Connection = Connect @@ -110,9 +141,11 @@ def get_client_info(): # for MySQLdb compatibility __version__ = get_client_info() + def thread_safe(): return True # match MySQLdb.thread_safe() + def install_as_MySQLdb(): """ After this function is called, any application that imports MySQLdb or @@ -122,16 +155,50 @@ def install_as_MySQLdb(): __all__ = [ - 'BINARY', 'Binary', 'Connect', 'Connection', 'DATE', 'Date', - 'Time', 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks', - 'DataError', 'DatabaseError', 'Error', 'FIELD_TYPE', 'IntegrityError', - 'InterfaceError', 'InternalError', 'MySQLError', 'NULL', 'NUMBER', - 'NotSupportedError', 'DBAPISet', 'OperationalError', 'ProgrammingError', - 'ROWID', 'STRING', 'TIME', 'TIMESTAMP', 'Warning', 'apilevel', 'connect', - 'connections', 'constants', 'converters', 'cursors', - 'escape_dict', 'escape_sequence', 'escape_string', 'get_client_info', - 'paramstyle', 'threadsafety', 'version_info', - + "BINARY", + "Binary", + "Connect", + "Connection", + "DATE", + "Date", + "Time", + "Timestamp", + "DateFromTicks", + "TimeFromTicks", + "TimestampFromTicks", + "DataError", + "DatabaseError", + "Error", + "FIELD_TYPE", + "IntegrityError", + "InterfaceError", + "InternalError", + "MySQLError", + "NULL", + "NUMBER", + "NotSupportedError", + "DBAPISet", + "OperationalError", + "ProgrammingError", + "ROWID", + "STRING", + "TIME", + "TIMESTAMP", + "Warning", + "apilevel", + "connect", + "connections", + "constants", + "converters", + "cursors", + "escape_dict", + "escape_sequence", + "escape_string", + "get_client_info", + "paramstyle", + "threadsafety", + "version_info", "install_as_MySQLdb", - "NULL", "__version__", + "NULL", + "__version__", ] diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 77caeafd8..d16a08959 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -9,6 +9,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization, hashes from cryptography.hazmat.primitives.asymmetric import padding + _have_cryptography = True except ImportError: _have_cryptography = False @@ -22,7 +23,7 @@ DEBUG = False SCRAMBLE_LENGTH = 20 -sha1_new = partial(hashlib.new, 'sha1') +sha1_new = partial(hashlib.new, "sha1") # mysql_native_password @@ -32,7 +33,7 @@ def scramble_native_password(password, message): """Scramble used for mysql_native_password""" if not password: - return b'' + return b"" stage1 = sha1_new(password).digest() stage2 = sha1_new(stage1).digest() @@ -59,7 +60,6 @@ def _my_crypt(message1, message2): class RandStruct_323: - def __init__(self, seed1, seed2): self.max_value = 0x3FFFFFFF self.seed1 = seed1 % self.max_value @@ -73,8 +73,10 @@ def my_rnd(self): def scramble_old_password(password, message): """Scramble for old_password""" - warnings.warn("old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" - "old password support will be removed in future PyMySQL version") + warnings.warn( + "old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" + "old password support will be removed in future PyMySQL version" + ) hash_pass = _hash_password_323(password) hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) hash_pass_n = struct.unpack(">LL", hash_pass) @@ -100,7 +102,7 @@ def _hash_password_323(password): nr2 = 0x12345671 # x in py3 is numbers, p27 is chars - for c in [byte2int(x) for x in password if x not in (' ', '\t', 32, 9)]: + for c in [byte2int(x) for x in password if x not in (" ", "\t", 32, 9)]: nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF add = (add + c) & 0xFFFFFFFF @@ -120,9 +122,12 @@ def _init_nacl(): global _nacl_bindings try: from nacl import bindings + _nacl_bindings = bindings except ImportError: - raise RuntimeError("'pynacl' package is required for ed25519_password auth method") + raise RuntimeError( + "'pynacl' package is required for ed25519_password auth method" + ) def _scalar_clamp(s32): @@ -185,7 +190,7 @@ def _xor_password(password, salt): # See https://github.com/mysql/mysql-server/blob/7d10c82196c8e45554f27c00681474a9fb86d137/sql/auth/sha2_password.cc#L939-L945 salt = salt[:SCRAMBLE_LENGTH] password_bytes = bytearray(password) - #salt = bytearray(salt) # for PY2 compat. + # salt = bytearray(salt) # for PY2 compat. salt_len = len(salt) for i in range(len(password_bytes)): password_bytes[i] ^= salt[i % salt_len] @@ -198,8 +203,10 @@ def sha2_rsa_encrypt(password, salt, public_key): Used for sha256_password and caching_sha2_password. """ if not _have_cryptography: - raise RuntimeError("'cryptography' package is required for sha256_password or caching_sha2_password auth methods") - message = _xor_password(password + b'\0', salt) + raise RuntimeError( + "'cryptography' package is required for sha256_password or caching_sha2_password auth methods" + ) + message = _xor_password(password + b"\0", salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) return rsa_key.encrypt( message, @@ -215,7 +222,7 @@ def sha256_password_auth(conn, pkt): if conn._secure: if DEBUG: print("sha256: Sending plain password") - data = conn.password + b'\0' + data = conn.password + b"\0" return _roundtrip(conn, data) if pkt.is_auth_switch_request(): @@ -224,12 +231,12 @@ def sha256_password_auth(conn, pkt): # Request server public key if DEBUG: print("sha256: Requesting server public key") - pkt = _roundtrip(conn, b'\1') + pkt = _roundtrip(conn, b"\1") if pkt.is_extra_auth_data(): conn.server_public_key = pkt._data[1:] if DEBUG: - print("Received public key:\n", conn.server_public_key.decode('ascii')) + print("Received public key:\n", conn.server_public_key.decode("ascii")) if conn.password: if not conn.server_public_key: @@ -237,7 +244,7 @@ def sha256_password_auth(conn, pkt): data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) else: - data = b'' + data = b"" return _roundtrip(conn, data) @@ -249,7 +256,7 @@ def scramble_caching_sha2(password, nonce): XOR(SHA256(password), SHA256(SHA256(SHA256(password)), nonce)) """ if not password: - return b'' + return b"" p1 = hashlib.sha256(password).digest() p2 = hashlib.sha256(p1).digest() @@ -265,7 +272,7 @@ def scramble_caching_sha2(password, nonce): def caching_sha2_password_auth(conn, pkt): # No password fast path if not conn.password: - return _roundtrip(conn, b'') + return _roundtrip(conn, b"") if pkt.is_auth_switch_request(): # Try from fast auth @@ -305,10 +312,10 @@ def caching_sha2_password_auth(conn, pkt): if conn._secure: if DEBUG: print("caching sha2: Sending plain password via secure connection") - return _roundtrip(conn, conn.password + b'\0') + return _roundtrip(conn, conn.password + b"\0") if not conn.server_public_key: - pkt = _roundtrip(conn, b'\x02') # Request public key + pkt = _roundtrip(conn, b"\x02") # Request public key if not pkt.is_extra_auth_data(): raise OperationalError( "caching sha2: Unknown packet for public key: %s" % pkt._data[:1] @@ -316,7 +323,7 @@ def caching_sha2_password_auth(conn, pkt): conn.server_public_key = pkt._data[1:] if DEBUG: - print(conn.server_public_key.decode('ascii')) + print(conn.server_public_key.decode("ascii")) data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) pkt = _roundtrip(conn, data) diff --git a/pymysql/_socketio.py b/pymysql/_socketio.py index 6a11d42e4..6b2d65a33 100644 --- a/pymysql/_socketio.py +++ b/pymysql/_socketio.py @@ -8,11 +8,12 @@ import io import errno -__all__ = ['SocketIO'] +__all__ = ["SocketIO"] EINTR = errno.EINTR _blocking_errnos = (errno.EAGAIN, errno.EWOULDBLOCK) + class SocketIO(io.RawIOBase): """Raw I/O implementation for stream sockets. @@ -85,29 +86,25 @@ def write(self, b): raise def readable(self): - """True if the SocketIO is open for reading. - """ + """True if the SocketIO is open for reading.""" if self.closed: raise ValueError("I/O operation on closed socket.") return self._reading def writable(self): - """True if the SocketIO is open for writing. - """ + """True if the SocketIO is open for writing.""" if self.closed: raise ValueError("I/O operation on closed socket.") return self._writing def seekable(self): - """True if the SocketIO is open for seeking. - """ + """True if the SocketIO is open for seeking.""" if self.closed: raise ValueError("I/O operation on closed socket.") return super().seekable() def fileno(self): - """Return the file descriptor of the underlying socket. - """ + """Return the file descriptor of the underlying socket.""" self._checkClosed() return self._sock.fileno() @@ -131,4 +128,3 @@ def close(self): io.RawIOBase.close(self) self._sock._decref_socketios() self._sock = None - diff --git a/pymysql/charset.py b/pymysql/charset.py index 3ef3ea461..ac87c53dd 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -1,31 +1,29 @@ -MBLENGTH = { - 8:1, - 33:3, - 88:2, - 91:2 - } +MBLENGTH = {8: 1, 33: 3, 88: 2, 91: 2} class Charset: def __init__(self, id, name, collation, is_default): self.id, self.name, self.collation = id, name, collation - self.is_default = is_default == 'Yes' + self.is_default = is_default == "Yes" def __repr__(self): return "Charset(id=%s, name=%r, collation=%r)" % ( - self.id, self.name, self.collation) + self.id, + self.name, + self.collation, + ) @property def encoding(self): name = self.name - if name in ('utf8mb4', 'utf8mb3'): - return 'utf8' - if name == 'latin1': - return 'cp1252' - if name == 'koi8r': - return 'koi8_r' - if name == 'koi8u': - return 'koi8_u' + if name in ("utf8mb4", "utf8mb3"): + return "utf8" + if name == "latin1": + return "cp1252" + if name == "koi8r": + return "koi8_r" + if name == "koi8u": + return "koi8_u" return name @property @@ -49,6 +47,7 @@ def by_id(self, id): def by_name(self, name): return self._by_name.get(name.lower()) + _charsets = Charsets() """ Generated with: @@ -62,149 +61,149 @@ def by_name(self, name): " """ -_charsets.add(Charset(1, 'big5', 'big5_chinese_ci', 'Yes')) -_charsets.add(Charset(2, 'latin2', 'latin2_czech_cs', '')) -_charsets.add(Charset(3, 'dec8', 'dec8_swedish_ci', 'Yes')) -_charsets.add(Charset(4, 'cp850', 'cp850_general_ci', 'Yes')) -_charsets.add(Charset(5, 'latin1', 'latin1_german1_ci', '')) -_charsets.add(Charset(6, 'hp8', 'hp8_english_ci', 'Yes')) -_charsets.add(Charset(7, 'koi8r', 'koi8r_general_ci', 'Yes')) -_charsets.add(Charset(8, 'latin1', 'latin1_swedish_ci', 'Yes')) -_charsets.add(Charset(9, 'latin2', 'latin2_general_ci', 'Yes')) -_charsets.add(Charset(10, 'swe7', 'swe7_swedish_ci', 'Yes')) -_charsets.add(Charset(11, 'ascii', 'ascii_general_ci', 'Yes')) -_charsets.add(Charset(12, 'ujis', 'ujis_japanese_ci', 'Yes')) -_charsets.add(Charset(13, 'sjis', 'sjis_japanese_ci', 'Yes')) -_charsets.add(Charset(14, 'cp1251', 'cp1251_bulgarian_ci', '')) -_charsets.add(Charset(15, 'latin1', 'latin1_danish_ci', '')) -_charsets.add(Charset(16, 'hebrew', 'hebrew_general_ci', 'Yes')) -_charsets.add(Charset(18, 'tis620', 'tis620_thai_ci', 'Yes')) -_charsets.add(Charset(19, 'euckr', 'euckr_korean_ci', 'Yes')) -_charsets.add(Charset(20, 'latin7', 'latin7_estonian_cs', '')) -_charsets.add(Charset(21, 'latin2', 'latin2_hungarian_ci', '')) -_charsets.add(Charset(22, 'koi8u', 'koi8u_general_ci', 'Yes')) -_charsets.add(Charset(23, 'cp1251', 'cp1251_ukrainian_ci', '')) -_charsets.add(Charset(24, 'gb2312', 'gb2312_chinese_ci', 'Yes')) -_charsets.add(Charset(25, 'greek', 'greek_general_ci', 'Yes')) -_charsets.add(Charset(26, 'cp1250', 'cp1250_general_ci', 'Yes')) -_charsets.add(Charset(27, 'latin2', 'latin2_croatian_ci', '')) -_charsets.add(Charset(28, 'gbk', 'gbk_chinese_ci', 'Yes')) -_charsets.add(Charset(29, 'cp1257', 'cp1257_lithuanian_ci', '')) -_charsets.add(Charset(30, 'latin5', 'latin5_turkish_ci', 'Yes')) -_charsets.add(Charset(31, 'latin1', 'latin1_german2_ci', '')) -_charsets.add(Charset(32, 'armscii8', 'armscii8_general_ci', 'Yes')) -_charsets.add(Charset(33, 'utf8', 'utf8_general_ci', 'Yes')) -_charsets.add(Charset(34, 'cp1250', 'cp1250_czech_cs', '')) -_charsets.add(Charset(36, 'cp866', 'cp866_general_ci', 'Yes')) -_charsets.add(Charset(37, 'keybcs2', 'keybcs2_general_ci', 'Yes')) -_charsets.add(Charset(38, 'macce', 'macce_general_ci', 'Yes')) -_charsets.add(Charset(39, 'macroman', 'macroman_general_ci', 'Yes')) -_charsets.add(Charset(40, 'cp852', 'cp852_general_ci', 'Yes')) -_charsets.add(Charset(41, 'latin7', 'latin7_general_ci', 'Yes')) -_charsets.add(Charset(42, 'latin7', 'latin7_general_cs', '')) -_charsets.add(Charset(43, 'macce', 'macce_bin', '')) -_charsets.add(Charset(44, 'cp1250', 'cp1250_croatian_ci', '')) -_charsets.add(Charset(45, 'utf8mb4', 'utf8mb4_general_ci', 'Yes')) -_charsets.add(Charset(46, 'utf8mb4', 'utf8mb4_bin', '')) -_charsets.add(Charset(47, 'latin1', 'latin1_bin', '')) -_charsets.add(Charset(48, 'latin1', 'latin1_general_ci', '')) -_charsets.add(Charset(49, 'latin1', 'latin1_general_cs', '')) -_charsets.add(Charset(50, 'cp1251', 'cp1251_bin', '')) -_charsets.add(Charset(51, 'cp1251', 'cp1251_general_ci', 'Yes')) -_charsets.add(Charset(52, 'cp1251', 'cp1251_general_cs', '')) -_charsets.add(Charset(53, 'macroman', 'macroman_bin', '')) -_charsets.add(Charset(57, 'cp1256', 'cp1256_general_ci', 'Yes')) -_charsets.add(Charset(58, 'cp1257', 'cp1257_bin', '')) -_charsets.add(Charset(59, 'cp1257', 'cp1257_general_ci', 'Yes')) -_charsets.add(Charset(63, 'binary', 'binary', 'Yes')) -_charsets.add(Charset(64, 'armscii8', 'armscii8_bin', '')) -_charsets.add(Charset(65, 'ascii', 'ascii_bin', '')) -_charsets.add(Charset(66, 'cp1250', 'cp1250_bin', '')) -_charsets.add(Charset(67, 'cp1256', 'cp1256_bin', '')) -_charsets.add(Charset(68, 'cp866', 'cp866_bin', '')) -_charsets.add(Charset(69, 'dec8', 'dec8_bin', '')) -_charsets.add(Charset(70, 'greek', 'greek_bin', '')) -_charsets.add(Charset(71, 'hebrew', 'hebrew_bin', '')) -_charsets.add(Charset(72, 'hp8', 'hp8_bin', '')) -_charsets.add(Charset(73, 'keybcs2', 'keybcs2_bin', '')) -_charsets.add(Charset(74, 'koi8r', 'koi8r_bin', '')) -_charsets.add(Charset(75, 'koi8u', 'koi8u_bin', '')) -_charsets.add(Charset(76, 'utf8', 'utf8_tolower_ci', '')) -_charsets.add(Charset(77, 'latin2', 'latin2_bin', '')) -_charsets.add(Charset(78, 'latin5', 'latin5_bin', '')) -_charsets.add(Charset(79, 'latin7', 'latin7_bin', '')) -_charsets.add(Charset(80, 'cp850', 'cp850_bin', '')) -_charsets.add(Charset(81, 'cp852', 'cp852_bin', '')) -_charsets.add(Charset(82, 'swe7', 'swe7_bin', '')) -_charsets.add(Charset(83, 'utf8', 'utf8_bin', '')) -_charsets.add(Charset(84, 'big5', 'big5_bin', '')) -_charsets.add(Charset(85, 'euckr', 'euckr_bin', '')) -_charsets.add(Charset(86, 'gb2312', 'gb2312_bin', '')) -_charsets.add(Charset(87, 'gbk', 'gbk_bin', '')) -_charsets.add(Charset(88, 'sjis', 'sjis_bin', '')) -_charsets.add(Charset(89, 'tis620', 'tis620_bin', '')) -_charsets.add(Charset(91, 'ujis', 'ujis_bin', '')) -_charsets.add(Charset(92, 'geostd8', 'geostd8_general_ci', 'Yes')) -_charsets.add(Charset(93, 'geostd8', 'geostd8_bin', '')) -_charsets.add(Charset(94, 'latin1', 'latin1_spanish_ci', '')) -_charsets.add(Charset(95, 'cp932', 'cp932_japanese_ci', 'Yes')) -_charsets.add(Charset(96, 'cp932', 'cp932_bin', '')) -_charsets.add(Charset(97, 'eucjpms', 'eucjpms_japanese_ci', 'Yes')) -_charsets.add(Charset(98, 'eucjpms', 'eucjpms_bin', '')) -_charsets.add(Charset(99, 'cp1250', 'cp1250_polish_ci', '')) -_charsets.add(Charset(192, 'utf8', 'utf8_unicode_ci', '')) -_charsets.add(Charset(193, 'utf8', 'utf8_icelandic_ci', '')) -_charsets.add(Charset(194, 'utf8', 'utf8_latvian_ci', '')) -_charsets.add(Charset(195, 'utf8', 'utf8_romanian_ci', '')) -_charsets.add(Charset(196, 'utf8', 'utf8_slovenian_ci', '')) -_charsets.add(Charset(197, 'utf8', 'utf8_polish_ci', '')) -_charsets.add(Charset(198, 'utf8', 'utf8_estonian_ci', '')) -_charsets.add(Charset(199, 'utf8', 'utf8_spanish_ci', '')) -_charsets.add(Charset(200, 'utf8', 'utf8_swedish_ci', '')) -_charsets.add(Charset(201, 'utf8', 'utf8_turkish_ci', '')) -_charsets.add(Charset(202, 'utf8', 'utf8_czech_ci', '')) -_charsets.add(Charset(203, 'utf8', 'utf8_danish_ci', '')) -_charsets.add(Charset(204, 'utf8', 'utf8_lithuanian_ci', '')) -_charsets.add(Charset(205, 'utf8', 'utf8_slovak_ci', '')) -_charsets.add(Charset(206, 'utf8', 'utf8_spanish2_ci', '')) -_charsets.add(Charset(207, 'utf8', 'utf8_roman_ci', '')) -_charsets.add(Charset(208, 'utf8', 'utf8_persian_ci', '')) -_charsets.add(Charset(209, 'utf8', 'utf8_esperanto_ci', '')) -_charsets.add(Charset(210, 'utf8', 'utf8_hungarian_ci', '')) -_charsets.add(Charset(211, 'utf8', 'utf8_sinhala_ci', '')) -_charsets.add(Charset(212, 'utf8', 'utf8_german2_ci', '')) -_charsets.add(Charset(213, 'utf8', 'utf8_croatian_ci', '')) -_charsets.add(Charset(214, 'utf8', 'utf8_unicode_520_ci', '')) -_charsets.add(Charset(215, 'utf8', 'utf8_vietnamese_ci', '')) -_charsets.add(Charset(223, 'utf8', 'utf8_general_mysql500_ci', '')) -_charsets.add(Charset(224, 'utf8mb4', 'utf8mb4_unicode_ci', '')) -_charsets.add(Charset(225, 'utf8mb4', 'utf8mb4_icelandic_ci', '')) -_charsets.add(Charset(226, 'utf8mb4', 'utf8mb4_latvian_ci', '')) -_charsets.add(Charset(227, 'utf8mb4', 'utf8mb4_romanian_ci', '')) -_charsets.add(Charset(228, 'utf8mb4', 'utf8mb4_slovenian_ci', '')) -_charsets.add(Charset(229, 'utf8mb4', 'utf8mb4_polish_ci', '')) -_charsets.add(Charset(230, 'utf8mb4', 'utf8mb4_estonian_ci', '')) -_charsets.add(Charset(231, 'utf8mb4', 'utf8mb4_spanish_ci', '')) -_charsets.add(Charset(232, 'utf8mb4', 'utf8mb4_swedish_ci', '')) -_charsets.add(Charset(233, 'utf8mb4', 'utf8mb4_turkish_ci', '')) -_charsets.add(Charset(234, 'utf8mb4', 'utf8mb4_czech_ci', '')) -_charsets.add(Charset(235, 'utf8mb4', 'utf8mb4_danish_ci', '')) -_charsets.add(Charset(236, 'utf8mb4', 'utf8mb4_lithuanian_ci', '')) -_charsets.add(Charset(237, 'utf8mb4', 'utf8mb4_slovak_ci', '')) -_charsets.add(Charset(238, 'utf8mb4', 'utf8mb4_spanish2_ci', '')) -_charsets.add(Charset(239, 'utf8mb4', 'utf8mb4_roman_ci', '')) -_charsets.add(Charset(240, 'utf8mb4', 'utf8mb4_persian_ci', '')) -_charsets.add(Charset(241, 'utf8mb4', 'utf8mb4_esperanto_ci', '')) -_charsets.add(Charset(242, 'utf8mb4', 'utf8mb4_hungarian_ci', '')) -_charsets.add(Charset(243, 'utf8mb4', 'utf8mb4_sinhala_ci', '')) -_charsets.add(Charset(244, 'utf8mb4', 'utf8mb4_german2_ci', '')) -_charsets.add(Charset(245, 'utf8mb4', 'utf8mb4_croatian_ci', '')) -_charsets.add(Charset(246, 'utf8mb4', 'utf8mb4_unicode_520_ci', '')) -_charsets.add(Charset(247, 'utf8mb4', 'utf8mb4_vietnamese_ci', '')) -_charsets.add(Charset(248, 'gb18030', 'gb18030_chinese_ci', 'Yes')) -_charsets.add(Charset(249, 'gb18030', 'gb18030_bin', '')) -_charsets.add(Charset(250, 'gb18030', 'gb18030_unicode_520_ci', '')) -_charsets.add(Charset(255, 'utf8mb4', 'utf8mb4_0900_ai_ci', '')) +_charsets.add(Charset(1, "big5", "big5_chinese_ci", "Yes")) +_charsets.add(Charset(2, "latin2", "latin2_czech_cs", "")) +_charsets.add(Charset(3, "dec8", "dec8_swedish_ci", "Yes")) +_charsets.add(Charset(4, "cp850", "cp850_general_ci", "Yes")) +_charsets.add(Charset(5, "latin1", "latin1_german1_ci", "")) +_charsets.add(Charset(6, "hp8", "hp8_english_ci", "Yes")) +_charsets.add(Charset(7, "koi8r", "koi8r_general_ci", "Yes")) +_charsets.add(Charset(8, "latin1", "latin1_swedish_ci", "Yes")) +_charsets.add(Charset(9, "latin2", "latin2_general_ci", "Yes")) +_charsets.add(Charset(10, "swe7", "swe7_swedish_ci", "Yes")) +_charsets.add(Charset(11, "ascii", "ascii_general_ci", "Yes")) +_charsets.add(Charset(12, "ujis", "ujis_japanese_ci", "Yes")) +_charsets.add(Charset(13, "sjis", "sjis_japanese_ci", "Yes")) +_charsets.add(Charset(14, "cp1251", "cp1251_bulgarian_ci", "")) +_charsets.add(Charset(15, "latin1", "latin1_danish_ci", "")) +_charsets.add(Charset(16, "hebrew", "hebrew_general_ci", "Yes")) +_charsets.add(Charset(18, "tis620", "tis620_thai_ci", "Yes")) +_charsets.add(Charset(19, "euckr", "euckr_korean_ci", "Yes")) +_charsets.add(Charset(20, "latin7", "latin7_estonian_cs", "")) +_charsets.add(Charset(21, "latin2", "latin2_hungarian_ci", "")) +_charsets.add(Charset(22, "koi8u", "koi8u_general_ci", "Yes")) +_charsets.add(Charset(23, "cp1251", "cp1251_ukrainian_ci", "")) +_charsets.add(Charset(24, "gb2312", "gb2312_chinese_ci", "Yes")) +_charsets.add(Charset(25, "greek", "greek_general_ci", "Yes")) +_charsets.add(Charset(26, "cp1250", "cp1250_general_ci", "Yes")) +_charsets.add(Charset(27, "latin2", "latin2_croatian_ci", "")) +_charsets.add(Charset(28, "gbk", "gbk_chinese_ci", "Yes")) +_charsets.add(Charset(29, "cp1257", "cp1257_lithuanian_ci", "")) +_charsets.add(Charset(30, "latin5", "latin5_turkish_ci", "Yes")) +_charsets.add(Charset(31, "latin1", "latin1_german2_ci", "")) +_charsets.add(Charset(32, "armscii8", "armscii8_general_ci", "Yes")) +_charsets.add(Charset(33, "utf8", "utf8_general_ci", "Yes")) +_charsets.add(Charset(34, "cp1250", "cp1250_czech_cs", "")) +_charsets.add(Charset(36, "cp866", "cp866_general_ci", "Yes")) +_charsets.add(Charset(37, "keybcs2", "keybcs2_general_ci", "Yes")) +_charsets.add(Charset(38, "macce", "macce_general_ci", "Yes")) +_charsets.add(Charset(39, "macroman", "macroman_general_ci", "Yes")) +_charsets.add(Charset(40, "cp852", "cp852_general_ci", "Yes")) +_charsets.add(Charset(41, "latin7", "latin7_general_ci", "Yes")) +_charsets.add(Charset(42, "latin7", "latin7_general_cs", "")) +_charsets.add(Charset(43, "macce", "macce_bin", "")) +_charsets.add(Charset(44, "cp1250", "cp1250_croatian_ci", "")) +_charsets.add(Charset(45, "utf8mb4", "utf8mb4_general_ci", "Yes")) +_charsets.add(Charset(46, "utf8mb4", "utf8mb4_bin", "")) +_charsets.add(Charset(47, "latin1", "latin1_bin", "")) +_charsets.add(Charset(48, "latin1", "latin1_general_ci", "")) +_charsets.add(Charset(49, "latin1", "latin1_general_cs", "")) +_charsets.add(Charset(50, "cp1251", "cp1251_bin", "")) +_charsets.add(Charset(51, "cp1251", "cp1251_general_ci", "Yes")) +_charsets.add(Charset(52, "cp1251", "cp1251_general_cs", "")) +_charsets.add(Charset(53, "macroman", "macroman_bin", "")) +_charsets.add(Charset(57, "cp1256", "cp1256_general_ci", "Yes")) +_charsets.add(Charset(58, "cp1257", "cp1257_bin", "")) +_charsets.add(Charset(59, "cp1257", "cp1257_general_ci", "Yes")) +_charsets.add(Charset(63, "binary", "binary", "Yes")) +_charsets.add(Charset(64, "armscii8", "armscii8_bin", "")) +_charsets.add(Charset(65, "ascii", "ascii_bin", "")) +_charsets.add(Charset(66, "cp1250", "cp1250_bin", "")) +_charsets.add(Charset(67, "cp1256", "cp1256_bin", "")) +_charsets.add(Charset(68, "cp866", "cp866_bin", "")) +_charsets.add(Charset(69, "dec8", "dec8_bin", "")) +_charsets.add(Charset(70, "greek", "greek_bin", "")) +_charsets.add(Charset(71, "hebrew", "hebrew_bin", "")) +_charsets.add(Charset(72, "hp8", "hp8_bin", "")) +_charsets.add(Charset(73, "keybcs2", "keybcs2_bin", "")) +_charsets.add(Charset(74, "koi8r", "koi8r_bin", "")) +_charsets.add(Charset(75, "koi8u", "koi8u_bin", "")) +_charsets.add(Charset(76, "utf8", "utf8_tolower_ci", "")) +_charsets.add(Charset(77, "latin2", "latin2_bin", "")) +_charsets.add(Charset(78, "latin5", "latin5_bin", "")) +_charsets.add(Charset(79, "latin7", "latin7_bin", "")) +_charsets.add(Charset(80, "cp850", "cp850_bin", "")) +_charsets.add(Charset(81, "cp852", "cp852_bin", "")) +_charsets.add(Charset(82, "swe7", "swe7_bin", "")) +_charsets.add(Charset(83, "utf8", "utf8_bin", "")) +_charsets.add(Charset(84, "big5", "big5_bin", "")) +_charsets.add(Charset(85, "euckr", "euckr_bin", "")) +_charsets.add(Charset(86, "gb2312", "gb2312_bin", "")) +_charsets.add(Charset(87, "gbk", "gbk_bin", "")) +_charsets.add(Charset(88, "sjis", "sjis_bin", "")) +_charsets.add(Charset(89, "tis620", "tis620_bin", "")) +_charsets.add(Charset(91, "ujis", "ujis_bin", "")) +_charsets.add(Charset(92, "geostd8", "geostd8_general_ci", "Yes")) +_charsets.add(Charset(93, "geostd8", "geostd8_bin", "")) +_charsets.add(Charset(94, "latin1", "latin1_spanish_ci", "")) +_charsets.add(Charset(95, "cp932", "cp932_japanese_ci", "Yes")) +_charsets.add(Charset(96, "cp932", "cp932_bin", "")) +_charsets.add(Charset(97, "eucjpms", "eucjpms_japanese_ci", "Yes")) +_charsets.add(Charset(98, "eucjpms", "eucjpms_bin", "")) +_charsets.add(Charset(99, "cp1250", "cp1250_polish_ci", "")) +_charsets.add(Charset(192, "utf8", "utf8_unicode_ci", "")) +_charsets.add(Charset(193, "utf8", "utf8_icelandic_ci", "")) +_charsets.add(Charset(194, "utf8", "utf8_latvian_ci", "")) +_charsets.add(Charset(195, "utf8", "utf8_romanian_ci", "")) +_charsets.add(Charset(196, "utf8", "utf8_slovenian_ci", "")) +_charsets.add(Charset(197, "utf8", "utf8_polish_ci", "")) +_charsets.add(Charset(198, "utf8", "utf8_estonian_ci", "")) +_charsets.add(Charset(199, "utf8", "utf8_spanish_ci", "")) +_charsets.add(Charset(200, "utf8", "utf8_swedish_ci", "")) +_charsets.add(Charset(201, "utf8", "utf8_turkish_ci", "")) +_charsets.add(Charset(202, "utf8", "utf8_czech_ci", "")) +_charsets.add(Charset(203, "utf8", "utf8_danish_ci", "")) +_charsets.add(Charset(204, "utf8", "utf8_lithuanian_ci", "")) +_charsets.add(Charset(205, "utf8", "utf8_slovak_ci", "")) +_charsets.add(Charset(206, "utf8", "utf8_spanish2_ci", "")) +_charsets.add(Charset(207, "utf8", "utf8_roman_ci", "")) +_charsets.add(Charset(208, "utf8", "utf8_persian_ci", "")) +_charsets.add(Charset(209, "utf8", "utf8_esperanto_ci", "")) +_charsets.add(Charset(210, "utf8", "utf8_hungarian_ci", "")) +_charsets.add(Charset(211, "utf8", "utf8_sinhala_ci", "")) +_charsets.add(Charset(212, "utf8", "utf8_german2_ci", "")) +_charsets.add(Charset(213, "utf8", "utf8_croatian_ci", "")) +_charsets.add(Charset(214, "utf8", "utf8_unicode_520_ci", "")) +_charsets.add(Charset(215, "utf8", "utf8_vietnamese_ci", "")) +_charsets.add(Charset(223, "utf8", "utf8_general_mysql500_ci", "")) +_charsets.add(Charset(224, "utf8mb4", "utf8mb4_unicode_ci", "")) +_charsets.add(Charset(225, "utf8mb4", "utf8mb4_icelandic_ci", "")) +_charsets.add(Charset(226, "utf8mb4", "utf8mb4_latvian_ci", "")) +_charsets.add(Charset(227, "utf8mb4", "utf8mb4_romanian_ci", "")) +_charsets.add(Charset(228, "utf8mb4", "utf8mb4_slovenian_ci", "")) +_charsets.add(Charset(229, "utf8mb4", "utf8mb4_polish_ci", "")) +_charsets.add(Charset(230, "utf8mb4", "utf8mb4_estonian_ci", "")) +_charsets.add(Charset(231, "utf8mb4", "utf8mb4_spanish_ci", "")) +_charsets.add(Charset(232, "utf8mb4", "utf8mb4_swedish_ci", "")) +_charsets.add(Charset(233, "utf8mb4", "utf8mb4_turkish_ci", "")) +_charsets.add(Charset(234, "utf8mb4", "utf8mb4_czech_ci", "")) +_charsets.add(Charset(235, "utf8mb4", "utf8mb4_danish_ci", "")) +_charsets.add(Charset(236, "utf8mb4", "utf8mb4_lithuanian_ci", "")) +_charsets.add(Charset(237, "utf8mb4", "utf8mb4_slovak_ci", "")) +_charsets.add(Charset(238, "utf8mb4", "utf8mb4_spanish2_ci", "")) +_charsets.add(Charset(239, "utf8mb4", "utf8mb4_roman_ci", "")) +_charsets.add(Charset(240, "utf8mb4", "utf8mb4_persian_ci", "")) +_charsets.add(Charset(241, "utf8mb4", "utf8mb4_esperanto_ci", "")) +_charsets.add(Charset(242, "utf8mb4", "utf8mb4_hungarian_ci", "")) +_charsets.add(Charset(243, "utf8mb4", "utf8mb4_sinhala_ci", "")) +_charsets.add(Charset(244, "utf8mb4", "utf8mb4_german2_ci", "")) +_charsets.add(Charset(245, "utf8mb4", "utf8mb4_croatian_ci", "")) +_charsets.add(Charset(246, "utf8mb4", "utf8mb4_unicode_520_ci", "")) +_charsets.add(Charset(247, "utf8mb4", "utf8mb4_vietnamese_ci", "")) +_charsets.add(Charset(248, "gb18030", "gb18030_chinese_ci", "Yes")) +_charsets.add(Charset(249, "gb18030", "gb18030_bin", "")) +_charsets.add(Charset(250, "gb18030", "gb18030_unicode_520_ci", "")) +_charsets.add(Charset(255, "utf8mb4", "utf8mb4_0900_ai_ci", "")) charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id diff --git a/pymysql/connections.py b/pymysql/connections.py index 6fd15e132..dc69868be 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -18,14 +18,19 @@ from .cursors import Cursor from .optionfile import Parser from .protocol import ( - dump_packet, MysqlPacket, FieldDescriptorPacket, OKPacketWrapper, - EOFPacketWrapper, LoadLocalPacketWrapper + dump_packet, + MysqlPacket, + FieldDescriptorPacket, + OKPacketWrapper, + EOFPacketWrapper, + LoadLocalPacketWrapper, ) from .util import byte2int, int2byte from . import err, VERSION_STRING try: import ssl + SSL_ENABLED = True except ImportError: ssl = None @@ -33,6 +38,7 @@ try: import getpass + DEFAULT_USER = getpass.getuser() del getpass except (ImportError, KeyError): @@ -43,8 +49,10 @@ _py_version = sys.version_info[:2] + def _fast_surrogateescape(s): - return s.decode('ascii', 'surrogateescape') + return s.decode("ascii", "surrogateescape") + def _makefile(sock, mode): return sock.makefile(mode) @@ -63,29 +71,34 @@ def _makefile(sock, mode): } -DEFAULT_CHARSET = 'utf8mb4' +DEFAULT_CHARSET = "utf8mb4" -MAX_PACKET_LEN = 2**24-1 +MAX_PACKET_LEN = 2 ** 24 - 1 def pack_int24(n): - return struct.pack(' 2: use_unicode = True @@ -184,7 +224,9 @@ def __init__(self, host=None, user=None, password="", password = passwd if compress or named_pipe: - raise NotImplementedError("compress and named_pipe arguments are not supported") + raise NotImplementedError( + "compress and named_pipe arguments are not supported" + ) self._local_infile = bool(local_infile) if self._local_infile: @@ -233,12 +275,14 @@ def _config(key, arg): ssl = { "ca": ssl_ca, "check_hostname": bool(ssl_verify_identity), - "verify_mode": ssl_verify_cert if ssl_verify_cert is not None else False, + "verify_mode": ssl_verify_cert + if ssl_verify_cert is not None + else False, } if ssl_cert is not None: ssl["cert"] = ssl_cert if ssl_key is not None: - ssl["key" ] = ssl_key + ssl["key"] = ssl_key if ssl: if not SSL_ENABLED: raise NotImplementedError("ssl module not found") @@ -253,7 +297,7 @@ def _config(key, arg): self.user = user or DEFAULT_USER self.password = password or b"" if isinstance(self.password, str): - self.password = self.password.encode('latin1') + self.password = self.password.encode("latin1") self.db = database self.unix_socket = unix_socket self.bind_address = bind_address @@ -307,9 +351,9 @@ def _config(key, arg): self.server_public_key = server_public_key self._connect_attrs = { - '_client_name': 'pymysql', - '_pid': str(os.getpid()), - '_client_version': VERSION_STRING, + "_client_name": "pymysql", + "_pid": str(os.getpid()), + "_client_version": VERSION_STRING, } if program_name: @@ -319,23 +363,23 @@ def _config(key, arg): self._sock = None else: self.connect() - + def __enter__(self): return self - + def __exit__(self, *exc_info): del exc_info self.close() - + def _create_ssl_ctx(self, sslp): if isinstance(sslp, ssl.SSLContext): return sslp - ca = sslp.get('ca') - capath = sslp.get('capath') + ca = sslp.get("ca") + capath = sslp.get("capath") hasnoca = ca is None and capath is None ctx = ssl.create_default_context(cafile=ca, capath=capath) - ctx.check_hostname = not hasnoca and sslp.get('check_hostname', True) - verify_mode_value = sslp.get('verify_mode') + ctx.check_hostname = not hasnoca and sslp.get("check_hostname", True) + verify_mode_value = sslp.get("verify_mode") if verify_mode_value is None: ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED elif isinstance(verify_mode_value, bool): @@ -351,10 +395,10 @@ def _create_ssl_ctx(self, sslp): ctx.verify_mode = ssl.CERT_REQUIRED else: ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED - if 'cert' in sslp: - ctx.load_cert_chain(sslp['cert'], keyfile=sslp.get('key')) - if 'cipher' in sslp: - ctx.set_ciphers(sslp['cipher']) + if "cert" in sslp: + ctx.load_cert_chain(sslp["cert"], keyfile=sslp.get("key")) + if "cipher" in sslp: + ctx.set_ciphers(sslp["cipher"]) ctx.options |= ssl.OP_NO_SSLv2 ctx.options |= ssl.OP_NO_SSLv3 return ctx @@ -373,7 +417,7 @@ def close(self): self._closed = True if self._sock is None: return - send_data = struct.pack('= 5: + if int(self.server_version.split(".", 1)[0]) >= 5: self.client_flag |= CLIENT.MULTI_RESULTS if self.user is None: @@ -800,28 +851,30 @@ def _request_authentication(self): if isinstance(self.user, str): self.user = self.user.encode(self.encoding) - data_init = struct.pack('=5.0) - data += authresp + b'\0' + data += authresp + b"\0" if self.db and self.server_capabilities & CLIENT.CONNECT_WITH_DB: if isinstance(self.db, str): self.db = self.db.encode(self.encoding) - data += self.db + b'\0' + data += self.db + b"\0" if self.server_capabilities & CLIENT.PLUGIN_AUTH: - data += (plugin_name or b'') + b'\0' + data += (plugin_name or b"") + b"\0" if self.server_capabilities & CLIENT.CONNECT_ATTRS: - connect_attrs = b'' + connect_attrs = b"" for k, v in self._connect_attrs.items(): - k = k.encode('utf-8') - connect_attrs += struct.pack('B', len(k)) + k - v = v.encode('utf-8') - connect_attrs += struct.pack('B', len(v)) + v - data += struct.pack('B', len(connect_attrs)) + connect_attrs + k = k.encode("utf-8") + connect_attrs += struct.pack("B", len(k)) + k + v = v.encode("utf-8") + connect_attrs += struct.pack("B", len(v)) + v + data += struct.pack("B", len(connect_attrs)) + connect_attrs self.write_packet(data) auth_packet = self._read_packet() @@ -868,15 +921,19 @@ def _request_authentication(self): # if authentication method isn't accepted the first byte # will have the octet 254 if auth_packet.is_auth_switch_request(): - if DEBUG: print("received auth switch") + if DEBUG: + print("received auth switch") # https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest - auth_packet.read_uint8() # 0xfe packet identifier + auth_packet.read_uint8() # 0xfe packet identifier plugin_name = auth_packet.read_string() - if self.server_capabilities & CLIENT.PLUGIN_AUTH and plugin_name is not None: + if ( + self.server_capabilities & CLIENT.PLUGIN_AUTH + and plugin_name is not None + ): auth_packet = self._process_auth(plugin_name, auth_packet) else: # send legacy handshake - data = _auth.scramble_old_password(self.password, self.salt) + b'\0' + data = _auth.scramble_old_password(self.password, self.salt) + b"\0" self.write_packet(data) auth_packet = self._read_packet() elif auth_packet.is_extra_auth_data(): @@ -888,9 +945,12 @@ def _request_authentication(self): elif self._auth_plugin_name == "sha256_password": auth_packet = _auth.sha256_password_auth(self, auth_packet) else: - raise err.OperationalError("Received extra packet for auth method %r", self._auth_plugin_name) + raise err.OperationalError( + "Received extra packet for auth method %r", self._auth_plugin_name + ) - if DEBUG: print("Succeed to auth") + if DEBUG: + print("Succeed to auth") def _process_auth(self, plugin_name, auth_packet): handler = self._get_auth_plugin_handler(plugin_name) @@ -898,22 +958,29 @@ def _process_auth(self, plugin_name, auth_packet): try: return handler.authenticate(auth_packet) except AttributeError: - if plugin_name != b'dialog': - raise err.OperationalError(2059, "Authentication plugin '%s'" - " not loaded: - %r missing authenticate method" % (plugin_name, type(handler))) + if plugin_name != b"dialog": + raise err.OperationalError( + 2059, + "Authentication plugin '%s'" + " not loaded: - %r missing authenticate method" + % (plugin_name, type(handler)), + ) if plugin_name == b"caching_sha2_password": return _auth.caching_sha2_password_auth(self, auth_packet) elif plugin_name == b"sha256_password": return _auth.sha256_password_auth(self, auth_packet) elif plugin_name == b"mysql_native_password": data = _auth.scramble_native_password(self.password, auth_packet.read_all()) - elif plugin_name == b'client_ed25519': + elif plugin_name == b"client_ed25519": data = _auth.ed25519_password(self.password, auth_packet.read_all()) elif plugin_name == b"mysql_old_password": - data = _auth.scramble_old_password(self.password, auth_packet.read_all()) + b'\0' + data = ( + _auth.scramble_old_password(self.password, auth_packet.read_all()) + + b"\0" + ) elif plugin_name == b"mysql_clear_password": # https://dev.mysql.com/doc/internals/en/clear-text-authentication.html - data = self.password + b'\0' + data = self.password + b"\0" elif plugin_name == b"dialog": pkt = auth_packet while True: @@ -923,27 +990,41 @@ def _process_auth(self, plugin_name, auth_packet): prompt = pkt.read_all() if prompt == b"Password: ": - self.write_packet(self.password + b'\0') + self.write_packet(self.password + b"\0") elif handler: - resp = 'no response - TypeError within plugin.prompt method' + resp = "no response - TypeError within plugin.prompt method" try: resp = handler.prompt(echo, prompt) - self.write_packet(resp + b'\0') + self.write_packet(resp + b"\0") except AttributeError: - raise err.OperationalError(2059, "Authentication plugin '%s'" \ - " not loaded: - %r missing prompt method" % (plugin_name, handler)) + raise err.OperationalError( + 2059, + "Authentication plugin '%s'" + " not loaded: - %r missing prompt method" + % (plugin_name, handler), + ) except TypeError: - raise err.OperationalError(2061, "Authentication plugin '%s'" \ - " %r didn't respond with string. Returned '%r' to prompt %r" % (plugin_name, handler, resp, prompt)) + raise err.OperationalError( + 2061, + "Authentication plugin '%s'" + " %r didn't respond with string. Returned '%r' to prompt %r" + % (plugin_name, handler, resp, prompt), + ) else: - raise err.OperationalError(2059, "Authentication plugin '%s' (%r) not configured" % (plugin_name, handler)) + raise err.OperationalError( + 2059, + "Authentication plugin '%s' (%r) not configured" + % (plugin_name, handler), + ) pkt = self._read_packet() pkt.check_error() if pkt.is_ok_packet() or last: break return pkt else: - raise err.OperationalError(2059, "Authentication plugin '%s' not configured" % plugin_name) + raise err.OperationalError( + 2059, "Authentication plugin '%s' not configured" % plugin_name + ) self.write_packet(data) pkt = self._read_packet() @@ -953,13 +1034,17 @@ def _process_auth(self, plugin_name, auth_packet): def _get_auth_plugin_handler(self, plugin_name): plugin_class = self._auth_plugin_map.get(plugin_name) if not plugin_class and isinstance(plugin_name, bytes): - plugin_class = self._auth_plugin_map.get(plugin_name.decode('ascii')) + plugin_class = self._auth_plugin_map.get(plugin_name.decode("ascii")) if plugin_class: try: handler = plugin_class(self) except TypeError: - raise err.OperationalError(2059, "Authentication plugin '%s'" - " not loaded: - %r cannot be constructed with connection object" % (plugin_name, plugin_class)) + raise err.OperationalError( + 2059, + "Authentication plugin '%s'" + " not loaded: - %r cannot be constructed with connection object" + % (plugin_name, plugin_class), + ) else: handler = None return handler @@ -982,24 +1067,24 @@ def _get_server_information(self): packet = self._read_packet() data = packet.get_all_data() - self.protocol_version = byte2int(data[i:i+1]) + self.protocol_version = byte2int(data[i : i + 1]) i += 1 - server_end = data.find(b'\0', i) - self.server_version = data[i:server_end].decode('latin1') + server_end = data.find(b"\0", i) + self.server_version = data[i:server_end].decode("latin1") i = server_end + 1 - self.server_thread_id = struct.unpack('= i + 6: - lang, stat, cap_h, salt_len = struct.unpack('= i + salt_len: # salt_len includes auth_plugin_data_part_1 and filler - self.salt += data[i:i+salt_len] + self.salt += data[i : i + salt_len] i += salt_len - i+=1 + i += 1 # AUTH PLUGIN NAME may appear here. if self.server_capabilities & CLIENT.PLUGIN_AUTH and len(data) >= i: # Due to Bug#59453 the auth-plugin-name is missing the terminating @@ -1033,12 +1120,12 @@ def _get_server_information(self): # ref: https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake # didn't use version checks as mariadb is corrected and reports # earlier than those two. - server_end = data.find(b'\0', i) - if server_end < 0: # pragma: no cover - very specific upstream bug + server_end = data.find(b"\0", i) + if server_end < 0: # pragma: no cover - very specific upstream bug # not found \0 and last field so take it all - self._auth_plugin_name = data[i:].decode('utf-8') + self._auth_plugin_name = data[i:].decode("utf-8") else: - self._auth_plugin_name = data[i:server_end].decode('utf-8') + self._auth_plugin_name = data[i:server_end].decode("utf-8") def get_server_info(self): return self.server_version @@ -1056,7 +1143,6 @@ def get_server_info(self): class MySQLResult: - def __init__(self, connection): """ :type connection: Connection @@ -1127,7 +1213,8 @@ def _read_ok_packet(self, first_packet): def _read_load_local_packet(self, first_packet): if not self.connection._local_infile: raise RuntimeError( - "**WARN**: Received LOAD_LOCAL packet but local_infile option is false.") + "**WARN**: Received LOAD_LOCAL packet but local_infile option is false." + ) load_packet = LoadLocalPacketWrapper(first_packet) sender = LoadLocalFile(load_packet.filename, self.connection) try: @@ -1137,14 +1224,16 @@ def _read_load_local_packet(self, first_packet): raise ok_packet = self.connection._read_packet() - if not ok_packet.is_ok_packet(): # pragma: no cover - upstream induced protocol error + if ( + not ok_packet.is_ok_packet() + ): # pragma: no cover - upstream induced protocol error raise err.OperationalError(2014, "Commands Out of Sync") self._read_ok_packet(ok_packet) def _check_packet_is_eof(self, packet): if not packet.is_eof_packet(): return False - #TODO: Support CLIENT.DEPRECATE_EOF + # TODO: Support CLIENT.DEPRECATE_EOF # 1) Add DEPRECATE_EOF to CAPABILITIES # 2) Mask CAPABILITIES with server_capabilities # 3) if server_capabilities & CLIENT.DEPRECATE_EOF: use OKPacketWrapper instead of EOFPacketWrapper @@ -1211,7 +1300,8 @@ def _read_row_from_packet(self, packet): if data is not None: if encoding is not None: data = data.decode(encoding) - if DEBUG: print("DEBUG: DATA = ", data) + if DEBUG: + print("DEBUG: DATA = ", data) if converter is not None: data = converter(data) row.append(data) @@ -1246,17 +1336,18 @@ def _get_descriptions(self): encoding = conn_encoding else: # Integers, Dates and Times, and other basic data is encoded in ascii - encoding = 'ascii' + encoding = "ascii" else: encoding = None converter = self.connection.decoders.get(field_type) if converter is converters.through: converter = None - if DEBUG: print("DEBUG: field={}, converter={}".format(field, converter)) + if DEBUG: + print("DEBUG: field={}, converter={}".format(field, converter)) self.converters.append((encoding, converter)) eof_packet = self.connection._read_packet() - assert eof_packet.is_eof_packet(), 'Protocol error, expecting EOF' + assert eof_packet.is_eof_packet(), "Protocol error, expecting EOF" self.description = tuple(description) @@ -1268,19 +1359,23 @@ def __init__(self, filename, connection): def send_data(self): """Send data packets from the local file to the server""" if not self.connection._sock: - raise err.InterfaceError(0, '') + raise err.InterfaceError(0, "") conn = self.connection try: - with open(self.filename, 'rb') as open_file: - packet_size = min(conn.max_allowed_packet, 16*1024) # 16KB is efficient enough + with open(self.filename, "rb") as open_file: + packet_size = min( + conn.max_allowed_packet, 16 * 1024 + ) # 16KB is efficient enough while True: chunk = open_file.read(packet_size) if not chunk: break conn.write_packet(chunk) except IOError: - raise err.OperationalError(1017, "Can't find file '{0}'".format(self.filename)) + raise err.OperationalError( + 1017, "Can't find file '{0}'".format(self.filename) + ) finally: # send the empty packet to signify we are done sending data - conn.write_packet(b'') + conn.write_packet(b"") diff --git a/pymysql/constants/CLIENT.py b/pymysql/constants/CLIENT.py index b42f1523c..34fe57a5d 100644 --- a/pymysql/constants/CLIENT.py +++ b/pymysql/constants/CLIENT.py @@ -21,9 +21,16 @@ CONNECT_ATTRS = 1 << 20 PLUGIN_AUTH_LENENC_CLIENT_DATA = 1 << 21 CAPABILITIES = ( - LONG_PASSWORD | LONG_FLAG | PROTOCOL_41 | TRANSACTIONS - | SECURE_CONNECTION | MULTI_RESULTS - | PLUGIN_AUTH | PLUGIN_AUTH_LENENC_CLIENT_DATA | CONNECT_ATTRS) + LONG_PASSWORD + | LONG_FLAG + | PROTOCOL_41 + | TRANSACTIONS + | SECURE_CONNECTION + | MULTI_RESULTS + | PLUGIN_AUTH + | PLUGIN_AUTH_LENENC_CLIENT_DATA + | CONNECT_ATTRS +) # Not done yet HANDLE_EXPIRED_PASSWORDS = 1 << 22 diff --git a/pymysql/constants/COMMAND.py b/pymysql/constants/COMMAND.py index 1da275533..2d98850b8 100644 --- a/pymysql/constants/COMMAND.py +++ b/pymysql/constants/COMMAND.py @@ -1,4 +1,3 @@ - COM_SLEEP = 0x00 COM_QUIT = 0x01 COM_INIT_DB = 0x02 @@ -9,12 +8,12 @@ COM_REFRESH = 0x07 COM_SHUTDOWN = 0x08 COM_STATISTICS = 0x09 -COM_PROCESS_INFO = 0x0a -COM_CONNECT = 0x0b -COM_PROCESS_KILL = 0x0c -COM_DEBUG = 0x0d -COM_PING = 0x0e -COM_TIME = 0x0f +COM_PROCESS_INFO = 0x0A +COM_CONNECT = 0x0B +COM_PROCESS_KILL = 0x0C +COM_DEBUG = 0x0D +COM_PING = 0x0E +COM_TIME = 0x0F COM_DELAYED_INSERT = 0x10 COM_CHANGE_USER = 0x11 COM_BINLOG_DUMP = 0x12 @@ -25,9 +24,9 @@ COM_STMT_EXECUTE = 0x17 COM_STMT_SEND_LONG_DATA = 0x18 COM_STMT_CLOSE = 0x19 -COM_STMT_RESET = 0x1a -COM_SET_OPTION = 0x1b -COM_STMT_FETCH = 0x1c -COM_DAEMON = 0x1d -COM_BINLOG_DUMP_GTID = 0x1e -COM_END = 0x1f +COM_STMT_RESET = 0x1A +COM_SET_OPTION = 0x1B +COM_STMT_FETCH = 0x1C +COM_DAEMON = 0x1D +COM_BINLOG_DUMP_GTID = 0x1E +COM_END = 0x1F diff --git a/pymysql/constants/CR.py b/pymysql/constants/CR.py index 48ca956ec..25579a7c6 100644 --- a/pymysql/constants/CR.py +++ b/pymysql/constants/CR.py @@ -1,68 +1,68 @@ # flake8: noqa # errmsg.h -CR_ERROR_FIRST = 2000 -CR_UNKNOWN_ERROR = 2000 -CR_SOCKET_CREATE_ERROR = 2001 -CR_CONNECTION_ERROR = 2002 -CR_CONN_HOST_ERROR = 2003 -CR_IPSOCK_ERROR = 2004 -CR_UNKNOWN_HOST = 2005 -CR_SERVER_GONE_ERROR = 2006 -CR_VERSION_ERROR = 2007 -CR_OUT_OF_MEMORY = 2008 -CR_WRONG_HOST_INFO = 2009 +CR_ERROR_FIRST = 2000 +CR_UNKNOWN_ERROR = 2000 +CR_SOCKET_CREATE_ERROR = 2001 +CR_CONNECTION_ERROR = 2002 +CR_CONN_HOST_ERROR = 2003 +CR_IPSOCK_ERROR = 2004 +CR_UNKNOWN_HOST = 2005 +CR_SERVER_GONE_ERROR = 2006 +CR_VERSION_ERROR = 2007 +CR_OUT_OF_MEMORY = 2008 +CR_WRONG_HOST_INFO = 2009 CR_LOCALHOST_CONNECTION = 2010 -CR_TCP_CONNECTION = 2011 +CR_TCP_CONNECTION = 2011 CR_SERVER_HANDSHAKE_ERR = 2012 -CR_SERVER_LOST = 2013 +CR_SERVER_LOST = 2013 CR_COMMANDS_OUT_OF_SYNC = 2014 CR_NAMEDPIPE_CONNECTION = 2015 -CR_NAMEDPIPEWAIT_ERROR = 2016 -CR_NAMEDPIPEOPEN_ERROR = 2017 +CR_NAMEDPIPEWAIT_ERROR = 2016 +CR_NAMEDPIPEOPEN_ERROR = 2017 CR_NAMEDPIPESETSTATE_ERROR = 2018 -CR_CANT_READ_CHARSET = 2019 +CR_CANT_READ_CHARSET = 2019 CR_NET_PACKET_TOO_LARGE = 2020 -CR_EMBEDDED_CONNECTION = 2021 -CR_PROBE_SLAVE_STATUS = 2022 -CR_PROBE_SLAVE_HOSTS = 2023 -CR_PROBE_SLAVE_CONNECT = 2024 +CR_EMBEDDED_CONNECTION = 2021 +CR_PROBE_SLAVE_STATUS = 2022 +CR_PROBE_SLAVE_HOSTS = 2023 +CR_PROBE_SLAVE_CONNECT = 2024 CR_PROBE_MASTER_CONNECT = 2025 CR_SSL_CONNECTION_ERROR = 2026 -CR_MALFORMED_PACKET = 2027 -CR_WRONG_LICENSE = 2028 +CR_MALFORMED_PACKET = 2027 +CR_WRONG_LICENSE = 2028 -CR_NULL_POINTER = 2029 -CR_NO_PREPARE_STMT = 2030 -CR_PARAMS_NOT_BOUND = 2031 -CR_DATA_TRUNCATED = 2032 +CR_NULL_POINTER = 2029 +CR_NO_PREPARE_STMT = 2030 +CR_PARAMS_NOT_BOUND = 2031 +CR_DATA_TRUNCATED = 2032 CR_NO_PARAMETERS_EXISTS = 2033 CR_INVALID_PARAMETER_NO = 2034 -CR_INVALID_BUFFER_USE = 2035 +CR_INVALID_BUFFER_USE = 2035 CR_UNSUPPORTED_PARAM_TYPE = 2036 -CR_SHARED_MEMORY_CONNECTION = 2037 -CR_SHARED_MEMORY_CONNECT_REQUEST_ERROR = 2038 -CR_SHARED_MEMORY_CONNECT_ANSWER_ERROR = 2039 +CR_SHARED_MEMORY_CONNECTION = 2037 +CR_SHARED_MEMORY_CONNECT_REQUEST_ERROR = 2038 +CR_SHARED_MEMORY_CONNECT_ANSWER_ERROR = 2039 CR_SHARED_MEMORY_CONNECT_FILE_MAP_ERROR = 2040 -CR_SHARED_MEMORY_CONNECT_MAP_ERROR = 2041 -CR_SHARED_MEMORY_FILE_MAP_ERROR = 2042 -CR_SHARED_MEMORY_MAP_ERROR = 2043 -CR_SHARED_MEMORY_EVENT_ERROR = 2044 +CR_SHARED_MEMORY_CONNECT_MAP_ERROR = 2041 +CR_SHARED_MEMORY_FILE_MAP_ERROR = 2042 +CR_SHARED_MEMORY_MAP_ERROR = 2043 +CR_SHARED_MEMORY_EVENT_ERROR = 2044 CR_SHARED_MEMORY_CONNECT_ABANDONED_ERROR = 2045 -CR_SHARED_MEMORY_CONNECT_SET_ERROR = 2046 -CR_CONN_UNKNOW_PROTOCOL = 2047 -CR_INVALID_CONN_HANDLE = 2048 -CR_SECURE_AUTH = 2049 -CR_FETCH_CANCELED = 2050 -CR_NO_DATA = 2051 -CR_NO_STMT_METADATA = 2052 -CR_NO_RESULT_SET = 2053 -CR_NOT_IMPLEMENTED = 2054 -CR_SERVER_LOST_EXTENDED = 2055 -CR_STMT_CLOSED = 2056 -CR_NEW_STMT_METADATA = 2057 -CR_ALREADY_CONNECTED = 2058 -CR_AUTH_PLUGIN_CANNOT_LOAD = 2059 -CR_DUPLICATE_CONNECTION_ATTR = 2060 -CR_AUTH_PLUGIN_ERR = 2061 +CR_SHARED_MEMORY_CONNECT_SET_ERROR = 2046 +CR_CONN_UNKNOW_PROTOCOL = 2047 +CR_INVALID_CONN_HANDLE = 2048 +CR_SECURE_AUTH = 2049 +CR_FETCH_CANCELED = 2050 +CR_NO_DATA = 2051 +CR_NO_STMT_METADATA = 2052 +CR_NO_RESULT_SET = 2053 +CR_NOT_IMPLEMENTED = 2054 +CR_SERVER_LOST_EXTENDED = 2055 +CR_STMT_CLOSED = 2056 +CR_NEW_STMT_METADATA = 2057 +CR_ALREADY_CONNECTED = 2058 +CR_AUTH_PLUGIN_CANNOT_LOAD = 2059 +CR_DUPLICATE_CONNECTION_ATTR = 2060 +CR_AUTH_PLUGIN_ERR = 2061 CR_ERROR_LAST = 2061 diff --git a/pymysql/constants/FIELD_TYPE.py b/pymysql/constants/FIELD_TYPE.py index 51bd5143b..b8b448660 100644 --- a/pymysql/constants/FIELD_TYPE.py +++ b/pymysql/constants/FIELD_TYPE.py @@ -1,5 +1,3 @@ - - DECIMAL = 0 TINY = 1 SHORT = 2 diff --git a/pymysql/constants/SERVER_STATUS.py b/pymysql/constants/SERVER_STATUS.py index 6f5d56630..8f8d77688 100644 --- a/pymysql/constants/SERVER_STATUS.py +++ b/pymysql/constants/SERVER_STATUS.py @@ -1,4 +1,3 @@ - SERVER_STATUS_IN_TRANS = 1 SERVER_STATUS_AUTOCOMMIT = 2 SERVER_MORE_RESULTS_EXISTS = 8 diff --git a/pymysql/converters.py b/pymysql/converters.py index 6d1fc9ee0..113dd298e 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -25,6 +25,7 @@ def escape_item(val, charset, mapping=None): val = encoder(val, mapping) return val + def escape_dict(val, charset, mapping=None): n = {} for k, v in val.items(): @@ -32,6 +33,7 @@ def escape_dict(val, charset, mapping=None): n[k] = quoted return n + def escape_sequence(val, charset, mapping=None): n = [] for item in val: @@ -39,32 +41,38 @@ def escape_sequence(val, charset, mapping=None): n.append(quoted) return "(" + ",".join(n) + ")" + def escape_set(val, charset, mapping=None): - return ','.join([escape_item(x, charset, mapping) for x in val]) + return ",".join([escape_item(x, charset, mapping) for x in val]) + def escape_bool(value, mapping=None): return str(int(value)) + def escape_int(value, mapping=None): return str(value) + def escape_float(value, mapping=None): s = repr(value) - if s in ('inf', 'nan'): + if s in ("inf", "nan"): raise ProgrammingError("%s can not be used with MySQL" % s) - if 'e' not in s: - s += 'e0' + if "e" not in s: + s += "e0" return s + _escape_table = [chr(x) for x in range(128)] -_escape_table[0] = u'\\0' -_escape_table[ord('\\')] = u'\\\\' -_escape_table[ord('\n')] = u'\\n' -_escape_table[ord('\r')] = u'\\r' -_escape_table[ord('\032')] = u'\\Z' +_escape_table[0] = u"\\0" +_escape_table[ord("\\")] = u"\\\\" +_escape_table[ord("\n")] = u"\\n" +_escape_table[ord("\r")] = u"\\r" +_escape_table[ord("\032")] = u"\\Z" _escape_table[ord('"')] = u'\\"' _escape_table[ord("'")] = u"\\'" + def escape_string(value, mapping=None): """escapes *value* without adding quote. @@ -74,18 +82,22 @@ def escape_string(value, mapping=None): def escape_bytes_prefixed(value, mapping=None): - return "_binary'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) + return "_binary'%s'" % value.decode("ascii", "surrogateescape").translate( + _escape_table + ) def escape_bytes(value, mapping=None): - return "'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) + return "'%s'" % value.decode("ascii", "surrogateescape").translate(_escape_table) def escape_str(value, mapping=None): return "'%s'" % escape_string(str(value), mapping) + def escape_None(value, mapping=None): - return 'NULL' + return "NULL" + def escape_timedelta(obj, mapping=None): seconds = int(obj.seconds) % 60 @@ -97,6 +109,7 @@ def escape_timedelta(obj, mapping=None): fmt = "'{0:02d}:{1:02d}:{2:02d}'" return fmt.format(hours, minutes, seconds, obj.microseconds) + def escape_time(obj, mapping=None): if obj.microsecond: fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" @@ -104,6 +117,7 @@ def escape_time(obj, mapping=None): fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}'" return fmt.format(obj) + def escape_datetime(obj, mapping=None): if obj.microsecond: fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" @@ -111,10 +125,12 @@ def escape_datetime(obj, mapping=None): fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}'" return fmt.format(obj) + def escape_date(obj, mapping=None): fmt = "'{0.year:04}-{0.month:02}-{0.day:02}'" return fmt.format(obj) + def escape_struct_time(obj, mapping=None): return escape_datetime(datetime.datetime(*obj[:6])) @@ -127,10 +143,13 @@ def _convert_second_fraction(s): if not s: return 0 # Pad zeros to ensure the fraction length in microseconds - s = s.ljust(6, '0') + s = s.ljust(6, "0") return int(s[:6]) -DATETIME_RE = re.compile(r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") + +DATETIME_RE = re.compile( + r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?" +) def convert_datetime(obj): @@ -150,7 +169,7 @@ def convert_datetime(obj): """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") m = DATETIME_RE.match(obj) if not m: @@ -159,10 +178,11 @@ def convert_datetime(obj): try: groups = list(m.groups()) groups[-1] = _convert_second_fraction(groups[-1]) - return datetime.datetime(*[ int(x) for x in groups ]) + return datetime.datetime(*[int(x) for x in groups]) except ValueError: return convert_date(obj) + TIMEDELTA_RE = re.compile(r"(-)?(\d{1,3}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") @@ -184,7 +204,7 @@ def convert_timedelta(obj): be parsed correctly by this function. """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") m = TIMEDELTA_RE.match(obj) if not m: @@ -196,16 +216,20 @@ def convert_timedelta(obj): negate = -1 if groups[0] else 1 hours, minutes, seconds, microseconds = groups[1:] - tdelta = datetime.timedelta( - hours = int(hours), - minutes = int(minutes), - seconds = int(seconds), - microseconds = int(microseconds) - ) * negate + tdelta = ( + datetime.timedelta( + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds), + microseconds=int(microseconds), + ) + * negate + ) return tdelta except ValueError: return obj + TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") @@ -232,7 +256,7 @@ def convert_time(obj): use set this function as the converter for FIELD_TYPE.TIME. """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") m = TIME_RE.match(obj) if not m: @@ -242,8 +266,12 @@ def convert_time(obj): groups = list(m.groups()) groups[-1] = _convert_second_fraction(groups[-1]) hours, minutes, seconds, microseconds = groups - return datetime.time(hour=int(hours), minute=int(minutes), - second=int(seconds), microsecond=int(microseconds)) + return datetime.time( + hour=int(hours), + minute=int(minutes), + second=int(seconds), + microsecond=int(microseconds), + ) except ValueError: return obj @@ -263,9 +291,9 @@ def convert_date(obj): """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") try: - return datetime.date(*[ int(x) for x in obj.split('-', 2) ]) + return datetime.date(*[int(x) for x in obj.split("-", 2)]) except ValueError: return obj @@ -274,7 +302,7 @@ def through(x): return x -#def convert_bit(b): +# def convert_bit(b): # b = "\x00" * (8 - len(b)) + b # pad w/ zeroes # return struct.unpack(">Q", b)[0] # diff --git a/pymysql/cursors.py b/pymysql/cursors.py index a8c528367..68ac78e7f 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -6,10 +6,11 @@ #: executemany only supports simple bulk insert. #: You can use it to load large dataset. RE_INSERT_VALUES = re.compile( - r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" + - r"(\(\s*(?:%s|%\(.+\)s)\s*(?:,\s*(?:%s|%\(.+\)s)\s*)*\))" + - r"(\s*(?:ON DUPLICATE.*)?);?\s*\Z", - re.IGNORECASE | re.DOTALL) + r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" + + r"(\(\s*(?:%s|%\(.+\)s)\s*(?:,\s*(?:%s|%\(.+\)s)\s*)*\))" + + r"(\s*(?:ON DUPLICATE.*)?);?\s*\Z", + re.IGNORECASE | re.DOTALL, +) class Cursor: @@ -167,16 +168,23 @@ def executemany(self, query, args): if m: q_prefix = m.group(1) % () q_values = m.group(2).rstrip() - q_postfix = m.group(3) or '' - assert q_values[0] == '(' and q_values[-1] == ')' - return self._do_execute_many(q_prefix, q_values, q_postfix, args, - self.max_stmt_length, - self._get_db().encoding) + q_postfix = m.group(3) or "" + assert q_values[0] == "(" and q_values[-1] == ")" + return self._do_execute_many( + q_prefix, + q_values, + q_postfix, + args, + self.max_stmt_length, + self._get_db().encoding, + ) self.rowcount = sum(self.execute(query, arg) for arg in args) return self.rowcount - def _do_execute_many(self, prefix, values, postfix, args, max_stmt_length, encoding): + def _do_execute_many( + self, prefix, values, postfix, args, max_stmt_length, encoding + ): conn = self._get_db() escape = self._escape_args if isinstance(prefix, str): @@ -187,18 +195,18 @@ def _do_execute_many(self, prefix, values, postfix, args, max_stmt_length, encod args = iter(args) v = values % escape(next(args), conn) if isinstance(v, str): - v = v.encode(encoding, 'surrogateescape') + v = v.encode(encoding, "surrogateescape") sql += v rows = 0 for arg in args: v = values % escape(arg, conn) if isinstance(v, str): - v = v.encode(encoding, 'surrogateescape') + v = v.encode(encoding, "surrogateescape") if len(sql) + len(v) + len(postfix) + 1 > max_stmt_length: rows += self.execute(sql + postfix) sql = bytearray(prefix) else: - sql += b',' + sql += b"," sql += v rows += self.execute(sql + postfix) self.rowcount = rows @@ -234,14 +242,19 @@ def callproc(self, procname, args=()): """ conn = self._get_db() if args: - fmt = '@_{0}_%d=%s'.format(procname) - self._query('SET %s' % ','.join(fmt % (index, conn.escape(arg)) - for index, arg in enumerate(args))) + fmt = "@_{0}_%d=%s".format(procname) + self._query( + "SET %s" + % ",".join( + fmt % (index, conn.escape(arg)) for index, arg in enumerate(args) + ) + ) self.nextset() - q = "CALL %s(%s)" % (procname, - ','.join(['@_%s_%d' % (procname, i) - for i in range(len(args))])) + q = "CALL %s(%s)" % ( + procname, + ",".join(["@_%s_%d" % (procname, i) for i in range(len(args))]), + ) self._query(q) self._executed = q return args @@ -261,7 +274,7 @@ def fetchmany(self, size=None): if self._rows is None: return () end = self.rownumber + (size or self.arraysize) - result = self._rows[self.rownumber:end] + result = self._rows[self.rownumber : end] self.rownumber = min(end, len(self._rows)) return result @@ -271,17 +284,17 @@ def fetchall(self): if self._rows is None: return () if self.rownumber: - result = self._rows[self.rownumber:] + result = self._rows[self.rownumber :] else: result = self._rows self.rownumber = len(self._rows) return result - def scroll(self, value, mode='relative'): + def scroll(self, value, mode="relative"): self._check_executed() - if mode == 'relative': + if mode == "relative": r = self.rownumber + value - elif mode == 'absolute': + elif mode == "absolute": r = value else: raise err.ProgrammingError("unknown scroll mode %s" % mode) @@ -343,7 +356,7 @@ def _do_get_result(self): for f in self._result.fields: name = f.name if name in fields: - name = f.table_name + '.' + name + name = f.table_name + "." + name fields.append(name) self._fields = fields @@ -453,21 +466,23 @@ def fetchmany(self, size=None): self.rownumber += 1 return rows - def scroll(self, value, mode='relative'): + def scroll(self, value, mode="relative"): self._check_executed() - if mode == 'relative': + if mode == "relative": if value < 0: raise err.NotSupportedError( - "Backwards scrolling not supported by this cursor") + "Backwards scrolling not supported by this cursor" + ) for _ in range(value): self.read_next() self.rownumber += value - elif mode == 'absolute': + elif mode == "absolute": if value < self.rownumber: raise err.NotSupportedError( - "Backwards scrolling not supported by this cursor") + "Backwards scrolling not supported by this cursor" + ) end = value - self.rownumber for _ in range(end): diff --git a/pymysql/err.py b/pymysql/err.py index 94100cfea..3da5b166f 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -74,33 +74,69 @@ def _map_error(exc, *errors): error_map[error] = exc -_map_error(ProgrammingError, ER.DB_CREATE_EXISTS, ER.SYNTAX_ERROR, - ER.PARSE_ERROR, ER.NO_SUCH_TABLE, ER.WRONG_DB_NAME, - ER.WRONG_TABLE_NAME, ER.FIELD_SPECIFIED_TWICE, - ER.INVALID_GROUP_FUNC_USE, ER.UNSUPPORTED_EXTENSION, - ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION, - ER.WRONG_DB_NAME, ER.WRONG_COLUMN_NAME, - ) -_map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL, - ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL, - ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW, ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, - ER.ILLEGAL_VALUE_FOR_TYPE) -_map_error(IntegrityError, ER.DUP_ENTRY, ER.NO_REFERENCED_ROW, - ER.NO_REFERENCED_ROW_2, ER.ROW_IS_REFERENCED, ER.ROW_IS_REFERENCED_2, - ER.CANNOT_ADD_FOREIGN, ER.BAD_NULL_ERROR) -_map_error(NotSupportedError, ER.WARNING_NOT_COMPLETE_ROLLBACK, - ER.NOT_SUPPORTED_YET, ER.FEATURE_DISABLED, ER.UNKNOWN_STORAGE_ENGINE) -_map_error(OperationalError, ER.DBACCESS_DENIED_ERROR, ER.ACCESS_DENIED_ERROR, - ER.CON_COUNT_ERROR, ER.TABLEACCESS_DENIED_ERROR, - ER.COLUMNACCESS_DENIED_ERROR, ER.CONSTRAINT_FAILED, ER.LOCK_DEADLOCK) +_map_error( + ProgrammingError, + ER.DB_CREATE_EXISTS, + ER.SYNTAX_ERROR, + ER.PARSE_ERROR, + ER.NO_SUCH_TABLE, + ER.WRONG_DB_NAME, + ER.WRONG_TABLE_NAME, + ER.FIELD_SPECIFIED_TWICE, + ER.INVALID_GROUP_FUNC_USE, + ER.UNSUPPORTED_EXTENSION, + ER.TABLE_MUST_HAVE_COLUMNS, + ER.CANT_DO_THIS_DURING_AN_TRANSACTION, + ER.WRONG_DB_NAME, + ER.WRONG_COLUMN_NAME, +) +_map_error( + DataError, + ER.WARN_DATA_TRUNCATED, + ER.WARN_NULL_TO_NOTNULL, + ER.WARN_DATA_OUT_OF_RANGE, + ER.NO_DEFAULT, + ER.PRIMARY_CANT_HAVE_NULL, + ER.DATA_TOO_LONG, + ER.DATETIME_FUNCTION_OVERFLOW, + ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, + ER.ILLEGAL_VALUE_FOR_TYPE, +) +_map_error( + IntegrityError, + ER.DUP_ENTRY, + ER.NO_REFERENCED_ROW, + ER.NO_REFERENCED_ROW_2, + ER.ROW_IS_REFERENCED, + ER.ROW_IS_REFERENCED_2, + ER.CANNOT_ADD_FOREIGN, + ER.BAD_NULL_ERROR, +) +_map_error( + NotSupportedError, + ER.WARNING_NOT_COMPLETE_ROLLBACK, + ER.NOT_SUPPORTED_YET, + ER.FEATURE_DISABLED, + ER.UNKNOWN_STORAGE_ENGINE, +) +_map_error( + OperationalError, + ER.DBACCESS_DENIED_ERROR, + ER.ACCESS_DENIED_ERROR, + ER.CON_COUNT_ERROR, + ER.TABLEACCESS_DENIED_ERROR, + ER.COLUMNACCESS_DENIED_ERROR, + ER.CONSTRAINT_FAILED, + ER.LOCK_DEADLOCK, +) del _map_error, ER def raise_mysql_exception(data): - errno = struct.unpack('= 2 and value[0] == value[-1] == quote: return value[1:-1] diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 541475ad6..24b3f23e8 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -25,7 +25,7 @@ def printable(data): if isinstance(data, int): return chr(data) return data - return '.' + return "." try: print("packet length:", len(data)) @@ -35,11 +35,14 @@ def printable(data): print("-" * 66) except ValueError: pass - dump_data = [data[i:i+16] for i in range(0, min(len(data), 256), 16)] + dump_data = [data[i : i + 16] for i in range(0, min(len(data), 256), 16)] for d in dump_data: - print(' '.join("{:02X}".format(byte2int(x)) for x in d) + - ' ' * (16 - len(d)) + ' ' * 2 + - ''.join(printable(x) for x in d)) + print( + " ".join("{:02X}".format(byte2int(x)) for x in d) + + " " * (16 - len(d)) + + " " * 2 + + "".join(printable(x) for x in d) + ) print("-" * 66) print() @@ -49,7 +52,8 @@ class MysqlPacket: Provides an interface for reading/parsing the packet results. """ - __slots__ = ('_position', '_data') + + __slots__ = ("_position", "_data") def __init__(self, data, encoding): self._position = 0 @@ -60,11 +64,13 @@ def get_all_data(self): def read(self, size): """Read the first 'size' bytes in packet and advance cursor past them.""" - result = self._data[self._position:(self._position+size)] + result = self._data[self._position : (self._position + size)] if len(result) != size: - error = ('Result length not requested length:\n' - 'Expected=%s. Actual=%s. Position: %s. Data Length: %s' - % (size, len(result), self._position, len(self._data))) + error = ( + "Result length not requested length:\n" + "Expected=%s. Actual=%s. Position: %s. Data Length: %s" + % (size, len(result), self._position, len(self._data)) + ) if DEBUG: print(error) self.dump() @@ -77,7 +83,7 @@ def read_all(self): (Subsequent read() will return errors.) """ - result = self._data[self._position:] + result = self._data[self._position :] self._position = None # ensure no subsequent read() return result @@ -85,8 +91,10 @@ def advance(self, length): """Advance the cursor in data buffer 'length' bytes.""" new_position = self._position + length if new_position < 0 or new_position > len(self._data): - raise Exception('Invalid advance amount (%s) for cursor. ' - 'Position=%s' % (length, new_position)) + raise Exception( + "Invalid advance amount (%s) for cursor. " + "Position=%s" % (length, new_position) + ) self._position = new_position def rewind(self, position=0): @@ -104,7 +112,7 @@ def get_bytes(self, position, length=1): No error checking is done. If requesting outside end of buffer an empty string (or string shorter than 'length') may be returned! """ - return self._data[position:(position+length)] + return self._data[position : (position + length)] def read_uint8(self): result = self._data[self._position] @@ -112,30 +120,30 @@ def read_uint8(self): return result def read_uint16(self): - result = struct.unpack_from('= 7 + return self._data[0:1] == b"\0" and len(self._data) >= 7 def is_eof_packet(self): # http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet # Caution: \xFE may be LengthEncodedInteger. # If \xFE is LengthEncodedInteger header, 8bytes followed. - return self._data[0:1] == b'\xfe' and len(self._data) < 9 + return self._data[0:1] == b"\xfe" and len(self._data) < 9 def is_auth_switch_request(self): # http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest - return self._data[0:1] == b'\xfe' + return self._data[0:1] == b"\xfe" def is_extra_auth_data(self): # https://dev.mysql.com/doc/internals/en/successful-authentication.html - return self._data[0:1] == b'\x01' + return self._data[0:1] == b"\x01" def is_resultset_packet(self): field_count = ord(self._data[0:1]) return 1 <= field_count <= 250 def is_load_local_packet(self): - return self._data[0:1] == b'\xfb' + return self._data[0:1] == b"\xfb" def is_error_packet(self): - return self._data[0:1] == b'\xff' + return self._data[0:1] == b"\xff" def check_error(self): if self.is_error_packet(): @@ -211,7 +219,8 @@ def raise_for_error(self): self.rewind() self.advance(1) # field_count == error (we already know that) errno = self.read_uint16() - if DEBUG: print("errno =", errno) + if DEBUG: + print("errno =", errno) err.raise_mysql_exception(self._data) def dump(self): @@ -240,8 +249,13 @@ def _parse_field_descriptor(self, encoding): self.org_table = self.read_length_coded_string().decode(encoding) self.name = self.read_length_coded_string().decode(encoding) self.org_name = self.read_length_coded_string().decode(encoding) - self.charsetnr, self.length, self.type_code, self.flags, self.scale = ( - self.read_struct('= version_tuple @@ -53,10 +59,12 @@ def connect(self, **params): p = self.databases[0].copy() p.update(params) conn = pymysql.connect(**p) + @self.addCleanup def teardown(): if conn.open: conn.close() + return conn def _teardown_connections(self): diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index 122882e65..581a0c4ae 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -6,9 +6,9 @@ class TestDictCursor(base.PyMySQLTestCase): - bob = {'name': 'bob', 'age': 21, 'DOB': datetime.datetime(1990, 2, 6, 23, 4, 56)} - jim = {'name': 'jim', 'age': 56, 'DOB': datetime.datetime(1955, 5, 9, 13, 12, 45)} - fred = {'name': 'fred', 'age': 100, 'DOB': datetime.datetime(1911, 9, 12, 1, 1, 1)} + bob = {"name": "bob", "age": 21, "DOB": datetime.datetime(1990, 2, 6, 23, 4, 56)} + jim = {"name": "jim", "age": 56, "DOB": datetime.datetime(1955, 5, 9, 13, 12, 45)} + fred = {"name": "fred", "age": 100, "DOB": datetime.datetime(1911, 9, 12, 1, 1, 1)} cursor_type = pymysql.cursors.DictCursor @@ -23,10 +23,14 @@ def setUp(self): c.execute("drop table if exists dictcursor") # include in filterwarnings since for unbuffered dict cursor warning for lack of table # will only be propagated at start of next execute() call - c.execute("""CREATE TABLE dictcursor (name char(20), age int , DOB datetime)""") - data = [("bob", 21, "1990-02-06 23:04:56"), - ("jim", 56, "1955-05-09 13:12:45"), - ("fred", 100, "1911-09-12 01:01:01")] + c.execute( + """CREATE TABLE dictcursor (name char(20), age int , DOB datetime)""" + ) + data = [ + ("bob", 21, "1990-02-06 23:04:56"), + ("jim", 56, "1955-05-09 13:12:45"), + ("fred", 100, "1911-09-12 01:01:01"), + ] c.executemany("insert into dictcursor values (%s,%s,%s)", data) def tearDown(self): @@ -39,13 +43,13 @@ def _ensure_cursor_expired(self, cursor): def test_DictCursor(self): bob, jim, fred = self.bob.copy(), self.jim.copy(), self.fred.copy() - #all assert test compare to the structure as would come out from MySQLdb + # all assert test compare to the structure as would come out from MySQLdb conn = self.conn c = conn.cursor(self.cursor_type) # try an update which should return no rows c.execute("update dictcursor set age=20 where name='bob'") - bob['age'] = 20 + bob["age"] = 20 # pull back the single row dict for bob and check c.execute("SELECT * from dictcursor where name='bob'") r = c.fetchone() @@ -55,19 +59,23 @@ def test_DictCursor(self): # same again, but via fetchall => tuple) c.execute("SELECT * from dictcursor where name='bob'") r = c.fetchall() - self.assertEqual([bob], r, "fetch a 1 row result via fetchall failed via DictCursor") + self.assertEqual( + [bob], r, "fetch a 1 row result via fetchall failed via DictCursor" + ) # same test again but iterate over the c.execute("SELECT * from dictcursor where name='bob'") for r in c: - self.assertEqual(bob, r, "fetch a 1 row result via iteration failed via DictCursor") + self.assertEqual( + bob, r, "fetch a 1 row result via iteration failed via DictCursor" + ) # get all 3 row via fetchall c.execute("SELECT * from dictcursor") r = c.fetchall() - self.assertEqual([bob,jim,fred], r, "fetchall failed via DictCursor") - #same test again but do a list comprehension + self.assertEqual([bob, jim, fred], r, "fetchall failed via DictCursor") + # same test again but do a list comprehension c.execute("SELECT * from dictcursor") r = list(c) - self.assertEqual([bob,jim,fred], r, "DictCursor should be iterable") + self.assertEqual([bob, jim, fred], r, "DictCursor should be iterable") # get all 2 row via fetchmany c.execute("SELECT * from dictcursor") r = c.fetchmany(2) @@ -75,12 +83,13 @@ def test_DictCursor(self): self._ensure_cursor_expired(c) def test_custom_dict(self): - class MyDict(dict): pass + class MyDict(dict): + pass class MyDictCursor(self.cursor_type): dict_type = MyDict - keys = ['name', 'age', 'DOB'] + keys = ["name", "age", "DOB"] bob = MyDict([(k, self.bob[k]) for k in keys]) jim = MyDict([(k, self.jim[k]) for k in keys]) fred = MyDict([(k, self.fred[k]) for k in keys]) @@ -93,18 +102,15 @@ class MyDictCursor(self.cursor_type): cur.execute("SELECT * FROM dictcursor") r = cur.fetchall() - self.assertEqual([bob, jim, fred], r, - "fetchall failed via MyDictCursor") + self.assertEqual([bob, jim, fred], r, "fetchall failed via MyDictCursor") cur.execute("SELECT * FROM dictcursor") r = list(cur) - self.assertEqual([bob, jim, fred], r, - "list failed via MyDictCursor") + self.assertEqual([bob, jim, fred], r, "list failed via MyDictCursor") cur.execute("SELECT * FROM dictcursor") r = cur.fetchmany(2) - self.assertEqual([bob, jim], r, - "list failed via MyDictCursor") + self.assertEqual([bob, jim], r, "list failed via MyDictCursor") self._ensure_cursor_expired(cur) @@ -114,6 +120,8 @@ class TestSSDictCursor(TestDictCursor): def _ensure_cursor_expired(self, cursor): list(cursor.fetchall_unbuffered()) + if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index 2b0de78a1..a68a77698 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -6,7 +6,7 @@ from pymysql.constants import CLIENT except Exception: # For local testing from top-level directory, without installing - sys.path.append('../pymysql') + sys.path.append("../pymysql") from pymysql.tests import base import pymysql.cursors from pymysql.constants import CLIENT @@ -18,35 +18,38 @@ def test_SSCursor(self): conn = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) data = [ - ('America', '', 'America/Jamaica'), - ('America', '', 'America/Los_Angeles'), - ('America', '', 'America/Lima'), - ('America', '', 'America/New_York'), - ('America', '', 'America/Menominee'), - ('America', '', 'America/Havana'), - ('America', '', 'America/El_Salvador'), - ('America', '', 'America/Costa_Rica'), - ('America', '', 'America/Denver'), - ('America', '', 'America/Detroit'),] + ("America", "", "America/Jamaica"), + ("America", "", "America/Los_Angeles"), + ("America", "", "America/Lima"), + ("America", "", "America/New_York"), + ("America", "", "America/Menominee"), + ("America", "", "America/Havana"), + ("America", "", "America/El_Salvador"), + ("America", "", "America/Costa_Rica"), + ("America", "", "America/Denver"), + ("America", "", "America/Detroit"), + ] cursor = conn.cursor(pymysql.cursors.SSCursor) # Create table - cursor.execute('CREATE TABLE tz_data (' - 'region VARCHAR(64),' - 'zone VARCHAR(64),' - 'name VARCHAR(64))') + cursor.execute( + "CREATE TABLE tz_data (" + "region VARCHAR(64)," + "zone VARCHAR(64)," + "name VARCHAR(64))" + ) conn.begin() # Test INSERT for i in data: - cursor.execute('INSERT INTO tz_data VALUES (%s, %s, %s)', i) - self.assertEqual(conn.affected_rows(), 1, 'affected_rows does not match') + cursor.execute("INSERT INTO tz_data VALUES (%s, %s, %s)", i) + self.assertEqual(conn.affected_rows(), 1, "affected_rows does not match") conn.commit() # Test fetchone() iter = 0 - cursor.execute('SELECT * FROM tz_data') + cursor.execute("SELECT * FROM tz_data") while True: row = cursor.fetchone() if row is None: @@ -54,26 +57,35 @@ def test_SSCursor(self): iter += 1 # Test cursor.rowcount - self.assertEqual(cursor.rowcount, affected_rows, - 'cursor.rowcount != %s' % (str(affected_rows))) + self.assertEqual( + cursor.rowcount, + affected_rows, + "cursor.rowcount != %s" % (str(affected_rows)), + ) # Test cursor.rownumber - self.assertEqual(cursor.rownumber, iter, - 'cursor.rowcount != %s' % (str(iter))) + self.assertEqual( + cursor.rownumber, iter, "cursor.rowcount != %s" % (str(iter)) + ) # Test row came out the same as it went in - self.assertEqual((row in data), True, - 'Row not found in source data') + self.assertEqual((row in data), True, "Row not found in source data") # Test fetchall - cursor.execute('SELECT * FROM tz_data') - self.assertEqual(len(cursor.fetchall()), len(data), - 'fetchall failed. Number of rows does not match') + cursor.execute("SELECT * FROM tz_data") + self.assertEqual( + len(cursor.fetchall()), + len(data), + "fetchall failed. Number of rows does not match", + ) # Test fetchmany - cursor.execute('SELECT * FROM tz_data') - self.assertEqual(len(cursor.fetchmany(2)), 2, - 'fetchmany failed. Number of rows does not match') + cursor.execute("SELECT * FROM tz_data") + self.assertEqual( + len(cursor.fetchmany(2)), + 2, + "fetchmany failed. Number of rows does not match", + ) # So MySQLdb won't throw "Commands out of sync" while True: @@ -82,30 +94,38 @@ def test_SSCursor(self): break # Test update, affected_rows() - cursor.execute('UPDATE tz_data SET zone = %s', ['Foo']) + cursor.execute("UPDATE tz_data SET zone = %s", ["Foo"]) conn.commit() - self.assertEqual(cursor.rowcount, len(data), - 'Update failed. affected_rows != %s' % (str(len(data)))) + self.assertEqual( + cursor.rowcount, + len(data), + "Update failed. affected_rows != %s" % (str(len(data))), + ) # Test executemany - cursor.executemany('INSERT INTO tz_data VALUES (%s, %s, %s)', data) - self.assertEqual(cursor.rowcount, len(data), - 'executemany failed. cursor.rowcount != %s' % (str(len(data)))) + cursor.executemany("INSERT INTO tz_data VALUES (%s, %s, %s)", data) + self.assertEqual( + cursor.rowcount, + len(data), + "executemany failed. cursor.rowcount != %s" % (str(len(data))), + ) # Test multiple datasets - cursor.execute('SELECT 1; SELECT 2; SELECT 3') - self.assertListEqual(list(cursor), [(1, )]) + cursor.execute("SELECT 1; SELECT 2; SELECT 3") + self.assertListEqual(list(cursor), [(1,)]) self.assertTrue(cursor.nextset()) - self.assertListEqual(list(cursor), [(2, )]) + self.assertListEqual(list(cursor), [(2,)]) self.assertTrue(cursor.nextset()) - self.assertListEqual(list(cursor), [(3, )]) + self.assertListEqual(list(cursor), [(3,)]) self.assertFalse(cursor.nextset()) - cursor.execute('DROP TABLE IF EXISTS tz_data') + cursor.execute("DROP TABLE IF EXISTS tz_data") cursor.close() + __all__ = ["TestSSCursor"] if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 840c48604..f8e622e66 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -18,23 +18,46 @@ def test_datatypes(self): """ test every data type """ conn = self.connect() c = conn.cursor() - c.execute("create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)") + c.execute( + "create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)" + ) try: # insert values - v = (True, -3, 123456789012, 5.7, "hello'\" world", u"Espa\xc3\xb1ol", "binary\x00data".encode(conn.encoding), datetime.date(1988,2,2), datetime.datetime(2014, 5, 15, 7, 45, 57), datetime.timedelta(5,6), datetime.time(16,32), time.localtime()) - c.execute("insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", v) + v = ( + True, + -3, + 123456789012, + 5.7, + "hello'\" world", + u"Espa\xc3\xb1ol", + "binary\x00data".encode(conn.encoding), + datetime.date(1988, 2, 2), + datetime.datetime(2014, 5, 15, 7, 45, 57), + datetime.timedelta(5, 6), + datetime.time(16, 32), + time.localtime(), + ) + c.execute( + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + v, + ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") r = c.fetchone() self.assertEqual(util.int2byte(1), r[0]) self.assertEqual(v[1:10], r[1:10]) - self.assertEqual(datetime.timedelta(0, 60 * (v[10].hour * 60 + v[10].minute)), r[10]) + self.assertEqual( + datetime.timedelta(0, 60 * (v[10].hour * 60 + v[10].minute)), r[10] + ) self.assertEqual(datetime.datetime(*v[-1][:6]), r[-1]) c.execute("delete from test_datatypes") # check nulls - c.execute("insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", [None] * 12) + c.execute( + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + [None] * 12, + ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") r = c.fetchone() self.assertEqual(tuple([None] * 12), r) @@ -43,11 +66,15 @@ def test_datatypes(self): # check sequences type for seq_type in (tuple, list, set, frozenset): - c.execute("insert into test_datatypes (i, l) values (2,4), (6,8), (10,12)") - seq = seq_type([2,6]) - c.execute("select l from test_datatypes where i in %s order by i", (seq,)) + c.execute( + "insert into test_datatypes (i, l) values (2,4), (6,8), (10,12)" + ) + seq = seq_type([2, 6]) + c.execute( + "select l from test_datatypes where i in %s order by i", (seq,) + ) r = c.fetchall() - self.assertEqual(((4,),(8,)), r) + self.assertEqual(((4,), (8,)), r) c.execute("delete from test_datatypes") finally: @@ -59,9 +86,12 @@ def test_dict(self): c = conn.cursor() c.execute("create table test_dict (a integer, b integer, c integer)") try: - c.execute("insert into test_dict (a,b,c) values (%(a)s, %(b)s, %(c)s)", {"a":1,"b":2,"c":3}) + c.execute( + "insert into test_dict (a,b,c) values (%(a)s, %(b)s, %(c)s)", + {"a": 1, "b": 2, "c": 3}, + ) c.execute("select a,b,c from test_dict") - self.assertEqual((1,2,3), c.fetchone()) + self.assertEqual((1, 2, 3), c.fetchone()) finally: c.execute("drop table test_dict") @@ -94,7 +124,8 @@ def test_binary(self): data = bytes(bytearray(range(255))) conn = self.connect() self.safe_create_table( - conn, "test_binary", "create table test_binary (b binary(255))") + conn, "test_binary", "create table test_binary (b binary(255))" + ) with conn.cursor() as c: c.execute("insert into test_binary (b) values (_binary %s)", (data,)) @@ -105,8 +136,7 @@ def test_blob(self): """test blob data""" data = bytes(bytearray(range(256)) * 4) conn = self.connect() - self.safe_create_table( - conn, "test_blob", "create table test_blob (b blob)") + self.safe_create_table(conn, "test_blob", "create table test_blob (b blob)") with conn.cursor() as c: c.execute("insert into test_blob (b) values (_binary %s)", (data,)) @@ -118,23 +148,29 @@ def test_untyped(self): conn = self.connect() c = conn.cursor() c.execute("select null,''") - self.assertEqual((None,u''), c.fetchone()) + self.assertEqual((None, u""), c.fetchone()) c.execute("select '',null") - self.assertEqual((u'',None), c.fetchone()) + self.assertEqual((u"", None), c.fetchone()) def test_timedelta(self): """ test timedelta conversion """ conn = self.connect() c = conn.cursor() - c.execute("select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')") - self.assertEqual((datetime.timedelta(0, 45000), - datetime.timedelta(0, 83579), - datetime.timedelta(0, 83579, 51000), - -datetime.timedelta(0, 45000), - -datetime.timedelta(0, 83579), - -datetime.timedelta(0, 83579, 51000), - -datetime.timedelta(0, 1800)), - c.fetchone()) + c.execute( + "select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')" + ) + self.assertEqual( + ( + datetime.timedelta(0, 45000), + datetime.timedelta(0, 83579), + datetime.timedelta(0, 83579, 51000), + -datetime.timedelta(0, 45000), + -datetime.timedelta(0, 83579), + -datetime.timedelta(0, 83579, 51000), + -datetime.timedelta(0, 1800), + ), + c.fetchone(), + ) def test_datetime_microseconds(self): """ test datetime conversion w microseconds""" @@ -146,10 +182,7 @@ def test_datetime_microseconds(self): dt = datetime.datetime(2013, 11, 12, 9, 9, 9, 123450) c.execute("create table test_datetime (id int, ts datetime(6))") try: - c.execute( - "insert into test_datetime values (%s, %s)", - (1, dt) - ) + c.execute("insert into test_datetime values (%s, %s)", (1, dt)) c.execute("select ts from test_datetime") self.assertEqual((dt,), c.fetchone()) finally: @@ -162,7 +195,7 @@ class TestCursor(base.PyMySQLTestCase): # compatible with the DB-API 2.0 spec and has not broken # any unit tests for anything we've tried. - #def test_description(self): + # def test_description(self): # """ test description attribute """ # # result is from MySQLdb module # r = (('Host', 254, 11, 60, 60, 0, 0), @@ -227,22 +260,22 @@ def test_aggregates(self): conn = self.connect() c = conn.cursor() try: - c.execute('create table test_aggregates (i integer)') + c.execute("create table test_aggregates (i integer)") for i in range(0, 10): - c.execute('insert into test_aggregates (i) values (%s)', (i,)) - c.execute('select sum(i) from test_aggregates') - r, = c.fetchone() - self.assertEqual(sum(range(0,10)), r) + c.execute("insert into test_aggregates (i) values (%s)", (i,)) + c.execute("select sum(i) from test_aggregates") + (r,) = c.fetchone() + self.assertEqual(sum(range(0, 10)), r) finally: - c.execute('drop table test_aggregates') + c.execute("drop table test_aggregates") def test_single_tuple(self): """ test a single tuple """ conn = self.connect() c = conn.cursor() self.safe_create_table( - conn, 'mystuff', - "create table mystuff (id integer primary key)") + conn, "mystuff", "create table mystuff (id integer primary key)" + ) c.execute("insert into mystuff (id) values (1)") c.execute("insert into mystuff (id) values (2)") c.execute("select id from mystuff where id in %s", ((1,),)) @@ -256,12 +289,16 @@ def test_json(self): if not self.mysql_server_is(conn, (5, 7, 0)): pytest.skip("JSON type is not supported on MySQL <= 5.6") - self.safe_create_table(conn, "test_json", """\ + self.safe_create_table( + conn, + "test_json", + """\ create table test_json ( id int not null, json JSON not null, primary key (id) -);""") +);""", + ) cur = conn.cursor() json_str = u'{"hello": "こんãĢãĄã¯"}' @@ -285,7 +322,10 @@ def setUp(self): c = conn.cursor(self.cursor_type) # create a table ane some data to query - self.safe_create_table(conn, 'bulkinsert', """\ + self.safe_create_table( + conn, + "bulkinsert", + """\ CREATE TABLE bulkinsert ( id int, @@ -294,7 +334,8 @@ def setUp(self): height int, PRIMARY KEY (id) ) -""") +""", + ) def _verify_records(self, data): conn = self.connect() @@ -308,27 +349,38 @@ def test_bulk_insert(self): cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] - cursor.executemany("insert into bulkinsert (id, name, age, height) " - "values (%s,%s,%s,%s)", data) + cursor.executemany( + "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + data, + ) self.assertEqual( - cursor._last_executed, bytearray( - b"insert into bulkinsert (id, name, age, height) values " - b"(0,'bob',21,123),(1,'jim',56,45),(2,'fred',100,180)")) - cursor.execute('commit') + cursor._last_executed, + bytearray( + b"insert into bulkinsert (id, name, age, height) values " + b"(0,'bob',21,123),(1,'jim',56,45),(2,'fred',100,180)" + ), + ) + cursor.execute("commit") self._verify_records(data) def test_bulk_insert_multiline_statement(self): conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] - cursor.executemany("""insert + cursor.executemany( + """insert into bulkinsert (id, name, age, height) values (%s, %s , %s, %s ) - """, data) - self.assertEqual(cursor._last_executed.strip(), bytearray(b"""insert + """, + data, + ) + self.assertEqual( + cursor._last_executed.strip(), + bytearray( + b"""insert into bulkinsert (id, name, age, height) values (0, @@ -337,17 +389,21 @@ def test_bulk_insert_multiline_statement(self): 'jim' , 56, 45 ),(2, 'fred' , 100, -180 )""")) - cursor.execute('commit') +180 )""" + ), + ) + cursor.execute("commit") self._verify_records(data) def test_bulk_insert_single_record(self): conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123)] - cursor.executemany("insert into bulkinsert (id, name, age, height) " - "values (%s,%s,%s,%s)", data) - cursor.execute('commit') + cursor.executemany( + "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + data, + ) + cursor.execute("commit") self._verify_records(data) def test_issue_288(self): @@ -355,15 +411,21 @@ def test_issue_288(self): conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] - cursor.executemany("""insert + cursor.executemany( + """insert into bulkinsert (id, name, age, height) values (%s, %s , %s, %s ) on duplicate key update age = values(age) - """, data) - self.assertEqual(cursor._last_executed.strip(), bytearray(b"""insert + """, + data, + ) + self.assertEqual( + cursor._last_executed.strip(), + bytearray( + b"""insert into bulkinsert (id, name, age, height) values (0, @@ -373,6 +435,8 @@ def test_issue_288(self): 45 ),(2, 'fred' , 100, 180 ) on duplicate key update -age = values(age)""")) - cursor.execute('commit') +age = values(age)""" + ), + ) + cursor.execute("commit") self._verify_records(data) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index db36c3e69..abd30e0ba 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -54,34 +54,37 @@ class TestAuthentication(base.PyMySQLTestCase): sha256_password_found = False import os - osuser = os.environ.get('USER') + + osuser = os.environ.get("USER") # socket auth requires the current user and for the connection to be a socket # rest do grants @localhost due to incomplete logic - TODO change to @% then db = base.PyMySQLTestCase.databases[0].copy() - socket_auth = db.get('unix_socket') is not None \ - and db.get('host') in ('localhost', '127.0.0.1') + socket_auth = db.get("unix_socket") is not None and db.get("host") in ( + "localhost", + "127.0.0.1", + ) cur = pymysql.connect(**db).cursor() - del db['user'] + del db["user"] cur.execute("SHOW PLUGINS") for r in cur: - if (r[1], r[2]) != (u'ACTIVE', u'AUTHENTICATION'): + if (r[1], r[2]) != (u"ACTIVE", u"AUTHENTICATION"): continue - if r[3] == u'auth_socket.so' or r[0] == u'unix_socket': + if r[3] == u"auth_socket.so" or r[0] == u"unix_socket": socket_plugin_name = r[0] socket_found = True - elif r[3] == u'dialog_examples.so': - if r[0] == 'two_questions': - two_questions_found = True - elif r[0] == 'three_attempts': - three_attempts_found = True - elif r[0] == u'pam': + elif r[3] == u"dialog_examples.so": + if r[0] == "two_questions": + two_questions_found = True + elif r[0] == "three_attempts": + three_attempts_found = True + elif r[0] == u"pam": pam_found = True - pam_plugin_name = r[3].split('.')[0] - if pam_plugin_name == 'auth_pam': - pam_plugin_name = 'pam' + pam_plugin_name = r[3].split(".")[0] + if pam_plugin_name == "auth_pam": + pam_plugin_name = "pam" # MySQL: authentication_pam # https://dev.mysql.com/doc/refman/5.5/en/pam-authentication-plugin.html @@ -89,11 +92,11 @@ class TestAuthentication(base.PyMySQLTestCase): # https://mariadb.com/kb/en/mariadb/pam-authentication-plugin/ # Names differ but functionality is close - elif r[0] == u'mysql_old_password': + elif r[0] == u"mysql_old_password": mysql_old_password_found = True - elif r[0] == u'sha256_password': + elif r[0] == u"sha256_password": sha256_password_found = True - #else: + # else: # print("plugin: %r" % r[0]) def test_plugin(self): @@ -101,9 +104,11 @@ def test_plugin(self): if not self.mysql_server_is(conn, (5, 5, 0)): pytest.skip("MySQL-5.5 required for plugins") cur = conn.cursor() - cur.execute("select plugin from mysql.user where concat(user, '@', host)=current_user()") + cur.execute( + "select plugin from mysql.user where concat(user, '@', host)=current_user()" + ) for r in cur: - self.assertIn(conn._auth_plugin_name, (r[0], 'mysql_native_password')) + self.assertIn(conn._auth_plugin_name, (r[0], "mysql_native_password")) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif(socket_found, reason="socket plugin already installed") @@ -113,17 +118,17 @@ def testSocketAuthInstallPlugin(self): try: cur.execute("install plugin auth_socket soname 'auth_socket.so'") TestAuthentication.socket_found = True - self.socket_plugin_name = 'auth_socket' + self.socket_plugin_name = "auth_socket" self.realtestSocketAuth() except pymysql.err.InternalError: try: cur.execute("install soname 'auth_socket'") TestAuthentication.socket_found = True - self.socket_plugin_name = 'unix_socket' + self.socket_plugin_name = "unix_socket" self.realtestSocketAuth() except pymysql.err.InternalError: TestAuthentication.socket_found = False - pytest.skip('we couldn\'t install the socket plugin') + pytest.skip("we couldn't install the socket plugin") finally: if TestAuthentication.socket_found: cur.execute("uninstall plugin %s" % self.socket_plugin_name) @@ -134,27 +139,30 @@ def testSocketAuth(self): self.realtestSocketAuth() def realtestSocketAuth(self): - with TempUser(self.connect().cursor(), TestAuthentication.osuser + '@localhost', - self.databases[0]['db'], self.socket_plugin_name) as u: + with TempUser( + self.connect().cursor(), + TestAuthentication.osuser + "@localhost", + self.databases[0]["db"], + self.socket_plugin_name, + ) as u: c = pymysql.connect(user=TestAuthentication.osuser, **self.db) class Dialog: - fail=False + fail = False def __init__(self, con): - self.fail=TestAuthentication.Dialog.fail + self.fail = TestAuthentication.Dialog.fail pass def prompt(self, echo, prompt): if self.fail: - self.fail=False - return b'bad guess at a password' + self.fail = False + return b"bad guess at a password" return self.m.get(prompt) class DialogHandler: - def __init__(self, con): - self.con=con + self.con = con def authenticate(self, pkt): while True: @@ -163,10 +171,10 @@ def authenticate(self, pkt): last = (flag & 0x01) == 0x01 prompt = pkt.read_all() - if prompt == b'Password, please:': - self.con.write_packet(b'stillnotverysecret\0') + if prompt == b"Password, please:": + self.con.write_packet(b"stillnotverysecret\0") else: - self.con.write_packet(b'no idea what to do with this prompt\0') + self.con.write_packet(b"no idea what to do with this prompt\0") pkt = self.con._read_packet() pkt.check_error() if pkt.is_ok_packet() or last: @@ -175,11 +183,12 @@ def authenticate(self, pkt): class DefectiveHandler: def __init__(self, con): - self.con=con - + self.con = con @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(two_questions_found, reason="two_questions plugin already installed") + @pytest.mark.skipif( + two_questions_found, reason="two_questions plugin already installed" + ) def testDialogAuthTwoQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -188,7 +197,7 @@ def testDialogAuthTwoQuestionsInstallPlugin(self): TestAuthentication.two_questions_found = True self.realTestDialogAuthTwoQuestions() except pymysql.err.InternalError: - pytest.skip('we couldn\'t install the two_questions plugin') + pytest.skip("we couldn't install the two_questions plugin") finally: if TestAuthentication.two_questions_found: cur.execute("uninstall plugin two_questions") @@ -199,17 +208,30 @@ def testDialogAuthTwoQuestions(self): self.realTestDialogAuthTwoQuestions() def realTestDialogAuthTwoQuestions(self): - TestAuthentication.Dialog.fail=False - TestAuthentication.Dialog.m = {b'Password, please:': b'notverysecret', - b'Are you sure ?': b'yes, of course'} - with TempUser(self.connect().cursor(), 'pymysql_2q@localhost', - self.databases[0]['db'], 'two_questions', 'notverysecret') as u: + TestAuthentication.Dialog.fail = False + TestAuthentication.Dialog.m = { + b"Password, please:": b"notverysecret", + b"Are you sure ?": b"yes, of course", + } + with TempUser( + self.connect().cursor(), + "pymysql_2q@localhost", + self.databases[0]["db"], + "two_questions", + "notverysecret", + ) as u: with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_2q', **self.db) - pymysql.connect(user='pymysql_2q', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) + pymysql.connect(user="pymysql_2q", **self.db) + pymysql.connect( + user="pymysql_2q", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(three_attempts_found, reason="three_attempts plugin already installed") + @pytest.mark.skipif( + three_attempts_found, reason="three_attempts plugin already installed" + ) def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -218,7 +240,7 @@ def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): TestAuthentication.three_attempts_found = True self.realTestDialogAuthThreeAttempts() except pymysql.err.InternalError: - pytest.skip('we couldn\'t install the three_attempts plugin') + pytest.skip("we couldn't install the three_attempts plugin") finally: if TestAuthentication.three_attempts_found: cur.execute("uninstall plugin three_attempts") @@ -229,30 +251,67 @@ def testDialogAuthThreeAttempts(self): self.realTestDialogAuthThreeAttempts() def realTestDialogAuthThreeAttempts(self): - TestAuthentication.Dialog.m = {b'Password, please:': b'stillnotverysecret'} - TestAuthentication.Dialog.fail=True # fail just once. We've got three attempts after all - with TempUser(self.connect().cursor(), 'pymysql_3a@localhost', - self.databases[0]['db'], 'three_attempts', 'stillnotverysecret') as u: - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.DialogHandler}, **self.db) + TestAuthentication.Dialog.m = {b"Password, please:": b"stillnotverysecret"} + TestAuthentication.Dialog.fail = ( + True # fail just once. We've got three attempts after all + ) + with TempUser( + self.connect().cursor(), + "pymysql_3a@localhost", + self.databases[0]["db"], + "three_attempts", + "stillnotverysecret", + ) as u: + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.DialogHandler}, + **self.db + ) with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': object}, **self.db) + pymysql.connect( + user="pymysql_3a", auth_plugin_map={b"dialog": object}, **self.db + ) with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.DefectiveHandler}, **self.db) + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.DefectiveHandler}, + **self.db + ) with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'notdialogplugin': TestAuthentication.Dialog}, **self.db) - TestAuthentication.Dialog.m = {b'Password, please:': b'I do not know'} + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"notdialogplugin": TestAuthentication.Dialog}, + **self.db + ) + TestAuthentication.Dialog.m = {b"Password, please:": b"I do not know"} with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - TestAuthentication.Dialog.m = {b'Password, please:': None} + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) + TestAuthentication.Dialog.m = {b"Password, please:": None} with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif(pam_found, reason="pam plugin already installed") - @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") - @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") + @pytest.mark.skipif( + os.environ.get("PASSWORD") is None, reason="PASSWORD env var required" + ) + @pytest.mark.skipif( + os.environ.get("PAMSERVICE") is None, reason="PAMSERVICE env var required" + ) def testPamAuthInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -261,133 +320,162 @@ def testPamAuthInstallPlugin(self): TestAuthentication.pam_found = True self.realTestPamAuth() except pymysql.err.InternalError: - pytest.skip('we couldn\'t install the auth_pam plugin') + pytest.skip("we couldn't install the auth_pam plugin") finally: if TestAuthentication.pam_found: cur.execute("uninstall plugin pam") - @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif(not pam_found, reason="no pam plugin") - @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") - @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") + @pytest.mark.skipif( + os.environ.get("PASSWORD") is None, reason="PASSWORD env var required" + ) + @pytest.mark.skipif( + os.environ.get("PAMSERVICE") is None, reason="PAMSERVICE env var required" + ) def testPamAuth(self): self.realTestPamAuth() def realTestPamAuth(self): db = self.db.copy() import os - db['password'] = os.environ.get('PASSWORD') + + db["password"] = os.environ.get("PASSWORD") cur = self.connect().cursor() try: - cur.execute('show grants for ' + TestAuthentication.osuser + '@localhost') + cur.execute("show grants for " + TestAuthentication.osuser + "@localhost") grants = cur.fetchone()[0] - cur.execute('drop user ' + TestAuthentication.osuser + '@localhost') + cur.execute("drop user " + TestAuthentication.osuser + "@localhost") except pymysql.OperationalError as e: # assuming the user doesn't exist which is ok too self.assertEqual(1045, e.args[0]) grants = None - with TempUser(cur, TestAuthentication.osuser + '@localhost', - self.databases[0]['db'], 'pam', os.environ.get('PAMSERVICE')) as u: + with TempUser( + cur, + TestAuthentication.osuser + "@localhost", + self.databases[0]["db"], + "pam", + os.environ.get("PAMSERVICE"), + ) as u: try: c = pymysql.connect(user=TestAuthentication.osuser, **db) - db['password'] = 'very bad guess at password' + db["password"] = "very bad guess at password" with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user=TestAuthentication.osuser, - auth_plugin_map={b'mysql_cleartext_password': TestAuthentication.DefectiveHandler}, - **self.db) + pymysql.connect( + user=TestAuthentication.osuser, + auth_plugin_map={ + b"mysql_cleartext_password": TestAuthentication.DefectiveHandler + }, + **self.db + ) except pymysql.OperationalError as e: self.assertEqual(1045, e.args[0]) # we had 'bad guess at password' work with pam. Well at least we get a permission denied here with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user=TestAuthentication.osuser, - auth_plugin_map={b'mysql_cleartext_password': TestAuthentication.DefectiveHandler}, - **self.db) + pymysql.connect( + user=TestAuthentication.osuser, + auth_plugin_map={ + b"mysql_cleartext_password": TestAuthentication.DefectiveHandler + }, + **self.db + ) if grants: # recreate the user cur.execute(grants) # select old_password("crummy p\tassword"); - #| old_password("crummy p\tassword") | - #| 2a01785203b08770 | + # | old_password("crummy p\tassword") | + # | 2a01785203b08770 | @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(not mysql_old_password_found, reason="no mysql_old_password plugin") + @pytest.mark.skipif( + not mysql_old_password_found, reason="no mysql_old_password plugin" + ) def testMySQLOldPasswordAuth(self): conn = self.connect() if self.mysql_server_is(conn, (5, 7, 0)): - pytest.skip('Old passwords aren\'t supported in 5.7') + pytest.skip("Old passwords aren't supported in 5.7") # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") # from login in MySQL-5.6 if self.mysql_server_is(conn, (5, 6, 0)): - pytest.skip('Old passwords don\'t authenticate in 5.6') + pytest.skip("Old passwords don't authenticate in 5.6") db = self.db.copy() - db['password'] = "crummy p\tassword" + db["password"] = "crummy p\tassword" c = conn.cursor() # deprecated in 5.6 - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) + c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) else: - c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) + c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) v = c.fetchone()[0] - self.assertEqual(v, '2a01785203b08770') + self.assertEqual(v, "2a01785203b08770") # only works in MariaDB and MySQL-5.6 - can't separate out by version - #if self.mysql_server_is(self.connect(), (5, 5, 0)): + # if self.mysql_server_is(self.connect(), (5, 5, 0)): # with TempUser(c, 'old_pass_user@localhost', # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: # cur = pymysql.connect(user='old_pass_user', **db).cursor() # cur.execute("SELECT VERSION()") c.execute("SELECT @@secure_auth") secure_auth_setting = c.fetchone()[0] - c.execute('set old_passwords=1') + c.execute("set old_passwords=1") # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: - c.execute('set global secure_auth=0') + c.execute("set global secure_auth=0") else: - c.execute('set global secure_auth=0') - with TempUser(c, 'old_pass_user@localhost', - self.databases[0]['db'], password=db['password']) as u: - cur = pymysql.connect(user='old_pass_user', **db).cursor() + c.execute("set global secure_auth=0") + with TempUser( + c, + "old_pass_user@localhost", + self.databases[0]["db"], + password=db["password"], + ) as u: + cur = pymysql.connect(user="old_pass_user", **db).cursor() cur.execute("SELECT VERSION()") - c.execute('set global secure_auth=%r' % secure_auth_setting) + c.execute("set global secure_auth=%r" % secure_auth_setting) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(not sha256_password_found, reason="no sha256 password authentication plugin found") + @pytest.mark.skipif( + not sha256_password_found, + reason="no sha256 password authentication plugin found", + ) def testAuthSHA256(self): conn = self.connect() c = conn.cursor() - with TempUser(c, 'pymysql_sha256@localhost', - self.databases[0]['db'], 'sha256_password') as u: + with TempUser( + c, "pymysql_sha256@localhost", self.databases[0]["db"], "sha256_password" + ) as u: if self.mysql_server_is(conn, (5, 7, 0)): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") else: - c.execute('SET old_passwords = 2') - c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' = PASSWORD('Sh@256Pa33')") + c.execute("SET old_passwords = 2") + c.execute( + "SET PASSWORD FOR 'pymysql_sha256'@'localhost' = PASSWORD('Sh@256Pa33')" + ) c.execute("FLUSH PRIVILEGES") db = self.db.copy() - db['password'] = "Sh@256Pa33" - # Although SHA256 is supported, need the configuration of public key of the mysql server. Currently will get error by this test. + db["password"] = "Sh@256Pa33" + # Although SHA256 is supported, need the configuration of public key of the mysql server. Currently will get error by this test. with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_sha256', **db) + pymysql.connect(user="pymysql_sha256", **db) -class TestConnection(base.PyMySQLTestCase): +class TestConnection(base.PyMySQLTestCase): def test_utf8mb4(self): """This test requires MySQL >= 5.5""" arg = self.databases[0].copy() - arg['charset'] = 'utf8mb4' + arg["charset"] = "utf8mb4" conn = pymysql.connect(**arg) def test_largedata(self): """Large query and response (>=16MB)""" cur = self.connect().cursor() cur.execute("SELECT @@max_allowed_packet") - if cur.fetchone()[0] < 16*1024*1024 + 10: + if cur.fetchone()[0] < 16 * 1024 * 1024 + 10: print("Set max_allowed_packet to bigger than 17MB") return - t = 'a' * (16*1024*1024) + t = "a" * (16 * 1024 * 1024) cur.execute("SELECT '" + t + "'") assert cur.fetchone()[0] == t @@ -406,15 +494,15 @@ def test_autocommit(self): def test_select_db(self): con = self.connect() - current_db = self.databases[0]['db'] - other_db = self.databases[1]['db'] + current_db = self.databases[0]["db"] + other_db = self.databases[1]["db"] cur = con.cursor() - cur.execute('SELECT database()') + cur.execute("SELECT database()") self.assertEqual(cur.fetchone()[0], current_db) con.select_db(other_db) - cur.execute('SELECT database()') + cur.execute("SELECT database()") self.assertEqual(cur.fetchone()[0], other_db) def test_connection_gone_away(self): @@ -429,29 +517,30 @@ def test_connection_gone_away(self): with self.assertRaises(pymysql.OperationalError) as cm: cur.execute("SELECT 1+1") # error occures while reading, not writing because of socket buffer. - #self.assertEqual(cm.exception.args[0], 2006) + # self.assertEqual(cm.exception.args[0], 2006) self.assertIn(cm.exception.args[0], (2006, 2013)) def test_init_command(self): conn = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) c = conn.cursor() c.execute('select "foobar";') - self.assertEqual(('foobar',), c.fetchone()) + self.assertEqual(("foobar",), c.fetchone()) conn.close() with self.assertRaises(pymysql.err.Error): conn.ping(reconnect=False) def test_read_default_group(self): conn = self.connect( - read_default_group='client', + read_default_group="client", ) self.assertTrue(conn.open) def test_set_charset(self): c = self.connect() - c.set_charset('utf8mb4') + c.set_charset("utf8mb4") # TODO validate setting here def test_defer_connect(self): @@ -460,12 +549,13 @@ def test_defer_connect(self): d = self.databases[0].copy() try: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(d['unix_socket']) + sock.connect(d["unix_socket"]) except KeyError: sock.close() sock = socket.create_connection( - (d.get('host', 'localhost'), d.get('port', 3306))) - for k in ['unix_socket', 'host', 'port']: + (d.get("host", "localhost"), d.get("port", 3306)) + ) + for k in ["unix_socket", "host", "port"]: try: del d[k] except KeyError: @@ -479,9 +569,12 @@ def test_defer_connect(self): def test_ssl_connect(self): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl={ "ca": "ca", @@ -497,9 +590,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_called_with("cipher") dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl={ "ca": "ca", @@ -514,9 +610,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca="ca", ) @@ -527,9 +626,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca="ca", ssl_cert="cert", @@ -543,9 +645,12 @@ def test_ssl_connect(self): for ssl_verify_cert in (True, "1", "yes", "true"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_cert="cert", ssl_key="key", @@ -554,14 +659,19 @@ def test_ssl_connect(self): assert create_default_context.called assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", keyfile="key" + ) dummy_ssl_context.set_ciphers.assert_not_called for ssl_verify_cert in (None, False, "0", "no", "false"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_cert="cert", ssl_key="key", @@ -570,15 +680,20 @@ def test_ssl_connect(self): assert create_default_context.called assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", keyfile="key" + ) dummy_ssl_context.set_ciphers.assert_not_called for ssl_ca in ("ca", None): for ssl_verify_cert in ("foo", "bar", ""): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca=ssl_ca, ssl_cert="cert", @@ -587,14 +702,21 @@ def test_ssl_connect(self): ) assert create_default_context.called assert not dummy_ssl_context.check_hostname - assert dummy_ssl_context.verify_mode == (ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE), (ssl_ca, ssl_verify_cert) - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + assert dummy_ssl_context.verify_mode == ( + ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE + ), (ssl_ca, ssl_verify_cert) + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", keyfile="key" + ) dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca="ca", ssl_cert="cert", @@ -608,9 +730,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_disabled=True, ssl={ @@ -622,9 +747,12 @@ def test_ssl_connect(self): assert not create_default_context.called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_disabled=True, ssl_ca="ca", @@ -679,7 +807,7 @@ class Custom(str): pass mapping = {str: pymysql.escape_string} - self.assertEqual(con.escape(Custom('foobar'), mapping), "'foobar'") + self.assertEqual(con.escape(Custom("foobar"), mapping), "'foobar'") def test_escape_no_default(self): con = self.connect() @@ -693,7 +821,7 @@ def test_escape_dict_value(self): mapping = con.encoders.copy() mapping[Foo] = escape_foo - self.assertEqual(con.escape({'foo': Foo()}, mapping), {'foo': "bar"}) + self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) def test_escape_list_item(self): con = self.connect() @@ -706,7 +834,8 @@ def test_escape_list_item(self): def test_previous_cursor_not_closed(self): con = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) cur1 = con.cursor() cur1.execute("SELECT 1; SELECT 2") cur2 = con.cursor() diff --git a/pymysql/tests/test_converters.py b/pymysql/tests/test_converters.py index c2c9b6bf1..dc194a9e2 100644 --- a/pymysql/tests/test_converters.py +++ b/pymysql/tests/test_converters.py @@ -7,34 +7,30 @@ class TestConverter(TestCase): - def test_escape_string(self): - self.assertEqual( - converters.escape_string(u"foo\nbar"), - u"foo\\nbar" - ) + self.assertEqual(converters.escape_string(u"foo\nbar"), u"foo\\nbar") def test_convert_datetime(self): expected = datetime.datetime(2007, 2, 24, 23, 6, 20) - dt = converters.convert_datetime('2007-02-24 23:06:20') + dt = converters.convert_datetime("2007-02-24 23:06:20") self.assertEqual(dt, expected) def test_convert_datetime_with_fsp(self): expected = datetime.datetime(2007, 2, 24, 23, 6, 20, 511581) - dt = converters.convert_datetime('2007-02-24 23:06:20.511581') + dt = converters.convert_datetime("2007-02-24 23:06:20.511581") self.assertEqual(dt, expected) def _test_convert_timedelta(self, with_negate=False, with_fsp=False): - d = {'hours': 789, 'minutes': 12, 'seconds': 34} - s = '%(hours)s:%(minutes)s:%(seconds)s' % d + d = {"hours": 789, "minutes": 12, "seconds": 34} + s = "%(hours)s:%(minutes)s:%(seconds)s" % d if with_fsp: - d['microseconds'] = 511581 - s += '.%(microseconds)s' % d + d["microseconds"] = 511581 + s += ".%(microseconds)s" % d expected = datetime.timedelta(**d) if with_negate: expected = -expected - s = '-' + s + s = "-" + s tdelta = converters.convert_timedelta(s) self.assertEqual(tdelta, expected) @@ -49,10 +45,10 @@ def test_convert_timedelta_with_fsp(self): def test_convert_time(self): expected = datetime.time(23, 6, 20) - time_obj = converters.convert_time('23:06:20') + time_obj = converters.convert_time("23:06:20") self.assertEqual(time_obj, expected) def test_convert_time_with_fsp(self): expected = datetime.time(23, 6, 20, 511581) - time_obj = converters.convert_time('23:06:20.511581') + time_obj = converters.convert_time("23:06:20.511581") self.assertEqual(time_obj, expected) diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 4c9174f5b..783caf88b 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -3,6 +3,7 @@ from pymysql.tests import base import pymysql.cursors + class CursorTest(base.PyMySQLTestCase): def setUp(self): super(CursorTest, self).setUp() @@ -10,12 +11,14 @@ def setUp(self): conn = self.connect() self.safe_create_table( conn, - "test", "create table test (data varchar(10))", + "test", + "create table test (data varchar(10))", ) cursor = conn.cursor() cursor.execute( "insert into test (data) values " - "('row1'), ('row2'), ('row3'), ('row4'), ('row5')") + "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" + ) cursor.close() self.test_connection = pymysql.connect(**self.databases[0]) self.addCleanup(self.test_connection.close) @@ -51,55 +54,78 @@ def test_cleanup_rows_buffered(self): c2 = conn.cursor() c2.execute("select 1") - self.assertEqual( - c2.fetchone(), (1,) - ) + self.assertEqual(c2.fetchone(), (1,)) self.assertIsNone(c2.fetchone()) def test_executemany(self): conn = self.test_connection cursor = conn.cursor(pymysql.cursors.Cursor) - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%s, %s)") - self.assertIsNotNone(m, 'error parse %s') - self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%s, %s)" + ) + self.assertIsNotNone(m, "error parse %s") + self.assertEqual(m.group(3), "", "group 3 not blank, bug in RE_INSERT_VALUES?") - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id)s, %(name)s)") - self.assertIsNotNone(m, 'error parse %(name)s') - self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%(id)s, %(name)s)" + ) + self.assertIsNotNone(m, "error parse %(name)s") + self.assertEqual(m.group(3), "", "group 3 not blank, bug in RE_INSERT_VALUES?") - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s)") - self.assertIsNotNone(m, 'error parse %(id_name)s') - self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s)" + ) + self.assertIsNotNone(m, "error parse %(id_name)s") + self.assertEqual(m.group(3), "", "group 3 not blank, bug in RE_INSERT_VALUES?") - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s) ON duplicate update") - self.assertIsNotNone(m, 'error parse %(id_name)s') - self.assertEqual(m.group(3), ' ON duplicate update', 'group 3 not ON duplicate update, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s) ON duplicate update" + ) + self.assertIsNotNone(m, "error parse %(id_name)s") + self.assertEqual( + m.group(3), + " ON duplicate update", + "group 3 not ON duplicate update, bug in RE_INSERT_VALUES?", + ) # https://github.com/PyMySQL/PyMySQL/pull/597 - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO bloup(foo, bar)VALUES(%s, %s)") + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO bloup(foo, bar)VALUES(%s, %s)" + ) assert m is not None # cursor._executed must bee "insert into test (data) values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)" # list args data = range(10) cursor.executemany("insert into test (data) values (%s)", data) - self.assertTrue(cursor._executed.endswith(b",(7),(8),(9)"), 'execute many with %s not in one query') + self.assertTrue( + cursor._executed.endswith(b",(7),(8),(9)"), + "execute many with %s not in one query", + ) # dict args - data_dict = [{'data': i} for i in range(10)] + data_dict = [{"data": i} for i in range(10)] cursor.executemany("insert into test (data) values (%(data)s)", data_dict) - self.assertTrue(cursor._executed.endswith(b",(7),(8),(9)"), 'execute many with %(data)s not in one query') + self.assertTrue( + cursor._executed.endswith(b",(7),(8),(9)"), + "execute many with %(data)s not in one query", + ) # %% in column set - cursor.execute("""\ + cursor.execute( + """\ CREATE TABLE percent_test ( `A%` INTEGER, - `B%` INTEGER)""") + `B%` INTEGER)""" + ) try: q = "INSERT INTO percent_test (`A%%`, `B%%`) VALUES (%s, %s)" self.assertIsNotNone(pymysql.cursors.RE_INSERT_VALUES.match(q)) cursor.executemany(q, [(3, 4), (5, 6)]) - self.assertTrue(cursor._executed.endswith(b"(3, 4),(5, 6)"), "executemany with %% not in one query") + self.assertTrue( + cursor._executed.endswith(b"(3, 4),(5, 6)"), + "executemany with %% not in one query", + ) finally: cursor.execute("DROP TABLE IF EXISTS percent_test") diff --git a/pymysql/tests/test_err.py b/pymysql/tests/test_err.py index bb6a5c499..6b54c6d04 100644 --- a/pymysql/tests/test_err.py +++ b/pymysql/tests/test_err.py @@ -7,9 +7,8 @@ class TestRaiseException(unittest.TestCase): - def test_raise_mysql_exception(self): data = b"\xff\x15\x04#28000Access denied" with self.assertRaises(err.OperationalError) as cm: err.raise_mysql_exception(data) - self.assertEqual(cm.exception.args, (1045, 'Access denied')) + self.assertEqual(cm.exception.args, (1045, "Access denied")) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 2e11ddb5b..95765e544 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -11,6 +11,7 @@ __all__ = ["TestOldIssues", "TestNewIssues", "TestGitHubIssues"] + class TestOldIssues(base.PyMySQLTestCase): def test_issue_3(self): """ undefined methods datetime_or_None, date_or_None """ @@ -21,7 +22,10 @@ def test_issue_3(self): c.execute("drop table if exists issue3") c.execute("create table issue3 (d date, t time, dt datetime, ts timestamp)") try: - c.execute("insert into issue3 (d, t, dt, ts) values (%s,%s,%s,%s)", (None, None, None, None)) + c.execute( + "insert into issue3 (d, t, dt, ts) values (%s,%s,%s,%s)", + (None, None, None, None), + ) c.execute("select d from issue3") self.assertEqual(None, c.fetchone()[0]) c.execute("select t from issue3") @@ -29,7 +33,11 @@ def test_issue_3(self): c.execute("select dt from issue3") self.assertEqual(None, c.fetchone()[0]) c.execute("select ts from issue3") - self.assertIn(type(c.fetchone()[0]), (type(None), datetime.datetime), 'expected Python type None or datetime from SQL timestamp') + self.assertIn( + type(c.fetchone()[0]), + (type(None), datetime.datetime), + "expected Python type None or datetime from SQL timestamp", + ) finally: c.execute("drop table issue3") @@ -58,7 +66,7 @@ def test_issue_6(self): """ exception: TypeError: ord() expected a character, but string of length 0 found """ # ToDo: this test requires access to db 'mysql'. kwargs = self.databases[0].copy() - kwargs['db'] = "mysql" + kwargs["db"] = "mysql" conn = pymysql.connect(**kwargs) c = conn.cursor() c.execute("select * from user") @@ -71,10 +79,12 @@ def test_issue_8(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists test") - c.execute("""CREATE TABLE `test` (`station` int NOT NULL DEFAULT '0', `dh` + c.execute( + """CREATE TABLE `test` (`station` int NOT NULL DEFAULT '0', `dh` datetime NOT NULL DEFAULT '2015-01-01 00:00:00', `echeance` int NOT NULL DEFAULT '0', `me` double DEFAULT NULL, `mo` double DEFAULT NULL, PRIMARY -KEY (`station`,`dh`,`echeance`)) ENGINE=MyISAM DEFAULT CHARSET=latin1;""") +KEY (`station`,`dh`,`echeance`)) ENGINE=MyISAM DEFAULT CHARSET=latin1;""" + ) try: self.assertEqual(0, c.execute("SELECT * FROM test")) c.execute("ALTER TABLE `test` ADD INDEX `idx_station` (`station`)") @@ -92,7 +102,7 @@ def test_issue_13(self): try: cur.execute("create table issue13 (t text)") # ticket says 18k - size = 18*1024 + size = 18 * 1024 cur.execute("insert into issue13 (t) values (%s)", ("x" * size,)) cur.execute("select t from issue13") # use assertTrue so that obscenely huge error messages don't print @@ -110,9 +120,9 @@ def test_issue_15(self): c.execute("drop table if exists issue15") c.execute("create table issue15 (t varchar(32))") try: - c.execute("insert into issue15 (t) values (%s)", (u'\xe4\xf6\xfc',)) + c.execute("insert into issue15 (t) values (%s)", (u"\xe4\xf6\xfc",)) c.execute("select t from issue15") - self.assertEqual(u'\xe4\xf6\xfc', c.fetchone()[0]) + self.assertEqual(u"\xe4\xf6\xfc", c.fetchone()[0]) finally: c.execute("drop table issue15") @@ -123,15 +133,21 @@ def test_issue_16(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists issue16") - c.execute("create table issue16 (name varchar(32) primary key, email varchar(32))") + c.execute( + "create table issue16 (name varchar(32) primary key, email varchar(32))" + ) try: - c.execute("insert into issue16 (name, email) values ('pete', 'floydophone')") + c.execute( + "insert into issue16 (name, email) values ('pete', 'floydophone')" + ) c.execute("select email from issue16 where name=%s", ("pete",)) self.assertEqual("floydophone", c.fetchone()[0]) finally: c.execute("drop table issue16") - @pytest.mark.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") + @pytest.mark.skip( + "test_issue_17() requires a custom, legacy MySQL configuration and will not be run." + ) def test_issue_17(self): """could not connect mysql use passwod""" conn = self.connect() @@ -146,7 +162,10 @@ def test_issue_17(self): c.execute("drop table if exists issue17") c.execute("create table issue17 (x varchar(32) primary key)") c.execute("insert into issue17 (x) values ('hello, world!')") - c.execute("grant all privileges on %s.issue17 to 'issue17user'@'%%' identified by '1234'" % db) + c.execute( + "grant all privileges on %s.issue17 to 'issue17user'@'%%' identified by '1234'" + % db + ) conn.commit() conn2 = pymysql.connect(host=host, user="issue17user", passwd="1234", db=db) @@ -156,6 +175,7 @@ def test_issue_17(self): finally: c.execute("drop table issue17") + class TestNewIssues(base.PyMySQLTestCase): def test_issue_34(self): try: @@ -168,8 +188,9 @@ def test_issue_34(self): def test_issue_33(self): conn = pymysql.connect(charset="utf8", **self.databases[0]) - self.safe_create_table(conn, u'hei\xdfe', - u'create table hei\xdfe (name varchar(32))') + self.safe_create_table( + conn, u"hei\xdfe", u"create table hei\xdfe (name varchar(32))" + ) c = conn.cursor() c.execute(u"insert into hei\xdfe (name) values ('Pi\xdfata')") c.execute(u"select name from hei\xdfe") @@ -233,7 +254,7 @@ def test_issue_37(self): def test_issue_38(self): conn = self.connect() c = conn.cursor() - datum = "a" * 1024 * 1023 # reduced size for most default mysql installs + datum = "a" * 1024 * 1023 # reduced size for most default mysql installs try: with warnings.catch_warnings(): @@ -251,7 +272,7 @@ def disabled_test_issue_54(self): warnings.filterwarnings("ignore") c.execute("drop table if exists issue54") big_sql = "select * from issue54 where " - big_sql += " and ".join("%d=%d" % (i,i) for i in range(0, 100000)) + big_sql += " and ".join("%d=%d" % (i, i) for i in range(0, 100000)) try: c.execute("create table issue54 (id integer primary key)") @@ -261,6 +282,7 @@ def disabled_test_issue_54(self): finally: c.execute("drop table issue54") + class TestGitHubIssues(base.PyMySQLTestCase): def test_issue_66(self): """ 'Connection' object has no attribute 'insert_id' """ @@ -271,7 +293,9 @@ def test_issue_66(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists issue66") - c.execute("create table issue66 (id integer primary key auto_increment, x integer)") + c.execute( + "create table issue66 (id integer primary key auto_increment, x integer)" + ) c.execute("insert into issue66 (x) values (1)") c.execute("insert into issue66 (x) values (1)") self.assertEqual(2, conn.insert_id()) @@ -290,17 +314,17 @@ def test_issue_79(self): c.execute("""CREATE TABLE a (id int, value int)""") c.execute("""CREATE TABLE b (id int, value int)""") - a=(1,11) - b=(1,22) + a = (1, 11) + b = (1, 22) try: c.execute("insert into a values (%s, %s)", a) c.execute("insert into b values (%s, %s)", b) c.execute("SELECT * FROM a inner join b on a.id = b.id") r = c.fetchall()[0] - self.assertEqual(r['id'], 1) - self.assertEqual(r['value'], 11) - self.assertEqual(r['b.value'], 22) + self.assertEqual(r["id"], 1) + self.assertEqual(r["value"], 11) + self.assertEqual(r["b.value"], 22) finally: c.execute("drop table a") c.execute("drop table b") @@ -312,10 +336,12 @@ def test_issue_95(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") cur.execute("DROP PROCEDURE IF EXISTS `foo`") - cur.execute("""CREATE PROCEDURE `foo` () + cur.execute( + """CREATE PROCEDURE `foo` () BEGIN SELECT 1; - END""") + END""" + ) try: cur.execute("""CALL foo()""") cur.execute("""SELECT 1""") @@ -355,40 +381,42 @@ def test_issue_175(self): conn = self.connect() cur = conn.cursor() for length in (200, 300): - columns = ', '.join('c{0} integer'.format(i) for i in range(length)) - sql = 'create table test_field_count ({0})'.format(columns) + columns = ", ".join("c{0} integer".format(i) for i in range(length)) + sql = "create table test_field_count ({0})".format(columns) try: cur.execute(sql) - cur.execute('select * from test_field_count') + cur.execute("select * from test_field_count") assert len(cur.description) == length finally: with warnings.catch_warnings(): warnings.filterwarnings("ignore") - cur.execute('drop table if exists test_field_count') + cur.execute("drop table if exists test_field_count") def test_issue_321(self): """ Test iterable as query argument. """ conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( - conn, "issue321", - "create table issue321 (value_1 varchar(1), value_2 varchar(1))") + conn, + "issue321", + "create table issue321 (value_1 varchar(1), value_2 varchar(1))", + ) sql_insert = "insert into issue321 (value_1, value_2) values (%s, %s)" - sql_dict_insert = ("insert into issue321 (value_1, value_2) " - "values (%(value_1)s, %(value_2)s)") - sql_select = ("select * from issue321 where " - "value_1 in %s and value_2=%s") + sql_dict_insert = ( + "insert into issue321 (value_1, value_2) " + "values (%(value_1)s, %(value_2)s)" + ) + sql_select = "select * from issue321 where " "value_1 in %s and value_2=%s" data = [ - [(u"a", ), u"\u0430"], + [(u"a",), u"\u0430"], [[u"b"], u"\u0430"], - {"value_1": [[u"c"]], "value_2": u"\u0430"} + {"value_1": [[u"c"]], "value_2": u"\u0430"}, ] cur = conn.cursor() self.assertEqual(cur.execute(sql_insert, data[0]), 1) self.assertEqual(cur.execute(sql_insert, data[1]), 1) self.assertEqual(cur.execute(sql_dict_insert, data[2]), 1) - self.assertEqual( - cur.execute(sql_select, [(u"a", u"b", u"c"), u"\u0430"]), 3) + self.assertEqual(cur.execute(sql_select, [(u"a", u"b", u"c"), u"\u0430"]), 3) self.assertEqual(cur.fetchone(), (u"a", u"\u0430")) self.assertEqual(cur.fetchone(), (u"b", u"\u0430")) self.assertEqual(cur.fetchone(), (u"c", u"\u0430")) @@ -397,9 +425,11 @@ def test_issue_364(self): """ Test mixed unicode/binary arguments in executemany. """ conn = pymysql.connect(charset="utf8mb4", **self.databases[0]) self.safe_create_table( - conn, "issue364", + conn, + "issue364", "create table issue364 (value_1 binary(3), value_2 varchar(3)) " - "engine=InnoDB default charset=utf8mb4") + "engine=InnoDB default charset=utf8mb4", + ) sql = "insert into issue364 (value_1, value_2) values (_binary %s, %s)" usql = u"insert into issue364 (value_1, value_2) values (_binary %s, %s)" @@ -427,11 +457,13 @@ def test_issue_363(self): """ Test binary / geometry types. """ conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( - conn, "issue363", + conn, + "issue363", "CREATE TABLE issue363 ( " "id INTEGER PRIMARY KEY, geom LINESTRING NOT NULL /*!80003 SRID 0 */, " "SPATIAL KEY geom (geom)) " - "ENGINE=MyISAM") + "ENGINE=MyISAM", + ) cur = conn.cursor() # From MySQL 5.7, ST_GeomFromText is added and GeomFromText is deprecated. @@ -443,26 +475,32 @@ def test_issue_363(self): geom_from_text = "GeomFromText" geom_as_text = "AsText" geom_as_bin = "AsBinary" - query = ("INSERT INTO issue363 (id, geom) VALUES" - "(1998, %s('LINESTRING(1.1 1.1,2.2 2.2)'))" % geom_from_text) + query = ( + "INSERT INTO issue363 (id, geom) VALUES" + "(1998, %s('LINESTRING(1.1 1.1,2.2 2.2)'))" % geom_from_text + ) cur.execute(query) # select WKT query = "SELECT %s(geom) FROM issue363" % geom_as_text cur.execute(query) row = cur.fetchone() - self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)", )) + self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)",)) # select WKB query = "SELECT %s(geom) FROM issue363" % geom_as_bin cur.execute(query) row = cur.fetchone() - self.assertEqual(row, - (b"\x01\x02\x00\x00\x00\x02\x00\x00\x00" - b"\x9a\x99\x99\x99\x99\x99\xf1?" - b"\x9a\x99\x99\x99\x99\x99\xf1?" - b"\x9a\x99\x99\x99\x99\x99\x01@" - b"\x9a\x99\x99\x99\x99\x99\x01@", )) + self.assertEqual( + row, + ( + b"\x01\x02\x00\x00\x00\x02\x00\x00\x00" + b"\x9a\x99\x99\x99\x99\x99\xf1?" + b"\x9a\x99\x99\x99\x99\x99\xf1?" + b"\x9a\x99\x99\x99\x99\x99\x01@" + b"\x9a\x99\x99\x99\x99\x99\x01@", + ), + ) # select internal binary cur.execute("SELECT geom FROM issue363") diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index 30186e3a5..bb8563055 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -16,8 +16,10 @@ def test_no_file(self): self.assertRaises( OperationalError, c.execute, - ("LOAD DATA LOCAL INFILE 'no_data.txt' INTO TABLE " - "test_load_local fields terminated by ','") + ( + "LOAD DATA LOCAL INFILE 'no_data.txt' INTO TABLE " + "test_load_local fields terminated by ','" + ), ) finally: c.execute("DROP TABLE test_load_local") @@ -28,13 +30,15 @@ def test_load_file(self): conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") - filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'data', - 'load_local_data.txt') + filename = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "data", "load_local_data.txt" + ) try: c.execute( - ("LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + - "test_load_local FIELDS TERMINATED BY ','").format(filename) + ( + "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + + "test_load_local FIELDS TERMINATED BY ','" + ).format(filename) ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -46,13 +50,15 @@ def test_unbuffered_load_file(self): conn = self.connect() c = conn.cursor(cursors.SSCursor) c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") - filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'data', - 'load_local_data.txt') + filename = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "data", "load_local_data.txt" + ) try: c.execute( - ("LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + - "test_load_local FIELDS TERMINATED BY ','").format(filename) + ( + "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + + "test_load_local FIELDS TERMINATED BY ','" + ).format(filename) ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -66,4 +72,5 @@ def test_unbuffered_load_file(self): if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index d5467b11b..2679edd55 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -7,11 +7,11 @@ class TestNextset(base.PyMySQLTestCase): - def test_nextset(self): con = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) cur = con.cursor() cur.execute("SELECT 1; SELECT 2;") self.assertEqual([(1,)], list(cur)) @@ -71,14 +71,14 @@ def test_multi_cursor(self): def test_multi_statement_warnings(self): con = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) cursor = con.cursor() try: - cursor.execute('DROP TABLE IF EXISTS a; ' - 'DROP TABLE IF EXISTS b;') + cursor.execute("DROP TABLE IF EXISTS a; " "DROP TABLE IF EXISTS b;") except TypeError: self.fail() - #TODO: How about SSCursor and nextset? + # TODO: How about SSCursor and nextset? # It's very hard to implement correctly... diff --git a/pymysql/tests/test_optionfile.py b/pymysql/tests/test_optionfile.py index 81bd1fe4b..39bd47c46 100644 --- a/pymysql/tests/test_optionfile.py +++ b/pymysql/tests/test_optionfile.py @@ -3,20 +3,19 @@ from pymysql.optionfile import Parser -__all__ = ['TestParser'] +__all__ = ["TestParser"] -_cfg_file = (r""" +_cfg_file = r""" [default] string = foo quoted = "bar" single_quoted = 'foobar' skip-slave-start -""") +""" class TestParser(TestCase): - def test_string(self): parser = Parser() parser.read_file(StringIO(_cfg_file)) diff --git a/pymysql/tests/thirdparty/__init__.py b/pymysql/tests/thirdparty/__init__.py index 7a613478d..d5f053711 100644 --- a/pymysql/tests/thirdparty/__init__.py +++ b/pymysql/tests/thirdparty/__init__.py @@ -2,4 +2,5 @@ if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py index e4237c69a..57c42ce7a 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py @@ -4,4 +4,5 @@ if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index e261a78eb..ffead0caf 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -22,7 +22,7 @@ def setUp(self): db = self.db_module.connect(*self.connect_args, **self.connect_kwargs) self.connection = db self.cursor = db.cursor() - self.BLOBText = ''.join([chr(i) for i in range(256)] * 100); + self.BLOBText = "".join([chr(i) for i in range(256)] * 100) self.BLOBUText = "".join(chr(i) for i in range(16834)) data = bytearray(range(256)) * 16 self.BLOBBinary = self.db_module.Binary(data) @@ -32,17 +32,22 @@ def setUp(self): def tearDown(self): if self.leak_test: import gc + del self.cursor orphans = gc.collect() - self.assertFalse(orphans, "%d orphaned objects found after deleting cursor" % orphans) + self.assertFalse( + orphans, "%d orphaned objects found after deleting cursor" % orphans + ) del self.connection orphans = gc.collect() - self.assertFalse(orphans, "%d orphaned objects found after deleting connection" % orphans) + self.assertFalse( + orphans, "%d orphaned objects found after deleting connection" % orphans + ) def table_exists(self, name): try: - self.cursor.execute('select * from %s where 1=0' % name) + self.cursor.execute("select * from %s where 1=0" % name) except Exception: return False else: @@ -54,7 +59,7 @@ def quote_identifier(self, ident): def new_table_name(self): i = id(self.cursor) while True: - name = self.quote_identifier('tb%08x' % i) + name = self.quote_identifier("tb%08x" % i) if not self.table_exists(name): return name i = i + 1 @@ -68,25 +73,27 @@ def create_table(self, columndefs): into the table. """ self.table = self.new_table_name() - self.cursor.execute('CREATE TABLE %s (%s) %s' % - (self.table, - ',\n'.join(columndefs), - self.create_table_extra)) + self.cursor.execute( + "CREATE TABLE %s (%s) %s" + % (self.table, ",\n".join(columndefs), self.create_table_extra) + ) def check_data_integrity(self, columndefs, generator): # insert self.create_table(columndefs) - insert_statement = ('INSERT INTO %s VALUES (%s)' % - (self.table, - ','.join(['%s'] * len(columndefs)))) - data = [ [ generator(i,j) for j in range(len(columndefs)) ] - for i in range(self.rows) ] + insert_statement = "INSERT INTO %s VALUES (%s)" % ( + self.table, + ",".join(["%s"] * len(columndefs)), + ) + data = [ + [generator(i, j) for j in range(len(columndefs))] for i in range(self.rows) + ] if self.debug: print(data) self.cursor.executemany(insert_statement, data) self.connection.commit() # verify - self.cursor.execute('select * from %s' % self.table) + self.cursor.execute("select * from %s" % self.table) l = self.cursor.fetchall() if self.debug: print(l) @@ -94,62 +101,74 @@ def check_data_integrity(self, columndefs, generator): try: for i in range(self.rows): for j in range(len(columndefs)): - self.assertEqual(l[i][j], generator(i,j)) + self.assertEqual(l[i][j], generator(i, j)) finally: if not self.debug: - self.cursor.execute('drop table %s' % (self.table)) + self.cursor.execute("drop table %s" % (self.table)) def test_transactions(self): - columndefs = ( 'col1 INT', 'col2 VARCHAR(255)') + columndefs = ("col1 INT", "col2 VARCHAR(255)") + def generator(row, col): - if col == 0: return row - else: return ('%i' % (row%10))*255 + if col == 0: + return row + else: + return ("%i" % (row % 10)) * 255 + self.create_table(columndefs) - insert_statement = ('INSERT INTO %s VALUES (%s)' % - (self.table, - ','.join(['%s'] * len(columndefs)))) - data = [ [ generator(i,j) for j in range(len(columndefs)) ] - for i in range(self.rows) ] + insert_statement = "INSERT INTO %s VALUES (%s)" % ( + self.table, + ",".join(["%s"] * len(columndefs)), + ) + data = [ + [generator(i, j) for j in range(len(columndefs))] for i in range(self.rows) + ] self.cursor.executemany(insert_statement, data) # verify self.connection.commit() - self.cursor.execute('select * from %s' % self.table) + self.cursor.execute("select * from %s" % self.table) l = self.cursor.fetchall() self.assertEqual(len(l), self.rows) for i in range(self.rows): for j in range(len(columndefs)): - self.assertEqual(l[i][j], generator(i,j)) - delete_statement = 'delete from %s where col1=%%s' % self.table + self.assertEqual(l[i][j], generator(i, j)) + delete_statement = "delete from %s where col1=%%s" % self.table self.cursor.execute(delete_statement, (0,)) - self.cursor.execute('select col1 from %s where col1=%s' % \ - (self.table, 0)) + self.cursor.execute("select col1 from %s where col1=%s" % (self.table, 0)) l = self.cursor.fetchall() self.assertFalse(l, "DELETE didn't work") self.connection.rollback() - self.cursor.execute('select col1 from %s where col1=%s' % \ - (self.table, 0)) + self.cursor.execute("select col1 from %s where col1=%s" % (self.table, 0)) l = self.cursor.fetchall() self.assertTrue(len(l) == 1, "ROLLBACK didn't work") - self.cursor.execute('drop table %s' % (self.table)) + self.cursor.execute("drop table %s" % (self.table)) def test_truncation(self): - columndefs = ( 'col1 INT', 'col2 VARCHAR(255)') + columndefs = ("col1 INT", "col2 VARCHAR(255)") + def generator(row, col): - if col == 0: return row - else: return ('%i' % (row%10))*((255-self.rows//2)+row) + if col == 0: + return row + else: + return ("%i" % (row % 10)) * ((255 - self.rows // 2) + row) + self.create_table(columndefs) - insert_statement = ('INSERT INTO %s VALUES (%s)' % - (self.table, - ','.join(['%s'] * len(columndefs)))) + insert_statement = "INSERT INTO %s VALUES (%s)" % ( + self.table, + ",".join(["%s"] * len(columndefs)), + ) try: - self.cursor.execute(insert_statement, (0, '0'*256)) + self.cursor.execute(insert_statement, (0, "0" * 256)) except Warning: - if self.debug: print(self.cursor.messages) + if self.debug: + print(self.cursor.messages) except self.connection.DataError: pass else: - self.fail("Over-long column did not generate warnings/exception with single insert") + self.fail( + "Over-long column did not generate warnings/exception with single insert" + ) self.connection.rollback() @@ -157,132 +176,136 @@ def generator(row, col): for i in range(self.rows): data = [] for j in range(len(columndefs)): - data.append(generator(i,j)) - self.cursor.execute(insert_statement,tuple(data)) + data.append(generator(i, j)) + self.cursor.execute(insert_statement, tuple(data)) except Warning: - if self.debug: print(self.cursor.messages) + if self.debug: + print(self.cursor.messages) except self.connection.DataError: pass else: - self.fail("Over-long columns did not generate warnings/exception with execute()") + self.fail( + "Over-long columns did not generate warnings/exception with execute()" + ) self.connection.rollback() try: - data = [ [ generator(i,j) for j in range(len(columndefs)) ] - for i in range(self.rows) ] + data = [ + [generator(i, j) for j in range(len(columndefs))] + for i in range(self.rows) + ] self.cursor.executemany(insert_statement, data) except Warning: - if self.debug: print(self.cursor.messages) + if self.debug: + print(self.cursor.messages) except self.connection.DataError: pass else: - self.fail("Over-long columns did not generate warnings/exception with executemany()") + self.fail( + "Over-long columns did not generate warnings/exception with executemany()" + ) self.connection.rollback() - self.cursor.execute('drop table %s' % (self.table)) + self.cursor.execute("drop table %s" % (self.table)) def test_CHAR(self): # Character data - def generator(row,col): - return ('%i' % ((row+col) % 10)) * 255 - self.check_data_integrity( - ('col1 char(255)','col2 char(255)'), - generator) + def generator(row, col): + return ("%i" % ((row + col) % 10)) * 255 + + self.check_data_integrity(("col1 char(255)", "col2 char(255)"), generator) def test_INT(self): # Number data - def generator(row,col): - return row*row - self.check_data_integrity( - ('col1 INT',), - generator) + def generator(row, col): + return row * row + + self.check_data_integrity(("col1 INT",), generator) def test_DECIMAL(self): # DECIMAL - def generator(row,col): + def generator(row, col): from decimal import Decimal + return Decimal("%d.%02d" % (row, col)) - self.check_data_integrity( - ('col1 DECIMAL(5,2)',), - generator) + + self.check_data_integrity(("col1 DECIMAL(5,2)",), generator) def test_DATE(self): ticks = time() - def generator(row,col): - return self.db_module.DateFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 DATE',), - generator) + + def generator(row, col): + return self.db_module.DateFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 DATE",), generator) def test_TIME(self): ticks = time() - def generator(row,col): - return self.db_module.TimeFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 TIME',), - generator) + + def generator(row, col): + return self.db_module.TimeFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 TIME",), generator) def test_DATETIME(self): ticks = time() - def generator(row,col): - return self.db_module.TimestampFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 DATETIME',), - generator) + + def generator(row, col): + return self.db_module.TimestampFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 DATETIME",), generator) def test_TIMESTAMP(self): ticks = time() - def generator(row,col): - return self.db_module.TimestampFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 TIMESTAMP',), - generator) + + def generator(row, col): + return self.db_module.TimestampFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 TIMESTAMP",), generator) def test_fractional_TIMESTAMP(self): ticks = time() - def generator(row,col): - return self.db_module.TimestampFromTicks(ticks+row*86400-col*1313+row*0.7*col/3.0) - self.check_data_integrity( - ('col1 TIMESTAMP',), - generator) + + def generator(row, col): + return self.db_module.TimestampFromTicks( + ticks + row * 86400 - col * 1313 + row * 0.7 * col / 3.0 + ) + + self.check_data_integrity(("col1 TIMESTAMP",), generator) def test_LONG(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBUText # 'BLOB Text ' * 1024 - self.check_data_integrity( - ('col1 INT', 'col2 LONG'), - generator) + return self.BLOBUText # 'BLOB Text ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 LONG"), generator) def test_TEXT(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBUText[:5192] # 'BLOB Text ' * 1024 - self.check_data_integrity( - ('col1 INT', 'col2 TEXT'), - generator) + return self.BLOBUText[:5192] # 'BLOB Text ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 TEXT"), generator) def test_LONG_BYTE(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBBinary # 'BLOB\000Binary ' * 1024 - self.check_data_integrity( - ('col1 INT','col2 LONG BYTE'), - generator) + return self.BLOBBinary # 'BLOB\000Binary ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 LONG BYTE"), generator) def test_BLOB(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBBinary # 'BLOB\000Binary ' * 1024 - self.check_data_integrity( - ('col1 INT','col2 BLOB'), - generator) + return self.BLOBBinary # 'BLOB\000Binary ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 BLOB"), generator) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 1cc202e23..6766aff32 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -1,4 +1,4 @@ -''' Python DB API 2.0 driver compliance unit test suite. +""" Python DB API 2.0 driver compliance unit test suite. This software is Public Domain and may be used without restrictions. @@ -8,11 +8,11 @@ this is turning out to be a thoroughly unwholesome unit test." -- Ian Bicking -''' +""" -__rcs_id__ = '$Id$' -__version__ = '$Revision$'[11:-2] -__author__ = 'Stuart Bishop ' +__rcs_id__ = "$Id$" +__version__ = "$Revision$"[11:-2] +__author__ = "Stuart Bishop " import time import unittest @@ -63,65 +63,66 @@ # - Fix bugs in test_setoutputsize_basic and test_setinputsizes # + class DatabaseAPI20Test(unittest.TestCase): - ''' Test a database self.driver for DB API 2.0 compatibility. - This implementation tests Gadfly, but the TestCase - is structured so that other self.drivers can subclass this - test case to ensure compiliance with the DB-API. It is - expected that this TestCase may be expanded in the future - if ambiguities or edge conditions are discovered. + """Test a database self.driver for DB API 2.0 compatibility. + This implementation tests Gadfly, but the TestCase + is structured so that other self.drivers can subclass this + test case to ensure compiliance with the DB-API. It is + expected that this TestCase may be expanded in the future + if ambiguities or edge conditions are discovered. - The 'Optional Extensions' are not yet being tested. + The 'Optional Extensions' are not yet being tested. - self.drivers should subclass this test, overriding setUp, tearDown, - self.driver, connect_args and connect_kw_args. Class specification - should be as follows: + self.drivers should subclass this test, overriding setUp, tearDown, + self.driver, connect_args and connect_kw_args. Class specification + should be as follows: - import dbapi20 - class mytest(dbapi20.DatabaseAPI20Test): - [...] + import dbapi20 + class mytest(dbapi20.DatabaseAPI20Test): + [...] - Don't 'import DatabaseAPI20Test from dbapi20', or you will - confuse the unit tester - just 'import dbapi20'. - ''' + Don't 'import DatabaseAPI20Test from dbapi20', or you will + confuse the unit tester - just 'import dbapi20'. + """ # The self.driver module. This should be the module where the 'connect' # method is to be found driver = None - connect_args = () # List of arguments to pass to connect - connect_kw_args = {} # Keyword arguments for connect - table_prefix = 'dbapi20test_' # If you need to specify a prefix for tables + connect_args = () # List of arguments to pass to connect + connect_kw_args = {} # Keyword arguments for connect + table_prefix = "dbapi20test_" # If you need to specify a prefix for tables - ddl1 = 'create table %sbooze (name varchar(20))' % table_prefix - ddl2 = 'create table %sbarflys (name varchar(20))' % table_prefix - xddl1 = 'drop table %sbooze' % table_prefix - xddl2 = 'drop table %sbarflys' % table_prefix + ddl1 = "create table %sbooze (name varchar(20))" % table_prefix + ddl2 = "create table %sbarflys (name varchar(20))" % table_prefix + xddl1 = "drop table %sbooze" % table_prefix + xddl2 = "drop table %sbarflys" % table_prefix - lowerfunc = 'lower' # Name of stored procedure to convert string->lowercase + lowerfunc = "lower" # Name of stored procedure to convert string->lowercase # Some drivers may need to override these helpers, for example adding # a 'commit' after the execute. - def executeDDL1(self,cursor): + def executeDDL1(self, cursor): cursor.execute(self.ddl1) - def executeDDL2(self,cursor): + def executeDDL2(self, cursor): cursor.execute(self.ddl2) def setUp(self): - ''' self.drivers should override this method to perform required setup - if any is necessary, such as creating the database. - ''' + """self.drivers should override this method to perform required setup + if any is necessary, such as creating the database. + """ pass def tearDown(self): - ''' self.drivers should override this method to perform required cleanup - if any is necessary, such as deleting the test database. - The default drops the tables that may be created. - ''' + """self.drivers should override this method to perform required cleanup + if any is necessary, such as deleting the test database. + The default drops the tables that may be created. + """ con = self._connect() try: cur = con.cursor() - for ddl in (self.xddl1,self.xddl2): + for ddl in (self.xddl1, self.xddl2): try: cur.execute(ddl) con.commit() @@ -134,9 +135,7 @@ def tearDown(self): def _connect(self): try: - return self.driver.connect( - *self.connect_args,**self.connect_kw_args - ) + return self.driver.connect(*self.connect_args, **self.connect_kw_args) except AttributeError: self.fail("No connect method found in self.driver module") @@ -149,7 +148,7 @@ def test_apilevel(self): # Must exist apilevel = self.driver.apilevel # Must equal 2.0 - self.assertEqual(apilevel,'2.0') + self.assertEqual(apilevel, "2.0") except AttributeError: self.fail("Driver doesn't define apilevel") @@ -158,7 +157,7 @@ def test_threadsafety(self): # Must exist threadsafety = self.driver.threadsafety # Must be a valid value - self.assertTrue(threadsafety in (0,1,2,3)) + self.assertTrue(threadsafety in (0, 1, 2, 3)) except AttributeError: self.fail("Driver doesn't define threadsafety") @@ -167,38 +166,24 @@ def test_paramstyle(self): # Must exist paramstyle = self.driver.paramstyle # Must be a valid value - self.assertTrue(paramstyle in ( - 'qmark','numeric','named','format','pyformat' - )) + self.assertTrue( + paramstyle in ("qmark", "numeric", "named", "format", "pyformat") + ) except AttributeError: self.fail("Driver doesn't define paramstyle") def test_Exceptions(self): # Make sure required exceptions exist, and are in the # defined heirarchy. - self.assertTrue(issubclass(self.driver.Warning,Exception)) - self.assertTrue(issubclass(self.driver.Error,Exception)) - self.assertTrue( - issubclass(self.driver.InterfaceError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.DatabaseError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.OperationalError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.IntegrityError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.InternalError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.ProgrammingError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.NotSupportedError,self.driver.Error) - ) + self.assertTrue(issubclass(self.driver.Warning, Exception)) + self.assertTrue(issubclass(self.driver.Error, Exception)) + self.assertTrue(issubclass(self.driver.InterfaceError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.DatabaseError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.OperationalError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.IntegrityError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.InternalError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.ProgrammingError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.NotSupportedError, self.driver.Error)) def test_ExceptionsAsConnectionAttributes(self): # OPTIONAL EXTENSION @@ -219,7 +204,6 @@ def test_ExceptionsAsConnectionAttributes(self): self.assertTrue(con.ProgrammingError is drv.ProgrammingError) self.assertTrue(con.NotSupportedError is drv.NotSupportedError) - def test_commit(self): con = self._connect() try: @@ -232,7 +216,7 @@ def test_rollback(self): con = self._connect() # If rollback is defined, it should either work or throw # the documented exception - if hasattr(con,'rollback'): + if hasattr(con, "rollback"): try: con.rollback() except self.driver.NotSupportedError: @@ -253,14 +237,14 @@ def test_cursor_isolation(self): cur1 = con.cursor() cur2 = con.cursor() self.executeDDL1(cur1) - cur1.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) + cur1.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) cur2.execute("select name from %sbooze" % self.table_prefix) booze = cur2.fetchall() - self.assertEqual(len(booze),1) - self.assertEqual(len(booze[0]),1) - self.assertEqual(booze[0][0],'Victoria Bitter') + self.assertEqual(len(booze), 1) + self.assertEqual(len(booze[0]), 1) + self.assertEqual(booze[0][0], "Victoria Bitter") finally: con.close() @@ -269,31 +253,41 @@ def test_description(self): try: cur = con.cursor() self.executeDDL1(cur) - self.assertEqual(cur.description,None, - 'cursor.description should be none after executing a ' - 'statement that can return no rows (such as DDL)' - ) - cur.execute('select name from %sbooze' % self.table_prefix) - self.assertEqual(len(cur.description),1, - 'cursor.description describes too many columns' - ) - self.assertEqual(len(cur.description[0]),7, - 'cursor.description[x] tuples must have 7 elements' - ) - self.assertEqual(cur.description[0][0].lower(),'name', - 'cursor.description[x][0] must return column name' - ) - self.assertEqual(cur.description[0][1],self.driver.STRING, - 'cursor.description[x][1] must return column type. Got %r' - % cur.description[0][1] - ) + self.assertEqual( + cur.description, + None, + "cursor.description should be none after executing a " + "statement that can return no rows (such as DDL)", + ) + cur.execute("select name from %sbooze" % self.table_prefix) + self.assertEqual( + len(cur.description), 1, "cursor.description describes too many columns" + ) + self.assertEqual( + len(cur.description[0]), + 7, + "cursor.description[x] tuples must have 7 elements", + ) + self.assertEqual( + cur.description[0][0].lower(), + "name", + "cursor.description[x][0] must return column name", + ) + self.assertEqual( + cur.description[0][1], + self.driver.STRING, + "cursor.description[x][1] must return column type. Got %r" + % cur.description[0][1], + ) # Make sure self.description gets reset self.executeDDL2(cur) - self.assertEqual(cur.description,None, - 'cursor.description not being set to None when executing ' - 'no-result statements (eg. DDL)' - ) + self.assertEqual( + cur.description, + None, + "cursor.description not being set to None when executing " + "no-result statements (eg. DDL)", + ) finally: con.close() @@ -302,47 +296,49 @@ def test_rowcount(self): try: cur = con.cursor() self.executeDDL1(cur) - self.assertEqual(cur.rowcount,-1, - 'cursor.rowcount should be -1 after executing no-result ' - 'statements' - ) - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) - self.assertTrue(cur.rowcount in (-1,1), - 'cursor.rowcount should == number or rows inserted, or ' - 'set to -1 after executing an insert statement' - ) + self.assertEqual( + cur.rowcount, + -1, + "cursor.rowcount should be -1 after executing no-result " "statements", + ) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + self.assertTrue( + cur.rowcount in (-1, 1), + "cursor.rowcount should == number or rows inserted, or " + "set to -1 after executing an insert statement", + ) cur.execute("select name from %sbooze" % self.table_prefix) - self.assertTrue(cur.rowcount in (-1,1), - 'cursor.rowcount should == number of rows returned, or ' - 'set to -1 after executing a select statement' - ) + self.assertTrue( + cur.rowcount in (-1, 1), + "cursor.rowcount should == number of rows returned, or " + "set to -1 after executing a select statement", + ) self.executeDDL2(cur) - self.assertEqual(cur.rowcount,-1, - 'cursor.rowcount not being reset to -1 after executing ' - 'no-result statements' - ) + self.assertEqual( + cur.rowcount, + -1, + "cursor.rowcount not being reset to -1 after executing " + "no-result statements", + ) finally: con.close() - lower_func = 'lower' + lower_func = "lower" + def test_callproc(self): con = self._connect() try: cur = con.cursor() - if self.lower_func and hasattr(cur,'callproc'): - r = cur.callproc(self.lower_func,('FOO',)) - self.assertEqual(len(r),1) - self.assertEqual(r[0],'FOO') + if self.lower_func and hasattr(cur, "callproc"): + r = cur.callproc(self.lower_func, ("FOO",)) + self.assertEqual(len(r), 1) + self.assertEqual(r[0], "FOO") r = cur.fetchall() - self.assertEqual(len(r),1,'callproc produced no result set') - self.assertEqual(len(r[0]),1, - 'callproc produced invalid result set' - ) - self.assertEqual(r[0][0],'foo', - 'callproc produced invalid results' - ) + self.assertEqual(len(r), 1, "callproc produced no result set") + self.assertEqual(len(r[0]), 1, "callproc produced invalid result set") + self.assertEqual(r[0][0], "foo", "callproc produced invalid results") finally: con.close() @@ -355,14 +351,14 @@ def test_close(self): # cursor.execute should raise an Error if called after connection # closed - self.assertRaises(self.driver.Error,self.executeDDL1,cur) + self.assertRaises(self.driver.Error, self.executeDDL1, cur) # connection.commit should raise an Error if called after connection' # closed.' - self.assertRaises(self.driver.Error,con.commit) + self.assertRaises(self.driver.Error, con.commit) # connection.close should raise an Error if called more than once - self.assertRaises(self.driver.Error,con.close) + self.assertRaises(self.driver.Error, con.close) def test_execute(self): con = self._connect() @@ -372,105 +368,99 @@ def test_execute(self): finally: con.close() - def _paraminsert(self,cur): + def _paraminsert(self, cur): self.executeDDL1(cur) - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) - self.assertTrue(cur.rowcount in (-1,1)) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + self.assertTrue(cur.rowcount in (-1, 1)) - if self.driver.paramstyle == 'qmark': + if self.driver.paramstyle == "qmark": cur.execute( - 'insert into %sbooze values (?)' % self.table_prefix, - ("Cooper's",) - ) - elif self.driver.paramstyle == 'numeric': + "insert into %sbooze values (?)" % self.table_prefix, ("Cooper's",) + ) + elif self.driver.paramstyle == "numeric": cur.execute( - 'insert into %sbooze values (:1)' % self.table_prefix, - ("Cooper's",) - ) - elif self.driver.paramstyle == 'named': + "insert into %sbooze values (:1)" % self.table_prefix, ("Cooper's",) + ) + elif self.driver.paramstyle == "named": cur.execute( - 'insert into %sbooze values (:beer)' % self.table_prefix, - {'beer':"Cooper's"} - ) - elif self.driver.paramstyle == 'format': + "insert into %sbooze values (:beer)" % self.table_prefix, + {"beer": "Cooper's"}, + ) + elif self.driver.paramstyle == "format": cur.execute( - 'insert into %sbooze values (%%s)' % self.table_prefix, - ("Cooper's",) - ) - elif self.driver.paramstyle == 'pyformat': + "insert into %sbooze values (%%s)" % self.table_prefix, ("Cooper's",) + ) + elif self.driver.paramstyle == "pyformat": cur.execute( - 'insert into %sbooze values (%%(beer)s)' % self.table_prefix, - {'beer':"Cooper's"} - ) + "insert into %sbooze values (%%(beer)s)" % self.table_prefix, + {"beer": "Cooper's"}, + ) else: - self.fail('Invalid paramstyle') - self.assertTrue(cur.rowcount in (-1,1)) + self.fail("Invalid paramstyle") + self.assertTrue(cur.rowcount in (-1, 1)) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) res = cur.fetchall() - self.assertEqual(len(res),2,'cursor.fetchall returned too few rows') - beers = [res[0][0],res[1][0]] + self.assertEqual(len(res), 2, "cursor.fetchall returned too few rows") + beers = [res[0][0], res[1][0]] beers.sort() - self.assertEqual(beers[0],"Cooper's", - 'cursor.fetchall retrieved incorrect data, or data inserted ' - 'incorrectly' - ) - self.assertEqual(beers[1],"Victoria Bitter", - 'cursor.fetchall retrieved incorrect data, or data inserted ' - 'incorrectly' - ) + self.assertEqual( + beers[0], + "Cooper's", + "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + ) + self.assertEqual( + beers[1], + "Victoria Bitter", + "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + ) def test_executemany(self): con = self._connect() try: cur = con.cursor() self.executeDDL1(cur) - largs = [ ("Cooper's",) , ("Boag's",) ] - margs = [ {'beer': "Cooper's"}, {'beer': "Boag's"} ] - if self.driver.paramstyle == 'qmark': + largs = [("Cooper's",), ("Boag's",)] + margs = [{"beer": "Cooper's"}, {"beer": "Boag's"}] + if self.driver.paramstyle == "qmark": cur.executemany( - 'insert into %sbooze values (?)' % self.table_prefix, - largs - ) - elif self.driver.paramstyle == 'numeric': + "insert into %sbooze values (?)" % self.table_prefix, largs + ) + elif self.driver.paramstyle == "numeric": cur.executemany( - 'insert into %sbooze values (:1)' % self.table_prefix, - largs - ) - elif self.driver.paramstyle == 'named': + "insert into %sbooze values (:1)" % self.table_prefix, largs + ) + elif self.driver.paramstyle == "named": cur.executemany( - 'insert into %sbooze values (:beer)' % self.table_prefix, - margs - ) - elif self.driver.paramstyle == 'format': + "insert into %sbooze values (:beer)" % self.table_prefix, margs + ) + elif self.driver.paramstyle == "format": cur.executemany( - 'insert into %sbooze values (%%s)' % self.table_prefix, - largs - ) - elif self.driver.paramstyle == 'pyformat': + "insert into %sbooze values (%%s)" % self.table_prefix, largs + ) + elif self.driver.paramstyle == "pyformat": cur.executemany( - 'insert into %sbooze values (%%(beer)s)' % ( - self.table_prefix - ), - margs - ) - else: - self.fail('Unknown paramstyle') - self.assertTrue(cur.rowcount in (-1,2), - 'insert using cursor.executemany set cursor.rowcount to ' - 'incorrect value %r' % cur.rowcount + "insert into %sbooze values (%%(beer)s)" % (self.table_prefix), + margs, ) - cur.execute('select name from %sbooze' % self.table_prefix) + else: + self.fail("Unknown paramstyle") + self.assertTrue( + cur.rowcount in (-1, 2), + "insert using cursor.executemany set cursor.rowcount to " + "incorrect value %r" % cur.rowcount, + ) + cur.execute("select name from %sbooze" % self.table_prefix) res = cur.fetchall() - self.assertEqual(len(res),2, - 'cursor.fetchall retrieved incorrect number of rows' - ) - beers = [res[0][0],res[1][0]] + self.assertEqual( + len(res), 2, "cursor.fetchall retrieved incorrect number of rows" + ) + beers = [res[0][0], res[1][0]] beers.sort() - self.assertEqual(beers[0],"Boag's",'incorrect data retrieved') - self.assertEqual(beers[1],"Cooper's",'incorrect data retrieved') + self.assertEqual(beers[0], "Boag's", "incorrect data retrieved") + self.assertEqual(beers[1], "Cooper's", "incorrect data retrieved") finally: con.close() @@ -481,59 +471,62 @@ def test_fetchone(self): # cursor.fetchone should raise an Error if called before # executing a select-type query - self.assertRaises(self.driver.Error,cur.fetchone) + self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows self.executeDDL1(cur) - self.assertRaises(self.driver.Error,cur.fetchone) + self.assertRaises(self.driver.Error, cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) - self.assertEqual(cur.fetchone(),None, - 'cursor.fetchone should return None if a query retrieves ' - 'no rows' - ) - self.assertTrue(cur.rowcount in (-1,0)) + cur.execute("select name from %sbooze" % self.table_prefix) + self.assertEqual( + cur.fetchone(), + None, + "cursor.fetchone should return None if a query retrieves " "no rows", + ) + self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) - self.assertRaises(self.driver.Error,cur.fetchone) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + self.assertRaises(self.driver.Error, cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchone() - self.assertEqual(len(r),1, - 'cursor.fetchone should have retrieved a single row' - ) - self.assertEqual(r[0],'Victoria Bitter', - 'cursor.fetchone retrieved incorrect data' - ) - self.assertEqual(cur.fetchone(),None, - 'cursor.fetchone should return None if no more rows available' - ) - self.assertTrue(cur.rowcount in (-1,1)) + self.assertEqual( + len(r), 1, "cursor.fetchone should have retrieved a single row" + ) + self.assertEqual( + r[0], "Victoria Bitter", "cursor.fetchone retrieved incorrect data" + ) + self.assertEqual( + cur.fetchone(), + None, + "cursor.fetchone should return None if no more rows available", + ) + self.assertTrue(cur.rowcount in (-1, 1)) finally: con.close() samples = [ - 'Carlton Cold', - 'Carlton Draft', - 'Mountain Goat', - 'Redback', - 'Victoria Bitter', - 'XXXX' - ] + "Carlton Cold", + "Carlton Draft", + "Mountain Goat", + "Redback", + "Victoria Bitter", + "XXXX", + ] def _populate(self): - ''' Return a list of sql commands to setup the DB for the fetch - tests. - ''' + """Return a list of sql commands to setup the DB for the fetch + tests. + """ populate = [ - "insert into %sbooze values ('%s')" % (self.table_prefix,s) - for s in self.samples - ] + "insert into %sbooze values ('%s')" % (self.table_prefix, s) + for s in self.samples + ] return populate def test_fetchmany(self): @@ -542,78 +535,88 @@ def test_fetchmany(self): cur = con.cursor() # cursor.fetchmany should raise an Error if called without - #issuing a query - self.assertRaises(self.driver.Error,cur.fetchmany,4) + # issuing a query + self.assertRaises(self.driver.Error, cur.fetchmany, 4) self.executeDDL1(cur) for sql in self._populate(): cur.execute(sql) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchmany() - self.assertEqual(len(r),1, - 'cursor.fetchmany retrieved incorrect number of rows, ' - 'default of arraysize is one.' - ) - cur.arraysize=10 - r = cur.fetchmany(3) # Should get 3 rows - self.assertEqual(len(r),3, - 'cursor.fetchmany retrieved incorrect number of rows' - ) - r = cur.fetchmany(4) # Should get 2 more - self.assertEqual(len(r),2, - 'cursor.fetchmany retrieved incorrect number of rows' - ) - r = cur.fetchmany(4) # Should be an empty sequence - self.assertEqual(len(r),0, - 'cursor.fetchmany should return an empty sequence after ' - 'results are exhausted' + self.assertEqual( + len(r), + 1, + "cursor.fetchmany retrieved incorrect number of rows, " + "default of arraysize is one.", + ) + cur.arraysize = 10 + r = cur.fetchmany(3) # Should get 3 rows + self.assertEqual( + len(r), 3, "cursor.fetchmany retrieved incorrect number of rows" + ) + r = cur.fetchmany(4) # Should get 2 more + self.assertEqual( + len(r), 2, "cursor.fetchmany retrieved incorrect number of rows" + ) + r = cur.fetchmany(4) # Should be an empty sequence + self.assertEqual( + len(r), + 0, + "cursor.fetchmany should return an empty sequence after " + "results are exhausted", ) - self.assertTrue(cur.rowcount in (-1,6)) + self.assertTrue(cur.rowcount in (-1, 6)) # Same as above, using cursor.arraysize - cur.arraysize=4 - cur.execute('select name from %sbooze' % self.table_prefix) - r = cur.fetchmany() # Should get 4 rows - self.assertEqual(len(r),4, - 'cursor.arraysize not being honoured by fetchmany' - ) - r = cur.fetchmany() # Should get 2 more - self.assertEqual(len(r),2) - r = cur.fetchmany() # Should be an empty sequence - self.assertEqual(len(r),0) - self.assertTrue(cur.rowcount in (-1,6)) - - cur.arraysize=6 - cur.execute('select name from %sbooze' % self.table_prefix) - rows = cur.fetchmany() # Should get all rows - self.assertTrue(cur.rowcount in (-1,6)) - self.assertEqual(len(rows),6) - self.assertEqual(len(rows),6) + cur.arraysize = 4 + cur.execute("select name from %sbooze" % self.table_prefix) + r = cur.fetchmany() # Should get 4 rows + self.assertEqual( + len(r), 4, "cursor.arraysize not being honoured by fetchmany" + ) + r = cur.fetchmany() # Should get 2 more + self.assertEqual(len(r), 2) + r = cur.fetchmany() # Should be an empty sequence + self.assertEqual(len(r), 0) + self.assertTrue(cur.rowcount in (-1, 6)) + + cur.arraysize = 6 + cur.execute("select name from %sbooze" % self.table_prefix) + rows = cur.fetchmany() # Should get all rows + self.assertTrue(cur.rowcount in (-1, 6)) + self.assertEqual(len(rows), 6) + self.assertEqual(len(rows), 6) rows = [r[0] for r in rows] rows.sort() # Make sure we get the right data back out - for i in range(0,6): - self.assertEqual(rows[i],self.samples[i], - 'incorrect data retrieved by cursor.fetchmany' - ) - - rows = cur.fetchmany() # Should return an empty list - self.assertEqual(len(rows),0, - 'cursor.fetchmany should return an empty sequence if ' - 'called after the whole result set has been fetched' + for i in range(0, 6): + self.assertEqual( + rows[i], + self.samples[i], + "incorrect data retrieved by cursor.fetchmany", ) - self.assertTrue(cur.rowcount in (-1,6)) + + rows = cur.fetchmany() # Should return an empty list + self.assertEqual( + len(rows), + 0, + "cursor.fetchmany should return an empty sequence if " + "called after the whole result set has been fetched", + ) + self.assertTrue(cur.rowcount in (-1, 6)) self.executeDDL2(cur) - cur.execute('select name from %sbarflys' % self.table_prefix) - r = cur.fetchmany() # Should get empty sequence - self.assertEqual(len(r),0, - 'cursor.fetchmany should return an empty sequence if ' - 'query retrieved no rows' - ) - self.assertTrue(cur.rowcount in (-1,0)) + cur.execute("select name from %sbarflys" % self.table_prefix) + r = cur.fetchmany() # Should get empty sequence + self.assertEqual( + len(r), + 0, + "cursor.fetchmany should return an empty sequence if " + "query retrieved no rows", + ) + self.assertTrue(cur.rowcount in (-1, 0)) finally: con.close() @@ -633,36 +636,41 @@ def test_fetchall(self): # cursor.fetchall should raise an Error if called # after executing a a statement that cannot return rows - self.assertRaises(self.driver.Error,cur.fetchall) + self.assertRaises(self.driver.Error, cur.fetchall) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,len(self.samples))) - self.assertEqual(len(rows),len(self.samples), - 'cursor.fetchall did not retrieve all rows' - ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) + self.assertEqual( + len(rows), + len(self.samples), + "cursor.fetchall did not retrieve all rows", + ) rows = [r[0] for r in rows] rows.sort() - for i in range(0,len(self.samples)): - self.assertEqual(rows[i],self.samples[i], - 'cursor.fetchall retrieved incorrect rows' + for i in range(0, len(self.samples)): + self.assertEqual( + rows[i], self.samples[i], "cursor.fetchall retrieved incorrect rows" ) rows = cur.fetchall() self.assertEqual( - len(rows),0, - 'cursor.fetchall should return an empty list if called ' - 'after the whole result set has been fetched' - ) - self.assertTrue(cur.rowcount in (-1,len(self.samples))) + len(rows), + 0, + "cursor.fetchall should return an empty list if called " + "after the whole result set has been fetched", + ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) self.executeDDL2(cur) - cur.execute('select name from %sbarflys' % self.table_prefix) + cur.execute("select name from %sbarflys" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,0)) - self.assertEqual(len(rows),0, - 'cursor.fetchall should return an empty list if ' - 'a select query returns no rows' - ) + self.assertTrue(cur.rowcount in (-1, 0)) + self.assertEqual( + len(rows), + 0, + "cursor.fetchall should return an empty list if " + "a select query returns no rows", + ) finally: con.close() @@ -675,74 +683,74 @@ def test_mixedfetch(self): for sql in self._populate(): cur.execute(sql) - cur.execute('select name from %sbooze' % self.table_prefix) - rows1 = cur.fetchone() + cur.execute("select name from %sbooze" % self.table_prefix) + rows1 = cur.fetchone() rows23 = cur.fetchmany(2) - rows4 = cur.fetchone() + rows4 = cur.fetchone() rows56 = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,6)) - self.assertEqual(len(rows23),2, - 'fetchmany returned incorrect number of rows' - ) - self.assertEqual(len(rows56),2, - 'fetchall returned incorrect number of rows' - ) + self.assertTrue(cur.rowcount in (-1, 6)) + self.assertEqual( + len(rows23), 2, "fetchmany returned incorrect number of rows" + ) + self.assertEqual( + len(rows56), 2, "fetchall returned incorrect number of rows" + ) rows = [rows1[0]] - rows.extend([rows23[0][0],rows23[1][0]]) + rows.extend([rows23[0][0], rows23[1][0]]) rows.append(rows4[0]) - rows.extend([rows56[0][0],rows56[1][0]]) + rows.extend([rows56[0][0], rows56[1][0]]) rows.sort() - for i in range(0,len(self.samples)): - self.assertEqual(rows[i],self.samples[i], - 'incorrect data retrieved or inserted' - ) + for i in range(0, len(self.samples)): + self.assertEqual( + rows[i], self.samples[i], "incorrect data retrieved or inserted" + ) finally: con.close() - def help_nextset_setUp(self,cur): - ''' Should create a procedure called deleteme - that returns two result sets, first the - number of rows in booze then "name from booze" - ''' - raise NotImplementedError('Helper not implemented') - #sql=""" + def help_nextset_setUp(self, cur): + """Should create a procedure called deleteme + that returns two result sets, first the + number of rows in booze then "name from booze" + """ + raise NotImplementedError("Helper not implemented") + # sql=""" # create procedure deleteme as # begin # select count(*) from booze # select name from booze # end - #""" - #cur.execute(sql) + # """ + # cur.execute(sql) - def help_nextset_tearDown(self,cur): - 'If cleaning up is needed after nextSetTest' - raise NotImplementedError('Helper not implemented') - #cur.execute("drop procedure deleteme") + def help_nextset_tearDown(self, cur): + "If cleaning up is needed after nextSetTest" + raise NotImplementedError("Helper not implemented") + # cur.execute("drop procedure deleteme") def test_nextset(self): con = self._connect() try: cur = con.cursor() - if not hasattr(cur,'nextset'): + if not hasattr(cur, "nextset"): return try: self.executeDDL1(cur) - sql=self._populate() + sql = self._populate() for sql in self._populate(): cur.execute(sql) self.help_nextset_setUp(cur) - cur.callproc('deleteme') - numberofrows=cur.fetchone() - assert numberofrows[0]== len(self.samples) + cur.callproc("deleteme") + numberofrows = cur.fetchone() + assert numberofrows[0] == len(self.samples) assert cur.nextset() - names=cur.fetchall() + names = cur.fetchall() assert len(names) == len(self.samples) - s=cur.nextset() - assert s == None,'No more return sets, should return None' + s = cur.nextset() + assert s == None, "No more return sets, should return None" finally: self.help_nextset_tearDown(cur) @@ -750,16 +758,16 @@ def test_nextset(self): con.close() def test_nextset(self): - raise NotImplementedError('Drivers need to override this test') + raise NotImplementedError("Drivers need to override this test") def test_arraysize(self): # Not much here - rest of the tests for this are in test_fetchmany con = self._connect() try: cur = con.cursor() - self.assertTrue(hasattr(cur,'arraysize'), - 'cursor.arraysize must be defined' - ) + self.assertTrue( + hasattr(cur, "arraysize"), "cursor.arraysize must be defined" + ) finally: con.close() @@ -767,8 +775,8 @@ def test_setinputsizes(self): con = self._connect() try: cur = con.cursor() - cur.setinputsizes( (25,) ) - self._paraminsert(cur) # Make sure cursor still works + cur.setinputsizes((25,)) + self._paraminsert(cur) # Make sure cursor still works finally: con.close() @@ -778,74 +786,70 @@ def test_setoutputsize_basic(self): try: cur = con.cursor() cur.setoutputsize(1000) - cur.setoutputsize(2000,0) - self._paraminsert(cur) # Make sure the cursor still works + cur.setoutputsize(2000, 0) + self._paraminsert(cur) # Make sure the cursor still works finally: con.close() def test_setoutputsize(self): # Real test for setoutputsize is driver dependant - raise NotImplementedError('Driver need to override this test') + raise NotImplementedError("Driver need to override this test") def test_None(self): con = self._connect() try: cur = con.cursor() self.executeDDL1(cur) - cur.execute('insert into %sbooze values (NULL)' % self.table_prefix) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("insert into %sbooze values (NULL)" % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchall() - self.assertEqual(len(r),1) - self.assertEqual(len(r[0]),1) - self.assertEqual(r[0][0],None,'NULL value not returned as None') + self.assertEqual(len(r), 1) + self.assertEqual(len(r[0]), 1) + self.assertEqual(r[0][0], None, "NULL value not returned as None") finally: con.close() def test_Date(self): - d1 = self.driver.Date(2002,12,25) - d2 = self.driver.DateFromTicks(time.mktime((2002,12,25,0,0,0,0,0,0))) + d1 = self.driver.Date(2002, 12, 25) + d2 = self.driver.DateFromTicks(time.mktime((2002, 12, 25, 0, 0, 0, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(d1),str(d2)) def test_Time(self): - t1 = self.driver.Time(13,45,30) - t2 = self.driver.TimeFromTicks(time.mktime((2001,1,1,13,45,30,0,0,0))) + t1 = self.driver.Time(13, 45, 30) + t2 = self.driver.TimeFromTicks(time.mktime((2001, 1, 1, 13, 45, 30, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Timestamp(self): - t1 = self.driver.Timestamp(2002,12,25,13,45,30) + t1 = self.driver.Timestamp(2002, 12, 25, 13, 45, 30) t2 = self.driver.TimestampFromTicks( - time.mktime((2002,12,25,13,45,30,0,0,0)) - ) + time.mktime((2002, 12, 25, 13, 45, 30, 0, 0, 0)) + ) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Binary(self): - b = self.driver.Binary(b'Something') - b = self.driver.Binary(b'') + b = self.driver.Binary(b"Something") + b = self.driver.Binary(b"") def test_STRING(self): - self.assertTrue(hasattr(self.driver,'STRING'), - 'module.STRING must be defined' - ) + self.assertTrue(hasattr(self.driver, "STRING"), "module.STRING must be defined") def test_BINARY(self): - self.assertTrue(hasattr(self.driver,'BINARY'), - 'module.BINARY must be defined.' - ) + self.assertTrue( + hasattr(self.driver, "BINARY"), "module.BINARY must be defined." + ) def test_NUMBER(self): - self.assertTrue(hasattr(self.driver,'NUMBER'), - 'module.NUMBER must be defined.' - ) + self.assertTrue( + hasattr(self.driver, "NUMBER"), "module.NUMBER must be defined." + ) def test_DATETIME(self): - self.assertTrue(hasattr(self.driver,'DATETIME'), - 'module.DATETIME must be defined.' - ) + self.assertTrue( + hasattr(self.driver, "DATETIME"), "module.DATETIME must be defined." + ) def test_ROWID(self): - self.assertTrue(hasattr(self.driver,'ROWID'), - 'module.ROWID must be defined.' - ) + self.assertTrue(hasattr(self.driver, "ROWID"), "module.ROWID must be defined.") diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 8c1dd535a..139089ab1 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -4,16 +4,23 @@ from pymysql.tests import base import warnings -warnings.filterwarnings('error') +warnings.filterwarnings("error") + class test_MySQLdb(capabilities.DatabaseTest): db_module = pymysql connect_args = () connect_kwargs = base.PyMySQLTestCase.databases[0].copy() - connect_kwargs.update(dict(read_default_file='~/.my.cnf', - use_unicode=True, binary_prefix=True, - charset='utf8mb4', sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL")) + connect_kwargs.update( + dict( + read_default_file="~/.my.cnf", + use_unicode=True, + binary_prefix=True, + charset="utf8mb4", + sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL", + ) + ) leak_test = False @@ -22,64 +29,70 @@ def quote_identifier(self, ident): def test_TIME(self): from datetime import timedelta - def generator(row,col): - return timedelta(0, row*8000) - self.check_data_integrity( - ('col1 TIME',), - generator) + + def generator(row, col): + return timedelta(0, row * 8000) + + self.check_data_integrity(("col1 TIME",), generator) def test_TINYINT(self): # Number data - def generator(row,col): - v = (row*row) % 256 + def generator(row, col): + v = (row * row) % 256 if v > 127: - v = v-256 + v = v - 256 return v - self.check_data_integrity( - ('col1 TINYINT',), - generator) + + self.check_data_integrity(("col1 TINYINT",), generator) def test_stored_procedures(self): db = self.connection c = self.cursor try: - self.create_table(('pos INT', 'tree CHAR(20)')) - c.executemany("INSERT INTO %s (pos,tree) VALUES (%%s,%%s)" % self.table, - list(enumerate('ash birch cedar larch pine'.split()))) + self.create_table(("pos INT", "tree CHAR(20)")) + c.executemany( + "INSERT INTO %s (pos,tree) VALUES (%%s,%%s)" % self.table, + list(enumerate("ash birch cedar larch pine".split())), + ) db.commit() - c.execute(""" + c.execute( + """ CREATE PROCEDURE test_sp(IN t VARCHAR(255)) BEGIN SELECT pos FROM %s WHERE tree = t; END - """ % self.table) + """ + % self.table + ) db.commit() - c.callproc('test_sp', ('larch',)) + c.callproc("test_sp", ("larch",)) rows = c.fetchall() self.assertEqual(len(rows), 1) self.assertEqual(rows[0][0], 3) c.nextset() finally: c.execute("DROP PROCEDURE IF EXISTS test_sp") - c.execute('drop table %s' % (self.table)) + c.execute("drop table %s" % (self.table)) def test_small_CHAR(self): # Character data - def generator(row,col): - i = ((row+1)*(col+1)+62)%256 - if i == 62: return '' - if i == 63: return None + def generator(row, col): + i = ((row + 1) * (col + 1) + 62) % 256 + if i == 62: + return "" + if i == 63: + return None return chr(i) - self.check_data_integrity( - ('col1 char(1)','col2 char(1)'), - generator) + + self.check_data_integrity(("col1 char(1)", "col2 char(1)"), generator) def test_bug_2671682(self): from pymysql.constants import ER + try: - self.cursor.execute("describe some_non_existent_table"); + self.cursor.execute("describe some_non_existent_table") except self.connection.ProgrammingError as msg: self.assertEqual(msg.args[0], ER.NO_SUCH_TABLE) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index 2c9a06005..e882c5eb3 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -9,13 +9,22 @@ class test_MySQLdb(dbapi20.DatabaseAPI20Test): driver = pymysql connect_args = () connect_kw_args = base.PyMySQLTestCase.databases[0].copy() - connect_kw_args.update(dict(read_default_file='~/.my.cnf', - charset='utf8', - sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL")) + connect_kw_args.update( + dict( + read_default_file="~/.my.cnf", + charset="utf8", + sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL", + ) + ) - def test_setoutputsize(self): pass - def test_setoutputsize_basic(self): pass - def test_nextset(self): pass + def test_setoutputsize(self): + pass + + def test_setoutputsize_basic(self): + pass + + def test_nextset(self): + pass """The tests on fetchone and fetchall and rowcount bogusly test for an exception if the statement cannot return a @@ -37,36 +46,41 @@ def test_fetchall(self): # cursor.fetchall should raise an Error if called # after executing a a statement that cannot return rows -## self.assertRaises(self.driver.Error,cur.fetchall) + ## self.assertRaises(self.driver.Error,cur.fetchall) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,len(self.samples))) - self.assertEqual(len(rows),len(self.samples), - 'cursor.fetchall did not retrieve all rows' - ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) + self.assertEqual( + len(rows), + len(self.samples), + "cursor.fetchall did not retrieve all rows", + ) rows = [r[0] for r in rows] rows.sort() - for i in range(0,len(self.samples)): - self.assertEqual(rows[i],self.samples[i], - 'cursor.fetchall retrieved incorrect rows' + for i in range(0, len(self.samples)): + self.assertEqual( + rows[i], self.samples[i], "cursor.fetchall retrieved incorrect rows" ) rows = cur.fetchall() self.assertEqual( - len(rows),0, - 'cursor.fetchall should return an empty list if called ' - 'after the whole result set has been fetched' - ) - self.assertTrue(cur.rowcount in (-1,len(self.samples))) + len(rows), + 0, + "cursor.fetchall should return an empty list if called " + "after the whole result set has been fetched", + ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) self.executeDDL2(cur) - cur.execute('select name from %sbarflys' % self.table_prefix) + cur.execute("select name from %sbarflys" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,0)) - self.assertEqual(len(rows),0, - 'cursor.fetchall should return an empty list if ' - 'a select query returns no rows' - ) + self.assertTrue(cur.rowcount in (-1, 0)) + self.assertEqual( + len(rows), + 0, + "cursor.fetchall should return an empty list if " + "a select query returns no rows", + ) finally: con.close() @@ -78,39 +92,40 @@ def test_fetchone(self): # cursor.fetchone should raise an Error if called before # executing a select-type query - self.assertRaises(self.driver.Error,cur.fetchone) + self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows self.executeDDL1(cur) -## self.assertRaises(self.driver.Error,cur.fetchone) + ## self.assertRaises(self.driver.Error,cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) - self.assertEqual(cur.fetchone(),None, - 'cursor.fetchone should return None if a query retrieves ' - 'no rows' - ) - self.assertTrue(cur.rowcount in (-1,0)) + cur.execute("select name from %sbooze" % self.table_prefix) + self.assertEqual( + cur.fetchone(), + None, + "cursor.fetchone should return None if a query retrieves " "no rows", + ) + self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) -## self.assertRaises(self.driver.Error,cur.fetchone) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + ## self.assertRaises(self.driver.Error,cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchone() - self.assertEqual(len(r),1, - 'cursor.fetchone should have retrieved a single row' - ) - self.assertEqual(r[0],'Victoria Bitter', - 'cursor.fetchone retrieved incorrect data' - ) -## self.assertEqual(cur.fetchone(),None, -## 'cursor.fetchone should return None if no more rows available' -## ) - self.assertTrue(cur.rowcount in (-1,1)) + self.assertEqual( + len(r), 1, "cursor.fetchone should have retrieved a single row" + ) + self.assertEqual( + r[0], "Victoria Bitter", "cursor.fetchone retrieved incorrect data" + ) + ## self.assertEqual(cur.fetchone(),None, + ## 'cursor.fetchone should return None if no more rows available' + ## ) + self.assertTrue(cur.rowcount in (-1, 1)) finally: con.close() @@ -120,81 +135,86 @@ def test_rowcount(self): try: cur = con.cursor() self.executeDDL1(cur) -## self.assertEqual(cur.rowcount,-1, -## 'cursor.rowcount should be -1 after executing no-result ' -## 'statements' -## ) - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) -## self.assertTrue(cur.rowcount in (-1,1), -## 'cursor.rowcount should == number or rows inserted, or ' -## 'set to -1 after executing an insert statement' -## ) + ## self.assertEqual(cur.rowcount,-1, + ## 'cursor.rowcount should be -1 after executing no-result ' + ## 'statements' + ## ) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + ## self.assertTrue(cur.rowcount in (-1,1), + ## 'cursor.rowcount should == number or rows inserted, or ' + ## 'set to -1 after executing an insert statement' + ## ) cur.execute("select name from %sbooze" % self.table_prefix) - self.assertTrue(cur.rowcount in (-1,1), - 'cursor.rowcount should == number of rows returned, or ' - 'set to -1 after executing a select statement' - ) + self.assertTrue( + cur.rowcount in (-1, 1), + "cursor.rowcount should == number of rows returned, or " + "set to -1 after executing a select statement", + ) self.executeDDL2(cur) -## self.assertEqual(cur.rowcount,-1, -## 'cursor.rowcount not being reset to -1 after executing ' -## 'no-result statements' -## ) + ## self.assertEqual(cur.rowcount,-1, + ## 'cursor.rowcount not being reset to -1 after executing ' + ## 'no-result statements' + ## ) finally: con.close() def test_callproc(self): - pass # performed in test_MySQL_capabilities - - def help_nextset_setUp(self,cur): - ''' Should create a procedure called deleteme - that returns two result sets, first the - number of rows in booze then "name from booze" - ''' - sql=""" + pass # performed in test_MySQL_capabilities + + def help_nextset_setUp(self, cur): + """Should create a procedure called deleteme + that returns two result sets, first the + number of rows in booze then "name from booze" + """ + sql = """ create procedure deleteme() begin select count(*) from %(tp)sbooze; select name from %(tp)sbooze; end - """ % dict(tp=self.table_prefix) + """ % dict( + tp=self.table_prefix + ) cur.execute(sql) - def help_nextset_tearDown(self,cur): - 'If cleaning up is needed after nextSetTest' + def help_nextset_tearDown(self, cur): + "If cleaning up is needed after nextSetTest" cur.execute("drop procedure deleteme") def test_nextset(self): from warnings import warn + con = self._connect() try: cur = con.cursor() - if not hasattr(cur,'nextset'): + if not hasattr(cur, "nextset"): return try: self.executeDDL1(cur) - sql=self._populate() + sql = self._populate() for sql in self._populate(): cur.execute(sql) self.help_nextset_setUp(cur) - cur.callproc('deleteme') - numberofrows=cur.fetchone() - assert numberofrows[0]== len(self.samples) + cur.callproc("deleteme") + numberofrows = cur.fetchone() + assert numberofrows[0] == len(self.samples) assert cur.nextset() - names=cur.fetchall() + names = cur.fetchall() assert len(names) == len(self.samples) - s=cur.nextset() + s = cur.nextset() if s: empty = cur.fetchall() - self.assertEqual(len(empty), 0, - "non-empty result set after other result sets") - #warn("Incompatibility: MySQL returns an empty result set for the CALL itself", + self.assertEqual( + len(empty), 0, "non-empty result set after other result sets" + ) + # warn("Incompatibility: MySQL returns an empty result set for the CALL itself", # Warning) - #assert s == None,'No more return sets, should return None' + # assert s == None,'No more return sets, should return None' finally: self.help_nextset_tearDown(cur) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py index 747ea4b04..b8d4bb1e6 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py @@ -2,6 +2,7 @@ import unittest import pymysql + _mysql = pymysql from pymysql.constants import FIELD_TYPE from pymysql.tests import base @@ -26,7 +27,7 @@ class CoreModule(unittest.TestCase): def test_NULL(self): """Should have a NULL constant.""" - self.assertEqual(_mysql.NULL, 'NULL') + self.assertEqual(_mysql.NULL, "NULL") def test_version(self): """Version information sanity.""" @@ -55,36 +56,45 @@ def tearDown(self): def test_thread_id(self): tid = self.conn.thread_id() - self.assertTrue(isinstance(tid, int), - "thread_id didn't return an integral value.") + self.assertTrue( + isinstance(tid, int), "thread_id didn't return an integral value." + ) - self.assertRaises(TypeError, self.conn.thread_id, ('evil',), - "thread_id shouldn't accept arguments.") + self.assertRaises( + TypeError, + self.conn.thread_id, + ("evil",), + "thread_id shouldn't accept arguments.", + ) def test_affected_rows(self): - self.assertEqual(self.conn.affected_rows(), 0, - "Should return 0 before we do anything.") - + self.assertEqual( + self.conn.affected_rows(), 0, "Should return 0 before we do anything." + ) - #def test_debug(self): - ## FIXME Only actually tests if you lack SUPER - #self.assertRaises(pymysql.OperationalError, - #self.conn.dump_debug_info) + # def test_debug(self): + ## FIXME Only actually tests if you lack SUPER + # self.assertRaises(pymysql.OperationalError, + # self.conn.dump_debug_info) def test_charset_name(self): - self.assertTrue(isinstance(self.conn.character_set_name(), str), - "Should return a string.") + self.assertTrue( + isinstance(self.conn.character_set_name(), str), "Should return a string." + ) def test_host_info(self): assert isinstance(self.conn.get_host_info(), str), "should return a string" def test_proto_info(self): - self.assertTrue(isinstance(self.conn.get_proto_info(), int), - "Should return an int.") + self.assertTrue( + isinstance(self.conn.get_proto_info(), int), "Should return an int." + ) def test_server_info(self): - self.assertTrue(isinstance(self.conn.get_server_info(), str), - "Should return an str.") + self.assertTrue( + isinstance(self.conn.get_server_info(), str), "Should return an str." + ) + if __name__ == "__main__": unittest.main() diff --git a/pymysql/util.py b/pymysql/util.py index 04683f83c..1349ec7bb 100644 --- a/pymysql/util.py +++ b/pymysql/util.py @@ -10,4 +10,3 @@ def byte2int(b): def int2byte(i): return struct.pack("!B", i) - diff --git a/tests/test_auth.py b/tests/test_auth.py index 61957655e..e5e2a64e5 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -10,7 +10,7 @@ port = 3306 ca = os.path.expanduser("~/ca.pem") -ssl = {'ca': ca, 'check_hostname': False} +ssl = {"ca": ca, "check_hostname": False} pass_sha256 = "pass_sha256_01234567890123456789" pass_caching_sha2 = "pass_caching_sha2_01234567890123456789" @@ -27,12 +27,16 @@ def test_sha256_no_passowrd_ssl(): def test_sha256_password(): - con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_sha256", password=pass_sha256, host=host, port=port, ssl=None + ) con.close() def test_sha256_password_ssl(): - con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=ssl) + con = pymysql.connect( + user="user_sha256", password=pass_sha256, host=host, port=port, ssl=ssl + ) con.close() @@ -47,20 +51,44 @@ def test_caching_sha2_no_password_ssl(): def test_caching_sha2_password(): - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=None, + ) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=None, + ) con.query("FLUSH PRIVILEGES") con.close() def test_caching_sha2_password_ssl(): - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=ssl) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=ssl, + ) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=None, + ) con.query("FLUSH PRIVILEGES") con.close() diff --git a/tests/test_mariadb_auth.py b/tests/test_mariadb_auth.py index 2f336feca..b3a2719cd 100644 --- a/tests/test_mariadb_auth.py +++ b/tests/test_mariadb_auth.py @@ -15,8 +15,9 @@ def test_ed25519_no_password(): def test_ed25519_password(): # nosec - con = pymysql.connect(user="user_ed25519", password="pass_ed25519", - host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_ed25519", password="pass_ed25519", host=host, port=port, ssl=None + ) con.close() From 175a3e0bc826fbf0a1d3cf6f73aac46a01672bba Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:07:27 +0900 Subject: [PATCH 077/227] Remove _socketio --- pymysql/_socketio.py | 130 ------------------------------------------- 1 file changed, 130 deletions(-) delete mode 100644 pymysql/_socketio.py diff --git a/pymysql/_socketio.py b/pymysql/_socketio.py deleted file mode 100644 index 6b2d65a33..000000000 --- a/pymysql/_socketio.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -SocketIO imported from socket module in Python 3. - -Copyright (c) 2001-2013 Python Software Foundation; All Rights Reserved. -""" - -from socket import * -import io -import errno - -__all__ = ["SocketIO"] - -EINTR = errno.EINTR -_blocking_errnos = (errno.EAGAIN, errno.EWOULDBLOCK) - - -class SocketIO(io.RawIOBase): - - """Raw I/O implementation for stream sockets. - - This class supports the makefile() method on sockets. It provides - the raw I/O interface on top of a socket object. - """ - - # One might wonder why not let FileIO do the job instead. There are two - # main reasons why FileIO is not adapted: - # - it wouldn't work under Windows (where you can't used read() and - # write() on a socket handle) - # - it wouldn't work with socket timeouts (FileIO would ignore the - # timeout and consider the socket non-blocking) - - # XXX More docs - - def __init__(self, sock, mode): - if mode not in ("r", "w", "rw", "rb", "wb", "rwb"): - raise ValueError("invalid mode: %r" % mode) - io.RawIOBase.__init__(self) - self._sock = sock - if "b" not in mode: - mode += "b" - self._mode = mode - self._reading = "r" in mode - self._writing = "w" in mode - self._timeout_occurred = False - - def readinto(self, b): - """Read up to len(b) bytes into the writable buffer *b* and return - the number of bytes read. If the socket is non-blocking and no bytes - are available, None is returned. - - If *b* is non-empty, a 0 return value indicates that the connection - was shutdown at the other end. - """ - self._checkClosed() - self._checkReadable() - if self._timeout_occurred: - raise IOError("cannot read from timed out object") - while True: - try: - return self._sock.recv_into(b) - except timeout: - self._timeout_occurred = True - raise - except error as e: - n = e.args[0] - if n == EINTR: - continue - if n in _blocking_errnos: - return None - raise - - def write(self, b): - """Write the given bytes or bytearray object *b* to the socket - and return the number of bytes written. This can be less than - len(b) if not all data could be written. If the socket is - non-blocking and no bytes could be written None is returned. - """ - self._checkClosed() - self._checkWritable() - try: - return self._sock.send(b) - except error as e: - # XXX what about EINTR? - if e.args[0] in _blocking_errnos: - return None - raise - - def readable(self): - """True if the SocketIO is open for reading.""" - if self.closed: - raise ValueError("I/O operation on closed socket.") - return self._reading - - def writable(self): - """True if the SocketIO is open for writing.""" - if self.closed: - raise ValueError("I/O operation on closed socket.") - return self._writing - - def seekable(self): - """True if the SocketIO is open for seeking.""" - if self.closed: - raise ValueError("I/O operation on closed socket.") - return super().seekable() - - def fileno(self): - """Return the file descriptor of the underlying socket.""" - self._checkClosed() - return self._sock.fileno() - - @property - def name(self): - if not self.closed: - return self.fileno() - else: - return -1 - - @property - def mode(self): - return self._mode - - def close(self): - """Close the SocketIO object. This doesn't close the underlying - socket, except if all references to it have disappeared. - """ - if self.closed: - return - io.RawIOBase.close(self) - self._sock._decref_socketios() - self._sock = None From 3299afd1f1402b0df464d13333473005298ea387 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:09:58 +0900 Subject: [PATCH 078/227] Simplify --- pymysql/__init__.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 5b49262e9..790cb9fcb 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -25,6 +25,7 @@ from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string +from . import connections from .err import ( Warning, Error, @@ -109,20 +110,10 @@ def Binary(x): def Connect(*args, **kwargs): - """ - Connect to the database; see connections.Connection.__init__() for - more information. - """ - from .connections import Connection - - return Connection(*args, **kwargs) - + return connections.Connection(*args, **kwargs) -from . import connections as _orig_conn -if _orig_conn.Connection.__init__.__doc__ is not None: - Connect.__doc__ = _orig_conn.Connection.__init__.__doc__ -del _orig_conn +Connect.__doc__ = connections.Connection.__init__.__doc__ def get_client_info(): # for MySQLdb compatibility From 587a59670ea1e10e3cc36d73ad47484cb67ebe4f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:13:33 +0900 Subject: [PATCH 079/227] Update flake8 setting --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index db1af5452..9d74b3a8b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,6 @@ [flake8] -ignore = E226,E301,E701 +ignore = E203,E501,W503,E722 exclude = tests,build -max-line-length = 119 [bdist_wheel] universal = 1 From 62108f59fe7d517c1586c6506a04c2963e6fe5f7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:19:15 +0900 Subject: [PATCH 080/227] Update flake8 setting --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9d74b3a8b..8efb08501 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [flake8] ignore = E203,E501,W503,E722 -exclude = tests,build +exclude = tests,build,.venv,docs [bdist_wheel] universal = 1 From 4185f7fe95ee498e61abbca9e02402318874ffb1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:20:13 +0900 Subject: [PATCH 081/227] Actions: Add lint --- .github/workflows/lint.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/lint.yaml diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 000000000..894a2d7cd --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,17 @@ +name: Lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: psf/black@stable + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1 + - name: flake8 + run: | + pip install flake8 + flake8 pymysql From df14c55377867b7a5a159a3ed5f0280b1cf10aea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:23:22 +0900 Subject: [PATCH 082/227] black setup.py --- setup.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index e35e7b290..37dcbf953 100755 --- a/setup.py +++ b/setup.py @@ -4,38 +4,38 @@ version = "0.10.1" -with io.open('./README.rst', encoding='utf-8') as f: +with io.open("./README.rst", encoding="utf-8") as f: readme = f.read() setup( name="PyMySQL", version=version, - url='https://github.com/PyMySQL/PyMySQL/', + url="https://github.com/PyMySQL/PyMySQL/", project_urls={ "Documentation": "https://pymysql.readthedocs.io/", }, - description='Pure Python MySQL Driver', + description="Pure Python MySQL Driver", long_description=readme, - packages=find_packages(exclude=['tests*', 'pymysql.tests*']), + packages=find_packages(exclude=["tests*", "pymysql.tests*"]), extras_require={ "rsa": ["cryptography"], "ed25519": ["PyNaCl>=1.4.0"], }, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Topic :: Database', + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Database", ], keywords="MySQL", ) From 9dc65c04a0fb60054161bdd7f46fb5c3baf39949 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:37:50 +0900 Subject: [PATCH 083/227] reformat black --- .github/workflows/lint.yaml | 2 + docs/source/conf.py | 156 ++++++++++++++++++++---------------- example.py | 2 +- 3 files changed, 88 insertions(+), 72 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 894a2d7cd..a18040509 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -9,6 +9,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: psf/black@stable + with: + args: ". --diff --check" - name: Setup flake8 annotations uses: rbialon/flake8-annotations@v1 - name: flake8 diff --git a/docs/source/conf.py b/docs/source/conf.py index bbadcbed1..77d7073a8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,55 +18,55 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath("../../")) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'PyMySQL' -copyright = u'2016, Yutaka Matsubara and GitHub contributors' +project = u"PyMySQL" +copyright = u"2016, Yutaka Matsubara and GitHub contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.7' +version = "0.7" # The full version, including alpha/beta/rc tags. -release = '0.7.2' +release = "0.7.2" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -74,154 +74,157 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = "default" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'PyMySQLdoc' +htmlhelp_basename = "PyMySQLdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'PyMySQL.tex', u'PyMySQL Documentation', - u'Yutaka Matsubara and GitHub contributors', 'manual'), + ( + "index", + "PyMySQL.tex", + u"PyMySQL Documentation", + u"Yutaka Matsubara and GitHub contributors", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -229,12 +232,17 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'pymysql', u'PyMySQL Documentation', - [u'Yutaka Matsubara and GitHub contributors'], 1) + ( + "index", + "pymysql", + u"PyMySQL Documentation", + [u"Yutaka Matsubara and GitHub contributors"], + 1, + ) ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -243,23 +251,29 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'PyMySQL', u'PyMySQL Documentation', - u'Yutaka Matsubara and GitHub contributors', 'PyMySQL', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "PyMySQL", + u"PyMySQL Documentation", + u"Yutaka Matsubara and GitHub contributors", + "PyMySQL", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {"http://docs.python.org/": None} diff --git a/example.py b/example.py index 68582138d..d40e94ab8 100644 --- a/example.py +++ b/example.py @@ -3,7 +3,7 @@ import pymysql -conn = pymysql.connect(host='localhost', port=3306, user='root', passwd='', db='mysql') +conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="", db="mysql") cur = conn.cursor() From e28c96eef07471f288f7308c2db73dc47f595436 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:45:41 +0900 Subject: [PATCH 084/227] Update README --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 269928b82..f8a854a61 100644 --- a/README.rst +++ b/README.rst @@ -35,13 +35,13 @@ Requirements * Python -- one of the following: - - CPython_ : 2.7 and >= 3.5 - - PyPy_ : Latest version + - CPython_ : 3.6 and newer + - PyPy_ : Latest 3.x version * MySQL Server -- one of the following: - - MySQL_ >= 5.5 - - MariaDB_ >= 5.5 + - MySQL_ >= 5.6 + - MariaDB_ >= 10.0 .. _CPython: https://www.python.org/ .. _PyPy: https://pypy.org/ @@ -77,6 +77,7 @@ Documentation is available online: https://pymysql.readthedocs.io/ For support, please refer to the `StackOverflow `_. + Example ------- From 7f44cd71f253be32d79d72dd4193f7a8a3557e8d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:51:01 +0900 Subject: [PATCH 085/227] Actions: Use cache for pip --- .github/workflows/test.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 71cc4e828..5b35716f9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,9 +46,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.py }} + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-1 + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependency + run: | + pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls + - name: Set up MySQL run: | - sleep 10 mysql -h 127.0.0.1 -uroot -e "select version()" mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' @@ -59,7 +70,6 @@ jobs: - name: Run test run: | - pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls pytest -v --cov --cov-config .coveragerc pymysql - name: Run MySQL8 auth test From 96b7583e5cc4d476d8071893eec9a0f479e835ec Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:55:14 +0900 Subject: [PATCH 086/227] Fix circular import --- pymysql/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 790cb9fcb..451012c8f 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -25,7 +25,6 @@ from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string -from . import connections from .err import ( Warning, Error, @@ -58,6 +57,8 @@ apilevel = "2.0" paramstyle = "pyformat" +from . import connections # noqa: E402 + class DBAPISet(frozenset): def __ne__(self, other): From 0e5afb12bcaee74c59dc5edb0d211e0e87a4536b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:59:58 +0900 Subject: [PATCH 087/227] Actions: Wait MySQL --- .github/workflows/test.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5b35716f9..0253ab0c4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -60,7 +60,11 @@ jobs: - name: Set up MySQL run: | - mysql -h 127.0.0.1 -uroot -e "select version()" + while : + do + sleep 1 + mysql --protocol=tcp -e 'select version()' && break + done mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' mysql -h 127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' From 8810ea977fa638c1a4db6f3a3047dbd2d8cc0b2d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:05:55 +0900 Subject: [PATCH 088/227] Actions: Run Lint only when py files are changed --- .github/workflows/lint.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a18040509..887a8f261 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,6 +1,12 @@ name: Lint -on: [push, pull_request] +on: + push: + paths: + - '**.py' + pull_request: + paths: + - '**.py' jobs: lint: From b637c37d87f66b2fbb93bc341e551fb55d9eba49 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:09:33 +0900 Subject: [PATCH 089/227] Actions: fix --- .github/workflows/test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0253ab0c4..e43df4b27 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -63,13 +63,13 @@ jobs: while : do sleep 1 - mysql --protocol=tcp -e 'select version()' && break + mysql -h127.0.0.1 -uroot -e 'select version()' && break done - mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" - mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' - mysql -h 127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - mysql -h 127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" - mysql -h 127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" + mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" + mysql -h127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' + mysql -h127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' + mysql -h127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" + mysql -h127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" cp .travis/docker.json pymysql/tests/databases.json - name: Run test From acce32fb2d2c6c5a438d7237d4744f13822b76c6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:16:56 +0900 Subject: [PATCH 090/227] Remove .travis.yml --- .travis.yml | 59 ----------------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index aa1f0f341..000000000 --- a/.travis.yml +++ /dev/null @@ -1,59 +0,0 @@ -# vim: sw=2 ts=2 sts=2 expandtab - -dist: bionic -language: python -cache: pip - -services: - - docker - -matrix: - include: - - env: - - DB=mariadb:10.2 - python: "3.6" - - env: - - DB=mariadb:10.3 - - TEST_MARIADB_AUTH=yes - python: "pypy3" - - env: - - DB=mariadb:10.5 - - TEST_MARIADB_AUTH=yes - python: "3.7" - - env: - - DB=mysql:5.6 - python: "3.9" - - env: - - DB=mysql:5.7 - python: "3.7" - - env: - - DB=mysql:8.0 - - TEST_AUTH=yes - python: "3.8" - -# different py version from 5.6 and 5.7 as cache seems to be based on py version -# http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version -# really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't -install: - - pip install -U coveralls coverage cryptography PyNaCl pytest pytest-cov - -before_script: - - ./.travis/initializedb.sh - - python -VV - - rm -f ~/.my.cnf # set in .travis.initialize.db.sh for the above commands - we should be using database.json however - - export COVERALLS_PARALLEL=true - -script: - - pytest -v --cov --cov-config .coveragerc pymysql - - if [ "${TEST_AUTH}" = "yes" ]; - then pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - fi - - if [ "${TEST_MARIADB_AUTH}" = "yes" ]; - then pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py; - fi - - if [ ! -z "${DB}" ]; - then docker logs mysqld; - fi - -after_success: - - coveralls From 27c72285d82620d07707c38224e205b866ba9c99 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:18:32 +0900 Subject: [PATCH 091/227] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 95430ae82..fef58a827 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,35,36,37,38,py,py3} +envlist = py{36,37,38,39,py3} [testenv] commands = pytest -v pymysql/tests/ From 0b2dd7e85984d5624ba0c972463add6d5696417c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:20:40 +0900 Subject: [PATCH 092/227] Update example.py --- example.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/example.py b/example.py index d40e94ab8..c12f103b5 100644 --- a/example.py +++ b/example.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function - import pymysql conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="", db="mysql") @@ -10,7 +8,6 @@ cur.execute("SELECT Host,User FROM user") print(cur.description) - print() for row in cur: From 58b331e2b1bb9f096e17487fc9f9a616e02b161c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:38:36 +0900 Subject: [PATCH 093/227] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..b6a7238dd --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,67 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '34 7 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 3481889b140cd621ea4b49266b4ea327b8a146cc Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:38:45 +0900 Subject: [PATCH 094/227] Cleanup (#921) * Cleanup * black --- README.rst | 8 +++----- pymysql/connections.py | 25 +++++++------------------ setup.py | 6 +----- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/README.rst b/README.rst index f8a854a61..06f3ed7b1 100644 --- a/README.rst +++ b/README.rst @@ -90,7 +90,7 @@ The following examples make use of a simple table `email` varchar(255) COLLATE utf8_bin NOT NULL, `password` varchar(255) COLLATE utf8_bin NOT NULL, PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_bin AUTO_INCREMENT=1 ; @@ -103,10 +103,9 @@ The following examples make use of a simple table user='user', password='passwd', db='db', - charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) - try: + with connection: with connection.cursor() as cursor: # Create a new record sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" @@ -122,8 +121,7 @@ The following examples make use of a simple table cursor.execute(sql, ('webmaster@python.org',)) result = cursor.fetchone() print(result) - finally: - connection.close() + This example will print: diff --git a/pymysql/connections.py b/pymysql/connections.py index dc69868be..32bf509b7 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -47,17 +47,6 @@ DEBUG = False -_py_version = sys.version_info[:2] - - -def _fast_surrogateescape(s): - return s.decode("ascii", "surrogateescape") - - -def _makefile(sock, mode): - return sock.makefile(mode) - - TEXT_TYPES = { FIELD_TYPE.BIT, FIELD_TYPE.BLOB, @@ -76,12 +65,12 @@ def _makefile(sock, mode): MAX_PACKET_LEN = 2 ** 24 - 1 -def pack_int24(n): +def _pack_int24(n): return struct.pack("=5.0) diff --git a/setup.py b/setup.py index 37dcbf953..08aa62f7e 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -import io from setuptools import setup, find_packages version = "0.10.1" -with io.open("./README.rst", encoding="utf-8") as f: +with open("./README.rst", encoding="utf-8") as f: readme = f.read() setup( @@ -23,10 +22,7 @@ }, classifiers=[ "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From cd61e56190c3ec6ab82934d9475712cd7a170656 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:50:11 +0900 Subject: [PATCH 095/227] Remove old_password support (#922) --- pymysql/_auth.py | 63 ------------------------------------------------ 1 file changed, 63 deletions(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index d16a08959..33fd9df86 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -2,7 +2,6 @@ Implements auth methods """ from .err import OperationalError -from .util import byte2int, int2byte try: @@ -16,9 +15,6 @@ from functools import partial import hashlib -import io -import struct -import warnings DEBUG = False @@ -53,65 +49,6 @@ def _my_crypt(message1, message2): return bytes(result) -# old_passwords support ported from libmysql/password.c -# https://dev.mysql.com/doc/internals/en/old-password-authentication.html - -SCRAMBLE_LENGTH_323 = 8 - - -class RandStruct_323: - def __init__(self, seed1, seed2): - self.max_value = 0x3FFFFFFF - self.seed1 = seed1 % self.max_value - self.seed2 = seed2 % self.max_value - - def my_rnd(self): - self.seed1 = (self.seed1 * 3 + self.seed2) % self.max_value - self.seed2 = (self.seed1 + self.seed2 + 33) % self.max_value - return float(self.seed1) / float(self.max_value) - - -def scramble_old_password(password, message): - """Scramble for old_password""" - warnings.warn( - "old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" - "old password support will be removed in future PyMySQL version" - ) - hash_pass = _hash_password_323(password) - hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) - hash_pass_n = struct.unpack(">LL", hash_pass) - hash_message_n = struct.unpack(">LL", hash_message) - - rand_st = RandStruct_323( - hash_pass_n[0] ^ hash_message_n[0], hash_pass_n[1] ^ hash_message_n[1] - ) - outbuf = io.BytesIO() - for _ in range(min(SCRAMBLE_LENGTH_323, len(message))): - outbuf.write(int2byte(int(rand_st.my_rnd() * 31) + 64)) - extra = int2byte(int(rand_st.my_rnd() * 31)) - out = outbuf.getvalue() - outbuf = io.BytesIO() - for c in out: - outbuf.write(int2byte(byte2int(c) ^ byte2int(extra))) - return outbuf.getvalue() - - -def _hash_password_323(password): - nr = 1345345333 - add = 7 - nr2 = 0x12345671 - - # x in py3 is numbers, p27 is chars - for c in [byte2int(x) for x in password if x not in (" ", "\t", 32, 9)]: - nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF - nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF - add = (add + c) & 0xFFFFFFFF - - r1 = nr & ((1 << 31) - 1) # kill sign bits - r2 = nr2 & ((1 << 31) - 1) - return struct.pack(">LL", r1, r2) - - # MariaDB's client_ed25519-plugin # https://mariadb.com/kb/en/library/connection/#client_ed25519-plugin From 8d3e079aed805ba18fea61014a61b8042225ac5d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 14:15:18 +0900 Subject: [PATCH 096/227] Add LGTM badge --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 06f3ed7b1..324010efa 100644 --- a/README.rst +++ b/README.rst @@ -11,6 +11,9 @@ .. image:: https://img.shields.io/badge/license-MIT-blue.svg :target: https://github.com/PyMySQL/PyMySQL/blob/master/LICENSE +.. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 + :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python + PyMySQL ======= From 744da2f5b853702c27be0ab10dad3312bed11030 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 14:21:20 +0900 Subject: [PATCH 097/227] remove util.py (#923) * remove util.py * black * fix * fix --- pymysql/connections.py | 7 +++---- pymysql/protocol.py | 9 +++------ pymysql/tests/test_basic.py | 3 +-- pymysql/tests/test_nextset.py | 1 - pymysql/util.py | 12 ------------ 5 files changed, 7 insertions(+), 25 deletions(-) delete mode 100644 pymysql/util.py diff --git a/pymysql/connections.py b/pymysql/connections.py index 32bf509b7..63a8b3a9c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -25,7 +25,6 @@ EOFPacketWrapper, LoadLocalPacketWrapper, ) -from .util import byte2int, int2byte from . import err, VERSION_STRING try: @@ -76,7 +75,7 @@ def _lenenc_int(i): "Encoding %d is less than 0 - no representation in LengthEncodedInteger" % i ) elif i < 0xFB: - return int2byte(i) + return bytes([i]) elif i < (1 << 16): return b"\xfc" + struct.pack(" Date: Sun, 3 Jan 2021 14:41:50 +0900 Subject: [PATCH 098/227] Update docs (#924) --- README.rst | 2 +- docs/source/user/development.rst | 3 ++- docs/source/user/examples.rst | 7 +++---- docs/source/user/installation.rst | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 324010efa..82303d056 100644 --- a/README.rst +++ b/README.rst @@ -93,7 +93,7 @@ The following examples make use of a simple table `email` varchar(255) COLLATE utf8_bin NOT NULL, `password` varchar(255) COLLATE utf8_bin NOT NULL, PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_bin + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1 ; diff --git a/docs/source/user/development.rst b/docs/source/user/development.rst index 39c40e1a7..099073185 100644 --- a/docs/source/user/development.rst +++ b/docs/source/user/development.rst @@ -30,7 +30,8 @@ and edit the new file to match your MySQL configuration:: To run all the tests, execute the script ``runtests.py``:: - $ python runtests.py + $ pip install pytest + $ pytest -v pymysql A ``tox.ini`` file is also provided for conveniently running tests on multiple Python versions:: diff --git a/docs/source/user/examples.rst b/docs/source/user/examples.rst index 87af40c37..966d46bd7 100644 --- a/docs/source/user/examples.rst +++ b/docs/source/user/examples.rst @@ -18,7 +18,7 @@ The following examples make use of a simple table `email` varchar(255) COLLATE utf8_bin NOT NULL, `password` varchar(255) COLLATE utf8_bin NOT NULL, PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1 ; @@ -34,7 +34,7 @@ The following examples make use of a simple table charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) - try: + with connection: with connection.cursor() as cursor: # Create a new record sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" @@ -50,8 +50,7 @@ The following examples make use of a simple table cursor.execute(sql, ('webmaster@python.org',)) result = cursor.fetchone() print(result) - finally: - connection.close() + This example will print: diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index d95961c66..0fea27266 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -18,13 +18,13 @@ Requirements * Python -- one of the following: - - CPython_ >= 2.7 or >= 3.5 - - Latest PyPy_ + - CPython_ >= 3.6 + - Latest PyPy_ 3 * MySQL Server -- one of the following: - - MySQL_ >= 5.5 - - MariaDB_ >= 5.5 + - MySQL_ >= 5.6 + - MariaDB_ >= 10.0 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ From 1a6b82d461037fdecf0c22476bde8b86884c8831 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 17:33:14 +0900 Subject: [PATCH 099/227] Update CHANGELOG --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d1313aa6..cb6e73cbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changes +## v1.0.0 + +Release date: TBD + +Backward incompatible changes: + +* Python 2.7 and 3.5 are not supported. +* old_password (used by MySQL older than 4.1) is not supported. + +Other changes: + +* Connection supports context manager API. ``__exit__`` closes the connection. (#886) +* Add MySQL Connector/Python compatible TLS options (#903) + + ## v0.10.1 Release date: 2020-09-10 From f9489ed163a4196ba9218d268901a6240fffe755 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 17:37:37 +0900 Subject: [PATCH 100/227] Test with MariaDB 10.0 (#925) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e43df4b27..dd45bcab9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,7 +10,7 @@ jobs: strategy: matrix: include: - - db: "mariadb:10.2" + - db: "mariadb:10.0" py: "3.9" - db: "mariadb:10.3" From d9b67a397b8fa839d0ec9c812fd7c0fcffc0fd30 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 4 Jan 2021 15:06:47 +0900 Subject: [PATCH 101/227] Code cleanup (#927) * cleanup * 2to3 -f unicode * black --- pymysql/converters.py | 14 +++++++------- pymysql/protocol.py | 14 +++++++------- pymysql/tests/test_basic.py | 8 ++++---- pymysql/tests/test_connection.py | 12 ++++++------ pymysql/tests/test_converters.py | 2 +- pymysql/tests/test_issues.py | 30 +++++++++++++++--------------- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 113dd298e..d910f5c5c 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -64,13 +64,13 @@ def escape_float(value, mapping=None): _escape_table = [chr(x) for x in range(128)] -_escape_table[0] = u"\\0" -_escape_table[ord("\\")] = u"\\\\" -_escape_table[ord("\n")] = u"\\n" -_escape_table[ord("\r")] = u"\\r" -_escape_table[ord("\032")] = u"\\Z" -_escape_table[ord('"')] = u'\\"' -_escape_table[ord("'")] = u"\\'" +_escape_table[0] = "\\0" +_escape_table[ord("\\")] = "\\\\" +_escape_table[ord("\n")] = "\\n" +_escape_table[ord("\r")] = "\\r" +_escape_table[ord("\032")] = "\\Z" +_escape_table[ord('"')] = '\\"' +_escape_table[ord("'")] = "\\'" def escape_string(value, mapping=None): diff --git a/pymysql/protocol.py b/pymysql/protocol.py index aa5feade6..559ba624a 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -182,31 +182,31 @@ def read_struct(self, fmt): def is_ok_packet(self): # https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html - return self._data[0:1] == b"\0" and len(self._data) >= 7 + return self._data[0] == 0 and len(self._data) >= 7 def is_eof_packet(self): # http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet # Caution: \xFE may be LengthEncodedInteger. # If \xFE is LengthEncodedInteger header, 8bytes followed. - return self._data[0:1] == b"\xfe" and len(self._data) < 9 + return self._data[0] == 0xFE and len(self._data) < 9 def is_auth_switch_request(self): # http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest - return self._data[0:1] == b"\xfe" + return self._data[0] == 0xFE def is_extra_auth_data(self): # https://dev.mysql.com/doc/internals/en/successful-authentication.html - return self._data[0:1] == b"\x01" + return self._data[0] == 1 def is_resultset_packet(self): - field_count = ord(self._data[0:1]) + field_count = self._data[0] return 1 <= field_count <= 250 def is_load_local_packet(self): - return self._data[0:1] == b"\xfb" + return self._data[0] == 0xFB def is_error_packet(self): - return self._data[0:1] == b"\xff" + return self._data[0] == 0xFF def check_error(self): if self.is_error_packet(): diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index fc1953122..c2590bf2f 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -29,7 +29,7 @@ def test_datatypes(self): 123456789012, 5.7, "hello'\" world", - u"Espa\xc3\xb1ol", + "Espa\xc3\xb1ol", "binary\x00data".encode(conn.encoding), datetime.date(1988, 2, 2), datetime.datetime(2014, 5, 15, 7, 45, 57), @@ -147,9 +147,9 @@ def test_untyped(self): conn = self.connect() c = conn.cursor() c.execute("select null,''") - self.assertEqual((None, u""), c.fetchone()) + self.assertEqual((None, ""), c.fetchone()) c.execute("select '',null") - self.assertEqual((u"", None), c.fetchone()) + self.assertEqual(("", None), c.fetchone()) def test_timedelta(self): """ test timedelta conversion """ @@ -300,7 +300,7 @@ def test_json(self): ) cur = conn.cursor() - json_str = u'{"hello": "こんãĢãĄã¯"}' + json_str = '{"hello": "こんãĢãĄã¯"}' cur.execute("INSERT INTO test_json (id, `json`) values (42, %s)", (json_str,)) cur.execute("SELECT `json` from `test_json` WHERE `id`=42") res = cur.fetchone()[0] diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index abd30e0ba..8303083d1 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -70,17 +70,17 @@ class TestAuthentication(base.PyMySQLTestCase): del db["user"] cur.execute("SHOW PLUGINS") for r in cur: - if (r[1], r[2]) != (u"ACTIVE", u"AUTHENTICATION"): + if (r[1], r[2]) != ("ACTIVE", "AUTHENTICATION"): continue - if r[3] == u"auth_socket.so" or r[0] == u"unix_socket": + if r[3] == "auth_socket.so" or r[0] == "unix_socket": socket_plugin_name = r[0] socket_found = True - elif r[3] == u"dialog_examples.so": + elif r[3] == "dialog_examples.so": if r[0] == "two_questions": two_questions_found = True elif r[0] == "three_attempts": three_attempts_found = True - elif r[0] == u"pam": + elif r[0] == "pam": pam_found = True pam_plugin_name = r[3].split(".")[0] if pam_plugin_name == "auth_pam": @@ -92,9 +92,9 @@ class TestAuthentication(base.PyMySQLTestCase): # https://mariadb.com/kb/en/mariadb/pam-authentication-plugin/ # Names differ but functionality is close - elif r[0] == u"mysql_old_password": + elif r[0] == "mysql_old_password": mysql_old_password_found = True - elif r[0] == u"sha256_password": + elif r[0] == "sha256_password": sha256_password_found = True # else: # print("plugin: %r" % r[0]) diff --git a/pymysql/tests/test_converters.py b/pymysql/tests/test_converters.py index dc194a9e2..b36ee4b39 100644 --- a/pymysql/tests/test_converters.py +++ b/pymysql/tests/test_converters.py @@ -8,7 +8,7 @@ class TestConverter(TestCase): def test_escape_string(self): - self.assertEqual(converters.escape_string(u"foo\nbar"), u"foo\\nbar") + self.assertEqual(converters.escape_string("foo\nbar"), "foo\\nbar") def test_convert_datetime(self): expected = datetime.datetime(2007, 2, 24, 23, 6, 20) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 95765e544..77d37481d 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -120,9 +120,9 @@ def test_issue_15(self): c.execute("drop table if exists issue15") c.execute("create table issue15 (t varchar(32))") try: - c.execute("insert into issue15 (t) values (%s)", (u"\xe4\xf6\xfc",)) + c.execute("insert into issue15 (t) values (%s)", ("\xe4\xf6\xfc",)) c.execute("select t from issue15") - self.assertEqual(u"\xe4\xf6\xfc", c.fetchone()[0]) + self.assertEqual("\xe4\xf6\xfc", c.fetchone()[0]) finally: c.execute("drop table issue15") @@ -189,12 +189,12 @@ def test_issue_34(self): def test_issue_33(self): conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( - conn, u"hei\xdfe", u"create table hei\xdfe (name varchar(32))" + conn, "hei\xdfe", "create table hei\xdfe (name varchar(32))" ) c = conn.cursor() - c.execute(u"insert into hei\xdfe (name) values ('Pi\xdfata')") - c.execute(u"select name from hei\xdfe") - self.assertEqual(u"Pi\xdfata", c.fetchone()[0]) + c.execute("insert into hei\xdfe (name) values ('Pi\xdfata')") + c.execute("select name from hei\xdfe") + self.assertEqual("Pi\xdfata", c.fetchone()[0]) @pytest.mark.skip("This test requires manual intervention") def test_issue_35(self): @@ -408,18 +408,18 @@ def test_issue_321(self): ) sql_select = "select * from issue321 where " "value_1 in %s and value_2=%s" data = [ - [(u"a",), u"\u0430"], - [[u"b"], u"\u0430"], - {"value_1": [[u"c"]], "value_2": u"\u0430"}, + [("a",), "\u0430"], + [["b"], "\u0430"], + {"value_1": [["c"]], "value_2": "\u0430"}, ] cur = conn.cursor() self.assertEqual(cur.execute(sql_insert, data[0]), 1) self.assertEqual(cur.execute(sql_insert, data[1]), 1) self.assertEqual(cur.execute(sql_dict_insert, data[2]), 1) - self.assertEqual(cur.execute(sql_select, [(u"a", u"b", u"c"), u"\u0430"]), 3) - self.assertEqual(cur.fetchone(), (u"a", u"\u0430")) - self.assertEqual(cur.fetchone(), (u"b", u"\u0430")) - self.assertEqual(cur.fetchone(), (u"c", u"\u0430")) + self.assertEqual(cur.execute(sql_select, [("a", "b", "c"), "\u0430"]), 3) + self.assertEqual(cur.fetchone(), ("a", "\u0430")) + self.assertEqual(cur.fetchone(), ("b", "\u0430")) + self.assertEqual(cur.fetchone(), ("c", "\u0430")) def test_issue_364(self): """ Test mixed unicode/binary arguments in executemany. """ @@ -432,8 +432,8 @@ def test_issue_364(self): ) sql = "insert into issue364 (value_1, value_2) values (_binary %s, %s)" - usql = u"insert into issue364 (value_1, value_2) values (_binary %s, %s)" - values = [pymysql.Binary(b"\x00\xff\x00"), u"\xe4\xf6\xfc"] + usql = "insert into issue364 (value_1, value_2) values (_binary %s, %s)" + values = [pymysql.Binary(b"\x00\xff\x00"), "\xe4\xf6\xfc"] # test single insert and select cur = conn.cursor() From 3818ad0d4c802d1e190cd4b0bc2be746ab3fa1f0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 4 Jan 2021 15:26:03 +0900 Subject: [PATCH 102/227] Use f-string (#928) --- pymysql/connections.py | 6 ++---- pymysql/cursors.py | 2 +- pymysql/protocol.py | 8 ++------ pymysql/tests/test_load_local.py | 10 ++-------- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 63a8b3a9c..7bc87a52d 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1331,7 +1331,7 @@ def _get_descriptions(self): if converter is converters.through: converter = None if DEBUG: - print("DEBUG: field={}, converter={}".format(field, converter)) + print(f"DEBUG: field={field}, converter={converter}") self.converters.append((encoding, converter)) eof_packet = self.connection._read_packet() @@ -1361,9 +1361,7 @@ def send_data(self): break conn.write_packet(chunk) except IOError: - raise err.OperationalError( - 1017, "Can't find file '{0}'".format(self.filename) - ) + raise err.OperationalError(1017, f"Can't find file '{self.filename}'") finally: # send the empty packet to signify we are done sending data conn.write_packet(b"") diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 68ac78e7f..666970b98 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -242,7 +242,7 @@ def callproc(self, procname, args=()): """ conn = self._get_db() if args: - fmt = "@_{0}_%d=%s".format(procname) + fmt = f"@_{procname}_%d=%s" self._query( "SET %s" % ",".join( diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 559ba624a..41c816736 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -323,9 +323,7 @@ class EOFPacketWrapper: def __init__(self, from_packet): if not from_packet.is_eof_packet(): raise ValueError( - "Cannot create '{0}' object from invalid packet type".format( - self.__class__ - ) + f"Cannot create '{self.__class__}' object from invalid packet type" ) self.packet = from_packet @@ -348,9 +346,7 @@ class LoadLocalPacketWrapper: def __init__(self, from_packet): if not from_packet.is_load_local_packet(): raise ValueError( - "Cannot create '{0}' object from invalid packet type".format( - self.__class__ - ) + f"Cannot create '{self.__class__}' object from invalid packet type" ) self.packet = from_packet diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index bb8563055..b1b8128e4 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -35,10 +35,7 @@ def test_load_file(self): ) try: c.execute( - ( - "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " - + "test_load_local FIELDS TERMINATED BY ','" - ).format(filename) + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -55,10 +52,7 @@ def test_unbuffered_load_file(self): ) try: c.execute( - ( - "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " - + "test_load_local FIELDS TERMINATED BY ','" - ).format(filename) + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) From 255b5dd931cbe3f9dda846ae99bed6b0c0ecf778 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 4 Jan 2021 16:18:17 +0900 Subject: [PATCH 103/227] code cleanup (#929) --- pymysql/connections.py | 15 +++------------ pymysql/tests/test_connection.py | 4 ++-- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 7bc87a52d..99a9575a9 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -174,7 +174,7 @@ def __init__( sql_mode=None, read_default_file=None, conv=None, - use_unicode=None, + use_unicode=True, client_flag=0, cursorclass=Cursor, init_command=None, @@ -203,9 +203,6 @@ def __init__( ssl_verify_cert=None, ssl_verify_identity=None, ): - if use_unicode is None and sys.version_info[0] > 2: - use_unicode = True - if db is not None and database is None: database = db if passwd is not None and not password: @@ -298,15 +295,9 @@ def _config(key, arg): if write_timeout is not None and write_timeout <= 0: raise ValueError("write_timeout should be > 0") self._write_timeout = write_timeout - if charset: - self.charset = charset - self.use_unicode = True - else: - self.charset = DEFAULT_CHARSET - self.use_unicode = False - if use_unicode is not None: - self.use_unicode = use_unicode + self.charset = charset or DEFAULT_CHARSET + self.use_unicode = use_unicode self.encoding = charset_by_name(self.charset).encoding diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 8303083d1..d89d04e91 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -403,7 +403,7 @@ def testMySQLOldPasswordAuth(self): c = conn.cursor() # deprecated in 5.6 - if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): + if self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) else: @@ -420,7 +420,7 @@ def testMySQLOldPasswordAuth(self): secure_auth_setting = c.fetchone()[0] c.execute("set old_passwords=1") # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): + if self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: c.execute("set global secure_auth=0") else: From 511b6a2af6031b234cd3cadfbdef8807eec797af Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 5 Jan 2021 14:43:04 +0900 Subject: [PATCH 104/227] Use keyword only argument (#930) * Use keyword only argument for constructor. * Remove old password test --- pymysql/__init__.py | 8 +---- pymysql/connections.py | 21 ++++++++----- pymysql/tests/test_connection.py | 52 -------------------------------- 3 files changed, 14 insertions(+), 67 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 451012c8f..478fdf6ae 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -110,11 +110,7 @@ def Binary(x): return bytes(x) -def Connect(*args, **kwargs): - return connections.Connection(*args, **kwargs) - - -Connect.__doc__ = connections.Connection.__init__.__doc__ +Connect = connect = Connection = connections.Connection def get_client_info(): # for MySQLdb compatibility @@ -124,8 +120,6 @@ def get_client_info(): # for MySQLdb compatibility return ".".join(map(str, version)) -connect = Connection = Connect - # we include a doctored version_info here for MySQLdb compatibility version_info = (1, 4, 0, "final", 0) diff --git a/pymysql/connections.py b/pymysql/connections.py index 99a9575a9..141381fe3 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -120,7 +120,7 @@ class Connection: See converters. :param use_unicode: Whether or not to default to unicode strings. - This option defaults to true for Py3k. + This option defaults to true. :param client_flag: Custom flags to send to MySQL. Find potential values in constants.CLIENT. :param cursorclass: Custom cursor class to use. :param init_command: Initial SQL statement to run when connection is established. @@ -164,12 +164,13 @@ class Connection: def __init__( self, - host=None, user=None, password="", + host=None, database=None, - port=0, + *, unix_socket=None, + port=0, charset="", sql_mode=None, read_default_file=None, @@ -179,13 +180,8 @@ def __init__( cursorclass=Cursor, init_command=None, connect_timeout=10, - ssl=None, read_default_group=None, - compress=None, - named_pipe=None, autocommit=False, - db=None, - passwd=None, local_infile=False, max_allowed_packet=16 * 1024 * 1024, defer_connect=False, @@ -196,16 +192,25 @@ def __init__( binary_prefix=False, program_name=None, server_public_key=None, + ssl=None, ssl_ca=None, ssl_cert=None, ssl_disabled=None, ssl_key=None, ssl_verify_cert=None, ssl_verify_identity=None, + compress=None, # not supported + named_pipe=None, # not supported + passwd=None, # deprecated + db=None, # deprecated ): if db is not None and database is None: + warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) database = db if passwd is not None and not password: + warnings.warn( + "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 + ) password = passwd if compress or named_pipe: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index d89d04e91..afbf014fc 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -383,58 +383,6 @@ def realTestPamAuth(self): # recreate the user cur.execute(grants) - # select old_password("crummy p\tassword"); - # | old_password("crummy p\tassword") | - # | 2a01785203b08770 | - @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif( - not mysql_old_password_found, reason="no mysql_old_password plugin" - ) - def testMySQLOldPasswordAuth(self): - conn = self.connect() - if self.mysql_server_is(conn, (5, 7, 0)): - pytest.skip("Old passwords aren't supported in 5.7") - # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") - # from login in MySQL-5.6 - if self.mysql_server_is(conn, (5, 6, 0)): - pytest.skip("Old passwords don't authenticate in 5.6") - db = self.db.copy() - db["password"] = "crummy p\tassword" - c = conn.cursor() - - # deprecated in 5.6 - if self.mysql_server_is(conn, (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) - else: - c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) - v = c.fetchone()[0] - self.assertEqual(v, "2a01785203b08770") - # only works in MariaDB and MySQL-5.6 - can't separate out by version - # if self.mysql_server_is(self.connect(), (5, 5, 0)): - # with TempUser(c, 'old_pass_user@localhost', - # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: - # cur = pymysql.connect(user='old_pass_user', **db).cursor() - # cur.execute("SELECT VERSION()") - c.execute("SELECT @@secure_auth") - secure_auth_setting = c.fetchone()[0] - c.execute("set old_passwords=1") - # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if self.mysql_server_is(conn, (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("set global secure_auth=0") - else: - c.execute("set global secure_auth=0") - with TempUser( - c, - "old_pass_user@localhost", - self.databases[0]["db"], - password=db["password"], - ) as u: - cur = pymysql.connect(user="old_pass_user", **db).cursor() - cur.execute("SELECT VERSION()") - c.execute("set global secure_auth=%r" % secure_auth_setting) - @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif( not sha256_password_found, From f5cbb6dea0a77c5e3055a299ed9a5b458c29cb12 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 6 Jan 2021 17:16:02 +0900 Subject: [PATCH 105/227] Remvoe escape_* functions from pymysql.__all__ (#931) * .travis -> ci * remove initializedb.sh * Remvoe escape functions from __all__ * fix test * don't use deprecated keyword * fix tests * fix tests * black --- .github/workflows/test.yaml | 2 +- .travis/database.json | 4 --- .travis/docker.json | 4 --- .travis/initializedb.sh | 54 -------------------------------- ci/database.json | 4 +++ ci/docker.json | 4 +++ docs/source/user/development.rst | 4 +-- pymysql/__init__.py | 5 --- pymysql/tests/base.py | 4 +-- pymysql/tests/test_connection.py | 19 ++++++----- pymysql/tests/test_issues.py | 4 +-- 11 files changed, 26 insertions(+), 82 deletions(-) delete mode 100644 .travis/database.json delete mode 100644 .travis/docker.json delete mode 100755 .travis/initializedb.sh create mode 100644 ci/database.json create mode 100644 ci/docker.json diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dd45bcab9..8f53c28d1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -70,7 +70,7 @@ jobs: mysql -h127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' mysql -h127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" mysql -h127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - cp .travis/docker.json pymysql/tests/databases.json + cp ci/docker.json pymysql/tests/databases.json - name: Run test run: | diff --git a/.travis/database.json b/.travis/database.json deleted file mode 100644 index ab1f60a3a..000000000 --- a/.travis/database.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - {"host": "localhost", "unix_socket": "/var/run/mysqld/mysqld.sock", "user": "root", "passwd": "", "db": "test1", "use_unicode": true, "local_infile": true}, - {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "db": "test2" } -] diff --git a/.travis/docker.json b/.travis/docker.json deleted file mode 100644 index b851fb6da..000000000 --- a/.travis/docker.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - {"host": "127.0.0.1", "port": 3306, "user": "root", "passwd": "", "db": "test1", "use_unicode": true, "local_infile": true}, - {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "db": "test2" } -] diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh deleted file mode 100755 index 6991cfe60..000000000 --- a/.travis/initializedb.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -set -ex - -docker pull ${DB} -docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} - -mysql() { - docker exec -i mysqld mysql "${@}" -} -while : -do - sleep 3 - mysql --protocol=tcp -e 'select version()' && break -done -docker logs mysqld - -if [ $DB == 'mysql:8.0' ]; then - WITH_PLUGIN='with mysql_native_password' - mysql -e 'SET GLOBAL local_infile=on' - docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - - # Test user for auth test - mysql -e ' - CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", - nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", - nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" - PASSWORD EXPIRE NEVER;' - mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' -elif [[ $DB == mariadb:10.* ]] && [ ${DB#mariadb:10.} -ge 3 ]; then - mysql -e ' - INSTALL SONAME "auth_ed25519"; - CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' - # we need to pass the hashed password manually until 10.4, so hide it here - mysql -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql - mysql -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql - WITH_PLUGIN='' -else - WITH_PLUGIN='' -fi - -mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' -mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - -mysql -u root -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" -mysql -u root -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - -cp .travis/docker.json pymysql/tests/databases.json diff --git a/ci/database.json b/ci/database.json new file mode 100644 index 000000000..aad0bfb29 --- /dev/null +++ b/ci/database.json @@ -0,0 +1,4 @@ +[ + {"host": "localhost", "unix_socket": "/var/run/mysqld/mysqld.sock", "user": "root", "password": "", "database": "test1", "use_unicode": true, "local_infile": true}, + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" } +] diff --git a/ci/docker.json b/ci/docker.json new file mode 100644 index 000000000..34a5c7b7c --- /dev/null +++ b/ci/docker.json @@ -0,0 +1,4 @@ +[ + {"host": "127.0.0.1", "port": 3306, "user": "root", "password": "", "database": "test1", "use_unicode": true, "local_infile": true}, + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" } +] diff --git a/docs/source/user/development.rst b/docs/source/user/development.rst index 099073185..af0576222 100644 --- a/docs/source/user/development.rst +++ b/docs/source/user/development.rst @@ -22,10 +22,10 @@ If you would like to run the test suite, create a database for testing like this mysql -e 'create database test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' mysql -e 'create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' -Then, copy the file ``.travis/database.json`` to ``pymysql/tests/databases.json`` +Then, copy the file ``ci/database.json`` to ``pymysql/tests/databases.json`` and edit the new file to match your MySQL configuration:: - $ cp .travis/database.json pymysql/tests/databases.json + $ cp ci/database.json pymysql/tests/databases.json $ $EDITOR pymysql/tests/databases.json To run all the tests, execute the script ``runtests.py``:: diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 478fdf6ae..6473f48d1 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -24,7 +24,6 @@ import sys from .constants import FIELD_TYPE -from .converters import escape_dict, escape_sequence, escape_string from .err import ( Warning, Error, @@ -177,14 +176,10 @@ def install_as_MySQLdb(): "constants", "converters", "cursors", - "escape_dict", - "escape_sequence", - "escape_string", "get_client_info", "paramstyle", "threadsafety", "version_info", "install_as_MySQLdb", - "NULL", "__version__", ] diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index 16cd23c0b..6f93a8317 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -21,11 +21,11 @@ class PyMySQLTestCase(unittest.TestCase): "host": "localhost", "user": "root", "passwd": "", - "db": "test1", + "database": "test1", "use_unicode": True, "local_infile": True, }, - {"host": "localhost", "user": "root", "passwd": "", "db": "test2"}, + {"host": "localhost", "user": "root", "passwd": "", "database": "test2"}, ] def mysql_server_is(self, conn, version_tuple): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index afbf014fc..be4006f60 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -142,7 +142,7 @@ def realtestSocketAuth(self): with TempUser( self.connect().cursor(), TestAuthentication.osuser + "@localhost", - self.databases[0]["db"], + self.databases[0]["database"], self.socket_plugin_name, ) as u: c = pymysql.connect(user=TestAuthentication.osuser, **self.db) @@ -216,7 +216,7 @@ def realTestDialogAuthTwoQuestions(self): with TempUser( self.connect().cursor(), "pymysql_2q@localhost", - self.databases[0]["db"], + self.databases[0]["database"], "two_questions", "notverysecret", ) as u: @@ -258,7 +258,7 @@ def realTestDialogAuthThreeAttempts(self): with TempUser( self.connect().cursor(), "pymysql_3a@localhost", - self.databases[0]["db"], + self.databases[0]["database"], "three_attempts", "stillnotverysecret", ) as u: @@ -353,7 +353,7 @@ def realTestPamAuth(self): with TempUser( cur, TestAuthentication.osuser + "@localhost", - self.databases[0]["db"], + self.databases[0]["database"], "pam", os.environ.get("PAMSERVICE"), ) as u: @@ -392,7 +392,10 @@ def testAuthSHA256(self): conn = self.connect() c = conn.cursor() with TempUser( - c, "pymysql_sha256@localhost", self.databases[0]["db"], "sha256_password" + c, + "pymysql_sha256@localhost", + self.databases[0]["database"], + "sha256_password", ) as u: if self.mysql_server_is(conn, (5, 7, 0)): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") @@ -442,8 +445,8 @@ def test_autocommit(self): def test_select_db(self): con = self.connect() - current_db = self.databases[0]["db"] - other_db = self.databases[1]["db"] + current_db = self.databases[0]["database"] + other_db = self.databases[1]["database"] cur = con.cursor() cur.execute("SELECT database()") @@ -754,7 +757,7 @@ def test_escape_fallback_encoder(self): class Custom(str): pass - mapping = {str: pymysql.escape_string} + mapping = {str: pymysql.converters.escape_string} self.assertEqual(con.escape(Custom("foobar"), mapping), "'foobar'") def test_escape_no_default(self): diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 77d37481d..b4ced4b06 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -66,7 +66,7 @@ def test_issue_6(self): """ exception: TypeError: ord() expected a character, but string of length 0 found """ # ToDo: this test requires access to db 'mysql'. kwargs = self.databases[0].copy() - kwargs["db"] = "mysql" + kwargs["database"] = "mysql" conn = pymysql.connect(**kwargs) c = conn.cursor() c.execute("select * from user") @@ -152,7 +152,7 @@ def test_issue_17(self): """could not connect mysql use passwod""" conn = self.connect() host = self.databases[0]["host"] - db = self.databases[0]["db"] + db = self.databases[0]["database"] c = conn.cursor() # grant access to a table to a user with a password From e24da41280af04e48423d00454fdd17343b63841 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 6 Jan 2021 21:58:04 +0900 Subject: [PATCH 106/227] Use `database` in examples. (#933) --- README.rst | 2 +- docs/source/user/examples.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 82303d056..46e60ff90 100644 --- a/README.rst +++ b/README.rst @@ -105,7 +105,7 @@ The following examples make use of a simple table connection = pymysql.connect(host='localhost', user='user', password='passwd', - db='db', + database='db', cursorclass=pymysql.cursors.DictCursor) with connection: diff --git a/docs/source/user/examples.rst b/docs/source/user/examples.rst index 966d46bd7..e9e02410d 100644 --- a/docs/source/user/examples.rst +++ b/docs/source/user/examples.rst @@ -30,7 +30,7 @@ The following examples make use of a simple table connection = pymysql.connect(host='localhost', user='user', password='passwd', - db='db', + database='db', charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) From 66e29fb789dd6a3c3c677c476ee9dc745efd2d04 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:25:20 +0900 Subject: [PATCH 107/227] Update CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6e73cbf..ccf1805e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,18 @@ Release date: TBD Backward incompatible changes: * Python 2.7 and 3.5 are not supported. -* old_password (used by MySQL older than 4.1) is not supported. +* ``connect()`` uses keyword-only arguments. User must use keyword argument. +* ``connect()`` kwargs ``db`` and ``passwd`` are now deprecated; Use ``database`` and ``password`` instead. +* old_password authentication method (used by MySQL older than 4.1) is not supported. +* MySQL 5.5 and MariaDB 5.5 are not officially supported, although it may still works. +* Removed ``escape_dict``, ``escape_sequence``, and ``escape_string`` from ``pymysql`` + module. They are still in ``pymysql.converters``. Other changes: * Connection supports context manager API. ``__exit__`` closes the connection. (#886) * Add MySQL Connector/Python compatible TLS options (#903) +* Major code cleanup; PyMySQL now uses black and flake8. ## v0.10.1 From 6e5d5bd94af056c66a1ed05de754a83f8628faea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:28:35 +0900 Subject: [PATCH 108/227] v1.0.0 --- CHANGELOG.md | 2 +- pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf1805e3..001b26318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v1.0.0 -Release date: TBD +Release date: 2021-01-07 Backward incompatible changes: diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 6473f48d1..455814684 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (0, 10, 1, None) +VERSION = (1, 0, 0, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 08aa62f7e..6e1f732cd 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from setuptools import setup, find_packages -version = "0.10.1" +version = "1.0.0" with open("./README.rst", encoding="utf-8") as f: readme = f.read() From f65351b1bd6c02eab07f20cbedada6ebfbf6d56d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:53:34 +0900 Subject: [PATCH 109/227] Do not create universal wheel --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8efb08501..b40802e4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,9 +2,6 @@ ignore = E203,E501,W503,E722 exclude = tests,build,.venv,docs -[bdist_wheel] -universal = 1 - [metadata] license = "MIT" license_files = LICENSE From 5a02e5780f615ac7793373d63c407b979c33cd1c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:59:12 +0900 Subject: [PATCH 110/227] remove badges --- README.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.rst b/README.rst index 46e60ff90..279181f16 100644 --- a/README.rst +++ b/README.rst @@ -2,15 +2,9 @@ :target: https://pymysql.readthedocs.io/ :alt: Documentation Status -.. image:: https://badge.fury.io/py/PyMySQL.svg - :target: https://badge.fury.io/py/PyMySQL - .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master -.. image:: https://img.shields.io/badge/license-MIT-blue.svg - :target: https://github.com/PyMySQL/PyMySQL/blob/master/LICENSE - .. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python From 5d1e27de3f35a936f7baf63036098d44f4a41a58 Mon Sep 17 00:00:00 2001 From: Nicusor Picatureanu <33037485+Nicusor97@users.noreply.github.com> Date: Thu, 7 Jan 2021 10:06:32 +0200 Subject: [PATCH 111/227] Set python_requires='>=3.6' (#936) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6e1f732cd..0224339e9 100755 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ description="Pure Python MySQL Driver", long_description=readme, packages=find_packages(exclude=["tests*", "pymysql.tests*"]), + python_requires=">=3.6", extras_require={ "rsa": ["cryptography"], "ed25519": ["PyNaCl>=1.4.0"], From 7c4700bd66b36e6e50e7f8c7df57635f0dafb006 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 17:55:31 +0900 Subject: [PATCH 112/227] Remove tox --- docs/source/user/development.rst | 5 ----- tox.ini | 9 --------- 2 files changed, 14 deletions(-) delete mode 100644 tox.ini diff --git a/docs/source/user/development.rst b/docs/source/user/development.rst index af0576222..1f8a2637f 100644 --- a/docs/source/user/development.rst +++ b/docs/source/user/development.rst @@ -32,8 +32,3 @@ To run all the tests, execute the script ``runtests.py``:: $ pip install pytest $ pytest -v pymysql - -A ``tox.ini`` file is also provided for conveniently running tests on multiple -Python versions:: - - $ tox diff --git a/tox.ini b/tox.ini deleted file mode 100644 index fef58a827..000000000 --- a/tox.ini +++ /dev/null @@ -1,9 +0,0 @@ -[tox] -envlist = py{36,37,38,39,py3} - -[testenv] -commands = pytest -v pymysql/tests/ -deps = coverage pytest -passenv = USER - PASSWORD - PAMSERVICE From 0acaa7f4fa4e2a9a30c835fc1be0b74eec3aaf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 8 Jan 2021 02:08:27 +0100 Subject: [PATCH 113/227] Use built-in unittest.mock (#938) Use built-in Python 3 unittest.mock instead of relying on mock package that is only necessary for ancient versions of Python. --- .github/workflows/test.yaml | 2 +- pymysql/tests/test_connection.py | 5 +++-- requirements-dev.txt | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f53c28d1..09846c943 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -56,7 +56,7 @@ jobs: - name: Install dependency run: | - pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls + pip install -U cryptography PyNaCl pytest pytest-cov coveralls - name: Set up MySQL run: | diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index be4006f60..75db73cd0 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,9 +1,10 @@ import datetime import ssl import sys -import time -import mock import pytest +import time +from unittest import mock + import pymysql from pymysql.tests import base from pymysql.constants import CLIENT diff --git a/requirements-dev.txt b/requirements-dev.txt index 69d3f68ae..d65512fbb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ cryptography PyNaCl>=1.4.0 pytest -mock From 2d36a195060b46e12f16d8b776468bab53ea6919 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 8 Jan 2021 10:38:14 +0900 Subject: [PATCH 114/227] Remove warning for db and passwd. (#940) * update doc * Remove warning. --- docs/source/user/examples.rst | 2 +- pymysql/connections.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/source/user/examples.rst b/docs/source/user/examples.rst index e9e02410d..3946db9b9 100644 --- a/docs/source/user/examples.rst +++ b/docs/source/user/examples.rst @@ -56,4 +56,4 @@ This example will print: .. code:: python - {'password': 'very-secret', 'id': 1} + {'id': 1, 'password': 'very-secret'} diff --git a/pymysql/connections.py b/pymysql/connections.py index 141381fe3..cb2035890 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -135,8 +135,6 @@ class Connection: :param ssl_verify_cert: Set to true to check the validity of server certificates :param ssl_verify_identity: Set to true to check the server's identity :param read_default_group: Group to read from in the configuration file. - :param compress: Not supported - :param named_pipe: Not supported :param autocommit: Autocommit mode. None means use server default. (default: False) :param local_infile: Boolean to enable the use of LOAD DATA LOCAL command. (default: False) :param max_allowed_packet: Max size of packet sent to server in bytes. (default: 16MB) @@ -149,9 +147,11 @@ class Connection: an argument. For the dialog plugin, a prompt(echo, prompt) method can be used (if no authenticate method) for returning a string from the user. (experimental) :param server_public_key: SHA256 authentication plugin public key value. (default: None) - :param db: Alias for database. (for compatibility to MySQLdb) - :param passwd: Alias for password. (for compatibility to MySQLdb) :param binary_prefix: Add _binary prefix on bytes and bytearray. (default: False) + :param compress: Not supported + :param named_pipe: Not supported + :param db: **DEPRECATED** Alias for database. + :param passwd: **DEPRECATED** Alias for password. See `Connection `_ in the specification. @@ -205,12 +205,16 @@ def __init__( db=None, # deprecated ): if db is not None and database is None: - warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) + # We will raise warining in 2022 or later. + # See https://github.com/PyMySQL/PyMySQL/issues/939 + # warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) database = db if passwd is not None and not password: - warnings.warn( - "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 - ) + # We will raise warining in 2022 or later. + # See https://github.com/PyMySQL/PyMySQL/issues/939 + # warnings.warn( + # "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 + # ) password = passwd if compress or named_pipe: From 5c6f8bcb741c32719a07e8c95eb8050cb9249511 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 8 Jan 2021 11:47:02 +0900 Subject: [PATCH 115/227] v1.0.1 --- CHANGELOG.md | 9 +++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 001b26318..beb4b2f9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changes +## v1.0.1 + +Release date: 2021-01-08 + +* Stop emitting DeprecationWarning for use of ``db`` and ``passwd``. + Note that they are still deprecated. (#939) +* Add ``python_requires=">=3.6"`` to setup.py. (#936) + + ## v1.0.0 Release date: 2021-01-07 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 455814684..ee59924a1 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 0, None) +VERSION = (1, 0, 1, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 0224339e9..f9962c75d 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from setuptools import setup, find_packages -version = "1.0.0" +version = "1.0.1" with open("./README.rst", encoding="utf-8") as f: readme = f.read() From abe83c262ea647a09e0f13587fa91d6a14a71598 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 8 Jan 2021 23:00:40 +0900 Subject: [PATCH 116/227] Make 4 more arguments to keyword-only. (#941) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index cb2035890..92b7a77e5 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -164,11 +164,11 @@ class Connection: def __init__( self, - user=None, + *, + user=None, # The first four arguments is based on DB-API 2.0 recommendation. password="", host=None, database=None, - *, unix_socket=None, port=0, charset="", From b12efdb6c1baa55e58a4384271e33a7351d554d5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 9 Jan 2021 20:32:51 +0900 Subject: [PATCH 117/227] v1.0.2 --- CHANGELOG.md | 8 ++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beb4b2f9c..9885af526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changes +## v1.0.2 + +Release date: 2021-01-09 + +* Fix `user`, `password`, `host`, `database` are still positional arguments. + All arguments of `connect()` are now keyword-only. (#941) + + ## v1.0.1 Release date: 2021-01-08 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index ee59924a1..5fe2aec54 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 1, None) +VERSION = (1, 0, 2, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index f9962c75d..1510a0cf8 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from setuptools import setup, find_packages -version = "1.0.1" +version = "1.0.2" with open("./README.rst", encoding="utf-8") as f: readme = f.read() From 1fd5292f33868f9f9c8b90e1e53f82dd4aa992b4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 18 Jan 2021 17:08:55 +0900 Subject: [PATCH 118/227] Update README.rst --- README.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.rst b/README.rst index 279181f16..f514d901e 100644 --- a/README.rst +++ b/README.rst @@ -17,13 +17,6 @@ PyMySQL This package contains a pure-Python MySQL client library, based on `PEP 249`_. -Most public APIs are compatible with mysqlclient and MySQLdb. - -NOTE: PyMySQL doesn't support low level APIs `_mysql` provides like `data_seek`, -`store_result`, and `use_result`. You should use high level APIs defined in `PEP 249`_. -But some APIs like `autocommit` and `ping` are supported because `PEP 249`_ doesn't cover -their usecase. - .. _`PEP 249`: https://www.python.org/dev/peps/pep-0249/ From 96d738a051673deff4d6b85d0d263c404e37e181 Mon Sep 17 00:00:00 2001 From: Rajat Jain Date: Tue, 19 Jan 2021 17:21:28 +0530 Subject: [PATCH 119/227] Remove Cursor._last_executed (#948) Fixes: #947. --- pymysql/cursors.py | 2 -- pymysql/tests/test_basic.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 666970b98..727a28e04 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -305,7 +305,6 @@ def scroll(self, value, mode="relative"): def _query(self, q): conn = self._get_db() - self._last_executed = q self._clear_result() conn.query(q) self._do_get_result() @@ -410,7 +409,6 @@ def close(self): def _query(self, q): conn = self._get_db() - self._last_executed = q self._clear_result() conn.query(q, unbuffered=True) self._do_get_result() diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index c2590bf2f..678ea9235 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -353,7 +353,7 @@ def test_bulk_insert(self): data, ) self.assertEqual( - cursor._last_executed, + cursor._executed, bytearray( b"insert into bulkinsert (id, name, age, height) values " b"(0,'bob',21,123),(1,'jim',56,45),(2,'fred',100,180)" @@ -377,7 +377,7 @@ def test_bulk_insert_multiline_statement(self): data, ) self.assertEqual( - cursor._last_executed.strip(), + cursor._executed.strip(), bytearray( b"""insert into bulkinsert (id, name, @@ -422,7 +422,7 @@ def test_issue_288(self): data, ) self.assertEqual( - cursor._last_executed.strip(), + cursor._executed.strip(), bytearray( b"""insert into bulkinsert (id, name, From 381e6aba21687cba18ca002db062f2fab3a04a9b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 19 Jan 2021 22:12:11 +0900 Subject: [PATCH 120/227] Actions: Fix 422 error on Coveralls (#949) * Actions: Update coveralls flag name * fix 422 error See https://github.com/TheKevJames/coveralls-python/issues/252 --- .github/workflows/test.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 09846c943..26b3f9c92 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,21 +106,23 @@ jobs: pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py - name: Report coverage - run: coveralls + run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_FLAG_NAME: ${{ matrix.test-name }} + COVERALLS_FLAG_NAME: ${{ matrix.py }}-${{ matrix.db }} COVERALLS_PARALLEL: true coveralls: name: Finish coveralls runs-on: ubuntu-20.04 needs: test - container: python:3-slim steps: + - uses: actions/setup-python@v2 + with: + python-version: 3.9 - name: Finished run: | - pip3 install --upgrade coveralls - coveralls --finish + pip install --upgrade coveralls + coveralls --finish --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 565dc36985a0d2c38a5a85cb4aa5b53e5c086f7c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 19 Jan 2021 22:25:28 +0900 Subject: [PATCH 121/227] Actions: Use cache in finish (#950) --- .github/workflows/test.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 26b3f9c92..158188cd1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -120,6 +120,14 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.9 + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: finish-pip-1 + restore-keys: | + finish-pip- + - name: Finished run: | pip install --upgrade coveralls From 5a11bab69075a5b9120877aa70f5b86f930809c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scheibe?= Date: Sun, 24 Jan 2021 04:00:02 +0100 Subject: [PATCH 122/227] Fix docstring for converter functions (#952) Co-authored-by: Rene Scheibe --- pymysql/converters.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index d910f5c5c..200cae5fa 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -155,16 +155,16 @@ def _convert_second_fraction(s): def convert_datetime(obj): """Returns a DATETIME or TIMESTAMP column value as a datetime object: - >>> datetime_or_None('2007-02-25 23:06:20') + >>> convert_datetime('2007-02-25 23:06:20') datetime.datetime(2007, 2, 25, 23, 6, 20) - >>> datetime_or_None('2007-02-25T23:06:20') + >>> convert_datetime('2007-02-25T23:06:20') datetime.datetime(2007, 2, 25, 23, 6, 20) Illegal values are returned as None: - >>> datetime_or_None('2007-02-31T23:06:20') is None + >>> convert_datetime('2007-02-31T23:06:20') is None True - >>> datetime_or_None('0000-00-00 00:00:00') is None + >>> convert_datetime('0000-00-00 00:00:00') is None True """ @@ -189,14 +189,14 @@ def convert_datetime(obj): def convert_timedelta(obj): """Returns a TIME column as a timedelta object: - >>> timedelta_or_None('25:06:17') + >>> convert_timedelta('25:06:17') datetime.timedelta(1, 3977) - >>> timedelta_or_None('-25:06:17') + >>> convert_timedelta('-25:06:17') datetime.timedelta(-2, 83177) Illegal values are returned as None: - >>> timedelta_or_None('random crap') is None + >>> convert_timedelta('random crap') is None True Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but @@ -236,14 +236,14 @@ def convert_timedelta(obj): def convert_time(obj): """Returns a TIME column as a time object: - >>> time_or_None('15:06:17') + >>> convert_time('15:06:17') datetime.time(15, 6, 17) Illegal values are returned as None: - >>> time_or_None('-25:06:17') is None + >>> convert_time('-25:06:17') is None True - >>> time_or_None('random crap') is None + >>> convert_time('random crap') is None True Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but @@ -279,14 +279,14 @@ def convert_time(obj): def convert_date(obj): """Returns a DATE column as a date object: - >>> date_or_None('2007-02-26') + >>> convert_date('2007-02-26') datetime.date(2007, 2, 26) Illegal values are returned as None: - >>> date_or_None('2007-02-31') is None + >>> convert_date('2007-02-31') is None True - >>> date_or_None('0000-00-00') is None + >>> convert_date('0000-00-00') is None True """ From 6ccbecc1a0dfd04065b081950d2d35b1dac0aaa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scheibe?= Date: Tue, 2 Feb 2021 07:23:09 +0100 Subject: [PATCH 123/227] Improve docstrings (#954) - dot at the end of descriptions - 3rd instead of 2nd person - more type information - minor rephrasing Co-authored-by: Rene Scheibe --- pymysql/connections.py | 46 +++++++++++++++++----------------- pymysql/cursors.py | 56 +++++++++++++++++++++++++++--------------- 2 files changed, 59 insertions(+), 43 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 92b7a77e5..b525014ca 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -99,18 +99,18 @@ class Connection: Establish a connection to the MySQL database. Accepts several arguments: - :param host: Host where the database server is located - :param user: Username to log in as + :param host: Host where the database server is located. + :param user: Username to log in as. :param password: Password to use. :param database: Database to use, None to not use a particular one. :param port: MySQL port to use, default is usually OK. (default: 3306) :param bind_address: When the client has multiple network interfaces, specify the interface from which to connect to the host. Argument can be a hostname or an IP address. - :param unix_socket: Optionally, you can use a unix socket rather than TCP/IP. + :param unix_socket: Use a unix socket rather than TCP/IP. :param read_timeout: The timeout for reading from the connection in seconds (default: None - no timeout) :param write_timeout: The timeout for writing to the connection in seconds (default: None - no timeout) - :param charset: Charset you want to use. + :param charset: Charset to use. :param sql_mode: Default SQL_MODE to use. :param read_default_file: Specifies my.cnf file to read these parameters from under the [client] section. @@ -124,16 +124,15 @@ class Connection: :param client_flag: Custom flags to send to MySQL. Find potential values in constants.CLIENT. :param cursorclass: Custom cursor class to use. :param init_command: Initial SQL statement to run when connection is established. - :param connect_timeout: Timeout before throwing an exception when connecting. + :param connect_timeout: The timeout for connecting to the database in seconds. (default: 10, min: 1, max: 31536000) - :param ssl: - A dict of arguments similar to mysql_ssl_set()'s parameters. - :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate - :param ssl_cert: Path to the file that contains a PEM-formatted client certificate - :param ssl_disabled: A boolean value that disables usage of TLS - :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate - :param ssl_verify_cert: Set to true to check the validity of server certificates - :param ssl_verify_identity: Set to true to check the server's identity + :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. + :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate. + :param ssl_cert: Path to the file that contains a PEM-formatted client certificate. + :param ssl_disabled: A boolean value that disables usage of TLS. + :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate. + :param ssl_verify_cert: Set to true to check the server certificate's validity. + :param ssl_verify_identity: Set to true to check the server's identity. :param read_default_group: Group to read from in the configuration file. :param autocommit: Autocommit mode. None means use server default. (default: False) :param local_infile: Boolean to enable the use of LOAD DATA LOCAL command. (default: False) @@ -148,8 +147,8 @@ class Connection: (if no authenticate method) for returning a string from the user. (experimental) :param server_public_key: SHA256 authentication plugin public key value. (default: None) :param binary_prefix: Add _binary prefix on bytes and bytearray. (default: False) - :param compress: Not supported - :param named_pipe: Not supported + :param compress: Not supported. + :param named_pipe: Not supported. :param db: **DEPRECATED** Alias for database. :param passwd: **DEPRECATED** Alias for password. @@ -415,11 +414,11 @@ def close(self): @property def open(self): - """Return True if the connection is open""" + """Return True if the connection is open.""" return self._sock is not None def _force_close(self): - """Close connection without QUIT message""" + """Close connection without QUIT message.""" if self._sock: try: self._sock.close() @@ -448,7 +447,7 @@ def _read_ok_packet(self): return ok def _send_autocommit_mode(self): - """Set whether or not to commit after every execute()""" + """Set whether or not to commit after every execute().""" self._execute_command( COMMAND.COM_QUERY, "SET AUTOCOMMIT = %s" % self.escape(self.autocommit_mode) ) @@ -496,7 +495,7 @@ def select_db(self, db): self._read_ok_packet() def escape(self, obj, mapping=None): - """Escape whatever value you pass to it. + """Escape whatever value is passed. Non-standard, for internal use; do not use this in your applications. """ @@ -510,7 +509,7 @@ def escape(self, obj, mapping=None): return converters.escape_item(obj, self.charset, mapping=mapping) def literal(self, obj): - """Alias for escape() + """Alias for escape(). Non-standard, for internal use; do not use this in your applications. """ @@ -530,9 +529,8 @@ def cursor(self, cursor=None): """ Create a new cursor to execute queries with. - :param cursor: The type of cursor to create; one of :py:class:`Cursor`, - :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. - None means use Cursor. + :param cursor: The type of cursor to create. None means use Cursor. + :type cursor: :py:class:`Cursor`, :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. """ if cursor: return cursor(self) @@ -565,6 +563,8 @@ def ping(self, reconnect=True): Check if the server is alive. :param reconnect: If the connection is closed, reconnect. + :type reconnect: boolean + :raise Error: If the connection is closed and reconnect=False. """ if self._sock is None: diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 727a28e04..2b5ccca90 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -15,7 +15,7 @@ class Cursor: """ - This is the object you use to interact with the database. + This is the object used to interact with the database. Do not create an instance of a Cursor yourself. Call connections.Connection.cursor(). @@ -79,7 +79,7 @@ def setoutputsizes(self, *args): """Does nothing, required by DB API.""" def _nextset(self, unbuffered=False): - """Get the next query set""" + """Get the next query set.""" conn = self._get_db() current_result = self._result if current_result is None or current_result is not conn._result: @@ -114,9 +114,18 @@ def _escape_args(self, args, conn): def mogrify(self, query, args=None): """ - Returns the exact string that is sent to the database by calling the + Returns the exact string that would be sent to the database by calling the execute() method. + :param query: Query to mogrify. + :type query: str + + :param args: Parameters used with query. (optional) + :type args: tuple, list or dict + + :return: The query with argument binding applied. + :rtype: str + This method follows the extension to the DB API 2.0 followed by Psycopg. """ conn = self._get_db() @@ -127,14 +136,15 @@ def mogrify(self, query, args=None): return query def execute(self, query, args=None): - """Execute a query + """Execute a query. - :param str query: Query to execute. + :param query: Query to execute. + :type query: str - :param args: parameters used with query. (optional) + :param args: Parameters used with query. (optional) :type args: tuple, list or dict - :return: Number of affected rows + :return: Number of affected rows. :rtype: int If args is a list or tuple, %s can be used as a placeholder in the query. @@ -150,12 +160,16 @@ def execute(self, query, args=None): return result def executemany(self, query, args): - # type: (str, list) -> int - """Run several data against one query + """Run several data against one query. + + :param query: Query to execute. + :type query: str + + :param args: Sequence of sequences or mappings. It is used as parameter. + :type args: tuple or list - :param query: query to execute on server - :param args: Sequence of sequences or mappings. It is used as parameter. :return: Number of rows affected, if any. + :rtype: int or None This method improves performance on multiple-row INSERT and REPLACE. Otherwise it is equivalent to looping over args with @@ -213,11 +227,13 @@ def _do_execute_many( return rows def callproc(self, procname, args=()): - """Execute stored procedure procname with args + """Execute stored procedure procname with args. - procname -- string, name of procedure to execute on server + :param procname: Name of procedure to execute on server. + :type procname: str - args -- Sequence of parameters to use with procedure + :param args: Sequence of parameters to use with procedure. + :type args: tuple or list Returns the original args. @@ -260,7 +276,7 @@ def callproc(self, procname, args=()): return args def fetchone(self): - """Fetch the next row""" + """Fetch the next row.""" self._check_executed() if self._rows is None or self.rownumber >= len(self._rows): return None @@ -269,7 +285,7 @@ def fetchone(self): return result def fetchmany(self, size=None): - """Fetch several rows""" + """Fetch several rows.""" self._check_executed() if self._rows is None: return () @@ -279,7 +295,7 @@ def fetchmany(self, size=None): return result def fetchall(self): - """Fetch all the rows""" + """Fetch all the rows.""" self._check_executed() if self._rows is None: return () @@ -418,11 +434,11 @@ def nextset(self): return self._nextset(unbuffered=True) def read_next(self): - """Read next row""" + """Read next row.""" return self._conv_row(self._result._read_rowdata_packet_unbuffered()) def fetchone(self): - """Fetch next row""" + """Fetch next row.""" self._check_executed() row = self.read_next() if row is None: @@ -450,7 +466,7 @@ def __iter__(self): return self.fetchall_unbuffered() def fetchmany(self, size=None): - """Fetch many""" + """Fetch many.""" self._check_executed() if size is None: size = self.arraysize From fb10477caf21122a89d7f216a0670d49dd2aa5d2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 27 Jun 2021 10:55:03 +0900 Subject: [PATCH 124/227] black --- pymysql/tests/test_basic.py | 16 ++++++++-------- pymysql/tests/test_connection.py | 18 +++++++++--------- pymysql/tests/test_issues.py | 32 ++++++++++++++++---------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 678ea9235..a0dea9c86 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -14,7 +14,7 @@ class TestConversion(base.PyMySQLTestCase): def test_datatypes(self): - """ test every data type """ + """test every data type""" conn = self.connect() c = conn.cursor() c.execute( @@ -80,7 +80,7 @@ def test_datatypes(self): c.execute("drop table test_datatypes") def test_dict(self): - """ test dict escaping """ + """test dict escaping""" conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a integer, b integer, c integer)") @@ -143,7 +143,7 @@ def test_blob(self): self.assertEqual(data, c.fetchone()[0]) def test_untyped(self): - """ test conversion of null, empty string """ + """test conversion of null, empty string""" conn = self.connect() c = conn.cursor() c.execute("select null,''") @@ -152,7 +152,7 @@ def test_untyped(self): self.assertEqual(("", None), c.fetchone()) def test_timedelta(self): - """ test timedelta conversion """ + """test timedelta conversion""" conn = self.connect() c = conn.cursor() c.execute( @@ -172,7 +172,7 @@ def test_timedelta(self): ) def test_datetime_microseconds(self): - """ test datetime conversion w microseconds""" + """test datetime conversion w microseconds""" conn = self.connect() if not self.mysql_server_is(conn, (5, 6, 4)): @@ -243,7 +243,7 @@ class TestCursor(base.PyMySQLTestCase): # self.assertEqual(r, c.description) def test_fetch_no_result(self): - """ test a fetchone() with no rows """ + """test a fetchone() with no rows""" conn = self.connect() c = conn.cursor() c.execute("create table test_nr (b varchar(32))") @@ -255,7 +255,7 @@ def test_fetch_no_result(self): c.execute("drop table test_nr") def test_aggregates(self): - """ test aggregate functions """ + """test aggregate functions""" conn = self.connect() c = conn.cursor() try: @@ -269,7 +269,7 @@ def test_aggregates(self): c.execute("drop table test_aggregates") def test_single_tuple(self): - """ test a single tuple """ + """test a single tuple""" conn = self.connect() c = conn.cursor() self.safe_create_table( diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 75db73cd0..a469be5a2 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -226,7 +226,7 @@ def realTestDialogAuthTwoQuestions(self): pymysql.connect( user="pymysql_2q", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @@ -266,12 +266,12 @@ def realTestDialogAuthThreeAttempts(self): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.DialogHandler}, - **self.db + **self.db, ) with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( @@ -282,27 +282,27 @@ def realTestDialogAuthThreeAttempts(self): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.DefectiveHandler}, - **self.db + **self.db, ) with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"notdialogplugin": TestAuthentication.Dialog}, - **self.db + **self.db, ) TestAuthentication.Dialog.m = {b"Password, please:": b"I do not know"} with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) TestAuthentication.Dialog.m = {b"Password, please:": None} with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @@ -367,7 +367,7 @@ def realTestPamAuth(self): auth_plugin_map={ b"mysql_cleartext_password": TestAuthentication.DefectiveHandler }, - **self.db + **self.db, ) except pymysql.OperationalError as e: self.assertEqual(1045, e.args[0]) @@ -378,7 +378,7 @@ def realTestPamAuth(self): auth_plugin_map={ b"mysql_cleartext_password": TestAuthentication.DefectiveHandler }, - **self.db + **self.db, ) if grants: # recreate the user diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index b4ced4b06..76d4b1334 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -14,7 +14,7 @@ class TestOldIssues(base.PyMySQLTestCase): def test_issue_3(self): - """ undefined methods datetime_or_None, date_or_None """ + """undefined methods datetime_or_None, date_or_None""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -42,7 +42,7 @@ def test_issue_3(self): c.execute("drop table issue3") def test_issue_4(self): - """ can't retrieve TIMESTAMP fields """ + """can't retrieve TIMESTAMP fields""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -57,13 +57,13 @@ def test_issue_4(self): c.execute("drop table issue4") def test_issue_5(self): - """ query on information_schema.tables fails """ + """query on information_schema.tables fails""" con = self.connect() cur = con.cursor() cur.execute("select * from information_schema.tables") def test_issue_6(self): - """ exception: TypeError: ord() expected a character, but string of length 0 found """ + """exception: TypeError: ord() expected a character, but string of length 0 found""" # ToDo: this test requires access to db 'mysql'. kwargs = self.databases[0].copy() kwargs["database"] = "mysql" @@ -73,7 +73,7 @@ def test_issue_6(self): conn.close() def test_issue_8(self): - """ Primary Key and Index error when selecting data """ + """Primary Key and Index error when selecting data""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -93,7 +93,7 @@ def test_issue_8(self): c.execute("drop table test") def test_issue_13(self): - """ can't handle large result fields """ + """can't handle large result fields""" conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): @@ -112,7 +112,7 @@ def test_issue_13(self): cur.execute("drop table issue13") def test_issue_15(self): - """ query should be expanded before perform character encoding """ + """query should be expanded before perform character encoding""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -127,7 +127,7 @@ def test_issue_15(self): c.execute("drop table issue15") def test_issue_16(self): - """ Patch for string and tuple escaping """ + """Patch for string and tuple escaping""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -285,7 +285,7 @@ def disabled_test_issue_54(self): class TestGitHubIssues(base.PyMySQLTestCase): def test_issue_66(self): - """ 'Connection' object has no attribute 'insert_id' """ + """'Connection' object has no attribute 'insert_id'""" conn = self.connect() c = conn.cursor() self.assertEqual(0, conn.insert_id()) @@ -303,7 +303,7 @@ def test_issue_66(self): c.execute("drop table issue66") def test_issue_79(self): - """ Duplicate field overwrites the previous one in the result of DictCursor """ + """Duplicate field overwrites the previous one in the result of DictCursor""" conn = self.connect() c = conn.cursor(pymysql.cursors.DictCursor) @@ -330,7 +330,7 @@ def test_issue_79(self): c.execute("drop table b") def test_issue_95(self): - """ Leftover trailing OK packet for "CALL my_sp" queries """ + """Leftover trailing OK packet for "CALL my_sp" queries""" conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): @@ -352,7 +352,7 @@ def test_issue_95(self): cur.execute("DROP PROCEDURE IF EXISTS `foo`") def test_issue_114(self): - """ autocommit is not set after reconnecting with ping() """ + """autocommit is not set after reconnecting with ping()""" conn = pymysql.connect(charset="utf8", **self.databases[0]) conn.autocommit(False) c = conn.cursor() @@ -377,7 +377,7 @@ def test_issue_114(self): conn.close() def test_issue_175(self): - """ The number of fields returned by server is read in wrong way """ + """The number of fields returned by server is read in wrong way""" conn = self.connect() cur = conn.cursor() for length in (200, 300): @@ -393,7 +393,7 @@ def test_issue_175(self): cur.execute("drop table if exists test_field_count") def test_issue_321(self): - """ Test iterable as query argument. """ + """Test iterable as query argument.""" conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( conn, @@ -422,7 +422,7 @@ def test_issue_321(self): self.assertEqual(cur.fetchone(), ("c", "\u0430")) def test_issue_364(self): - """ Test mixed unicode/binary arguments in executemany. """ + """Test mixed unicode/binary arguments in executemany.""" conn = pymysql.connect(charset="utf8mb4", **self.databases[0]) self.safe_create_table( conn, @@ -454,7 +454,7 @@ def test_issue_364(self): cur.executemany(usql, args=(values, values, values)) def test_issue_363(self): - """ Test binary / geometry types. """ + """Test binary / geometry types.""" conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( conn, From 46d17402afaa07369b954eee026f68c5b96207ba Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Jul 2021 12:44:50 +0900 Subject: [PATCH 125/227] Use dessant/lock-threads. --- .github/workflows/lock.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/lock.yml diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 000000000..1b25b4c79 --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,16 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write + +jobs: + action: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v2 + From d0cd254bb4886d04b74f868b4e63f2c595bebe2b Mon Sep 17 00:00:00 2001 From: Valentin Nechayev Date: Tue, 3 Aug 2021 08:57:21 +0300 Subject: [PATCH 126/227] Fix generating authentication response with long strings (#988) Connection attributes shall be encoded using lenenc-str approach for a separate string and the whole section. --- pymysql/connections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index b525014ca..00605dd9b 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -898,10 +898,10 @@ def _request_authentication(self): connect_attrs = b"" for k, v in self._connect_attrs.items(): k = k.encode("utf-8") - connect_attrs += struct.pack("B", len(k)) + k + connect_attrs += _lenenc_int(len(k)) + k v = v.encode("utf-8") - connect_attrs += struct.pack("B", len(v)) + v - data += struct.pack("B", len(connect_attrs)) + connect_attrs + connect_attrs += _lenenc_int(len(v)) + v + data += _lenenc_int(len(connect_attrs)) + connect_attrs self.write_packet(data) auth_packet = self._read_packet() From f0091e09889a3db2400f821bee6a411fa1822a44 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Aug 2021 15:06:22 +0900 Subject: [PATCH 127/227] Fix doctest in pymysql.converters (#994) Fixes #993 --- pymysql/converters.py | 44 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 200cae5fa..da63ceb7b 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -160,13 +160,12 @@ def convert_datetime(obj): >>> convert_datetime('2007-02-25T23:06:20') datetime.datetime(2007, 2, 25, 23, 6, 20) - Illegal values are returned as None: - - >>> convert_datetime('2007-02-31T23:06:20') is None - True - >>> convert_datetime('0000-00-00 00:00:00') is None - True + Illegal values are returned as str: + >>> convert_datetime('2007-02-31T23:06:20') + '2007-02-31T23:06:20' + >>> convert_datetime('0000-00-00 00:00:00') + '0000-00-00 00:00:00' """ if isinstance(obj, (bytes, bytearray)): obj = obj.decode("ascii") @@ -190,14 +189,14 @@ def convert_timedelta(obj): """Returns a TIME column as a timedelta object: >>> convert_timedelta('25:06:17') - datetime.timedelta(1, 3977) + datetime.timedelta(days=1, seconds=3977) >>> convert_timedelta('-25:06:17') - datetime.timedelta(-2, 83177) + datetime.timedelta(days=-2, seconds=82423) - Illegal values are returned as None: + Illegal values are returned as string: - >>> convert_timedelta('random crap') is None - True + >>> convert_timedelta('random crap') + 'random crap' Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but can accept values as (+|-)DD HH:MM:SS. The latter format will not @@ -239,12 +238,12 @@ def convert_time(obj): >>> convert_time('15:06:17') datetime.time(15, 6, 17) - Illegal values are returned as None: + Illegal values are returned as str: - >>> convert_time('-25:06:17') is None - True - >>> convert_time('random crap') is None - True + >>> convert_time('-25:06:17') + '-25:06:17' + >>> convert_time('random crap') + 'random crap' Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but can accept values as (+|-)DD HH:MM:SS. The latter format will not @@ -282,13 +281,12 @@ def convert_date(obj): >>> convert_date('2007-02-26') datetime.date(2007, 2, 26) - Illegal values are returned as None: - - >>> convert_date('2007-02-31') is None - True - >>> convert_date('0000-00-00') is None - True + Illegal values are returned as str: + >>> convert_date('2007-02-31') + '2007-02-31' + >>> convert_date('0000-00-00') + '0000-00-00' """ if isinstance(obj, (bytes, bytearray)): obj = obj.decode("ascii") @@ -362,3 +360,5 @@ def through(x): conversions = encoders.copy() conversions.update(decoders) Thing2Literal = escape_str + +# Run doctests with `pytest --doctest-modules pymysql/converters.py` From eba874bd771901b54440b40265b26b0597ea6146 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 4 Aug 2021 13:08:47 +0900 Subject: [PATCH 128/227] Actions: Run test with Python 3.10 (#996) --- .github/workflows/test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 158188cd1..6f6f97a58 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -31,6 +31,9 @@ jobs: py: "3.9" mysql_auth: true + - db: "mysql:8.0" + py: "3.10-dev" + services: mysql: image: "${{ matrix.db }}" From 33d165dc3087d298ed0e2d7c4e306ccfdab1ec2c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 28 Aug 2021 12:28:44 +0900 Subject: [PATCH 129/227] Fix calling undefined function (#1003) Fixes #981. --- pymysql/connections.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 00605dd9b..32b37bbf6 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -920,10 +920,7 @@ def _request_authentication(self): ): auth_packet = self._process_auth(plugin_name, auth_packet) else: - # send legacy handshake - data = _auth.scramble_old_password(self.password, self.salt) + b"\0" - self.write_packet(data) - auth_packet = self._read_packet() + raise err.OperationalError("received unknown auth swich request") elif auth_packet.is_extra_auth_data(): if DEBUG: print("received extra data") From 78f0cf99e5d5351df0821442e4dc35c49a6390c6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 28 Aug 2021 13:19:08 +0900 Subject: [PATCH 130/227] Stop showing handler name when hander is not set. (#1004) Fixes #987. --- pymysql/connections.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 32b37bbf6..199558ec5 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -998,8 +998,7 @@ def _process_auth(self, plugin_name, auth_packet): else: raise err.OperationalError( 2059, - "Authentication plugin '%s' (%r) not configured" - % (plugin_name, handler), + "Authentication plugin '%s' not configured" % (plugin_name,), ) pkt = self._read_packet() pkt.check_error() From f24cb9aa7295921bcd8f34f752c8a05b981d3125 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 2 Oct 2021 17:23:14 +1000 Subject: [PATCH 131/227] tests: container docker-entrypoint-initdb.d for ease of testing (#1009) This allows easier local testing in a container image. mysql (mysql in ubuntu) --comments is needed to push mariab comments to the server side for processing. --- .github/workflows/test.yaml | 30 +++------------------ ci/docker-entrypoint-initdb.d/README | 12 +++++++++ ci/docker-entrypoint-initdb.d/init.sql | 7 +++++ ci/docker-entrypoint-initdb.d/mariadb.sql | 2 ++ ci/docker-entrypoint-initdb.d/mysql.sql | 8 ++++++ pymysql/tests/test_connection.py | 32 +++++++++++++++++++++++ tests/test_mariadb_auth.py | 24 ----------------- 7 files changed, 65 insertions(+), 50 deletions(-) create mode 100644 ci/docker-entrypoint-initdb.d/README create mode 100644 ci/docker-entrypoint-initdb.d/init.sql create mode 100644 ci/docker-entrypoint-initdb.d/mariadb.sql create mode 100644 ci/docker-entrypoint-initdb.d/mysql.sql delete mode 100644 tests/test_mariadb_auth.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6f6f97a58..1269ad057 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,16 +10,14 @@ jobs: strategy: matrix: include: - - db: "mariadb:10.0" + - db: "mariadb:10.2" py: "3.9" - db: "mariadb:10.3" py: "3.8" - mariadb_auth: true - db: "mariadb:10.5" py: "3.7" - mariadb_auth: true - db: "mysql:5.6" py: "3.6" @@ -69,10 +67,9 @@ jobs: mysql -h127.0.0.1 -uroot -e 'select version()' && break done mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" - mysql -h127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' - mysql -h127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - mysql -h127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" - mysql -h127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" + mysql -h127.0.0.1 -uroot --comments < ci/docker-entrypoint-initdb.d/init.sql + mysql -h127.0.0.1 -uroot --comments < ci/docker-entrypoint-initdb.d/mysql.sql + mysql -h127.0.0.1 -uroot --comments < ci/docker-entrypoint-initdb.d/mariadb.sql cp ci/docker.json pymysql/tests/databases.json - name: Run test @@ -87,27 +84,8 @@ jobs: docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - mysql -uroot -h127.0.0.1 -e ' - CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", - nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", - nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" - PASSWORD EXPIRE NEVER; - GRANT RELOAD ON *.* TO user_caching_sha2;' pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - - name: Run MariaDB auth test - if: ${{ matrix.mariadb_auth }} - run: | - mysql -uroot -h127.0.0.1 -e ' - INSTALL SONAME "auth_ed25519"; - CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' - # we need to pass the hashed password manually until 10.4, so hide it here - mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql -uroot -h127.0.0.1 - mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql -uroot -h127.0.0.1 - pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py - - name: Report coverage run: coveralls --service=github env: diff --git a/ci/docker-entrypoint-initdb.d/README b/ci/docker-entrypoint-initdb.d/README new file mode 100644 index 000000000..6a54b93da --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/README @@ -0,0 +1,12 @@ +To test with a MariaDB or MySQL container image: + +docker run -d -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=1 \ + --name=mysqld -v ./ci/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:z \ + mysql:8.0.26 --local-infile=1 + +cp ci/docker.json pymysql/tests/databases.json + +pytest + + +Note: Some authentication tests that don't match the image version will fail. diff --git a/ci/docker-entrypoint-initdb.d/init.sql b/ci/docker-entrypoint-initdb.d/init.sql new file mode 100644 index 000000000..b741d41c5 --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/init.sql @@ -0,0 +1,7 @@ +create database test1 DEFAULT CHARACTER SET utf8mb4; +create database test2 DEFAULT CHARACTER SET utf8mb4; +create user test2 identified by 'some password'; +grant all on test2.* to test2; +create user test2@localhost identified by 'some password'; +grant all on test2.* to test2@localhost; + diff --git a/ci/docker-entrypoint-initdb.d/mariadb.sql b/ci/docker-entrypoint-initdb.d/mariadb.sql new file mode 100644 index 000000000..912d365a9 --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/mariadb.sql @@ -0,0 +1,2 @@ +/*M!100122 INSTALL SONAME "auth_ed25519" */; +/*M!100122 CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so" */; diff --git a/ci/docker-entrypoint-initdb.d/mysql.sql b/ci/docker-entrypoint-initdb.d/mysql.sql new file mode 100644 index 000000000..a4ba0927d --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/mysql.sql @@ -0,0 +1,8 @@ +/*!80001 CREATE USER + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", + nopass_sha256 IDENTIFIED WITH "sha256_password", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER */; + +/*!80001 GRANT RELOAD ON *.* TO user_caching_sha2 */; diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index a469be5a2..e95b75d6f 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -53,6 +53,7 @@ class TestAuthentication(base.PyMySQLTestCase): pam_found = False mysql_old_password_found = False sha256_password_found = False + ed25519_found = False import os @@ -97,6 +98,8 @@ class TestAuthentication(base.PyMySQLTestCase): mysql_old_password_found = True elif r[0] == "sha256_password": sha256_password_found = True + elif r[0] == "ed25519": + ed25519_found = True # else: # print("plugin: %r" % r[0]) @@ -412,6 +415,35 @@ def testAuthSHA256(self): with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user="pymysql_sha256", **db) + @pytest.mark.skipif(not ed25519_found, reason="no ed25519 authention plugin") + def testAuthEd25519(self): + db = self.db.copy() + del db["password"] + conn = self.connect() + c = conn.cursor() + c.execute("select ed25519_password(''), ed25519_password('ed25519_password')") + for r in c: + empty_pass = r[0].decode("ascii") + non_empty_pass = r[1].decode("ascii") + + with TempUser( + c, + "pymysql_ed25519", + self.databases[0]["database"], + "ed25519", + empty_pass, + ) as u: + pymysql.connect(user="pymysql_ed25519", password="", **db) + + with TempUser( + c, + "pymysql_ed25519", + self.databases[0]["database"], + "ed25519", + non_empty_pass, + ) as u: + pymysql.connect(user="pymysql_ed25519", password="ed25519_password", **db) + class TestConnection(base.PyMySQLTestCase): def test_utf8mb4(self): diff --git a/tests/test_mariadb_auth.py b/tests/test_mariadb_auth.py deleted file mode 100644 index b3a2719cd..000000000 --- a/tests/test_mariadb_auth.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Test for auth methods supported by MariaDB 10.3+""" - -import pymysql - -# pymysql.connections.DEBUG = True -# pymysql._auth.DEBUG = True - -host = "127.0.0.1" -port = 3306 - - -def test_ed25519_no_password(): - con = pymysql.connect(user="nopass_ed25519", host=host, port=port, ssl=None) - con.close() - - -def test_ed25519_password(): # nosec - con = pymysql.connect( - user="user_ed25519", password="pass_ed25519", host=host, port=port, ssl=None - ) - con.close() - - -# default mariadb docker images aren't configured with SSL From 534f4a6f53097384842b55ac7466a8033c0d1375 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 31 Jan 2022 05:32:17 +0100 Subject: [PATCH 132/227] fix typo in comment (#1024) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 199558ec5..bfe8b10ac 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -204,12 +204,12 @@ def __init__( db=None, # deprecated ): if db is not None and database is None: - # We will raise warining in 2022 or later. + # We will raise warning in 2022 or later. # See https://github.com/PyMySQL/PyMySQL/issues/939 # warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) database = db if passwd is not None and not password: - # We will raise warining in 2022 or later. + # We will raise warning in 2022 or later. # See https://github.com/PyMySQL/PyMySQL/issues/939 # warnings.warn( # "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 From 72f70c9ff81103b4a2e0b8531663a80d44595c2d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 31 Jan 2022 13:50:32 +0900 Subject: [PATCH 133/227] Update black version (#1026) --- docs/source/conf.py | 16 ++++++++-------- pymysql/connections.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 77d7073a8..a57a03c44 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -46,8 +46,8 @@ master_doc = "index" # General information about the project. -project = u"PyMySQL" -copyright = u"2016, Yutaka Matsubara and GitHub contributors" +project = "PyMySQL" +copyright = "2016, Yutaka Matsubara and GitHub contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -200,8 +200,8 @@ ( "index", "PyMySQL.tex", - u"PyMySQL Documentation", - u"Yutaka Matsubara and GitHub contributors", + "PyMySQL Documentation", + "Yutaka Matsubara and GitHub contributors", "manual", ), ] @@ -235,8 +235,8 @@ ( "index", "pymysql", - u"PyMySQL Documentation", - [u"Yutaka Matsubara and GitHub contributors"], + "PyMySQL Documentation", + ["Yutaka Matsubara and GitHub contributors"], 1, ) ] @@ -254,8 +254,8 @@ ( "index", "PyMySQL", - u"PyMySQL Documentation", - u"Yutaka Matsubara and GitHub contributors", + "PyMySQL Documentation", + "Yutaka Matsubara and GitHub contributors", "PyMySQL", "One line description of project.", "Miscellaneous", diff --git a/pymysql/connections.py b/pymysql/connections.py index bfe8b10ac..2edeb5083 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -61,7 +61,7 @@ DEFAULT_CHARSET = "utf8mb4" -MAX_PACKET_LEN = 2 ** 24 - 1 +MAX_PACKET_LEN = 2**24 - 1 def _pack_int24(n): From afbef5ea0d1bc4c5c2d5d15c5ce519ecdfd29a1d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 31 Jan 2022 14:35:31 +0900 Subject: [PATCH 134/227] Actions: Use actions/setup-python cache (#1027) --- .github/workflows/test.yaml | 25 +++++++++---------------- requirements-dev.txt | 2 ++ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1269ad057..2a9ff0a6c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,17 +47,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.py }} - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-1 - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' + cache-dependency-path: 'requirements-dev.txt' - name: Install dependency run: | - pip install -U cryptography PyNaCl pytest pytest-cov coveralls + pip install -U -r requirements-dev.txt - name: Set up MySQL run: | @@ -98,16 +93,14 @@ jobs: runs-on: ubuntu-20.04 needs: test steps: - - uses: actions/setup-python@v2 - with: - python-version: 3.9 + - name: requirements. + run: | + echo coveralls > requirements.txt - - uses: actions/cache@v2 + - uses: actions/setup-python@v2 with: - path: ~/.cache/pip - key: finish-pip-1 - restore-keys: | - finish-pip- + python-version: '3.9' + cache: 'pip' - name: Finished run: | diff --git a/requirements-dev.txt b/requirements-dev.txt index d65512fbb..13d7f7fb4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,5 @@ cryptography PyNaCl>=1.4.0 pytest +pytest-cov +coveralls From 2beebd92b8ad3fb59a93714c799450dbfebe3922 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 1 Feb 2022 01:04:50 +0100 Subject: [PATCH 135/227] update pymysql.constants.CR (#1029) values from https://github.com/mysql/mysql-server/blob/mysql-8.0.28/include/errmsg.h --- pymysql/constants/CR.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pymysql/constants/CR.py b/pymysql/constants/CR.py index 25579a7c6..deae977e5 100644 --- a/pymysql/constants/CR.py +++ b/pymysql/constants/CR.py @@ -65,4 +65,15 @@ CR_AUTH_PLUGIN_CANNOT_LOAD = 2059 CR_DUPLICATE_CONNECTION_ATTR = 2060 CR_AUTH_PLUGIN_ERR = 2061 -CR_ERROR_LAST = 2061 +CR_INSECURE_API_ERR = 2062 +CR_FILE_NAME_TOO_LONG = 2063 +CR_SSL_FIPS_MODE_ERR = 2064 +CR_DEPRECATED_COMPRESSION_NOT_SUPPORTED = 2065 +CR_COMPRESSION_WRONGLY_CONFIGURED = 2066 +CR_KERBEROS_USER_NOT_FOUND = 2067 +CR_LOAD_DATA_LOCAL_INFILE_REJECTED = 2068 +CR_LOAD_DATA_LOCAL_INFILE_REALPATH_FAIL = 2069 +CR_DNS_SRV_LOOKUP_FAILED = 2070 +CR_MANDATORY_TRACKER_NOT_FOUND = 2071 +CR_INVALID_FACTOR_NO = 2072 +CR_ERROR_LAST = 2072 From 3fb9dd9b1f88334bb8014969a7b7f7027632dcca Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 1 Feb 2022 04:57:02 +0100 Subject: [PATCH 136/227] Use constants (#1028) --- pymysql/connections.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 2edeb5083..04e3c53fb 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -13,7 +13,7 @@ from . import _auth from .charset import charset_by_name, charset_by_id -from .constants import CLIENT, COMMAND, CR, FIELD_TYPE, SERVER_STATUS +from .constants import CLIENT, COMMAND, CR, ER, FIELD_TYPE, SERVER_STATUS from . import converters from .cursors import Cursor from .optionfile import Parser @@ -441,7 +441,10 @@ def get_autocommit(self): def _read_ok_packet(self): pkt = self._read_packet() if not pkt.is_ok_packet(): - raise err.OperationalError(2014, "Command Out of Sync") + raise err.OperationalError( + CR.CR_COMMANDS_OUT_OF_SYNC, + "Command Out of Sync", + ) ok = OKPacketWrapper(pkt) self.server_status = ok.server_status return ok @@ -654,7 +657,8 @@ def connect(self, sock=None): if isinstance(e, (OSError, IOError, socket.error)): exc = err.OperationalError( - 2003, "Can't connect to MySQL server on %r (%s)" % (self.host, e) + CR.CR_CONN_HOST_ERROR, + "Can't connect to MySQL server on %r (%s)" % (self.host, e), ) # Keep original exception and traceback to investigate error. exc.original_exception = e @@ -945,7 +949,7 @@ def _process_auth(self, plugin_name, auth_packet): except AttributeError: if plugin_name != b"dialog": raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s'" " not loaded: - %r missing authenticate method" % (plugin_name, type(handler)), @@ -983,21 +987,21 @@ def _process_auth(self, plugin_name, auth_packet): self.write_packet(resp + b"\0") except AttributeError: raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s'" " not loaded: - %r missing prompt method" % (plugin_name, handler), ) except TypeError: raise err.OperationalError( - 2061, + CR.CR_AUTH_PLUGIN_ERR, "Authentication plugin '%s'" " %r didn't respond with string. Returned '%r' to prompt %r" % (plugin_name, handler, resp, prompt), ) else: raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s' not configured" % (plugin_name,), ) pkt = self._read_packet() @@ -1007,7 +1011,8 @@ def _process_auth(self, plugin_name, auth_packet): return pkt else: raise err.OperationalError( - 2059, "Authentication plugin '%s' not configured" % plugin_name + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, + "Authentication plugin '%s' not configured" % plugin_name, ) self.write_packet(data) @@ -1024,7 +1029,7 @@ def _get_auth_plugin_handler(self, plugin_name): handler = plugin_class(self) except TypeError: raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s'" " not loaded: - %r cannot be constructed with connection object" % (plugin_name, plugin_class), @@ -1211,7 +1216,10 @@ def _read_load_local_packet(self, first_packet): if ( not ok_packet.is_ok_packet() ): # pragma: no cover - upstream induced protocol error - raise err.OperationalError(2014, "Commands Out of Sync") + raise err.OperationalError( + CR.CR_COMMANDS_OUT_OF_SYNC, + "Commands Out of Sync", + ) self._read_ok_packet(ok_packet) def _check_packet_is_eof(self, packet): @@ -1357,7 +1365,10 @@ def send_data(self): break conn.write_packet(chunk) except IOError: - raise err.OperationalError(1017, f"Can't find file '{self.filename}'") + raise err.OperationalError( + ER.FILE_NOT_FOUND, + f"Can't find file '{self.filename}'", + ) finally: # send the empty packet to signify we are done sending data conn.write_packet(b"") From cebba92d338d89ac46381f3e1ca637416a77c0e2 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 6 Feb 2022 08:50:49 +0100 Subject: [PATCH 137/227] Improve GitHub workflow (#1031) - concurrency cancels builds in progress e.g. on pull requests - matrix jobs no longer fail fast, allowing to see failure reasons for all matrix jobs - coveralls no longer runs on forks, this would fail anyways --- .github/workflows/test.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2a9ff0a6c..d9b9e2afb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,10 +4,15 @@ on: push: pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: include: - db: "mariadb:10.2" @@ -82,6 +87,7 @@ jobs: pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - name: Report coverage + if: github.repository == 'PyMySQL/PyMySQL' run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -89,6 +95,7 @@ jobs: COVERALLS_PARALLEL: true coveralls: + if: github.repository == 'PyMySQL/PyMySQL' name: Finish coveralls runs-on: ubuntu-20.04 needs: test From 062384c26d10556529af91d0f0946e302b727d18 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 6 Feb 2022 08:52:15 +0100 Subject: [PATCH 138/227] Drop support of EOL Python and DB versions (#1030) - Python now requires 3.7+, reflected in python_requires - MySQL now requires 5.7+ in tests - MariaDB unchanged in tests, only dropped support in documentation - Added Python 3.11 to test matrix - Added MariaDB 10.7 to test matrix - DB version checks have been removed from various tests where no longer needed this also results in running a few tests on MariaDB which were previously only running on MySQL. --- .github/workflows/test.yaml | 8 ++++---- CHANGELOG.md | 9 +++++++++ README.rst | 6 +++--- docs/source/user/installation.rst | 6 +++--- pymysql/tests/base.py | 5 +++++ pymysql/tests/test_basic.py | 6 +++--- pymysql/tests/test_connection.py | 10 +--------- pymysql/tests/test_issues.py | 15 +++------------ setup.py | 5 +++-- 9 files changed, 34 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d9b9e2afb..0d2e99983 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -24,18 +24,18 @@ jobs: - db: "mariadb:10.5" py: "3.7" - - db: "mysql:5.6" - py: "3.6" + - db: "mariadb:10.7" + py: "3.11-dev" - db: "mysql:5.7" - py: "pypy-3.6" + py: "pypy-3.8" - db: "mysql:8.0" py: "3.9" mysql_auth: true - db: "mysql:8.0" - py: "3.10-dev" + py: "3.10" services: mysql: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9885af526..abf38b3ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changes +## v1.0.3 + +Release date: TBD + +* Dropped support of end of life MySQL version 5.6 +* Dropped support of end of life MariaDB versions below 10.2 +* Dropped support of end of life Python version 3.6 + + ## v1.0.2 Release date: 2021-01-09 diff --git a/README.rst b/README.rst index f514d901e..e7c9419ee 100644 --- a/README.rst +++ b/README.rst @@ -25,13 +25,13 @@ Requirements * Python -- one of the following: - - CPython_ : 3.6 and newer + - CPython_ : 3.7 and newer - PyPy_ : Latest 3.x version * MySQL Server -- one of the following: - - MySQL_ >= 5.6 - - MariaDB_ >= 10.0 + - MySQL_ >= 5.7 + - MariaDB_ >= 10.2 .. _CPython: https://www.python.org/ .. _PyPy: https://pypy.org/ diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index 0fea27266..c66aae3d4 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -18,13 +18,13 @@ Requirements * Python -- one of the following: - - CPython_ >= 3.6 + - CPython_ >= 3.7 - Latest PyPy_ 3 * MySQL Server -- one of the following: - - MySQL_ >= 5.6 - - MariaDB_ >= 10.0 + - MySQL_ >= 5.7 + - MariaDB_ >= 10.2 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index 6f93a8317..a87307a57 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -32,6 +32,11 @@ def mysql_server_is(self, conn, version_tuple): """Return True if the given connection is on the version given or greater. + This only checks the server version string provided when the + connection is established, therefore any check for a version tuple + greater than (5, 5, 5) will always fail on MariaDB, as it always + starts with 5.5.5, e.g. 5.5.5-10.7.1-MariaDB-1:10.7.1+maria~focal. + e.g.:: if self.mysql_server_is(conn, (5, 6, 4)): diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index a0dea9c86..d37d19762 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -175,8 +175,6 @@ def test_datetime_microseconds(self): """test datetime conversion w microseconds""" conn = self.connect() - if not self.mysql_server_is(conn, (5, 6, 4)): - pytest.skip("target backend does not support microseconds") c = conn.cursor() dt = datetime.datetime(2013, 11, 12, 9, 9, 9, 123450) c.execute("create table test_datetime (id int, ts datetime(6))") @@ -285,8 +283,10 @@ def test_json(self): args = self.databases[0].copy() args["charset"] = "utf8mb4" conn = pymysql.connect(**args) + # MariaDB only has limited JSON support, stores data as longtext + # https://mariadb.com/kb/en/json-data-type/ if not self.mysql_server_is(conn, (5, 7, 0)): - pytest.skip("JSON type is not supported on MySQL <= 5.6") + pytest.skip("JSON type is only supported on MySQL >= 5.7") self.safe_create_table( conn, diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index e95b75d6f..23a2aa047 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -105,8 +105,6 @@ class TestAuthentication(base.PyMySQLTestCase): def test_plugin(self): conn = self.connect() - if not self.mysql_server_is(conn, (5, 5, 0)): - pytest.skip("MySQL-5.5 required for plugins") cur = conn.cursor() cur.execute( "select plugin from mysql.user where concat(user, '@', host)=current_user()" @@ -401,13 +399,7 @@ def testAuthSHA256(self): self.databases[0]["database"], "sha256_password", ) as u: - if self.mysql_server_is(conn, (5, 7, 0)): - c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") - else: - c.execute("SET old_passwords = 2") - c.execute( - "SET PASSWORD FOR 'pymysql_sha256'@'localhost' = PASSWORD('Sh@256Pa33')" - ) + c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") c.execute("FLUSH PRIVILEGES") db = self.db.copy() db["password"] = "Sh@256Pa33" diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 76d4b1334..3ea2c2c46 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -466,29 +466,20 @@ def test_issue_363(self): ) cur = conn.cursor() - # From MySQL 5.7, ST_GeomFromText is added and GeomFromText is deprecated. - if self.mysql_server_is(conn, (5, 7, 0)): - geom_from_text = "ST_GeomFromText" - geom_as_text = "ST_AsText" - geom_as_bin = "ST_AsBinary" - else: - geom_from_text = "GeomFromText" - geom_as_text = "AsText" - geom_as_bin = "AsBinary" query = ( "INSERT INTO issue363 (id, geom) VALUES" - "(1998, %s('LINESTRING(1.1 1.1,2.2 2.2)'))" % geom_from_text + "(1998, ST_GeomFromText('LINESTRING(1.1 1.1,2.2 2.2)'))" ) cur.execute(query) # select WKT - query = "SELECT %s(geom) FROM issue363" % geom_as_text + query = "SELECT ST_AsText(geom) FROM issue363" cur.execute(query) row = cur.fetchone() self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)",)) # select WKB - query = "SELECT %s(geom) FROM issue363" % geom_as_bin + query = "SELECT ST_AsBinary(geom) FROM issue363" cur.execute(query) row = cur.fetchone() self.assertEqual( diff --git a/setup.py b/setup.py index 1510a0cf8..7cdc692fb 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ description="Pure Python MySQL Driver", long_description=readme, packages=find_packages(exclude=["tests*", "pymysql.tests*"]), - python_requires=">=3.6", + python_requires=">=3.7", extras_require={ "rsa": ["cryptography"], "ed25519": ["PyNaCl>=1.4.0"], @@ -24,10 +24,11 @@ classifiers=[ "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From ee88d0f0e6499ad3054edbf057e08abfe25993c4 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 6 Feb 2022 08:53:30 +0100 Subject: [PATCH 139/227] Fix coveralls branch in README.rst (#1034) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e7c9419ee..f1384c92f 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ :target: https://pymysql.readthedocs.io/ :alt: Documentation Status -.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github - :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master +.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github + :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=main .. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python From eb108a61669f8883426d35f153dc48c6348d4b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A4=80=EA=B7=9C?= Date: Tue, 22 Mar 2022 14:54:05 +0900 Subject: [PATCH 140/227] Fix minor typo in error message (#1038) --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 04e3c53fb..9de40dea5 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -924,7 +924,7 @@ def _request_authentication(self): ): auth_packet = self._process_auth(plugin_name, auth_packet) else: - raise err.OperationalError("received unknown auth swich request") + raise err.OperationalError("received unknown auth switch request") elif auth_packet.is_extra_auth_data(): if DEBUG: print("received extra data") From b9e07c5bb56806a167003ced8d3c5e704657e503 Mon Sep 17 00:00:00 2001 From: Daniel Golding Date: Sat, 16 Apr 2022 07:23:52 +0200 Subject: [PATCH 141/227] Document that the ssl connection parameter can be an SSLContext (#1045) --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 9de40dea5..94ea545fc 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -126,7 +126,7 @@ class Connection: :param init_command: Initial SQL statement to run when connection is established. :param connect_timeout: The timeout for connecting to the database in seconds. (default: 10, min: 1, max: 31536000) - :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. + :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters or an ssl.SSLContext. :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate. :param ssl_cert: Path to the file that contains a PEM-formatted client certificate. :param ssl_disabled: A boolean value that disables usage of TLS. From 72ee1f3804082442fcbc5c0b1a054ed5c284cd7d Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 14 Jun 2022 06:40:21 +0200 Subject: [PATCH 142/227] Update mariadb tests to 10.8, remove end of life mariadb 10.2 (#1049) --- .github/workflows/test.yaml | 6 +++--- CHANGELOG.md | 2 +- README.rst | 2 +- docs/source/user/installation.rst | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0d2e99983..e07a4c9b8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,9 +15,6 @@ jobs: fail-fast: false matrix: include: - - db: "mariadb:10.2" - py: "3.9" - - db: "mariadb:10.3" py: "3.8" @@ -27,6 +24,9 @@ jobs: - db: "mariadb:10.7" py: "3.11-dev" + - db: "mariadb:10.8" + py: "3.9" + - db: "mysql:5.7" py: "pypy-3.8" diff --git a/CHANGELOG.md b/CHANGELOG.md index abf38b3ff..5a4292449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Release date: TBD * Dropped support of end of life MySQL version 5.6 -* Dropped support of end of life MariaDB versions below 10.2 +* Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 diff --git a/README.rst b/README.rst index f1384c92f..318e94604 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Requirements * MySQL Server -- one of the following: - MySQL_ >= 5.7 - - MariaDB_ >= 10.2 + - MariaDB_ >= 10.3 .. _CPython: https://www.python.org/ .. _PyPy: https://pypy.org/ diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index c66aae3d4..9313f14d3 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -24,7 +24,7 @@ Requirements * MySQL Server -- one of the following: - MySQL_ >= 5.7 - - MariaDB_ >= 10.2 + - MariaDB_ >= 10.3 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ From 0ab388939ae96fa32acc59ebcc2e7b1a2a4da8c1 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 14 Jul 2022 07:57:13 +0200 Subject: [PATCH 143/227] Fix CodeQL target branch (#1054) master branch was renamed to main some time ago, leading to this action no longer working properly, at least for PRs --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b6a7238dd..94165437a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,10 +13,10 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [ main ] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ main ] schedule: - cron: '34 7 * * 2' From 7f47ac0184294b15a3b53cdcbe96b9895d0c6f4c Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 14 Jul 2022 07:57:25 +0200 Subject: [PATCH 144/227] Update CodeQL GitHub action to v2 (#1055) v1 has been deprecated: https://github.blog/changelog/2022-04-27-code-scanning-deprecation-of-codeql-action-v1/ --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 94165437a..d559b1cd2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -50,7 +50,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -64,4 +64,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From d1748350b9b6b4efdcead428fad2fbcdb7cfddd0 Mon Sep 17 00:00:00 2001 From: WangDi Date: Fri, 22 Jul 2022 13:12:12 +0800 Subject: [PATCH 145/227] tests: remove duplicate test (#1057) --- pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index e882c5eb3..9ac190f27 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -23,9 +23,6 @@ def test_setoutputsize(self): def test_setoutputsize_basic(self): pass - def test_nextset(self): - pass - """The tests on fetchone and fetchall and rowcount bogusly test for an exception if the statement cannot return a result set. MySQL always returns a result set; it's just that From dd47caae95011e79b9e2ee12549d23f05a7f839d Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 24 Aug 2022 04:50:30 +0200 Subject: [PATCH 146/227] Remove deprecated socket.error from Connection.connect exception handler (#1062) Since python 3.3, `socket.error` is a deprecated alias for OSError, which is already included. --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 94ea545fc..3265d32ee 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -655,7 +655,7 @@ def connect(self, sock=None): except: # noqa pass - if isinstance(e, (OSError, IOError, socket.error)): + if isinstance(e, (OSError, IOError)): exc = err.OperationalError( CR.CR_CONN_HOST_ERROR, "Can't connect to MySQL server on %r (%s)" % (self.host, e), From e77b21898ab46887067df981eaa19809533ec4bf Mon Sep 17 00:00:00 2001 From: Chuck Cadman <51368516+cdcadman@users.noreply.github.com> Date: Mon, 19 Sep 2022 00:06:49 -0700 Subject: [PATCH 147/227] Raise ProgrammingError on -inf in addition to inf (#1067) Co-authored-by: Chuck Cadman --- pymysql/converters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index da63ceb7b..2acc3e58d 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -56,7 +56,7 @@ def escape_int(value, mapping=None): def escape_float(value, mapping=None): s = repr(value) - if s in ("inf", "nan"): + if s in ("inf", "-inf", "nan"): raise ProgrammingError("%s can not be used with MySQL" % s) if "e" not in s: s += "e0" From 3dc1abbdaf7af99357c834c58f0e27f871ebe885 Mon Sep 17 00:00:00 2001 From: SergeantMenacingGarlic <87030047+SergeantMenacingGarlic@users.noreply.github.com> Date: Tue, 11 Oct 2022 03:06:18 -0400 Subject: [PATCH 148/227] Add unix socket test (#1061) --- .github/workflows/test.yaml | 9 +++++++++ ci/docker.json | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e07a4c9b8..5a8f6dab5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,9 +45,18 @@ jobs: env: MYSQL_ALLOW_EMPTY_PASSWORD: yes options: "--name=mysqld" + volumes: + - /run/mysqld:/run/mysqld steps: - uses: actions/checkout@v2 + + - name: Workaround MySQL container permissions + if: startsWith(matrix.db, 'mysql') + run: | + sudo chown 999:999 /run/mysqld + /usr/bin/docker ps --all --filter status=exited --no-trunc --format "{{.ID}}" | xargs -r /usr/bin/docker start + - name: Set up Python ${{ matrix.py }} uses: actions/setup-python@v2 with: diff --git a/ci/docker.json b/ci/docker.json index 34a5c7b7c..63d19a687 100644 --- a/ci/docker.json +++ b/ci/docker.json @@ -1,4 +1,5 @@ [ {"host": "127.0.0.1", "port": 3306, "user": "root", "password": "", "database": "test1", "use_unicode": true, "local_infile": true}, - {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" } + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" }, + {"host": "localhost", "port": 3306, "user": "test2", "password": "some password", "database": "test2", "unix_socket": "/run/mysqld/mysqld.sock"} ] From 90317924e8f4ae5af871d4ef32cfadf963a795f4 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 11 Nov 2022 03:27:42 +0100 Subject: [PATCH 149/227] Use Python 3.11 release instead of -dev in tests (#1076) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5a8f6dab5..39afc5791 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,7 +22,7 @@ jobs: py: "3.7" - db: "mariadb:10.7" - py: "3.11-dev" + py: "3.11" - db: "mariadb:10.8" py: "3.9" From ed56379dcc165f8810c8678c56bff7bb544a710f Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Fri, 11 Nov 2022 13:28:06 +1100 Subject: [PATCH 150/227] docs: Fix a few typos (#1053) --- pymysql/tests/test_connection.py | 2 +- pymysql/tests/test_issues.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 23a2aa047..94a8dea04 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -492,7 +492,7 @@ def test_connection_gone_away(self): time.sleep(2) with self.assertRaises(pymysql.OperationalError) as cm: cur.execute("SELECT 1+1") - # error occures while reading, not writing because of socket buffer. + # error occurs while reading, not writing because of socket buffer. # self.assertEqual(cm.exception.args[0], 2006) self.assertIn(cm.exception.args[0], (2006, 2013)) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 3ea2c2c46..733d56a16 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -149,7 +149,7 @@ def test_issue_16(self): "test_issue_17() requires a custom, legacy MySQL configuration and will not be run." ) def test_issue_17(self): - """could not connect mysql use passwod""" + """could not connect mysql use password""" conn = self.connect() host = self.databases[0]["host"] db = self.databases[0]["database"] From e3a1beba22234f419d68c6947d7a1a0bf5d2eae4 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 9 Jan 2023 09:36:10 +0100 Subject: [PATCH 151/227] flake8: Use max_line_length instead of ignoring E501 (#1081) --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b40802e4b..e487e5e74 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [flake8] -ignore = E203,E501,W503,E722 exclude = tests,build,.venv,docs +ignore = E203,W503,E722 +max_line_length=129 [metadata] license = "MIT" From e91d097029f90055237741b5e56f81933ec1c981 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 9 Jan 2023 13:10:32 +0100 Subject: [PATCH 152/227] Fix typos discovered by codespell (#1082) --- CHANGELOG.md | 2 +- pymysql/_auth.py | 2 +- pymysql/tests/test_DictCursor.py | 2 +- pymysql/tests/test_basic.py | 2 +- pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py | 12 ++++++------ .../thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a4292449..87c3f9e86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -204,7 +204,7 @@ Release date: 2016-08-30 Release date: 2016-07-29 * Fix SELECT JSON type cause UnicodeError -* Avoid float convertion while parsing microseconds +* Avoid float conversion while parsing microseconds * Warning has number * SSCursor supports warnings diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 33fd9df86..f6c9eb967 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -241,7 +241,7 @@ def caching_sha2_password_auth(conn, pkt): return pkt if n != 4: - raise OperationalError("caching sha2: Unknwon result for fast auth: %s" % n) + raise OperationalError("caching sha2: Unknown result for fast auth: %s" % n) if DEBUG: print("caching sha2: Trying full auth...") diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index 581a0c4ae..bbc87d032 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -17,7 +17,7 @@ def setUp(self): self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) - # create a table ane some data to query + # create a table and some data to query with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists dictcursor") diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index d37d19762..bc88e5a55 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -320,7 +320,7 @@ def setUp(self): self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) - # create a table ane some data to query + # create a table and some data to query self.safe_create_table( conn, "bulkinsert", diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 6766aff32..30620ce41 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -51,9 +51,9 @@ # - Now a subclass of TestCase, to avoid requiring the driver stub # to use multiple inheritance # - Reversed the polarity of buggy test in test_description -# - Test exception heirarchy correctly +# - Test exception hierarchy correctly # - self.populate is now self._populate(), so if a driver stub -# overrides self.ddl1 this change propogates +# overrides self.ddl1 this change propagates # - VARCHAR columns now have a width, which will hopefully make the # DDL even more portible (this will be reversed if it causes more problems) # - cursor.rowcount being checked after various execute and fetchXXX methods @@ -174,7 +174,7 @@ def test_paramstyle(self): def test_Exceptions(self): # Make sure required exceptions exist, and are in the - # defined heirarchy. + # defined hierarchy. self.assertTrue(issubclass(self.driver.Warning, Exception)) self.assertTrue(issubclass(self.driver.Error, Exception)) self.assertTrue(issubclass(self.driver.InterfaceError, self.driver.Error)) @@ -474,7 +474,7 @@ def test_fetchone(self): self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows self.executeDDL1(cur) self.assertRaises(self.driver.Error, cur.fetchone) @@ -487,7 +487,7 @@ def test_fetchone(self): self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows cur.execute( "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) ) @@ -792,7 +792,7 @@ def test_setoutputsize_basic(self): con.close() def test_setoutputsize(self): - # Real test for setoutputsize is driver dependant + # Real test for setoutputsize is driver dependent raise NotImplementedError("Driver need to override this test") def test_None(self): diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index 9ac190f27..bc1e1b2ea 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -92,7 +92,7 @@ def test_fetchone(self): self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows self.executeDDL1(cur) ## self.assertRaises(self.driver.Error,cur.fetchone) @@ -105,7 +105,7 @@ def test_fetchone(self): self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows cur.execute( "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) ) From 15c2e4c88bfffacce3cc7eaa5a89fdf25c58edea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Jan 2023 10:10:37 +0900 Subject: [PATCH 153/227] Action: Update to dessant/lock-threads@v4 --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 1b25b4c79..7806b7db5 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -12,5 +12,5 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v2 + - uses: dessant/lock-threads@v4 From 67af9a55b4f6fa9fe7d0cc13877b4f6016db3680 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Jan 2023 13:27:07 +0900 Subject: [PATCH 154/227] Action: Run 'Lock Threads' weekly. --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 7806b7db5..c8f2ca245 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -2,7 +2,7 @@ name: 'Lock Threads' on: schedule: - - cron: '0 0 * * *' + - cron: '9 30 * * 1' permissions: issues: write From d734f15bd8ed20a7442c6bac59d3894181cc326e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 14:35:02 +0900 Subject: [PATCH 155/227] Action: Add doctest (#1086) --- .github/workflows/test.yaml | 1 + pymysql/tests/test_basic.py | 1 - pymysql/tests/test_connection.py | 1 - pymysql/tests/thirdparty/test_MySQLdb/capabilities.py | 1 - .../tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py | 1 - 5 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 39afc5791..aee9e1bca 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -84,6 +84,7 @@ jobs: - name: Run test run: | pytest -v --cov --cov-config .coveragerc pymysql + pytest -v --cov --cov-config .coveragerc --doctest-modules pymysql/converters.py - name: Run MySQL8 auth test if: ${{ matrix.mysql_auth }} diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index bc88e5a55..8af07da09 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -312,7 +312,6 @@ def test_json(self): class TestBulkInserts(base.PyMySQLTestCase): - cursor_type = pymysql.cursors.DictCursor def setUp(self): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 94a8dea04..d6fb5e523 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -45,7 +45,6 @@ def __exit__(self, exc_type, exc_value, traceback): class TestAuthentication(base.PyMySQLTestCase): - socket_auth = False socket_found = False two_questions_found = False diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index ffead0caf..0276a558a 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -10,7 +10,6 @@ class DatabaseTest(unittest.TestCase): - db_module = None connect_args = () connect_kwargs = dict(use_unicode=True, charset="utf8mb4", binary_prefix=True) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 139089ab1..11bfdbe29 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -8,7 +8,6 @@ class test_MySQLdb(capabilities.DatabaseTest): - db_module = pymysql connect_args = () connect_kwargs = base.PyMySQLTestCase.databases[0].copy() From 958a195d20551821db34b0c6b2d79739bc5543cf Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 15:58:08 +0900 Subject: [PATCH 156/227] Action: Fix lock --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index c8f2ca245..5dde1354a 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -2,7 +2,7 @@ name: 'Lock Threads' on: schedule: - - cron: '9 30 * * 1' + - cron: '30 9 * * 1' permissions: issues: write From 6270177c19fcb29e9d48c5178f91601a0e1a1fb1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 16:58:15 +0900 Subject: [PATCH 157/227] README: Remove LGTM label --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 318e94604..592b295a5 100644 --- a/README.rst +++ b/README.rst @@ -5,9 +5,6 @@ .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=main -.. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 - :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python - PyMySQL ======= From 592c4d2cf29702d36ad56469d74de4510fb5a376 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 17:01:16 +0900 Subject: [PATCH 158/227] Action: Fix test coverage --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index aee9e1bca..2b334503a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -84,7 +84,7 @@ jobs: - name: Run test run: | pytest -v --cov --cov-config .coveragerc pymysql - pytest -v --cov --cov-config .coveragerc --doctest-modules pymysql/converters.py + pytest -v --cov-append --cov-config .coveragerc --doctest-modules pymysql/converters.py - name: Run MySQL8 auth test if: ${{ matrix.mysql_auth }} From ded5f5a2d20f6eb033ade4096e88e291e432740b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 Feb 2023 20:39:57 +0900 Subject: [PATCH 159/227] Use pyproject.toml (#1087) --- .flake8 | 4 ++++ pyproject.toml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 14 -------------- setup.py | 39 --------------------------------------- 4 files changed, 53 insertions(+), 53 deletions(-) create mode 100644 .flake8 create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100755 setup.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..3f1c38a3f --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +exclude = tests,build,.venv,docs +ignore = E203,W503,E722 +max_line_length=129 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..3793a8c1b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[project] +name = "PyMySQL" +version = "1.0.2" +description = "Pure Python MySQL Driver" +authors = [ + {name = "Inada Naoki", email = "songofacandy@gmail.com"}, + {name = "Yutaka Matsubara", email = "yutaka.matsubara@gmail.com"} +] +dependencies = [] + +requires-python = ">=3.7" +readme = "README.rst" +license = {text = "MIT License"} +keywords = ["MySQL"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Database", +] + +[project.optional-dependencies] +"rsa" = [ + "cryptography" +] +"ed25519" = [ + "PyNaCl>=1.4.0" +] + +[project.urls] +"Project" = "https://github.com/PyMySQL/PyMySQL" +"Documentation" = "https://pymysql.readthedocs.io/" + +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +namespaces = false +include = ["pymysql"] +exclude = ["tests*", "pymysql.tests*"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e487e5e74..000000000 --- a/setup.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[flake8] -exclude = tests,build,.venv,docs -ignore = E203,W503,E722 -max_line_length=129 - -[metadata] -license = "MIT" -license_files = LICENSE - -author=yutaka.matsubara -author_email=yutaka.matsubara@gmail.com - -maintainer=Inada Naoki -maintainer_email=songofacandy@gmail.com diff --git a/setup.py b/setup.py deleted file mode 100755 index 7cdc692fb..000000000 --- a/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup, find_packages - -version = "1.0.2" - -with open("./README.rst", encoding="utf-8") as f: - readme = f.read() - -setup( - name="PyMySQL", - version=version, - url="https://github.com/PyMySQL/PyMySQL/", - project_urls={ - "Documentation": "https://pymysql.readthedocs.io/", - }, - description="Pure Python MySQL Driver", - long_description=readme, - packages=find_packages(exclude=["tests*", "pymysql.tests*"]), - python_requires=">=3.7", - extras_require={ - "rsa": ["cryptography"], - "ed25519": ["PyNaCl>=1.4.0"], - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Topic :: Database", - ], - keywords="MySQL", -) From 5fa787694107c5a5dd7742852a0f830dc7bcf560 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 6 Feb 2023 12:40:18 +0100 Subject: [PATCH 160/227] Upgrade GitHub Actions (#1080) --- .github/workflows/lint.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 887a8f261..a3131ce25 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,10 +10,12 @@ on: jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x - uses: psf/black@stable with: args: ". --diff --check" From b1399c95bcde8ef73cbc3a6d4e8bf767094bbd9e Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 7 Feb 2023 00:55:20 +0100 Subject: [PATCH 161/227] Upgrade more GitHub Actions (#1088) Followup to #1080 --- .github/workflows/test.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2b334503a..993347f64 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,7 +10,7 @@ concurrency: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -49,7 +49,7 @@ jobs: - /run/mysqld:/run/mysqld steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Workaround MySQL container permissions if: startsWith(matrix.db, 'mysql') @@ -58,7 +58,7 @@ jobs: /usr/bin/docker ps --all --filter status=exited --no-trunc --format "{{.ID}}" | xargs -r /usr/bin/docker start - name: Set up Python ${{ matrix.py }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} cache: 'pip' @@ -66,7 +66,7 @@ jobs: - name: Install dependency run: | - pip install -U -r requirements-dev.txt + pip install --upgrade -r requirements-dev.txt - name: Set up MySQL run: | @@ -107,16 +107,16 @@ jobs: coveralls: if: github.repository == 'PyMySQL/PyMySQL' name: Finish coveralls - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: test steps: - name: requirements. run: | echo coveralls > requirements.txt - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.x' cache: 'pip' - name: Finished From d894ab5c045fd4bc86edbe8321454b86410e12c4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 22 Mar 2023 19:54:05 +0900 Subject: [PATCH 162/227] Convert README to Markdown (#1093) --- README.md | 105 +++++++++++++++++++++++++++++++++++++ README.rst | 138 ------------------------------------------------- pyproject.toml | 2 +- 3 files changed, 106 insertions(+), 139 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 000000000..dec840803 --- /dev/null +++ b/README.md @@ -0,0 +1,105 @@ +[![Documentation Status](https://readthedocs.org/projects/pymysql/badge/?version=latest)](https://pymysql.readthedocs.io/) +[![image](https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github)](https://coveralls.io/github/PyMySQL/PyMySQL?branch=main) + +# PyMySQL + +This package contains a pure-Python MySQL client library, based on [PEP +249](https://www.python.org/dev/peps/pep-0249/). + +## Requirements + +- Python -- one of the following: + - [CPython](https://www.python.org/) : 3.7 and newer + - [PyPy](https://pypy.org/) : Latest 3.x version +- MySQL Server -- one of the following: + - [MySQL](https://www.mysql.com/) \>= 5.7 + - [MariaDB](https://mariadb.org/) \>= 10.3 + +## Installation + +Package is uploaded on [PyPI](https://pypi.org/project/PyMySQL). + +You can install it with pip: + + $ python3 -m pip install PyMySQL + +To use "sha256_password" or "caching_sha2_password" for authenticate, +you need to install additional dependency: + + $ python3 -m pip install PyMySQL[rsa] + +To use MariaDB's "ed25519" authentication method, you need to install +additional dependency: + + $ python3 -m pip install PyMySQL[ed25519] + +## Documentation + +Documentation is available online: + +For support, please refer to the +[StackOverflow](https://stackoverflow.com/questions/tagged/pymysql). + +## Example + +The following examples make use of a simple table + +``` sql +CREATE TABLE `users` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `email` varchar(255) COLLATE utf8_bin NOT NULL, + `password` varchar(255) COLLATE utf8_bin NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +AUTO_INCREMENT=1 ; +``` + +``` python +import pymysql.cursors + +# Connect to the database +connection = pymysql.connect(host='localhost', + user='user', + password='passwd', + database='db', + cursorclass=pymysql.cursors.DictCursor) + +with connection: + with connection.cursor() as cursor: + # Create a new record + sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" + cursor.execute(sql, ('webmaster@python.org', 'very-secret')) + + # connection is not autocommit by default. So you must commit to save + # your changes. + connection.commit() + + with connection.cursor() as cursor: + # Read a single record + sql = "SELECT `id`, `password` FROM `users` WHERE `email`=%s" + cursor.execute(sql, ('webmaster@python.org',)) + result = cursor.fetchone() + print(result) +``` + +This example will print: + +``` python +{'password': 'very-secret', 'id': 1} +``` + +## Resources + +- DB-API 2.0: +- MySQL Reference Manuals: +- MySQL client/server protocol: + +- "Connector" channel in MySQL Community Slack: + +- PyMySQL mailing list: + + +## License + +PyMySQL is released under the MIT License. See LICENSE for more +information. diff --git a/README.rst b/README.rst deleted file mode 100644 index 592b295a5..000000000 --- a/README.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. image:: https://readthedocs.org/projects/pymysql/badge/?version=latest - :target: https://pymysql.readthedocs.io/ - :alt: Documentation Status - -.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github - :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=main - - -PyMySQL -======= - -.. contents:: Table of Contents - :local: - -This package contains a pure-Python MySQL client library, based on `PEP 249`_. - -.. _`PEP 249`: https://www.python.org/dev/peps/pep-0249/ - - -Requirements -------------- - -* Python -- one of the following: - - - CPython_ : 3.7 and newer - - PyPy_ : Latest 3.x version - -* MySQL Server -- one of the following: - - - MySQL_ >= 5.7 - - MariaDB_ >= 10.3 - -.. _CPython: https://www.python.org/ -.. _PyPy: https://pypy.org/ -.. _MySQL: https://www.mysql.com/ -.. _MariaDB: https://mariadb.org/ - - -Installation ------------- - -Package is uploaded on `PyPI `_. - -You can install it with pip:: - - $ python3 -m pip install PyMySQL - -To use "sha256_password" or "caching_sha2_password" for authenticate, -you need to install additional dependency:: - - $ python3 -m pip install PyMySQL[rsa] - -To use MariaDB's "ed25519" authentication method, you need to install -additional dependency:: - - $ python3 -m pip install PyMySQL[ed25519] - - -Documentation -------------- - -Documentation is available online: https://pymysql.readthedocs.io/ - -For support, please refer to the `StackOverflow -`_. - - -Example -------- - -The following examples make use of a simple table - -.. code:: sql - - CREATE TABLE `users` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `email` varchar(255) COLLATE utf8_bin NOT NULL, - `password` varchar(255) COLLATE utf8_bin NOT NULL, - PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin - AUTO_INCREMENT=1 ; - - -.. code:: python - - import pymysql.cursors - - # Connect to the database - connection = pymysql.connect(host='localhost', - user='user', - password='passwd', - database='db', - cursorclass=pymysql.cursors.DictCursor) - - with connection: - with connection.cursor() as cursor: - # Create a new record - sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" - cursor.execute(sql, ('webmaster@python.org', 'very-secret')) - - # connection is not autocommit by default. So you must commit to save - # your changes. - connection.commit() - - with connection.cursor() as cursor: - # Read a single record - sql = "SELECT `id`, `password` FROM `users` WHERE `email`=%s" - cursor.execute(sql, ('webmaster@python.org',)) - result = cursor.fetchone() - print(result) - - -This example will print: - -.. code:: python - - {'password': 'very-secret', 'id': 1} - - -Resources ---------- - -* DB-API 2.0: https://www.python.org/dev/peps/pep-0249/ - -* MySQL Reference Manuals: https://dev.mysql.com/doc/ - -* MySQL client/server protocol: - https://dev.mysql.com/doc/internals/en/client-server-protocol.html - -* "Connector" channel in MySQL Community Slack: - https://lefred.be/mysql-community-on-slack/ - -* PyMySQL mailing list: https://groups.google.com/forum/#!forum/pymysql-users - -License -------- - -PyMySQL is released under the MIT License. See LICENSE for more information. diff --git a/pyproject.toml b/pyproject.toml index 3793a8c1b..a0a36105c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors = [ dependencies = [] requires-python = ">=3.7" -readme = "README.rst" +readme = "README.md" license = {text = "MIT License"} keywords = ["MySQL"] classifiers = [ From adff5ee6bf62be0d1bbc7eb8cb49e310d258ad51 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 23 Mar 2023 18:11:35 +0900 Subject: [PATCH 163/227] Update MANIFEST.in --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index e9e1eebcb..e2e577a9d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst LICENSE CHANGELOG.md +include README.md LICENSE CHANGELOG.md From d0c2871192b9a53733f32158dade3ea2e1847eab Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 24 Mar 2023 01:41:54 +0900 Subject: [PATCH 164/227] Release v1.0.3rc1 (#1094) --- pymysql/__init__.py | 2 +- pyproject.toml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 5fe2aec54..291d5c6a9 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 2, None) +VERSION = (1, 0, 3, "rc1") if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/pyproject.toml b/pyproject.toml index a0a36105c..dbb82c8d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,5 @@ [project] name = "PyMySQL" -version = "1.0.2" description = "Pure Python MySQL Driver" authors = [ {name = "Inada Naoki", email = "songofacandy@gmail.com"}, @@ -26,6 +25,7 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Topic :: Database", ] +dynamic = ["version"] [project.optional-dependencies] "rsa" = [ @@ -47,3 +47,6 @@ build-backend = "setuptools.build_meta" namespaces = false include = ["pymysql"] exclude = ["tests*", "pymysql.tests*"] + +[tool.setuptools.dynamic] +version = {attr = "pymysql.VERSION"} From 35bf026a7fda258277548ab93195972aeb867322 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Mar 2023 13:59:34 +0900 Subject: [PATCH 165/227] Fix setuptools didn't include pymysql.constants (#1096) Fix #1095 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dbb82c8d7..0f043181a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] namespaces = false -include = ["pymysql"] +include = ["pymysql*"] exclude = ["tests*", "pymysql.tests*"] [tool.setuptools.dynamic] From 7b0e0eab5fe0293a24adcdbdf479043eef939793 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 28 Mar 2023 12:34:54 +0900 Subject: [PATCH 166/227] v1.0.3 (#1097) --- pymysql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 291d5c6a9..4b6cc2a99 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 3, "rc1") +VERSION = (1, 0, 3, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: From 930b25034f1a3b6e3a202e072675f163770b25cb Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 28 Mar 2023 12:53:08 +0900 Subject: [PATCH 167/227] Fix VERSION for dynamic version (#1098) --- pymysql/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 4b6cc2a99..c0039c3fe 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,11 +47,11 @@ ) -VERSION = (1, 0, 3, None) -if VERSION[3] is not None: +VERSION = (1, 0, 3) +if len(VERSION) > 3: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: - VERSION_STRING = "%d.%d.%d" % VERSION[:3] + VERSION_STRING = "%d.%d.%d" % VERSION threadsafety = 1 apilevel = "2.0" paramstyle = "pyformat" @@ -113,10 +113,7 @@ def Binary(x): def get_client_info(): # for MySQLdb compatibility - version = VERSION - if VERSION[3] is None: - version = VERSION[:3] - return ".".join(map(str, version)) + return VERSION_STRING # we include a doctored version_info here for MySQLdb compatibility From 57e2e93276c7b48e6ec5b99c1712e48661d92183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 28 Mar 2023 16:48:57 +0200 Subject: [PATCH 168/227] Remove redundant wheel dep from pyproject.toml (#1099) Remove the redundant `wheel` dependency, as it is added by the backend automatically. Listing it explicitly in the documentation was a historical mistake and has been fixed since, see: https://github.com/pypa/setuptools/commit/f7d30a9529378cf69054b5176249e5457aaf640a --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0f043181a..a67031b3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dynamic = ["version"] "Documentation" = "https://pymysql.readthedocs.io/" [build-system] -requires = ["setuptools>=61", "wheel"] +requires = ["setuptools>=61"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] From 885841f3fee416c222a75d83a81f74d3dcd71b51 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 31 Mar 2023 23:42:11 +0900 Subject: [PATCH 169/227] Add security policy --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..da9c516dd --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure. From 72e7c580515588f0646c3322c3dba63dbcc90810 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 1 May 2023 19:22:22 +0900 Subject: [PATCH 170/227] Run lock-threads only on PyMySQL/PyMySQL --- .github/workflows/lock.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 5dde1354a..780dd92d7 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -9,7 +9,8 @@ permissions: pull-requests: write jobs: - action: + lock-threads: + if: github.repository == 'PyMySQL/PyMySQL' runs-on: ubuntu-latest steps: - uses: dessant/lock-threads@v4 From 101f6e970cb2df47f1363bca590aab88a809804c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 22 May 2023 10:44:35 +0000 Subject: [PATCH 171/227] Update FUNDING.yml --- .github/FUNDING.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 89fc5cf8f..253a13aca 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,10 +1,10 @@ # These are supported funding model platforms -github: [methane] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: ["methane"] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +tidelift: "pypi/PyMySQL" # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username From a5e837f9de3b13abcef3500a1dc35fdbfa2f5784 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 23 May 2023 19:28:20 +0900 Subject: [PATCH 172/227] ci: Fix black options (#1109) --- .github/workflows/lint.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a3131ce25..9d9eafb05 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,7 +18,8 @@ jobs: python-version: 3.x - uses: psf/black@stable with: - args: ". --diff --check" + options: "--check --verbose" + src: "." - name: Setup flake8 annotations uses: rbialon/flake8-annotations@v1 - name: flake8 From 1448310e1400a87267f2707eadceab00af4dedad Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 23 May 2023 19:28:34 +0900 Subject: [PATCH 173/227] Remove unused function (#1108) --- pymysql/cursors.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 2b5ccca90..b36f473c7 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -95,13 +95,6 @@ def _nextset(self, unbuffered=False): def nextset(self): return self._nextset(False) - def _ensure_bytes(self, x, encoding=None): - if isinstance(x, str): - x = x.encode(encoding) - elif isinstance(x, (tuple, list)): - x = type(x)(self._ensure_bytes(v, encoding=encoding) for v in x) - return x - def _escape_args(self, args, conn): if isinstance(args, (tuple, list)): return tuple(conn.literal(arg) for arg in args) From 01ddf9d1b26d78d5d03e483d076544a5a50d7c47 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 23 May 2023 14:01:02 +0200 Subject: [PATCH 174/227] Expose `Cursor.warning_count` (#1056) In #774 automatic warnings were removed. This provides a way to check for existence of warnings without having to perform an additional query over the network. Co-authored-by: Inada Naoki --- CHANGELOG.md | 7 +++++++ pymysql/cursors.py | 5 +++++ pymysql/tests/test_SSCursor.py | 33 ++++++++++++++++++++++++++++++-- pymysql/tests/test_cursor.py | 20 +++++++++++++++++-- pymysql/tests/test_load_local.py | 32 +++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87c3f9e86..76fdb6a71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changes +## v1.1.0 + +Release date: TBD + +* Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) + ## v1.0.3 Release date: TBD @@ -7,6 +13,7 @@ Release date: TBD * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 +* Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) ## v1.0.2 diff --git a/pymysql/cursors.py b/pymysql/cursors.py index b36f473c7..e57fba76e 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -32,6 +32,7 @@ class Cursor: def __init__(self, connection): self.connection = connection + self.warning_count = 0 self.description = None self.rownumber = 0 self.rowcount = -1 @@ -324,6 +325,7 @@ def _clear_result(self): self._result = None self.rowcount = 0 + self.warning_count = 0 self.description = None self.lastrowid = None self._rows = None @@ -334,6 +336,7 @@ def _do_get_result(self): self._result = result = conn._result self.rowcount = result.affected_rows + self.warning_count = result.warning_count self.description = result.description self.lastrowid = result.insert_id self._rows = result.rows @@ -435,6 +438,7 @@ def fetchone(self): self._check_executed() row = self.read_next() if row is None: + self.warning_count = self._result.warning_count return None self.rownumber += 1 return row @@ -468,6 +472,7 @@ def fetchmany(self, size=None): for i in range(size): row = self.read_next() if row is None: + self.warning_count = self._result.warning_count break rows.append(row) self.rownumber += 1 diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index a68a77698..d19d3e5dc 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -3,13 +3,13 @@ try: from pymysql.tests import base import pymysql.cursors - from pymysql.constants import CLIENT + from pymysql.constants import CLIENT, ER except Exception: # For local testing from top-level directory, without installing sys.path.append("../pymysql") from pymysql.tests import base import pymysql.cursors - from pymysql.constants import CLIENT + from pymysql.constants import CLIENT, ER class TestSSCursor(base.PyMySQLTestCase): @@ -122,6 +122,35 @@ def test_SSCursor(self): cursor.execute("DROP TABLE IF EXISTS tz_data") cursor.close() + def test_warnings(self): + con = self.connect() + cur = con.cursor(pymysql.cursors.SSCursor) + cur.execute("DROP TABLE IF EXISTS `no_exists_table`") + self.assertEqual(cur.warning_count, 1) + + cur.execute("SHOW WARNINGS") + w = cur.fetchone() + self.assertEqual(w[1], ER.BAD_TABLE_ERROR) + self.assertIn( + "no_exists_table", + w[2], + ) + + # ensure unbuffered result is finished + self.assertIsNone(cur.fetchone()) + + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + self.assertIsNone(cur.fetchone()) + + self.assertEqual(cur.warning_count, 0) + + cur.execute("SELECT CAST('abc' AS SIGNED)") + # this ensures fully retrieving the unbuffered result + rows = cur.fetchmany(2) + self.assertEqual(len(rows), 1) + self.assertEqual(cur.warning_count, 1) + __all__ = ["TestSSCursor"] diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 783caf88b..63ecce02d 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -1,5 +1,4 @@ -import warnings - +from pymysql.constants import ER from pymysql.tests import base import pymysql.cursors @@ -129,3 +128,20 @@ def test_executemany(self): ) finally: cursor.execute("DROP TABLE IF EXISTS percent_test") + + def test_warnings(self): + con = self.connect() + cur = con.cursor() + cur.execute("DROP TABLE IF EXISTS `no_exists_table`") + self.assertEqual(cur.warning_count, 1) + + cur.execute("SHOW WARNINGS") + w = cur.fetchone() + self.assertEqual(w[1], ER.BAD_TABLE_ERROR) + self.assertIn( + "no_exists_table", + w[2], + ) + + cur.execute("SELECT 1") + self.assertEqual(cur.warning_count, 0) diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index b1b8128e4..194c5be9c 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -1,4 +1,5 @@ from pymysql import cursors, OperationalError, Warning +from pymysql.constants import ER from pymysql.tests import base import os @@ -63,6 +64,37 @@ def test_unbuffered_load_file(self): c = conn.cursor() c.execute("DROP TABLE test_load_local") + def test_load_warnings(self): + """Test load local infile produces the appropriate warnings""" + conn = self.connect() + c = conn.cursor() + c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") + filename = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "data", + "load_local_warn_data.txt", + ) + try: + c.execute( + ( + "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + + "test_load_local FIELDS TERMINATED BY ','" + ).format(filename) + ) + self.assertEqual(1, c.warning_count) + + c.execute("SHOW WARNINGS") + w = c.fetchone() + + self.assertEqual(ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, w[1]) + self.assertIn( + "incorrect integer value", + w[2].lower(), + ) + finally: + c.execute("DROP TABLE test_load_local") + c.close() + if __name__ == "__main__": import unittest From ea79b3216e948ca1095bc7802e798bc3eb9dd599 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 23 May 2023 14:18:40 +0200 Subject: [PATCH 175/227] Add constants and tests related to query timeouts (#1033) --- pymysql/constants/ER.py | 3 + pymysql/tests/base.py | 8 +++ pymysql/tests/test_SSCursor.py | 101 +++++++++++++++++++++++++++++---- pymysql/tests/test_cursor.py | 67 ++++++++++++++++++++++ 4 files changed, 168 insertions(+), 11 deletions(-) diff --git a/pymysql/constants/ER.py b/pymysql/constants/ER.py index ddcc4e907..98729d12d 100644 --- a/pymysql/constants/ER.py +++ b/pymysql/constants/ER.py @@ -470,5 +470,8 @@ WRONG_STRING_LENGTH = 1468 ERROR_LAST = 1468 +# MariaDB only +STATEMENT_TIMEOUT = 1969 +QUERY_TIMEOUT = 3024 # https://github.com/PyMySQL/PyMySQL/issues/607 CONSTRAINT_FAILED = 4025 diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index a87307a57..ff33bc4e4 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -49,6 +49,14 @@ def mysql_server_is(self, conn, version_tuple): ) return server_version_tuple >= version_tuple + def get_mysql_vendor(self, conn): + server_version = conn.get_server_info() + + if "MariaDB" in server_version: + return "mariadb" + + return "mysql" + _connections = None @property diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index d19d3e5dc..9cb5bafed 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -1,15 +1,8 @@ -import sys +import pytest -try: - from pymysql.tests import base - import pymysql.cursors - from pymysql.constants import CLIENT, ER -except Exception: - # For local testing from top-level directory, without installing - sys.path.append("../pymysql") - from pymysql.tests import base - import pymysql.cursors - from pymysql.constants import CLIENT, ER +from pymysql.tests import base +import pymysql.cursors +from pymysql.constants import CLIENT, ER class TestSSCursor(base.PyMySQLTestCase): @@ -122,6 +115,92 @@ def test_SSCursor(self): cursor.execute("DROP TABLE IF EXISTS tz_data") cursor.close() + def test_execution_time_limit(self): + # this method is similarly implemented in test_cursor + + conn = self.connect() + + # table creation and filling is SSCursor only as it's not provided by self.setUp() + self.safe_create_table( + conn, + "test", + "create table test (data varchar(10))", + ) + with conn.cursor() as cur: + cur.execute( + "insert into test (data) values " + "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" + ) + conn.commit() + + db_type = self.get_mysql_vendor(conn) + + with conn.cursor(pymysql.cursors.SSCursor) as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + + cur.execute(sql) + # unlike Cursor, SSCursor returns a list of tuples here + self.assertEqual( + cur.fetchall(), + [ + ("row1", 0), + ("row2", 0), + ("row3", 0), + ("row4", 0), + ("row5", 0), + ], + ) + + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + cur.execute(sql) + self.assertEqual(cur.fetchone(), ("row1", 0)) + + # this discards the previous unfinished query and raises an + # incomplete unbuffered query warning + with pytest.warns(UserWarning): + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + + # SSCursor will not read the EOF packet until we try to read + # another row. Skipping this will raise an incomplete unbuffered + # query warning in the next cur.execute(). + self.assertEqual(cur.fetchone(), None) + + if db_type == "mysql": + sql = "SELECT /*+ MAX_EXECUTION_TIME(1) */ data, sleep(1) FROM test" + else: + sql = "SET STATEMENT max_statement_time=0.001 FOR SELECT data, sleep(1) FROM test" + with pytest.raises(pymysql.err.OperationalError) as cm: + # in an unbuffered cursor the OperationalError may not show up + # until fetching the entire result + cur.execute(sql) + cur.fetchall() + + if db_type == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + self.assertEqual(cm.value.args[0], ER.QUERY_TIMEOUT) + else: + self.assertEqual(cm.value.args[0], ER.STATEMENT_TIMEOUT) + + # connection should still be fine at this point + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + def test_warnings(self): con = self.connect() cur = con.cursor(pymysql.cursors.SSCursor) diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 63ecce02d..66d968dfe 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -2,6 +2,8 @@ from pymysql.tests import base import pymysql.cursors +import pytest + class CursorTest(base.PyMySQLTestCase): def setUp(self): @@ -18,6 +20,7 @@ def setUp(self): "insert into test (data) values " "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" ) + conn.commit() cursor.close() self.test_connection = pymysql.connect(**self.databases[0]) self.addCleanup(self.test_connection.close) @@ -129,6 +132,70 @@ def test_executemany(self): finally: cursor.execute("DROP TABLE IF EXISTS percent_test") + def test_execution_time_limit(self): + # this method is similarly implemented in test_SScursor + + conn = self.test_connection + db_type = self.get_mysql_vendor(conn) + + with conn.cursor(pymysql.cursors.Cursor) as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + + cur.execute(sql) + # unlike SSCursor, Cursor returns a tuple of tuples here + self.assertEqual( + cur.fetchall(), + ( + ("row1", 0), + ("row2", 0), + ("row3", 0), + ("row4", 0), + ("row5", 0), + ), + ) + + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + cur.execute(sql) + self.assertEqual(cur.fetchone(), ("row1", 0)) + + # this discards the previous unfinished query + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + + if db_type == "mysql": + sql = "SELECT /*+ MAX_EXECUTION_TIME(1) */ data, sleep(1) FROM test" + else: + sql = "SET STATEMENT max_statement_time=0.001 FOR SELECT data, sleep(1) FROM test" + with pytest.raises(pymysql.err.OperationalError) as cm: + # in a buffered cursor this should reliably raise an + # OperationalError + cur.execute(sql) + + if db_type == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + self.assertEqual(cm.value.args[0], ER.QUERY_TIMEOUT) + else: + self.assertEqual(cm.value.args[0], ER.STATEMENT_TIMEOUT) + + # connection should still be fine at this point + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + def test_warnings(self): con = self.connect() cur = con.cursor() From 2ee4f706d34412a6d39417b92360bfa13ddc4e14 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 23 May 2023 21:46:00 +0900 Subject: [PATCH 176/227] Fix wrong merge --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76fdb6a71..ce74e84ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,6 @@ Release date: TBD * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 -* Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) ## v1.0.2 From 3cd76d7256416e3aa9575b3b9823c9491f92369c Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 23 May 2023 14:47:38 +0200 Subject: [PATCH 177/227] Fix SSCursor raising query timeout error on wrong query on MySQL DB (#1035) Fixes https://github.com/PyMySQL/PyMySQL/issues/1032#issuecomment-1030764742 --- CHANGELOG.md | 2 ++ pymysql/connections.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce74e84ba..6dc752253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,10 @@ Release date: TBD +* Fixed SSCursor raising OperationalError for query timeouts on wrong statement (#1032) * Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) + ## v1.0.3 Release date: TBD diff --git a/pymysql/connections.py b/pymysql/connections.py index 3265d32ee..f82b1951d 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1262,7 +1262,20 @@ def _finish_unbuffered_query(self): # in fact, no way to stop MySQL from sending all the data after # executing a query, so we just spin, and wait for an EOF packet. while self.unbuffered_active: - packet = self.connection._read_packet() + try: + packet = self.connection._read_packet() + except err.OperationalError as e: + if e.args[0] in ( + ER.QUERY_TIMEOUT, + ER.STATEMENT_TIMEOUT, + ): + # if the query timed out we can simply ignore this error + self.unbuffered_active = False + self.connection = None + return + + raise + if self._check_packet_is_eof(packet): self.unbuffered_active = False self.connection = None # release reference to kill cyclic reference. From a6f53dbffa5ee6986b0c48c32e43bd071a04217d Mon Sep 17 00:00:00 2001 From: Gonzalo Sanchez Date: Tue, 23 May 2023 12:43:50 -0300 Subject: [PATCH 178/227] Make Cursor an iterator (#995) Fix #992 Co-authored-by: Gonzalo Sanchez Co-authored-by: Inada Naoki --- pymysql/cursors.py | 11 +++++++---- pymysql/tests/test_cursor.py | 8 ++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index e57fba76e..d8a93c781 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -342,7 +342,13 @@ def _do_get_result(self): self._rows = result.rows def __iter__(self): - return iter(self.fetchone, None) + return self + + def __next__(self): + row = self.fetchone() + if row is None: + raise StopIteration + return row Warning = err.Warning Error = err.Error @@ -459,9 +465,6 @@ def fetchall_unbuffered(self): """ return iter(self.fetchone, None) - def __iter__(self): - return self.fetchall_unbuffered() - def fetchmany(self, size=None): """Fetch many.""" self._check_executed() diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 66d968dfe..16d297f68 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -25,6 +25,14 @@ def setUp(self): self.test_connection = pymysql.connect(**self.databases[0]) self.addCleanup(self.test_connection.close) + def test_cursor_is_iterator(self): + """Test that the cursor is an iterator""" + conn = self.test_connection + cursor = conn.cursor() + cursor.execute("select * from test") + self.assertEqual(cursor.__iter__(), cursor) + self.assertEqual(cursor.__next__(), ("row1",)) + def test_cleanup_rows_unbuffered(self): conn = self.test_connection cursor = conn.cursor(pymysql.cursors.SSCursor) From 4072c7fff9871f6eb811b9b4442bbb5411b6d01b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 01:17:31 +0900 Subject: [PATCH 179/227] ci: Update CodeQL workflow (#1110) --- .github/workflows/codeql-analysis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d559b1cd2..a4c434c5e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,21 +27,16 @@ jobs: strategy: fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: - languages: ${{ matrix.language }} + languages: "python" # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. From 2fe0b1293d1a24140f6d35f5ff37d7b5a46a28e1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 14:17:19 +0900 Subject: [PATCH 180/227] Use Ruff instead of flake8 (#1112) --- .flake8 | 4 - .github/workflows/lint.yaml | 13 +--- pymysql/_auth.py | 3 +- pymysql/connections.py | 15 ++-- pymysql/converters.py | 5 +- pymysql/tests/__init__.py | 19 ----- pymysql/tests/base.py | 1 - pymysql/tests/test_basic.py | 28 +++++-- pymysql/tests/test_connection.py | 75 +++++++------------ pymysql/tests/test_cursor.py | 3 +- pymysql/tests/test_issues.py | 2 - pymysql/tests/test_load_local.py | 8 +- .../tests/thirdparty/test_MySQLdb/__init__.py | 2 - .../thirdparty/test_MySQLdb/capabilities.py | 1 - .../tests/thirdparty/test_MySQLdb/dbapi20.py | 20 +++-- .../test_MySQLdb/test_MySQLdb_capabilities.py | 1 - .../test_MySQLdb/test_MySQLdb_dbapi20.py | 4 - .../test_MySQLdb/test_MySQLdb_nonstandard.py | 1 - pyproject.toml | 6 ++ 19 files changed, 89 insertions(+), 122 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 3f1c38a3f..000000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -exclude = tests,build,.venv,docs -ignore = E203,W503,E722 -max_line_length=129 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 9d9eafb05..77edb0c38 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -2,6 +2,7 @@ name: Lint on: push: + branches: ["main"] paths: - '**.py' pull_request: @@ -13,16 +14,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x + - uses: psf/black@stable with: options: "--check --verbose" src: "." - - name: Setup flake8 annotations - uses: rbialon/flake8-annotations@v1 - - name: flake8 - run: | - pip install flake8 - flake8 pymysql + + - uses: chartboost/ruff-action@v1 diff --git a/pymysql/_auth.py b/pymysql/_auth.py index f6c9eb967..99987b770 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -141,7 +141,8 @@ def sha2_rsa_encrypt(password, salt, public_key): """ if not _have_cryptography: raise RuntimeError( - "'cryptography' package is required for sha256_password or caching_sha2_password auth methods" + "'cryptography' package is required for sha256_password or" + + " caching_sha2_password auth methods" ) message = _xor_password(password + b"\0", salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) diff --git a/pymysql/connections.py b/pymysql/connections.py index f82b1951d..7bbc089f8 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -108,8 +108,10 @@ class Connection: the interface from which to connect to the host. Argument can be a hostname or an IP address. :param unix_socket: Use a unix socket rather than TCP/IP. - :param read_timeout: The timeout for reading from the connection in seconds (default: None - no timeout) - :param write_timeout: The timeout for writing to the connection in seconds (default: None - no timeout) + :param read_timeout: The timeout for reading from the connection in seconds. + (default: None - no timeout) + :param write_timeout: The timeout for writing to the connection in seconds. + (default: None - no timeout) :param charset: Charset to use. :param sql_mode: Default SQL_MODE to use. :param read_default_file: @@ -130,7 +132,8 @@ class Connection: :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate. :param ssl_cert: Path to the file that contains a PEM-formatted client certificate. :param ssl_disabled: A boolean value that disables usage of TLS. - :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate. + :param ssl_key: Path to the file that contains a PEM-formatted private key for + the client certificate. :param ssl_verify_cert: Set to true to check the server certificate's validity. :param ssl_verify_identity: Set to true to check the server's identity. :param read_default_group: Group to read from in the configuration file. @@ -533,7 +536,8 @@ def cursor(self, cursor=None): Create a new cursor to execute queries with. :param cursor: The type of cursor to create. None means use Cursor. - :type cursor: :py:class:`Cursor`, :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. + :type cursor: :py:class:`Cursor`, :py:class:`SSCursor`, :py:class:`DictCursor`, + or :py:class:`SSDictCursor`. """ if cursor: return cursor(self) @@ -1228,7 +1232,8 @@ def _check_packet_is_eof(self, packet): # TODO: Support CLIENT.DEPRECATE_EOF # 1) Add DEPRECATE_EOF to CAPABILITIES # 2) Mask CAPABILITIES with server_capabilities - # 3) if server_capabilities & CLIENT.DEPRECATE_EOF: use OKPacketWrapper instead of EOFPacketWrapper + # 3) if server_capabilities & CLIENT.DEPRECATE_EOF: + # use OKPacketWrapper instead of EOFPacketWrapper wp = EOFPacketWrapper(packet) self.warning_count = wp.warning_count self.has_next = wp.has_next diff --git a/pymysql/converters.py b/pymysql/converters.py index 2acc3e58d..1adac7528 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -120,7 +120,10 @@ def escape_time(obj, mapping=None): def escape_datetime(obj, mapping=None): if obj.microsecond: - fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" + fmt = ( + "'{0.year:04}-{0.month:02}-{0.day:02}" + + " {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" + ) else: fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}'" return fmt.format(obj) diff --git a/pymysql/tests/__init__.py b/pymysql/tests/__init__.py index fe3b1d0f5..e69de29bb 100644 --- a/pymysql/tests/__init__.py +++ b/pymysql/tests/__init__.py @@ -1,19 +0,0 @@ -# Sorted by alphabetical order -from pymysql.tests.test_DictCursor import * -from pymysql.tests.test_SSCursor import * -from pymysql.tests.test_basic import * -from pymysql.tests.test_connection import * -from pymysql.tests.test_converters import * -from pymysql.tests.test_cursor import * -from pymysql.tests.test_err import * -from pymysql.tests.test_issues import * -from pymysql.tests.test_load_local import * -from pymysql.tests.test_nextset import * -from pymysql.tests.test_optionfile import * - -from pymysql.tests.thirdparty import * - -if __name__ == "__main__": - import unittest - - unittest.main() diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index ff33bc4e4..b5094563a 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -1,4 +1,3 @@ -import gc import json import os import re diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 8af07da09..ecf043f67 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -6,7 +6,6 @@ import pymysql.cursors from pymysql.tests import base -from pymysql.err import ProgrammingError __all__ = ["TestConversion", "TestCursor", "TestBulkInserts"] @@ -18,7 +17,22 @@ def test_datatypes(self): conn = self.connect() c = conn.cursor() c.execute( - "create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)" + """ +create table test_datatypes ( + b bit, + i int, + l bigint, + f real, + s varchar(32), + u varchar(32), + bb blob, + d date, + dt datetime, + ts timestamp, + td time, + t time, + st datetime) +""" ) try: # insert values @@ -38,7 +52,8 @@ def test_datatypes(self): time.localtime(), ) c.execute( - "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values" + " (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", v, ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") @@ -54,7 +69,8 @@ def test_datatypes(self): # check nulls c.execute( - "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st)" + " values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", [None] * 12, ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") @@ -156,7 +172,8 @@ def test_timedelta(self): conn = self.connect() c = conn.cursor() c.execute( - "select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')" + "select time('12:30'), time('23:12:59'), time('23:12:59.05100')," + + " time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')" ) self.assertEqual( ( @@ -317,7 +334,6 @@ class TestBulkInserts(base.PyMySQLTestCase): def setUp(self): super(TestBulkInserts, self).setUp() self.conn = conn = self.connect() - c = conn.cursor(self.cursor_type) # create a table and some data to query self.safe_create_table( diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index d6fb5e523..bbaf3dec1 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,6 +1,5 @@ import datetime import ssl -import sys import pytest import time from unittest import mock @@ -145,8 +144,8 @@ def realtestSocketAuth(self): TestAuthentication.osuser + "@localhost", self.databases[0]["database"], self.socket_plugin_name, - ) as u: - c = pymysql.connect(user=TestAuthentication.osuser, **self.db) + ): + pymysql.connect(user=TestAuthentication.osuser, **self.db) class Dialog: fail = False @@ -168,7 +167,7 @@ def __init__(self, con): def authenticate(self, pkt): while True: flag = pkt.read_uint8() - echo = (flag & 0x06) == 0x02 + # echo = (flag & 0x06) == 0x02 last = (flag & 0x01) == 0x01 prompt = pkt.read_all() @@ -220,7 +219,7 @@ def realTestDialogAuthTwoQuestions(self): self.databases[0]["database"], "two_questions", "notverysecret", - ) as u: + ): with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user="pymysql_2q", **self.db) pymysql.connect( @@ -262,7 +261,7 @@ def realTestDialogAuthThreeAttempts(self): self.databases[0]["database"], "three_attempts", "stillnotverysecret", - ) as u: + ): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, @@ -357,9 +356,9 @@ def realTestPamAuth(self): self.databases[0]["database"], "pam", os.environ.get("PAMSERVICE"), - ) as u: + ): try: - c = pymysql.connect(user=TestAuthentication.osuser, **db) + pymysql.connect(user=TestAuthentication.osuser, **db) db["password"] = "very bad guess at password" with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( @@ -371,7 +370,8 @@ def realTestPamAuth(self): ) except pymysql.OperationalError as e: self.assertEqual(1045, e.args[0]) - # we had 'bad guess at password' work with pam. Well at least we get a permission denied here + # we had 'bad guess at password' work with pam. Well at least we get + # a permission denied here with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user=TestAuthentication.osuser, @@ -397,12 +397,13 @@ def testAuthSHA256(self): "pymysql_sha256@localhost", self.databases[0]["database"], "sha256_password", - ) as u: + ): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") c.execute("FLUSH PRIVILEGES") db = self.db.copy() db["password"] = "Sh@256Pa33" - # Although SHA256 is supported, need the configuration of public key of the mysql server. Currently will get error by this test. + # Although SHA256 is supported, need the configuration of public key of + # the mysql server. Currently will get error by this test. with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user="pymysql_sha256", **db) @@ -423,7 +424,7 @@ def testAuthEd25519(self): self.databases[0]["database"], "ed25519", empty_pass, - ) as u: + ): pymysql.connect(user="pymysql_ed25519", password="", **db) with TempUser( @@ -432,7 +433,7 @@ def testAuthEd25519(self): self.databases[0]["database"], "ed25519", non_empty_pass, - ) as u: + ): pymysql.connect(user="pymysql_ed25519", password="ed25519_password", **db) @@ -441,7 +442,7 @@ def test_utf8mb4(self): """This test requires MySQL >= 5.5""" arg = self.databases[0].copy() arg["charset"] = "utf8mb4" - conn = pymysql.connect(**arg) + pymysql.connect(**arg) def test_largedata(self): """Large query and response (>=16MB)""" @@ -544,9 +545,7 @@ def test_defer_connect(self): def test_ssl_connect(self): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -565,9 +564,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_called_with("cipher") dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -585,9 +582,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -601,9 +596,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -620,9 +613,7 @@ def test_ssl_connect(self): for ssl_verify_cert in (True, "1", "yes", "true"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -641,9 +632,7 @@ def test_ssl_connect(self): for ssl_verify_cert in (None, False, "0", "no", "false"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -663,9 +652,7 @@ def test_ssl_connect(self): for ssl_ca in ("ca", None): for ssl_verify_cert in ("foo", "bar", ""): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -686,9 +673,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -705,9 +690,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -722,9 +705,7 @@ def test_ssl_connect(self): assert not create_default_context.called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -762,21 +743,18 @@ def test_escape_string(self): def test_escape_builtin_encoders(self): con = self.connect() - cur = con.cursor() val = datetime.datetime(2012, 3, 4, 5, 6) self.assertEqual(con.escape(val, con.encoders), "'2012-03-04 05:06:00'") def test_escape_custom_object(self): con = self.connect() - cur = con.cursor() mapping = {Foo: escape_foo} self.assertEqual(con.escape(Foo(), mapping), "bar") def test_escape_fallback_encoder(self): con = self.connect() - cur = con.cursor() class Custom(str): pass @@ -786,13 +764,11 @@ class Custom(str): def test_escape_no_default(self): con = self.connect() - cur = con.cursor() self.assertRaises(TypeError, con.escape, 42, {}) def test_escape_dict_value(self): con = self.connect() - cur = con.cursor() mapping = con.encoders.copy() mapping[Foo] = escape_foo @@ -800,7 +776,6 @@ def test_escape_dict_value(self): def test_escape_list_item(self): con = self.connect() - cur = con.cursor() mapping = con.encoders.copy() mapping[Foo] = escape_foo diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 16d297f68..6666ab883 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -105,7 +105,8 @@ def test_executemany(self): ) assert m is not None - # cursor._executed must bee "insert into test (data) values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)" + # cursor._executed must bee "insert into test (data) + # values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)" # list args data = range(10) cursor.executemany("insert into test (data) values (%s)", data) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 733d56a16..7f361c94f 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -1,12 +1,10 @@ import datetime import time import warnings -import sys import pytest import pymysql -from pymysql import cursors from pymysql.tests import base __all__ = ["TestOldIssues", "TestNewIssues", "TestGitHubIssues"] diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index 194c5be9c..509221420 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -1,4 +1,4 @@ -from pymysql import cursors, OperationalError, Warning +from pymysql import cursors, OperationalError from pymysql.constants import ER from pymysql.tests import base @@ -36,7 +36,8 @@ def test_load_file(self): ) try: c.execute( - f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local" + + " FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -53,7 +54,8 @@ def test_unbuffered_load_file(self): ) try: c.execute( - f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local" + + " FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py index 57c42ce7a..501bfd2db 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py @@ -1,6 +1,4 @@ -from .test_MySQLdb_capabilities import test_MySQLdb as test_capabilities from .test_MySQLdb_nonstandard import * -from .test_MySQLdb_dbapi20 import test_MySQLdb as test_dbapi2 if __name__ == "__main__": import unittest diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index 0276a558a..bb47cc5f6 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -4,7 +4,6 @@ Adapted from a script by M-A Lemburg. """ -import sys from time import time import unittest diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 30620ce41..838512955 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -225,7 +225,7 @@ def test_rollback(self): def test_cursor(self): con = self._connect() try: - cur = con.cursor() + con.cursor() finally: con.close() @@ -810,28 +810,26 @@ def test_None(self): con.close() def test_Date(self): - d1 = self.driver.Date(2002, 12, 25) - d2 = self.driver.DateFromTicks(time.mktime((2002, 12, 25, 0, 0, 0, 0, 0, 0))) + self.driver.Date(2002, 12, 25) + self.driver.DateFromTicks(time.mktime((2002, 12, 25, 0, 0, 0, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(d1),str(d2)) def test_Time(self): - t1 = self.driver.Time(13, 45, 30) - t2 = self.driver.TimeFromTicks(time.mktime((2001, 1, 1, 13, 45, 30, 0, 0, 0))) + self.driver.Time(13, 45, 30) + self.driver.TimeFromTicks(time.mktime((2001, 1, 1, 13, 45, 30, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Timestamp(self): - t1 = self.driver.Timestamp(2002, 12, 25, 13, 45, 30) - t2 = self.driver.TimestampFromTicks( - time.mktime((2002, 12, 25, 13, 45, 30, 0, 0, 0)) - ) + self.driver.Timestamp(2002, 12, 25, 13, 45, 30) + self.driver.TimestampFromTicks(time.mktime((2002, 12, 25, 13, 45, 30, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Binary(self): - b = self.driver.Binary(b"Something") - b = self.driver.Binary(b"") + self.driver.Binary(b"Something") + self.driver.Binary(b"") def test_STRING(self): self.assertTrue(hasattr(self.driver, "STRING"), "module.STRING must be defined") diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 11bfdbe29..6a2894a5a 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -1,5 +1,4 @@ from . import capabilities -import unittest import pymysql from pymysql.tests import base import warnings diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index bc1e1b2ea..c68289fe8 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -2,8 +2,6 @@ import pymysql from pymysql.tests import base -import unittest - class test_MySQLdb(dbapi20.DatabaseAPI20Test): driver = pymysql @@ -181,8 +179,6 @@ def help_nextset_tearDown(self, cur): cur.execute("drop procedure deleteme") def test_nextset(self): - from warnings import warn - con = self._connect() try: cur = con.cursor() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py index b8d4bb1e6..1545fbb5e 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py @@ -1,4 +1,3 @@ -import sys import unittest import pymysql diff --git a/pyproject.toml b/pyproject.toml index a67031b3b..48fe3660d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,9 @@ exclude = ["tests*", "pymysql.tests*"] [tool.setuptools.dynamic] version = {attr = "pymysql.VERSION"} + +[tool.ruff] +line-length = 99 +exclude = [ + "pymysql/tests/thirdparty", +] From d02e090e7a4766584750720d058bcc8e46eec48f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 14:50:22 +0900 Subject: [PATCH 181/227] Use Codecov instead of coveralls. (#1113) --- .github/workflows/test.yaml | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 993347f64..bea7747c8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,6 +44,7 @@ jobs: - 3306:3306 env: MYSQL_ALLOW_EMPTY_PASSWORD: yes + MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: yes options: "--name=mysqld" volumes: - /run/mysqld:/run/mysqld @@ -96,32 +97,6 @@ jobs: docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - - name: Report coverage + - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - run: coveralls --service=github - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_FLAG_NAME: ${{ matrix.py }}-${{ matrix.db }} - COVERALLS_PARALLEL: true - - coveralls: - if: github.repository == 'PyMySQL/PyMySQL' - name: Finish coveralls - runs-on: ubuntu-latest - needs: test - steps: - - name: requirements. - run: | - echo coveralls > requirements.txt - - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - cache: 'pip' - - - name: Finished - run: | - pip install --upgrade coveralls - coveralls --finish --service=github - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: codecov/codecov-action@v3 From f5c0ac217b08e8a59f382bd252491de9f73d6f6a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 14:52:36 +0900 Subject: [PATCH 182/227] Update README codecov badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dec840803..6e6a6bf24 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Documentation Status](https://readthedocs.org/projects/pymysql/badge/?version=latest)](https://pymysql.readthedocs.io/) -[![image](https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github)](https://coveralls.io/github/PyMySQL/PyMySQL?branch=main) +[![codecov](https://codecov.io/gh/PyMySQL/PyMySQL/branch/main/graph/badge.svg?token=ppEuaNXBW4)](https://codecov.io/gh/PyMySQL/PyMySQL) # PyMySQL From b39a43ade46eaacb081615a82bdc14ef62974ccf Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 16:53:04 +0900 Subject: [PATCH 183/227] ci: Fix MySQL 8 build overwrite previous coverage --- .github/workflows/test.yaml | 2 +- pyproject.toml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bea7747c8..c3275ccab 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -95,7 +95,7 @@ jobs: docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - pytest -v --cov --cov-config .coveragerc tests/test_auth.py; + pytest -v --cov-append --cov-config .coveragerc tests/test_auth.py; - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' diff --git a/pyproject.toml b/pyproject.toml index 48fe3660d..18714779d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,3 +56,8 @@ line-length = 99 exclude = [ "pymysql/tests/thirdparty", ] + +[tool.pdm.dev-dependencies] +dev = [ + "pytest-cov>=4.0.0", +] From 92287000831deed476e6d4a8341c6210f984bda5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 22:24:24 +0900 Subject: [PATCH 184/227] optionfile: Replace `_` with `-` (#1114) Fix #1020 --- pymysql/optionfile.py | 3 +++ pymysql/tests/test_optionfile.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pymysql/optionfile.py b/pymysql/optionfile.py index 432621b72..c36f16255 100644 --- a/pymysql/optionfile.py +++ b/pymysql/optionfile.py @@ -13,6 +13,9 @@ def __remove_quotes(self, value): return value[1:-1] return value + def optionxform(self, key): + return key.lower().replace("_", "-") + def get(self, section, option): value = configparser.RawConfigParser.get(self, section, option) return self.__remove_quotes(value) diff --git a/pymysql/tests/test_optionfile.py b/pymysql/tests/test_optionfile.py index 39bd47c46..d13553dda 100644 --- a/pymysql/tests/test_optionfile.py +++ b/pymysql/tests/test_optionfile.py @@ -21,4 +21,4 @@ def test_string(self): parser.read_file(StringIO(_cfg_file)) self.assertEqual(parser.get("default", "string"), "foo") self.assertEqual(parser.get("default", "quoted"), "bar") - self.assertEqual(parser.get("default", "single_quoted"), "foobar") + self.assertEqual(parser.get("default", "single-quoted"), "foobar") From bfbc6a53db56d37993837ea59146995e7410b41b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 00:08:34 +0900 Subject: [PATCH 185/227] Cursor.fetchall() always return list. (#1115) Cursor.fetchmany() returns empty tuple when exhausted all rows. It is for Django compatibility. Fix #1042. --- pymysql/cursors.py | 8 +++++++- pymysql/tests/test_nextset.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index d8a93c781..e098e7deb 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -282,6 +282,8 @@ def fetchmany(self, size=None): """Fetch several rows.""" self._check_executed() if self._rows is None: + # Django expects () for EOF. + # https://github.com/django/django/blob/0c1518ee429b01c145cf5b34eab01b0b92f8c246/django/db/backends/mysql/features.py#L8 return () end = self.rownumber + (size or self.arraysize) result = self._rows[self.rownumber : end] @@ -292,7 +294,7 @@ def fetchall(self): """Fetch all the rows.""" self._check_executed() if self._rows is None: - return () + return [] if self.rownumber: result = self._rows[self.rownumber :] else: @@ -479,6 +481,10 @@ def fetchmany(self, size=None): break rows.append(row) self.rownumber += 1 + if not rows: + # Django expects () for EOF. + # https://github.com/django/django/blob/0c1518ee429b01c145cf5b34eab01b0b92f8c246/django/db/backends/mysql/features.py#L8 + return () return rows def scroll(self, value, mode="relative"): diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index 28972325d..4b6b2a779 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -38,7 +38,7 @@ def test_nextset_error(self): self.assertEqual([(i,)], list(cur)) with self.assertRaises(pymysql.ProgrammingError): cur.nextset() - self.assertEqual((), cur.fetchall()) + self.assertEqual([], cur.fetchall()) def test_ok_and_next(self): cur = self.connect(client_flag=CLIENT.MULTI_STATEMENTS).cursor() From bd3bd014999475242b5963b1af7990beaa6af6b5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 00:33:50 +0900 Subject: [PATCH 186/227] Fix LOAD DATA LOCAL INFILE write EOF packet on closed connection. (#1116) Fix #989 --- .gitignore | 1 + pymysql/connections.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 98f4d45c8..09a5654fb 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ /pymysql/tests/databases.json __pycache__ Pipfile.lock +pdm.lock diff --git a/pymysql/connections.py b/pymysql/connections.py index 7bbc089f8..ef3342aa1 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1370,7 +1370,7 @@ def send_data(self): """Send data packets from the local file to the server""" if not self.connection._sock: raise err.InterfaceError(0, "") - conn = self.connection + conn: Connection = self.connection try: with open(self.filename, "rb") as open_file: @@ -1388,5 +1388,6 @@ def send_data(self): f"Can't find file '{self.filename}'", ) finally: - # send the empty packet to signify we are done sending data - conn.write_packet(b"") + if not conn._closed: + # send the empty packet to signify we are done sending data + conn.write_packet(b"") From 9a694a16a3a98ebf53cd14a1361db6c9faadba8f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 00:52:43 +0900 Subject: [PATCH 187/227] Deprecate Cursor.Error access (#1117) Fix #1111. --- pymysql/cursors.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index e098e7deb..84564a086 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -1,4 +1,5 @@ import re +import warnings from . import err @@ -352,16 +353,29 @@ def __next__(self): raise StopIteration return row - Warning = err.Warning - Error = err.Error - InterfaceError = err.InterfaceError - DatabaseError = err.DatabaseError - DataError = err.DataError - OperationalError = err.OperationalError - IntegrityError = err.IntegrityError - InternalError = err.InternalError - ProgrammingError = err.ProgrammingError - NotSupportedError = err.NotSupportedError + def __getattr__(self, name): + # DB-API 2.0 optional extension says these errors can be accessed + # via Connection object. But MySQLdb had defined them on Cursor object. + if name in ( + "Warning", + "Error", + "InterfaceError", + "DatabaseError", + "DataError", + "OperationalError", + "IntegrityError", + "InternalError", + "ProgrammingError", + "NotSupportedError", + ): + # Deprecated since v1.1 + warnings.warn( + "PyMySQL errors hould be accessed from `pymysql` package", + DeprecationWarning, + stacklevel=2, + ) + return getattr(err, name) + raise AttributeError(name) class DictCursorMixin: From 103004d6ed59d8eef95fe069e8ca4f60d4965be3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 01:02:27 +0900 Subject: [PATCH 188/227] Run pyupgrade (#1118) --- pymysql/charset.py | 2 +- pymysql/connections.py | 20 +++++++++++--------- pymysql/cursors.py | 4 ++-- pymysql/protocol.py | 4 ++-- pymysql/tests/base.py | 4 ++-- pymysql/tests/test_DictCursor.py | 4 ++-- pymysql/tests/test_basic.py | 2 +- pymysql/tests/test_connection.py | 4 ++-- pymysql/tests/test_cursor.py | 2 +- pymysql/tests/test_issues.py | 4 ++-- 10 files changed, 26 insertions(+), 24 deletions(-) diff --git a/pymysql/charset.py b/pymysql/charset.py index ac87c53dd..cdc021644 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -7,7 +7,7 @@ def __init__(self, id, name, collation, is_default): self.is_default = is_default == "Yes" def __repr__(self): - return "Charset(id=%s, name=%r, collation=%r)" % ( + return "Charset(id={}, name={!r}, collation={!r})".format( self.id, self.name, self.collation, diff --git a/pymysql/connections.py b/pymysql/connections.py index ef3342aa1..d161e7896 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -528,7 +528,9 @@ def escape_string(self, s): def _quote_bytes(self, s): if self.server_status & SERVER_STATUS.SERVER_STATUS_NO_BACKSLASH_ESCAPES: - return "'%s'" % (s.replace(b"'", b"''").decode("ascii", "surrogateescape"),) + return "'{}'".format( + s.replace(b"'", b"''").decode("ascii", "surrogateescape") + ) return converters.escape_bytes(s) def cursor(self, cursor=None): @@ -621,7 +623,7 @@ def connect(self, sock=None): (self.host, self.port), self.connect_timeout, **kwargs ) break - except (OSError, IOError) as e: + except OSError as e: if e.errno == errno.EINTR: continue raise @@ -662,7 +664,7 @@ def connect(self, sock=None): if isinstance(e, (OSError, IOError)): exc = err.OperationalError( CR.CR_CONN_HOST_ERROR, - "Can't connect to MySQL server on %r (%s)" % (self.host, e), + f"Can't connect to MySQL server on {self.host!r} ({e})", ) # Keep original exception and traceback to investigate error. exc.original_exception = e @@ -739,13 +741,13 @@ def _read_bytes(self, num_bytes): try: data = self._rfile.read(num_bytes) break - except (IOError, OSError) as e: + except OSError as e: if e.errno == errno.EINTR: continue self._force_close() raise err.OperationalError( CR.CR_SERVER_LOST, - "Lost connection to MySQL server during query (%s)" % (e,), + f"Lost connection to MySQL server during query ({e})", ) except BaseException: # Don't convert unknown exception to MySQLError. @@ -762,10 +764,10 @@ def _write_bytes(self, data): self._sock.settimeout(self._write_timeout) try: self._sock.sendall(data) - except IOError as e: + except OSError as e: self._force_close() raise err.OperationalError( - CR.CR_SERVER_GONE_ERROR, "MySQL server has gone away (%r)" % (e,) + CR.CR_SERVER_GONE_ERROR, f"MySQL server has gone away ({e!r})" ) def _read_query_result(self, unbuffered=False): @@ -1006,7 +1008,7 @@ def _process_auth(self, plugin_name, auth_packet): else: raise err.OperationalError( CR.CR_AUTH_PLUGIN_CANNOT_LOAD, - "Authentication plugin '%s' not configured" % (plugin_name,), + f"Authentication plugin '{plugin_name}' not configured", ) pkt = self._read_packet() pkt.check_error() @@ -1382,7 +1384,7 @@ def send_data(self): if not chunk: break conn.write_packet(chunk) - except IOError: + except OSError: raise err.OperationalError( ER.FILE_NOT_FOUND, f"Can't find file '{self.filename}'", diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 84564a086..8be05ca23 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -262,7 +262,7 @@ def callproc(self, procname, args=()): ) self.nextset() - q = "CALL %s(%s)" % ( + q = "CALL {}({})".format( procname, ",".join(["@_%s_%d" % (procname, i) for i in range(len(args))]), ) @@ -383,7 +383,7 @@ class DictCursorMixin: dict_type = dict def _do_get_result(self): - super(DictCursorMixin, self)._do_get_result() + super()._do_get_result() fields = [] if self.description: for f in self._result.fields: diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 41c816736..2db92d39a 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -35,7 +35,7 @@ def printable(data): dump_data = [data[i : i + 16] for i in range(0, min(len(data), 256), 16)] for d in dump_data: print( - " ".join("{:02X}".format(x) for x in d) + " ".join(f"{x:02X}" for x in d) + " " * (16 - len(d)) + " " * 2 + "".join(printable(x) for x in d) @@ -275,7 +275,7 @@ def get_column_length(self): return self.length def __str__(self): - return "%s %r.%r.%r, type=%s, flags=%x" % ( + return "{} {!r}.{!r}.{!r}, type={}, flags={:x}".format( self.__class__, self.db, self.table_name, diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index b5094563a..6dfa9590a 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -98,7 +98,7 @@ def safe_create_table(self, connection, tablename, ddl, cleanup=True): with warnings.catch_warnings(): warnings.simplefilter("ignore") - cursor.execute("drop table if exists `%s`" % (tablename,)) + cursor.execute(f"drop table if exists `{tablename}`") cursor.execute(ddl) cursor.close() if cleanup: @@ -108,5 +108,5 @@ def drop_table(self, connection, tablename): cursor = connection.cursor() with warnings.catch_warnings(): warnings.simplefilter("ignore") - cursor.execute("drop table if exists `%s`" % (tablename,)) + cursor.execute(f"drop table if exists `{tablename}`") cursor.close() diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index bbc87d032..4e545792a 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -13,7 +13,7 @@ class TestDictCursor(base.PyMySQLTestCase): cursor_type = pymysql.cursors.DictCursor def setUp(self): - super(TestDictCursor, self).setUp() + super().setUp() self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) @@ -36,7 +36,7 @@ def setUp(self): def tearDown(self): c = self.conn.cursor() c.execute("drop table dictcursor") - super(TestDictCursor, self).tearDown() + super().tearDown() def _ensure_cursor_expired(self, cursor): pass diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index ecf043f67..e77605fdf 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -332,7 +332,7 @@ class TestBulkInserts(base.PyMySQLTestCase): cursor_type = pymysql.cursors.DictCursor def setUp(self): - super(TestBulkInserts, self).setUp() + super().setUp() self.conn = conn = self.connect() # create a table and some data to query diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index bbaf3dec1..869ff0f85 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -28,7 +28,7 @@ def __init__(self, c, user, db, auth=None, authdata=None, password=None): # already exists - TODO need to check the same plugin applies self._created = False try: - c.execute("GRANT SELECT ON %s.* TO %s" % (db, user)) + c.execute(f"GRANT SELECT ON {db}.* TO {user}") self._grant = True except pymysql.err.InternalError: self._grant = False @@ -38,7 +38,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): if self._grant: - self._c.execute("REVOKE SELECT ON %s.* FROM %s" % (self._db, self._user)) + self._c.execute(f"REVOKE SELECT ON {self._db}.* FROM {self._user}") if self._created: self._c.execute("DROP USER %s" % self._user) diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 6666ab883..b292c2068 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -7,7 +7,7 @@ class CursorTest(base.PyMySQLTestCase): def setUp(self): - super(CursorTest, self).setUp() + super().setUp() conn = self.connect() self.safe_create_table( diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 7f361c94f..3564d3a6e 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -379,8 +379,8 @@ def test_issue_175(self): conn = self.connect() cur = conn.cursor() for length in (200, 300): - columns = ", ".join("c{0} integer".format(i) for i in range(length)) - sql = "create table test_field_count ({0})".format(columns) + columns = ", ".join(f"c{i} integer" for i in range(length)) + sql = f"create table test_field_count ({columns})" try: cur.execute(sql) cur.execute("select * from test_field_count") From 69290924144f961167c257ae33959c46e298efd2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 02:01:00 +0900 Subject: [PATCH 189/227] Add `collation` option and `set_character_set()` to Connection (#1119) Send `SET NAMES` on every new connection to ensure charset/collation are correctly configured. Fix #1092 --- pymysql/connections.py | 43 +++++++++++++++++++++++++++++--- pymysql/tests/test_connection.py | 14 +++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index d161e7896..f4782939b 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -112,7 +112,8 @@ class Connection: (default: None - no timeout) :param write_timeout: The timeout for writing to the connection in seconds. (default: None - no timeout) - :param charset: Charset to use. + :param str charset: Charset to use. + :param str collation: Collation name to use. :param sql_mode: Default SQL_MODE to use. :param read_default_file: Specifies my.cnf file to read these parameters from under the [client] section. @@ -174,6 +175,7 @@ def __init__( unix_socket=None, port=0, charset="", + collation=None, sql_mode=None, read_default_file=None, conv=None, @@ -308,6 +310,7 @@ def _config(key, arg): self._write_timeout = write_timeout self.charset = charset or DEFAULT_CHARSET + self.collation = collation self.use_unicode = use_unicode self.encoding = charset_by_name(self.charset).encoding @@ -593,13 +596,32 @@ def ping(self, reconnect=True): raise def set_charset(self, charset): + """Deprecated. Use set_character_set() instead.""" + # This function has been implemented in old PyMySQL. + # But this name is different from MySQLdb. + # So we keep this function for compatibility and add + # new set_character_set() function. + self.set_character_set(charset) + + def set_character_set(self, charset, collation=None): + """ + Set charaset (and collation) + + Send "SET NAMES charset [COLLATE collation]" query. + Update Connection.encoding based on charset. + """ # Make sure charset is supported. encoding = charset_by_name(charset).encoding - self._execute_command(COMMAND.COM_QUERY, "SET NAMES %s" % self.escape(charset)) + if collation: + query = f"SET NAMES {charset} COLLATE {collation}" + else: + query = f"SET NAMES {charset}" + self._execute_command(COMMAND.COM_QUERY, query) self._read_packet() self.charset = charset self.encoding = encoding + self.collation = collation def connect(self, sock=None): self._closed = False @@ -641,15 +663,30 @@ def connect(self, sock=None): self._get_server_information() self._request_authentication() + # Send "SET NAMES" query on init for: + # - Ensure charaset (and collation) is set to the server. + # - collation_id in handshake packet may be ignored. + # - If collation is not specified, we don't know what is server's + # default collation for the charset. For example, default collation + # of utf8mb4 is: + # - MySQL 5.7, MariaDB 10.x: utf8mb4_general_ci + # - MySQL 8.0: utf8mb4_0900_ai_ci + # + # Reference: + # - https://github.com/PyMySQL/PyMySQL/issues/1092 + # - https://github.com/wagtail/wagtail/issues/9477 + # - https://zenn.dev/methane/articles/2023-mysql-collation (Japanese) + self.set_character_set(self.charset, self.collation) + if self.sql_mode is not None: c = self.cursor() c.execute("SET sql_mode=%s", (self.sql_mode,)) + c.close() if self.init_command is not None: c = self.cursor() c.execute(self.init_command) c.close() - self.commit() if self.autocommit_mode is not None: self.autocommit(self.autocommit_mode) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 869ff0f85..0803efc92 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -444,6 +444,20 @@ def test_utf8mb4(self): arg["charset"] = "utf8mb4" pymysql.connect(**arg) + def test_set_character_set(self): + con = self.connect() + cur = con.cursor() + + con.set_character_set("latin1") + cur.execute("SELECT @@character_set_connection") + self.assertEqual(cur.fetchone(), ("latin1",)) + self.assertEqual(con.encoding, "cp1252") + + con.set_character_set("utf8mb4", "utf8mb4_general_ci") + cur.execute("SELECT @@character_set_connection, @@collation_connection") + self.assertEqual(cur.fetchone(), ("utf8mb4", "utf8mb4_general_ci")) + self.assertEqual(con.encoding, "utf8") + def test_largedata(self): """Large query and response (>=16MB)""" cur = self.connect().cursor() From fee5df0397ae99af8def8225b450e25002b8cb13 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 14:55:46 +0900 Subject: [PATCH 190/227] CI: Run Django test (#1121) There are some known difference between them so we can not pass the test for now. Fix #1100 --- .github/workflows/django.yaml | 66 +++++++++++++++++++++++++++++++++++ ci/test_mysql.py | 47 +++++++++++++++++++++++++ pymysql/__init__.py | 51 +++++++++++++-------------- pymysql/connections.py | 4 +-- 4 files changed, 140 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/django.yaml create mode 100644 ci/test_mysql.py diff --git a/.github/workflows/django.yaml b/.github/workflows/django.yaml new file mode 100644 index 000000000..da664f856 --- /dev/null +++ b/.github/workflows/django.yaml @@ -0,0 +1,66 @@ +name: Django test + +on: + push: + # branches: ["main"] + # pull_request: + +jobs: + django-test: + name: "Run Django LTS test suite" + runs-on: ubuntu-latest + # There are some known difference between MySQLdb and PyMySQL. + continue-on-error: true + env: + PIP_NO_PYTHON_VERSION_WARNING: 1 + PIP_DISABLE_PIP_VERSION_CHECK: 1 + # DJANGO_VERSION: "3.2.19" + strategy: + fail-fast: false + matrix: + include: + # Django 3.2.9+ supports Python 3.10 + # https://docs.djangoproject.com/ja/3.2/releases/3.2/ + - django: "3.2.19" + python: "3.10" + + - django: "4.2.1" + python: "3.11" + + steps: + - name: Start MySQL + run: | + sudo systemctl start mysql.service + mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -uroot -proot mysql + mysql -uroot -proot -e "set global innodb_flush_log_at_trx_commit=0;" + mysql -uroot -proot -e "CREATE USER 'scott'@'%' IDENTIFIED BY 'tiger'; GRANT ALL ON *.* TO scott;" + mysql -uroot -proot -e "CREATE DATABASE django_default; CREATE DATABASE django_other;" + + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install mysqlclient + run: | + #pip install mysqlclient # Use stable version + pip install .[rsa] + + - name: Setup Django + run: | + sudo apt-get install libmemcached-dev + wget https://github.com/django/django/archive/${{ matrix.django }}.tar.gz + tar xf ${{ matrix.django }}.tar.gz + cp ci/test_mysql.py django-${{ matrix.django }}/tests/ + cd django-${{ matrix.django }} + pip install . -r tests/requirements/py3.txt + + - name: Run Django test + run: | + cd django-${{ matrix.django }}/tests/ + # test_runner does not using our test_mysql.py + # We can't run whole django test suite for now. + # Run olly backends test + DJANGO_SETTINGS_MODULE=test_mysql python runtests.py backends diff --git a/ci/test_mysql.py b/ci/test_mysql.py new file mode 100644 index 000000000..b97978a27 --- /dev/null +++ b/ci/test_mysql.py @@ -0,0 +1,47 @@ +# This is an example test settings file for use with the Django test suite. +# +# The 'sqlite3' backend requires only the ENGINE setting (an in- +# memory database will be used). All other backends will require a +# NAME and potentially authentication information. See the +# following section in the docs for more information: +# +# https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/ +# +# The different databases that Django supports behave differently in certain +# situations, so it is recommended to run the test suite against as many +# database backends as possible. You may want to create a separate settings +# file for each of the backends you test against. + +import pymysql + +pymysql.install_as_MySQLdb() + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": "django_default", + "HOST": "127.0.0.1", + "USER": "scott", + "PASSWORD": "tiger", + "TEST": {"CHARSET": "utf8mb3", "COLLATION": "utf8mb3_general_ci"}, + }, + "other": { + "ENGINE": "django.db.backends.mysql", + "NAME": "django_other", + "HOST": "127.0.0.1", + "USER": "scott", + "PASSWORD": "tiger", + "TEST": {"CHARSET": "utf8mb3", "COLLATION": "utf8mb3_general_ci"}, + }, +} + +SECRET_KEY = "django_tests_secret_key" + +# Use a fast hasher to speed up tests. +PASSWORD_HASHERS = [ + "django.contrib.auth.hashers.MD5PasswordHasher", +] + +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" + +USE_TZ = False diff --git a/pymysql/__init__.py b/pymysql/__init__.py index c0039c3fe..ab43c1a94 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -46,12 +46,30 @@ TimestampFromTicks, ) +# PyMySQL version. +# Used by setuptools. +VERSION = (1, 1, 0, "dev", 1) + +### for mysqlclient compatibility +### Django checks mysqlclient version. +version_info = (1, 4, 3, "final", 0) +__version__ = "1.4.3" + + +def get_client_info(): # for MySQLdb compatibility + return __version__ + + +def install_as_MySQLdb(): + """ + After this function is called, any application that imports MySQLdb + will unwittingly actually use pymysql. + """ + sys.modules["MySQLdb"] = sys.modules["pymysql"] + + +# end of mysqlclient compatibility code -VERSION = (1, 0, 3) -if len(VERSION) > 3: - VERSION_STRING = "%d.%d.%d_%s" % VERSION -else: - VERSION_STRING = "%d.%d.%d" % VERSION threadsafety = 1 apilevel = "2.0" paramstyle = "pyformat" @@ -109,31 +127,12 @@ def Binary(x): return bytes(x) -Connect = connect = Connection = connections.Connection - - -def get_client_info(): # for MySQLdb compatibility - return VERSION_STRING - - -# we include a doctored version_info here for MySQLdb compatibility -version_info = (1, 4, 0, "final", 0) - -NULL = "NULL" - -__version__ = get_client_info() - - def thread_safe(): return True # match MySQLdb.thread_safe() -def install_as_MySQLdb(): - """ - After this function is called, any application that imports MySQLdb or - _mysql will unwittingly actually use pymysql. - """ - sys.modules["MySQLdb"] = sys.modules["_mysql"] = sys.modules["pymysql"] +Connect = connect = Connection = connections.Connection +NULL = "NULL" __all__ = [ diff --git a/pymysql/connections.py b/pymysql/connections.py index f4782939b..6edac04c1 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -25,7 +25,7 @@ EOFPacketWrapper, LoadLocalPacketWrapper, ) -from . import err, VERSION_STRING +from . import err, __version__ try: import ssl @@ -346,7 +346,7 @@ def _config(key, arg): self._connect_attrs = { "_client_name": "pymysql", "_pid": str(os.getpid()), - "_client_version": VERSION_STRING, + "_client_version": __version__, } if program_name: From a5849526821c2d085b94e25ef0b2499ae04dad84 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 15:02:25 +0900 Subject: [PATCH 191/227] Update CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc752253..0e94843c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,17 @@ Release date: TBD ## v1.0.3 -Release date: TBD +Release date: 2023-03-28 * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 +* Removed _last_executed because of duplication with _executed by @rajat315315 in https://github.com/PyMySQL/PyMySQL/pull/948 +* Fix generating authentication response with long strings by @netch80 in https://github.com/PyMySQL/PyMySQL/pull/988 +* update pymysql.constants.CR by @Nothing4You in https://github.com/PyMySQL/PyMySQL/pull/1029 +* Document that the ssl connection parameter can be an SSLContext by @cakemanny in https://github.com/PyMySQL/PyMySQL/pull/1045 +* Raise ProgrammingError on -np.inf in addition to np.inf by @cdcadman in https://github.com/PyMySQL/PyMySQL/pull/1067 +* Use Python 3.11 release instead of -dev in tests by @Nothing4You in https://github.com/PyMySQL/PyMySQL/pull/1076 ## v1.0.2 From 2596bbb5b796aae5bb0759b403d6d28cc22b720c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 15:26:39 +0900 Subject: [PATCH 192/227] Release v1.1.0rc1 (#1122) --- CHANGELOG.md | 8 ++++++++ pymysql/__init__.py | 5 +++-- pymysql/connections.py | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e94843c2..dc5ff1612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ Release date: TBD * Fixed SSCursor raising OperationalError for query timeouts on wrong statement (#1032) * Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) +* Make Cursor iterator (#995) +* Support '_' in key name in my.cnf (#1114) +* `Cursor.fetchall()` returns empty list instead of tuple (#1115). Note that `Cursor.fetchmany()` still return empty tuple after reading all rows for compatibility with Django. +* Deprecate Error classes in Cursor class (#1117) +* Add `Connection.set_character_set(charset, collation=None)` (#1119) +* Deprecate `Connection.set_charset(charset)` (#1119) +* New connection always send "SET NAMES charset [COLLATE collation]" query. (#1119) + Since collation table is vary on MySQL server versions, collation in handshake is fragile. ## v1.0.3 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index ab43c1a94..b9971ff0f 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,8 +47,9 @@ ) # PyMySQL version. -# Used by setuptools. -VERSION = (1, 1, 0, "dev", 1) +# Used by setuptools and connection_attrs +VERSION = (1, 1, 0, "rc", 1) +VERSION_STRING = "1.1.0rc1" ### for mysqlclient compatibility ### Django checks mysqlclient version. diff --git a/pymysql/connections.py b/pymysql/connections.py index 6edac04c1..843bea5e0 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -25,7 +25,7 @@ EOFPacketWrapper, LoadLocalPacketWrapper, ) -from . import err, __version__ +from . import err, VERSION_STRING try: import ssl @@ -345,8 +345,8 @@ def _config(key, arg): self._connect_attrs = { "_client_name": "pymysql", + "_client_version": VERSION_STRING, "_pid": str(os.getpid()), - "_client_version": __version__, } if program_name: From 2df6c068b7a0dd733e72a068b3aca3e8738177ad Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 25 May 2023 17:36:52 +1000 Subject: [PATCH 193/227] Bump mariadb version (#1123) In README and GH actions. --- .github/workflows/test.yaml | 6 +++--- README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c3275ccab..6b1e0f321 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,16 +15,16 @@ jobs: fail-fast: false matrix: include: - - db: "mariadb:10.3" + - db: "mariadb:10.4" py: "3.8" - db: "mariadb:10.5" py: "3.7" - - db: "mariadb:10.7" + - db: "mariadb:10.6" py: "3.11" - - db: "mariadb:10.8" + - db: "mariadb:lts" py: "3.9" - db: "mysql:5.7" diff --git a/README.md b/README.md index 6e6a6bf24..32f5df2f4 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ This package contains a pure-Python MySQL client library, based on [PEP - [PyPy](https://pypy.org/) : Latest 3.x version - MySQL Server -- one of the following: - [MySQL](https://www.mysql.com/) \>= 5.7 - - [MariaDB](https://mariadb.org/) \>= 10.3 + - [MariaDB](https://mariadb.org/) \>= 10.4 ## Installation From f4c348fdcf4ac21a92be58b6f94e9d7a13826a38 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Jun 2023 13:55:38 +0900 Subject: [PATCH 194/227] Configure Renovate (#1124) --- renovate.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 000000000..39a2b6e9a --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ] +} From c3a12f683345a97a8cc8516cf2123a5836c38f7d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 15 Jun 2023 13:20:26 +0900 Subject: [PATCH 195/227] Make charset="utf8" use utf8mb4. (#1127) Use charset="utf8mb3" to use utf8mb3 instead. Fix #1126 --- pymysql/charset.py | 319 +++++++++++++++++----------------- pymysql/tests/test_charset.py | 25 +++ 2 files changed, 188 insertions(+), 156 deletions(-) create mode 100644 pymysql/tests/test_charset.py diff --git a/pymysql/charset.py b/pymysql/charset.py index cdc021644..b1c1ca8b8 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -1,16 +1,16 @@ +# Internal use only. Do not use directly. + MBLENGTH = {8: 1, 33: 3, 88: 2, 91: 2} class Charset: - def __init__(self, id, name, collation, is_default): + def __init__(self, id, name, collation, is_default=False): self.id, self.name, self.collation = id, name, collation - self.is_default = is_default == "Yes" + self.is_default = is_default def __repr__(self): - return "Charset(id={}, name={!r}, collation={!r})".format( - self.id, - self.name, - self.collation, + return ( + f"Charset(id={self.id}, name={self.name!r}, collation={self.collation!r})" ) @property @@ -45,165 +45,172 @@ def by_id(self, id): return self._by_id[id] def by_name(self, name): + if name == "utf8": + name = "utf8mb4" return self._by_name.get(name.lower()) _charsets = Charsets() +charset_by_name = _charsets.by_name +charset_by_id = _charsets.by_id + """ +TODO: update this script. + Generated with: mysql -N -s -e "select id, character_set_name, collation_name, is_default from information_schema.collations order by id;" | python -c "import sys for l in sys.stdin.readlines(): - id, name, collation, is_default = l.split(chr(9)) - print '_charsets.add(Charset(%s, \'%s\', \'%s\', \'%s\'))' \ - % (id, name, collation, is_default.strip()) -" - + id, name, collation, is_default = l.split(chr(9)) + if is_default.strip() == "Yes": + print('_charsets.add(Charset(%s, \'%s\', \'%s\', True))' \ + % (id, name, collation)) + else: + print('_charsets.add(Charset(%s, \'%s\', \'%s\'))' \ + % (id, name, collation, bool(is_default.strip())) """ -_charsets.add(Charset(1, "big5", "big5_chinese_ci", "Yes")) -_charsets.add(Charset(2, "latin2", "latin2_czech_cs", "")) -_charsets.add(Charset(3, "dec8", "dec8_swedish_ci", "Yes")) -_charsets.add(Charset(4, "cp850", "cp850_general_ci", "Yes")) -_charsets.add(Charset(5, "latin1", "latin1_german1_ci", "")) -_charsets.add(Charset(6, "hp8", "hp8_english_ci", "Yes")) -_charsets.add(Charset(7, "koi8r", "koi8r_general_ci", "Yes")) -_charsets.add(Charset(8, "latin1", "latin1_swedish_ci", "Yes")) -_charsets.add(Charset(9, "latin2", "latin2_general_ci", "Yes")) -_charsets.add(Charset(10, "swe7", "swe7_swedish_ci", "Yes")) -_charsets.add(Charset(11, "ascii", "ascii_general_ci", "Yes")) -_charsets.add(Charset(12, "ujis", "ujis_japanese_ci", "Yes")) -_charsets.add(Charset(13, "sjis", "sjis_japanese_ci", "Yes")) -_charsets.add(Charset(14, "cp1251", "cp1251_bulgarian_ci", "")) -_charsets.add(Charset(15, "latin1", "latin1_danish_ci", "")) -_charsets.add(Charset(16, "hebrew", "hebrew_general_ci", "Yes")) -_charsets.add(Charset(18, "tis620", "tis620_thai_ci", "Yes")) -_charsets.add(Charset(19, "euckr", "euckr_korean_ci", "Yes")) -_charsets.add(Charset(20, "latin7", "latin7_estonian_cs", "")) -_charsets.add(Charset(21, "latin2", "latin2_hungarian_ci", "")) -_charsets.add(Charset(22, "koi8u", "koi8u_general_ci", "Yes")) -_charsets.add(Charset(23, "cp1251", "cp1251_ukrainian_ci", "")) -_charsets.add(Charset(24, "gb2312", "gb2312_chinese_ci", "Yes")) -_charsets.add(Charset(25, "greek", "greek_general_ci", "Yes")) -_charsets.add(Charset(26, "cp1250", "cp1250_general_ci", "Yes")) -_charsets.add(Charset(27, "latin2", "latin2_croatian_ci", "")) -_charsets.add(Charset(28, "gbk", "gbk_chinese_ci", "Yes")) -_charsets.add(Charset(29, "cp1257", "cp1257_lithuanian_ci", "")) -_charsets.add(Charset(30, "latin5", "latin5_turkish_ci", "Yes")) -_charsets.add(Charset(31, "latin1", "latin1_german2_ci", "")) -_charsets.add(Charset(32, "armscii8", "armscii8_general_ci", "Yes")) -_charsets.add(Charset(33, "utf8", "utf8_general_ci", "Yes")) -_charsets.add(Charset(34, "cp1250", "cp1250_czech_cs", "")) -_charsets.add(Charset(36, "cp866", "cp866_general_ci", "Yes")) -_charsets.add(Charset(37, "keybcs2", "keybcs2_general_ci", "Yes")) -_charsets.add(Charset(38, "macce", "macce_general_ci", "Yes")) -_charsets.add(Charset(39, "macroman", "macroman_general_ci", "Yes")) -_charsets.add(Charset(40, "cp852", "cp852_general_ci", "Yes")) -_charsets.add(Charset(41, "latin7", "latin7_general_ci", "Yes")) -_charsets.add(Charset(42, "latin7", "latin7_general_cs", "")) -_charsets.add(Charset(43, "macce", "macce_bin", "")) -_charsets.add(Charset(44, "cp1250", "cp1250_croatian_ci", "")) -_charsets.add(Charset(45, "utf8mb4", "utf8mb4_general_ci", "Yes")) -_charsets.add(Charset(46, "utf8mb4", "utf8mb4_bin", "")) -_charsets.add(Charset(47, "latin1", "latin1_bin", "")) -_charsets.add(Charset(48, "latin1", "latin1_general_ci", "")) -_charsets.add(Charset(49, "latin1", "latin1_general_cs", "")) -_charsets.add(Charset(50, "cp1251", "cp1251_bin", "")) -_charsets.add(Charset(51, "cp1251", "cp1251_general_ci", "Yes")) -_charsets.add(Charset(52, "cp1251", "cp1251_general_cs", "")) -_charsets.add(Charset(53, "macroman", "macroman_bin", "")) -_charsets.add(Charset(57, "cp1256", "cp1256_general_ci", "Yes")) -_charsets.add(Charset(58, "cp1257", "cp1257_bin", "")) -_charsets.add(Charset(59, "cp1257", "cp1257_general_ci", "Yes")) -_charsets.add(Charset(63, "binary", "binary", "Yes")) -_charsets.add(Charset(64, "armscii8", "armscii8_bin", "")) -_charsets.add(Charset(65, "ascii", "ascii_bin", "")) -_charsets.add(Charset(66, "cp1250", "cp1250_bin", "")) -_charsets.add(Charset(67, "cp1256", "cp1256_bin", "")) -_charsets.add(Charset(68, "cp866", "cp866_bin", "")) -_charsets.add(Charset(69, "dec8", "dec8_bin", "")) -_charsets.add(Charset(70, "greek", "greek_bin", "")) -_charsets.add(Charset(71, "hebrew", "hebrew_bin", "")) -_charsets.add(Charset(72, "hp8", "hp8_bin", "")) -_charsets.add(Charset(73, "keybcs2", "keybcs2_bin", "")) -_charsets.add(Charset(74, "koi8r", "koi8r_bin", "")) -_charsets.add(Charset(75, "koi8u", "koi8u_bin", "")) -_charsets.add(Charset(76, "utf8", "utf8_tolower_ci", "")) -_charsets.add(Charset(77, "latin2", "latin2_bin", "")) -_charsets.add(Charset(78, "latin5", "latin5_bin", "")) -_charsets.add(Charset(79, "latin7", "latin7_bin", "")) -_charsets.add(Charset(80, "cp850", "cp850_bin", "")) -_charsets.add(Charset(81, "cp852", "cp852_bin", "")) -_charsets.add(Charset(82, "swe7", "swe7_bin", "")) -_charsets.add(Charset(83, "utf8", "utf8_bin", "")) -_charsets.add(Charset(84, "big5", "big5_bin", "")) -_charsets.add(Charset(85, "euckr", "euckr_bin", "")) -_charsets.add(Charset(86, "gb2312", "gb2312_bin", "")) -_charsets.add(Charset(87, "gbk", "gbk_bin", "")) -_charsets.add(Charset(88, "sjis", "sjis_bin", "")) -_charsets.add(Charset(89, "tis620", "tis620_bin", "")) -_charsets.add(Charset(91, "ujis", "ujis_bin", "")) -_charsets.add(Charset(92, "geostd8", "geostd8_general_ci", "Yes")) -_charsets.add(Charset(93, "geostd8", "geostd8_bin", "")) -_charsets.add(Charset(94, "latin1", "latin1_spanish_ci", "")) -_charsets.add(Charset(95, "cp932", "cp932_japanese_ci", "Yes")) -_charsets.add(Charset(96, "cp932", "cp932_bin", "")) -_charsets.add(Charset(97, "eucjpms", "eucjpms_japanese_ci", "Yes")) -_charsets.add(Charset(98, "eucjpms", "eucjpms_bin", "")) -_charsets.add(Charset(99, "cp1250", "cp1250_polish_ci", "")) -_charsets.add(Charset(192, "utf8", "utf8_unicode_ci", "")) -_charsets.add(Charset(193, "utf8", "utf8_icelandic_ci", "")) -_charsets.add(Charset(194, "utf8", "utf8_latvian_ci", "")) -_charsets.add(Charset(195, "utf8", "utf8_romanian_ci", "")) -_charsets.add(Charset(196, "utf8", "utf8_slovenian_ci", "")) -_charsets.add(Charset(197, "utf8", "utf8_polish_ci", "")) -_charsets.add(Charset(198, "utf8", "utf8_estonian_ci", "")) -_charsets.add(Charset(199, "utf8", "utf8_spanish_ci", "")) -_charsets.add(Charset(200, "utf8", "utf8_swedish_ci", "")) -_charsets.add(Charset(201, "utf8", "utf8_turkish_ci", "")) -_charsets.add(Charset(202, "utf8", "utf8_czech_ci", "")) -_charsets.add(Charset(203, "utf8", "utf8_danish_ci", "")) -_charsets.add(Charset(204, "utf8", "utf8_lithuanian_ci", "")) -_charsets.add(Charset(205, "utf8", "utf8_slovak_ci", "")) -_charsets.add(Charset(206, "utf8", "utf8_spanish2_ci", "")) -_charsets.add(Charset(207, "utf8", "utf8_roman_ci", "")) -_charsets.add(Charset(208, "utf8", "utf8_persian_ci", "")) -_charsets.add(Charset(209, "utf8", "utf8_esperanto_ci", "")) -_charsets.add(Charset(210, "utf8", "utf8_hungarian_ci", "")) -_charsets.add(Charset(211, "utf8", "utf8_sinhala_ci", "")) -_charsets.add(Charset(212, "utf8", "utf8_german2_ci", "")) -_charsets.add(Charset(213, "utf8", "utf8_croatian_ci", "")) -_charsets.add(Charset(214, "utf8", "utf8_unicode_520_ci", "")) -_charsets.add(Charset(215, "utf8", "utf8_vietnamese_ci", "")) -_charsets.add(Charset(223, "utf8", "utf8_general_mysql500_ci", "")) -_charsets.add(Charset(224, "utf8mb4", "utf8mb4_unicode_ci", "")) -_charsets.add(Charset(225, "utf8mb4", "utf8mb4_icelandic_ci", "")) -_charsets.add(Charset(226, "utf8mb4", "utf8mb4_latvian_ci", "")) -_charsets.add(Charset(227, "utf8mb4", "utf8mb4_romanian_ci", "")) -_charsets.add(Charset(228, "utf8mb4", "utf8mb4_slovenian_ci", "")) -_charsets.add(Charset(229, "utf8mb4", "utf8mb4_polish_ci", "")) -_charsets.add(Charset(230, "utf8mb4", "utf8mb4_estonian_ci", "")) -_charsets.add(Charset(231, "utf8mb4", "utf8mb4_spanish_ci", "")) -_charsets.add(Charset(232, "utf8mb4", "utf8mb4_swedish_ci", "")) -_charsets.add(Charset(233, "utf8mb4", "utf8mb4_turkish_ci", "")) -_charsets.add(Charset(234, "utf8mb4", "utf8mb4_czech_ci", "")) -_charsets.add(Charset(235, "utf8mb4", "utf8mb4_danish_ci", "")) -_charsets.add(Charset(236, "utf8mb4", "utf8mb4_lithuanian_ci", "")) -_charsets.add(Charset(237, "utf8mb4", "utf8mb4_slovak_ci", "")) -_charsets.add(Charset(238, "utf8mb4", "utf8mb4_spanish2_ci", "")) -_charsets.add(Charset(239, "utf8mb4", "utf8mb4_roman_ci", "")) -_charsets.add(Charset(240, "utf8mb4", "utf8mb4_persian_ci", "")) -_charsets.add(Charset(241, "utf8mb4", "utf8mb4_esperanto_ci", "")) -_charsets.add(Charset(242, "utf8mb4", "utf8mb4_hungarian_ci", "")) -_charsets.add(Charset(243, "utf8mb4", "utf8mb4_sinhala_ci", "")) -_charsets.add(Charset(244, "utf8mb4", "utf8mb4_german2_ci", "")) -_charsets.add(Charset(245, "utf8mb4", "utf8mb4_croatian_ci", "")) -_charsets.add(Charset(246, "utf8mb4", "utf8mb4_unicode_520_ci", "")) -_charsets.add(Charset(247, "utf8mb4", "utf8mb4_vietnamese_ci", "")) -_charsets.add(Charset(248, "gb18030", "gb18030_chinese_ci", "Yes")) -_charsets.add(Charset(249, "gb18030", "gb18030_bin", "")) -_charsets.add(Charset(250, "gb18030", "gb18030_unicode_520_ci", "")) -_charsets.add(Charset(255, "utf8mb4", "utf8mb4_0900_ai_ci", "")) -charset_by_name = _charsets.by_name -charset_by_id = _charsets.by_id +_charsets.add(Charset(1, "big5", "big5_chinese_ci", True)) +_charsets.add(Charset(2, "latin2", "latin2_czech_cs")) +_charsets.add(Charset(3, "dec8", "dec8_swedish_ci", True)) +_charsets.add(Charset(4, "cp850", "cp850_general_ci", True)) +_charsets.add(Charset(5, "latin1", "latin1_german1_ci")) +_charsets.add(Charset(6, "hp8", "hp8_english_ci", True)) +_charsets.add(Charset(7, "koi8r", "koi8r_general_ci", True)) +_charsets.add(Charset(8, "latin1", "latin1_swedish_ci", True)) +_charsets.add(Charset(9, "latin2", "latin2_general_ci", True)) +_charsets.add(Charset(10, "swe7", "swe7_swedish_ci", True)) +_charsets.add(Charset(11, "ascii", "ascii_general_ci", True)) +_charsets.add(Charset(12, "ujis", "ujis_japanese_ci", True)) +_charsets.add(Charset(13, "sjis", "sjis_japanese_ci", True)) +_charsets.add(Charset(14, "cp1251", "cp1251_bulgarian_ci")) +_charsets.add(Charset(15, "latin1", "latin1_danish_ci")) +_charsets.add(Charset(16, "hebrew", "hebrew_general_ci", True)) +_charsets.add(Charset(18, "tis620", "tis620_thai_ci", True)) +_charsets.add(Charset(19, "euckr", "euckr_korean_ci", True)) +_charsets.add(Charset(20, "latin7", "latin7_estonian_cs")) +_charsets.add(Charset(21, "latin2", "latin2_hungarian_ci")) +_charsets.add(Charset(22, "koi8u", "koi8u_general_ci", True)) +_charsets.add(Charset(23, "cp1251", "cp1251_ukrainian_ci")) +_charsets.add(Charset(24, "gb2312", "gb2312_chinese_ci", True)) +_charsets.add(Charset(25, "greek", "greek_general_ci", True)) +_charsets.add(Charset(26, "cp1250", "cp1250_general_ci", True)) +_charsets.add(Charset(27, "latin2", "latin2_croatian_ci")) +_charsets.add(Charset(28, "gbk", "gbk_chinese_ci", True)) +_charsets.add(Charset(29, "cp1257", "cp1257_lithuanian_ci")) +_charsets.add(Charset(30, "latin5", "latin5_turkish_ci", True)) +_charsets.add(Charset(31, "latin1", "latin1_german2_ci")) +_charsets.add(Charset(32, "armscii8", "armscii8_general_ci", True)) +_charsets.add(Charset(33, "utf8mb3", "utf8mb3_general_ci", True)) +_charsets.add(Charset(34, "cp1250", "cp1250_czech_cs")) +_charsets.add(Charset(36, "cp866", "cp866_general_ci", True)) +_charsets.add(Charset(37, "keybcs2", "keybcs2_general_ci", True)) +_charsets.add(Charset(38, "macce", "macce_general_ci", True)) +_charsets.add(Charset(39, "macroman", "macroman_general_ci", True)) +_charsets.add(Charset(40, "cp852", "cp852_general_ci", True)) +_charsets.add(Charset(41, "latin7", "latin7_general_ci", True)) +_charsets.add(Charset(42, "latin7", "latin7_general_cs")) +_charsets.add(Charset(43, "macce", "macce_bin")) +_charsets.add(Charset(44, "cp1250", "cp1250_croatian_ci")) +_charsets.add(Charset(45, "utf8mb4", "utf8mb4_general_ci", True)) +_charsets.add(Charset(46, "utf8mb4", "utf8mb4_bin")) +_charsets.add(Charset(47, "latin1", "latin1_bin")) +_charsets.add(Charset(48, "latin1", "latin1_general_ci")) +_charsets.add(Charset(49, "latin1", "latin1_general_cs")) +_charsets.add(Charset(50, "cp1251", "cp1251_bin")) +_charsets.add(Charset(51, "cp1251", "cp1251_general_ci", True)) +_charsets.add(Charset(52, "cp1251", "cp1251_general_cs")) +_charsets.add(Charset(53, "macroman", "macroman_bin")) +_charsets.add(Charset(57, "cp1256", "cp1256_general_ci", True)) +_charsets.add(Charset(58, "cp1257", "cp1257_bin")) +_charsets.add(Charset(59, "cp1257", "cp1257_general_ci", True)) +_charsets.add(Charset(63, "binary", "binary", True)) +_charsets.add(Charset(64, "armscii8", "armscii8_bin")) +_charsets.add(Charset(65, "ascii", "ascii_bin")) +_charsets.add(Charset(66, "cp1250", "cp1250_bin")) +_charsets.add(Charset(67, "cp1256", "cp1256_bin")) +_charsets.add(Charset(68, "cp866", "cp866_bin")) +_charsets.add(Charset(69, "dec8", "dec8_bin")) +_charsets.add(Charset(70, "greek", "greek_bin")) +_charsets.add(Charset(71, "hebrew", "hebrew_bin")) +_charsets.add(Charset(72, "hp8", "hp8_bin")) +_charsets.add(Charset(73, "keybcs2", "keybcs2_bin")) +_charsets.add(Charset(74, "koi8r", "koi8r_bin")) +_charsets.add(Charset(75, "koi8u", "koi8u_bin")) +_charsets.add(Charset(76, "utf8mb3", "utf8mb3_tolower_ci")) +_charsets.add(Charset(77, "latin2", "latin2_bin")) +_charsets.add(Charset(78, "latin5", "latin5_bin")) +_charsets.add(Charset(79, "latin7", "latin7_bin")) +_charsets.add(Charset(80, "cp850", "cp850_bin")) +_charsets.add(Charset(81, "cp852", "cp852_bin")) +_charsets.add(Charset(82, "swe7", "swe7_bin")) +_charsets.add(Charset(83, "utf8mb3", "utf8mb3_bin")) +_charsets.add(Charset(84, "big5", "big5_bin")) +_charsets.add(Charset(85, "euckr", "euckr_bin")) +_charsets.add(Charset(86, "gb2312", "gb2312_bin")) +_charsets.add(Charset(87, "gbk", "gbk_bin")) +_charsets.add(Charset(88, "sjis", "sjis_bin")) +_charsets.add(Charset(89, "tis620", "tis620_bin")) +_charsets.add(Charset(91, "ujis", "ujis_bin")) +_charsets.add(Charset(92, "geostd8", "geostd8_general_ci", True)) +_charsets.add(Charset(93, "geostd8", "geostd8_bin")) +_charsets.add(Charset(94, "latin1", "latin1_spanish_ci")) +_charsets.add(Charset(95, "cp932", "cp932_japanese_ci", True)) +_charsets.add(Charset(96, "cp932", "cp932_bin")) +_charsets.add(Charset(97, "eucjpms", "eucjpms_japanese_ci", True)) +_charsets.add(Charset(98, "eucjpms", "eucjpms_bin")) +_charsets.add(Charset(99, "cp1250", "cp1250_polish_ci")) +_charsets.add(Charset(192, "utf8mb3", "utf8mb3_unicode_ci")) +_charsets.add(Charset(193, "utf8mb3", "utf8mb3_icelandic_ci")) +_charsets.add(Charset(194, "utf8mb3", "utf8mb3_latvian_ci")) +_charsets.add(Charset(195, "utf8mb3", "utf8mb3_romanian_ci")) +_charsets.add(Charset(196, "utf8mb3", "utf8mb3_slovenian_ci")) +_charsets.add(Charset(197, "utf8mb3", "utf8mb3_polish_ci")) +_charsets.add(Charset(198, "utf8mb3", "utf8mb3_estonian_ci")) +_charsets.add(Charset(199, "utf8mb3", "utf8mb3_spanish_ci")) +_charsets.add(Charset(200, "utf8mb3", "utf8mb3_swedish_ci")) +_charsets.add(Charset(201, "utf8mb3", "utf8mb3_turkish_ci")) +_charsets.add(Charset(202, "utf8mb3", "utf8mb3_czech_ci")) +_charsets.add(Charset(203, "utf8mb3", "utf8mb3_danish_ci")) +_charsets.add(Charset(204, "utf8mb3", "utf8mb3_lithuanian_ci")) +_charsets.add(Charset(205, "utf8mb3", "utf8mb3_slovak_ci")) +_charsets.add(Charset(206, "utf8mb3", "utf8mb3_spanish2_ci")) +_charsets.add(Charset(207, "utf8mb3", "utf8mb3_roman_ci")) +_charsets.add(Charset(208, "utf8mb3", "utf8mb3_persian_ci")) +_charsets.add(Charset(209, "utf8mb3", "utf8mb3_esperanto_ci")) +_charsets.add(Charset(210, "utf8mb3", "utf8mb3_hungarian_ci")) +_charsets.add(Charset(211, "utf8mb3", "utf8mb3_sinhala_ci")) +_charsets.add(Charset(212, "utf8mb3", "utf8mb3_german2_ci")) +_charsets.add(Charset(213, "utf8mb3", "utf8mb3_croatian_ci")) +_charsets.add(Charset(214, "utf8mb3", "utf8mb3_unicode_520_ci")) +_charsets.add(Charset(215, "utf8mb3", "utf8mb3_vietnamese_ci")) +_charsets.add(Charset(223, "utf8mb3", "utf8mb3_general_mysql500_ci")) +_charsets.add(Charset(224, "utf8mb4", "utf8mb4_unicode_ci")) +_charsets.add(Charset(225, "utf8mb4", "utf8mb4_icelandic_ci")) +_charsets.add(Charset(226, "utf8mb4", "utf8mb4_latvian_ci")) +_charsets.add(Charset(227, "utf8mb4", "utf8mb4_romanian_ci")) +_charsets.add(Charset(228, "utf8mb4", "utf8mb4_slovenian_ci")) +_charsets.add(Charset(229, "utf8mb4", "utf8mb4_polish_ci")) +_charsets.add(Charset(230, "utf8mb4", "utf8mb4_estonian_ci")) +_charsets.add(Charset(231, "utf8mb4", "utf8mb4_spanish_ci")) +_charsets.add(Charset(232, "utf8mb4", "utf8mb4_swedish_ci")) +_charsets.add(Charset(233, "utf8mb4", "utf8mb4_turkish_ci")) +_charsets.add(Charset(234, "utf8mb4", "utf8mb4_czech_ci")) +_charsets.add(Charset(235, "utf8mb4", "utf8mb4_danish_ci")) +_charsets.add(Charset(236, "utf8mb4", "utf8mb4_lithuanian_ci")) +_charsets.add(Charset(237, "utf8mb4", "utf8mb4_slovak_ci")) +_charsets.add(Charset(238, "utf8mb4", "utf8mb4_spanish2_ci")) +_charsets.add(Charset(239, "utf8mb4", "utf8mb4_roman_ci")) +_charsets.add(Charset(240, "utf8mb4", "utf8mb4_persian_ci")) +_charsets.add(Charset(241, "utf8mb4", "utf8mb4_esperanto_ci")) +_charsets.add(Charset(242, "utf8mb4", "utf8mb4_hungarian_ci")) +_charsets.add(Charset(243, "utf8mb4", "utf8mb4_sinhala_ci")) +_charsets.add(Charset(244, "utf8mb4", "utf8mb4_german2_ci")) +_charsets.add(Charset(245, "utf8mb4", "utf8mb4_croatian_ci")) +_charsets.add(Charset(246, "utf8mb4", "utf8mb4_unicode_520_ci")) +_charsets.add(Charset(247, "utf8mb4", "utf8mb4_vietnamese_ci")) +_charsets.add(Charset(248, "gb18030", "gb18030_chinese_ci", True)) +_charsets.add(Charset(249, "gb18030", "gb18030_bin")) +_charsets.add(Charset(250, "gb18030", "gb18030_unicode_520_ci")) +_charsets.add(Charset(255, "utf8mb4", "utf8mb4_0900_ai_ci")) diff --git a/pymysql/tests/test_charset.py b/pymysql/tests/test_charset.py new file mode 100644 index 000000000..94e6e1559 --- /dev/null +++ b/pymysql/tests/test_charset.py @@ -0,0 +1,25 @@ +import pymysql.charset + + +def test_utf8(): + utf8mb3 = pymysql.charset.charset_by_name("utf8mb3") + assert utf8mb3.name == "utf8mb3" + assert utf8mb3.collation == "utf8mb3_general_ci" + assert ( + repr(utf8mb3) + == "Charset(id=33, name='utf8mb3', collation='utf8mb3_general_ci')" + ) + + # MySQL 8.0 changed the default collation for utf8mb4. + # But we use old default for compatibility. + utf8mb4 = pymysql.charset.charset_by_name("utf8mb4") + assert utf8mb4.name == "utf8mb4" + assert utf8mb4.collation == "utf8mb4_general_ci" + assert ( + repr(utf8mb4) + == "Charset(id=45, name='utf8mb4', collation='utf8mb4_general_ci')" + ) + + # utf8 is alias of utf8mb4 since MySQL 8.0, and PyMySQL v1.1. + utf8 = pymysql.charset.charset_by_name("utf8") + assert utf8 == utf8mb4 From fed7e8069bf09d3b4e819dc8c59d6b7096e4183f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 15 Jun 2023 13:31:21 +0900 Subject: [PATCH 196/227] Add codecov.yml (#1128) --- codecov.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..919adf200 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,7 @@ +# https://docs.codecov.com/docs/common-recipe-list +coverage: + status: + project: + default: + target: auto + threshold: 3% From f3f3477682a3bbc80eb0240034abcd288d7dda63 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 15 Jun 2023 16:58:17 +0900 Subject: [PATCH 197/227] Release v1.1.0rc2 (#1129) --- CHANGELOG.md | 1 + pymysql/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5ff1612..ea1d732a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Release date: TBD * Deprecate `Connection.set_charset(charset)` (#1119) * New connection always send "SET NAMES charset [COLLATE collation]" query. (#1119) Since collation table is vary on MySQL server versions, collation in handshake is fragile. +* Support `charset="utf8mb3"` option (#1127) ## v1.0.3 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index b9971ff0f..68d7043b6 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -48,8 +48,8 @@ # PyMySQL version. # Used by setuptools and connection_attrs -VERSION = (1, 1, 0, "rc", 1) -VERSION_STRING = "1.1.0rc1" +VERSION = (1, 1, 0, "rc", 2) +VERSION_STRING = "1.1.0rc2" ### for mysqlclient compatibility ### Django checks mysqlclient version. From 0803b539d4e370001fc93942643ab6843d3eb331 Mon Sep 17 00:00:00 2001 From: "codesee-maps[bot]" <86324825+codesee-maps[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 18:22:40 +0000 Subject: [PATCH 198/227] Install the CodeSee workflow. Learn more at https://docs.codesee.io --- .github/workflows/codesee-arch-diagram.yml | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/codesee-arch-diagram.yml diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml new file mode 100644 index 000000000..806d41d12 --- /dev/null +++ b/.github/workflows/codesee-arch-diagram.yml @@ -0,0 +1,23 @@ +# This workflow was added by CodeSee. Learn more at https://codesee.io/ +# This is v2.0 of this workflow file +on: + push: + branches: + - main + pull_request_target: + types: [opened, synchronize, reopened] + +name: CodeSee + +permissions: read-all + +jobs: + codesee: + runs-on: ubuntu-latest + continue-on-error: true + name: Analyze the repo with CodeSee + steps: + - uses: Codesee-io/codesee-action@v2 + with: + codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + codesee-url: https://app.codesee.io From fe856a55963eac53d5fd714d7de06328cab90293 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 26 Jun 2023 14:31:53 +0900 Subject: [PATCH 199/227] Release v1.1.0 (#1130) --- CHANGELOG.md | 13 ++++++++++--- pymysql/__init__.py | 8 ++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1d732a9..c62836706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,15 @@ # Changes +## Backward incompatible changes planned in the future. + +* Error classes in Cursor class will be removed after 2024-06 +* `Connection.set_charset(charset)` will be removed after 2024-06 +* `db` and `passwd` will emit DeprecationWarning in v1.2. See #933. + + ## v1.1.0 -Release date: TBD +Release date: 2023-06-26 * Fixed SSCursor raising OperationalError for query timeouts on wrong statement (#1032) * Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) @@ -10,7 +17,7 @@ Release date: TBD * Support '_' in key name in my.cnf (#1114) * `Cursor.fetchall()` returns empty list instead of tuple (#1115). Note that `Cursor.fetchmany()` still return empty tuple after reading all rows for compatibility with Django. * Deprecate Error classes in Cursor class (#1117) -* Add `Connection.set_character_set(charset, collation=None)` (#1119) +* Add `Connection.set_character_set(charset, collation=None)`. This method is compatible with mysqlclient. (#1119) * Deprecate `Connection.set_charset(charset)` (#1119) * New connection always send "SET NAMES charset [COLLATE collation]" query. (#1119) Since collation table is vary on MySQL server versions, collation in handshake is fragile. @@ -24,7 +31,7 @@ Release date: 2023-03-28 * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 -* Removed _last_executed because of duplication with _executed by @rajat315315 in https://github.com/PyMySQL/PyMySQL/pull/948 +* Removed `_last_executed` because of duplication with `_executed` by @rajat315315 in https://github.com/PyMySQL/PyMySQL/pull/948 * Fix generating authentication response with long strings by @netch80 in https://github.com/PyMySQL/PyMySQL/pull/988 * update pymysql.constants.CR by @Nothing4You in https://github.com/PyMySQL/PyMySQL/pull/1029 * Document that the ssl connection parameter can be an SSLContext by @cakemanny in https://github.com/PyMySQL/PyMySQL/pull/1045 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 68d7043b6..53625d375 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -48,13 +48,13 @@ # PyMySQL version. # Used by setuptools and connection_attrs -VERSION = (1, 1, 0, "rc", 2) -VERSION_STRING = "1.1.0rc2" +VERSION = (1, 1, 0, "final", 1) +VERSION_STRING = "1.1.0" ### for mysqlclient compatibility ### Django checks mysqlclient version. -version_info = (1, 4, 3, "final", 0) -__version__ = "1.4.3" +version_info = (1, 4, 6, "final", 1) +__version__ = "1.4.6" def get_client_info(): # for MySQLdb compatibility From dbf1ff52a695278cd80e179641f67bb6e2a83326 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 26 Jun 2023 14:33:12 +0900 Subject: [PATCH 200/227] Fix dynamic version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 18714779d..15df9f3ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ include = ["pymysql*"] exclude = ["tests*", "pymysql.tests*"] [tool.setuptools.dynamic] -version = {attr = "pymysql.VERSION"} +version = {attr = "pymysql.VERSION_STRING"} [tool.ruff] line-length = 99 From 6b10225c94087d47782049aafc8e12efa512337b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 1 Jul 2023 01:29:58 +0900 Subject: [PATCH 201/227] Disable renovate dashboard --- renovate.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index 39a2b6e9a..09e16da6b 100644 --- a/renovate.json +++ b/renovate.json @@ -2,5 +2,6 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:base" - ] + ], + "dependencyDashboard": false } From 8157da51e844f619eb693c5f5dd2758dca1d1c98 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 4 Sep 2023 02:04:26 -0600 Subject: [PATCH 202/227] Add support for Python 3.12 (#1134) --- .github/workflows/test.yaml | 7 +++++++ pyproject.toml | 1 + 2 files changed, 8 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6b1e0f321..1153b9e46 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,6 +8,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: test: runs-on: ubuntu-latest @@ -24,6 +27,9 @@ jobs: - db: "mariadb:10.6" py: "3.11" + - db: "mariadb:10.6" + py: "3.12" + - db: "mariadb:lts" py: "3.9" @@ -62,6 +68,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} + allow-prereleases: true cache: 'pip' cache-dependency-path: 'requirements-dev.txt' diff --git a/pyproject.toml b/pyproject.toml index 15df9f3ce..8e75058c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 9e956ad5212f533a0541ee4f5e9e676d8a11b6d6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:59:37 +0900 Subject: [PATCH 203/227] update actions/checkout action to v4 (#1136) --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/django.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a4c434c5e..13519f183 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/django.yaml b/.github/workflows/django.yaml index da664f856..395c64fda 100644 --- a/.github/workflows/django.yaml +++ b/.github/workflows/django.yaml @@ -36,7 +36,7 @@ jobs: mysql -uroot -proot -e "CREATE USER 'scott'@'%' IDENTIFIED BY 'tiger'; GRANT ALL ON *.* TO scott;" mysql -uroot -proot -e "CREATE DATABASE django_default; CREATE DATABASE django_other;" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 77edb0c38..c0c013b0a 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -13,7 +13,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1153b9e46..dcd1abea2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -56,7 +56,7 @@ jobs: - /run/mysqld:/run/mysqld steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Workaround MySQL container permissions if: startsWith(matrix.db, 'mysql') From c1d8063759a4a3968b0f7907e098554d9a8ad552 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 06:55:02 +0900 Subject: [PATCH 204/227] Update codecov/codecov-action action to v4 (#1137) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dcd1abea2..b28b63bd8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,4 +106,4 @@ jobs: - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 From 39bf50e057332d27b8ccf27f11d04467fa1e3904 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 14 Nov 2023 18:34:43 +0900 Subject: [PATCH 205/227] ci: use codecov@v3 (#1142) v4 is still beta. --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b28b63bd8..dcd1abea2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,4 +106,4 @@ jobs: - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v3 From 5820fa09844276477b3f6299341f9dc05d415526 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 14 Nov 2023 18:35:17 +0900 Subject: [PATCH 206/227] update CHANGELOG Add future changes. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c62836706..f371ef324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * Error classes in Cursor class will be removed after 2024-06 * `Connection.set_charset(charset)` will be removed after 2024-06 * `db` and `passwd` will emit DeprecationWarning in v1.2. See #933. - +* `Connection.ping(reconnect)` change the default to not reconnect. ## v1.1.0 From 8b514a4bc103852c8031fd4e0e634ae3d2c10c22 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 02:44:13 +0900 Subject: [PATCH 207/227] ci: update dessant/lock-threads action to v5 (#1141) --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 780dd92d7..21449e3b8 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -13,5 +13,5 @@ jobs: if: github.repository == 'PyMySQL/PyMySQL' runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v4 + - uses: dessant/lock-threads@v5 From 523f0949f33f481e4d41c920c2e1314faeae28ab Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 13:51:06 +0900 Subject: [PATCH 208/227] update document settings --- .readthedocs.yaml | 22 ++++ docs/Makefile | 24 ----- docs/make.bat | 242 ------------------------------------------ docs/source/conf.py | 3 +- docs/source/index.rst | 4 +- 5 files changed, 26 insertions(+), 269 deletions(-) create mode 100644 .readthedocs.yaml delete mode 100644 docs/make.bat diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..0ff559627 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/docs/Makefile b/docs/Makefile index d37255520..c1240d2ba 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -74,30 +74,6 @@ json: @echo @echo "Build finished; now you can process the JSON files." -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyMySQL.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyMySQL.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PyMySQL" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyMySQL" - @echo "# devhelp" - epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index dcd4287c6..000000000 --- a/docs/make.bat +++ /dev/null @@ -1,242 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source -set I18NSPHINXOPTS=%SPHINXOPTS% source -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PyMySQL.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PyMySQL.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/docs/source/conf.py b/docs/source/conf.py index a57a03c44..d346fbda8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -101,7 +101,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "default" +# html_theme = "default" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/source/index.rst b/docs/source/index.rst index 97633f1aa..e64b64238 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ -Welcome to PyMySQL's documentation! -=================================== +PyMySQL documentation +===================== .. toctree:: :maxdepth: 2 From f62893a2c284468091efc95e5b744abcf274dc34 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 13:57:25 +0900 Subject: [PATCH 209/227] update document settings --- docs/source/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index d346fbda8..410e9c74d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,7 +30,7 @@ # ones. extensions = [ "sphinx.ext.autodoc", - "sphinx.ext.intersphinx", + "sphinx_rtd_theme", ] # Add any paths that contain templates here, relative to this directory. @@ -47,7 +47,7 @@ # General information about the project. project = "PyMySQL" -copyright = "2016, Yutaka Matsubara and GitHub contributors" +copyright = "2023, Inada Naoki and GitHub contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -104,6 +104,7 @@ # html_theme = "default" html_theme = "sphinx_rtd_theme" + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. From eb0b6e3429fcd1971be56cae32ffe5780c1c9cb6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 13:57:25 +0900 Subject: [PATCH 210/227] update document settings --- docs/source/conf.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 410e9c74d..1eafbda8e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,7 +30,6 @@ # ones. extensions = [ "sphinx.ext.autodoc", - "sphinx_rtd_theme", ] # Add any paths that contain templates here, relative to this directory. @@ -101,9 +100,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# html_theme = "default" -html_theme = "sphinx_rtd_theme" - + html_theme = "default" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 7f96f9335181c5ae4992a097540348a2ae174cc6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:12:24 +0900 Subject: [PATCH 211/227] fix conf.py --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1eafbda8e..a8bee6c63 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -100,7 +100,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. - html_theme = "default" +html_theme = "default" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 910e5fc1e2bec0e80f75ac5c2d955686e3a5242c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:28:25 +0900 Subject: [PATCH 212/227] update document settings --- .readthedocs.yaml | 15 +++++---------- docs/requirements.txt | 2 ++ 2 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0ff559627..59fdb65df 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,22 +1,17 @@ # .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required version: 2 -# Set the version of Python and other tools you might need build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.12" + +python: + install: + - requirements: docs/requirements.txt # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/source/conf.py - -# We recommend specifying your dependencies to enable reproducible builds: -# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..8d45d0b68 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx~=7.2 +sphinx-rtd-theme~=1.3.0 From 0001c409524e4738a3e686c7faf65421281fbf4f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:34:27 +0900 Subject: [PATCH 213/227] doc: use rtd theme (#1143) --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a8bee6c63..78dc55ca8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -100,7 +100,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "default" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 1ed7cffc0335442235ac103ed458ae38f95b33b1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:39:24 +0900 Subject: [PATCH 214/227] fix ruff error --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8e75058c5..b9a3ef546 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ line-length = 99 exclude = [ "pymysql/tests/thirdparty", ] +ignore = ["E721"] [tool.pdm.dev-dependencies] dev = [ From 84d3f93701341ba34c352663d3a5fc22af2f3d32 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 17:24:51 +0900 Subject: [PATCH 215/227] use Ruff as formatter (#1144) --- .github/workflows/lint.yaml | 14 ++++++++------ pymysql/protocol.py | 4 ++-- pymysql/tests/test_SSCursor.py | 5 +---- pymysql/tests/test_basic.py | 6 +++--- pymysql/tests/test_cursor.py | 3 +-- pymysql/tests/test_issues.py | 5 ++--- pymysql/tests/test_nextset.py | 2 +- pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py | 8 ++++---- .../test_MySQLdb/test_MySQLdb_dbapi20.py | 2 +- pyproject.toml | 1 - 10 files changed, 23 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c0c013b0a..269211c25 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -13,11 +13,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: checkout + uses: actions/checkout@v4 - - uses: psf/black@stable - with: - options: "--check --verbose" - src: "." + - name: lint + uses: chartboost/ruff-action@v1 - - uses: chartboost/ruff-action@v1 + - name: check format + uses: chartboost/ruff-action@v1 + with: + args: "format --diff" diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 2db92d39a..340d9cf2c 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -89,8 +89,8 @@ def advance(self, length): new_position = self._position + length if new_position < 0 or new_position > len(self._data): raise Exception( - "Invalid advance amount (%s) for cursor. " - "Position=%s" % (length, new_position) + "Invalid advance amount (%s) for cursor. Position=%s" + % (length, new_position) ) self._position = new_position diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index 9cb5bafed..d5e6e2bce 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -27,10 +27,7 @@ def test_SSCursor(self): # Create table cursor.execute( - "CREATE TABLE tz_data (" - "region VARCHAR(64)," - "zone VARCHAR(64)," - "name VARCHAR(64))" + "CREATE TABLE tz_data (region VARCHAR(64), zone VARCHAR(64), name VARCHAR(64))" ) conn.begin() diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index e77605fdf..c60b0ccaa 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -364,7 +364,7 @@ def test_bulk_insert(self): data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] cursor.executemany( - "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + "insert into bulkinsert (id, name, age, height) values (%s,%s,%s,%s)", data, ) self.assertEqual( @@ -414,14 +414,14 @@ def test_bulk_insert_single_record(self): cursor = conn.cursor() data = [(0, "bob", 21, 123)] cursor.executemany( - "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + "insert into bulkinsert (id, name, age, height) values (%s,%s,%s,%s)", data, ) cursor.execute("commit") self._verify_records(data) def test_issue_288(self): - """executemany should work with "insert ... on update" """ + """executemany should work with "insert ... on update""" conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index b292c2068..2e267fb6a 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -17,8 +17,7 @@ def setUp(self): ) cursor = conn.cursor() cursor.execute( - "insert into test (data) values " - "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" + "insert into test (data) values ('row1'), ('row2'), ('row3'), ('row4'), ('row5')" ) conn.commit() cursor.close() diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 3564d3a6e..f1fe8dd48 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -401,10 +401,9 @@ def test_issue_321(self): sql_insert = "insert into issue321 (value_1, value_2) values (%s, %s)" sql_dict_insert = ( - "insert into issue321 (value_1, value_2) " - "values (%(value_1)s, %(value_2)s)" + "insert into issue321 (value_1, value_2) values (%(value_1)s, %(value_2)s)" ) - sql_select = "select * from issue321 where " "value_1 in %s and value_2=%s" + sql_select = "select * from issue321 where value_1 in %s and value_2=%s" data = [ [("a",), "\u0430"], [["b"], "\u0430"], diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index 4b6b2a779..a10f8d5b7 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -75,7 +75,7 @@ def test_multi_statement_warnings(self): cursor = con.cursor() try: - cursor.execute("DROP TABLE IF EXISTS a; " "DROP TABLE IF EXISTS b;") + cursor.execute("DROP TABLE IF EXISTS a; DROP TABLE IF EXISTS b;") except TypeError: self.fail() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 838512955..fff14b86f 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -299,7 +299,7 @@ def test_rowcount(self): self.assertEqual( cur.rowcount, -1, - "cursor.rowcount should be -1 after executing no-result " "statements", + "cursor.rowcount should be -1 after executing no-result statements", ) cur.execute( "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) @@ -409,12 +409,12 @@ def _paraminsert(self, cur): self.assertEqual( beers[0], "Cooper's", - "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + "cursor.fetchall retrieved incorrect data, or data inserted incorrectly", ) self.assertEqual( beers[1], "Victoria Bitter", - "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + "cursor.fetchall retrieved incorrect data, or data inserted incorrectly", ) def test_executemany(self): @@ -482,7 +482,7 @@ def test_fetchone(self): self.assertEqual( cur.fetchone(), None, - "cursor.fetchone should return None if a query retrieves " "no rows", + "cursor.fetchone should return None if a query retrieves no rows", ) self.assertTrue(cur.rowcount in (-1, 0)) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index c68289fe8..5c34d40d1 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -98,7 +98,7 @@ def test_fetchone(self): self.assertEqual( cur.fetchone(), None, - "cursor.fetchone should return None if a query retrieves " "no rows", + "cursor.fetchone should return None if a query retrieves no rows", ) self.assertTrue(cur.rowcount in (-1, 0)) diff --git a/pyproject.toml b/pyproject.toml index b9a3ef546..1c10b4b70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ exclude = ["tests*", "pymysql.tests*"] version = {attr = "pymysql.VERSION_STRING"} [tool.ruff] -line-length = 99 exclude = [ "pymysql/tests/thirdparty", ] From d206182822d61650eed8ccd167171ea6131113d1 Mon Sep 17 00:00:00 2001 From: Sergei Vaskov Date: Thu, 16 Nov 2023 12:39:02 +0200 Subject: [PATCH 216/227] Add ssl_key_password param (#1145) Add support for SSL private key password in Connection class to handle encrypted keys. Co-authored-by: Sergei Vaskov --- pymysql/connections.py | 10 +++- pymysql/tests/test_connection.py | 81 +++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 843bea5e0..7e12e1696 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -135,6 +135,7 @@ class Connection: :param ssl_disabled: A boolean value that disables usage of TLS. :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate. + :param ssl_key_password: The password for the client certificate private key. :param ssl_verify_cert: Set to true to check the server certificate's validity. :param ssl_verify_identity: Set to true to check the server's identity. :param read_default_group: Group to read from in the configuration file. @@ -201,6 +202,7 @@ def __init__( ssl_cert=None, ssl_disabled=None, ssl_key=None, + ssl_key_password=None, ssl_verify_cert=None, ssl_verify_identity=None, compress=None, # not supported @@ -262,7 +264,7 @@ def _config(key, arg): if not ssl: ssl = {} if isinstance(ssl, dict): - for key in ["ca", "capath", "cert", "key", "cipher"]: + for key in ["ca", "capath", "cert", "key", "password", "cipher"]: value = _config("ssl-" + key, ssl.get(key)) if value: ssl[key] = value @@ -281,6 +283,8 @@ def _config(key, arg): ssl["cert"] = ssl_cert if ssl_key is not None: ssl["key"] = ssl_key + if ssl_key_password is not None: + ssl["password"] = ssl_key_password if ssl: if not SSL_ENABLED: raise NotImplementedError("ssl module not found") @@ -389,7 +393,9 @@ def _create_ssl_ctx(self, sslp): else: ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED if "cert" in sslp: - ctx.load_cert_chain(sslp["cert"], keyfile=sslp.get("key")) + ctx.load_cert_chain( + sslp["cert"], keyfile=sslp.get("key"), password=sslp.get("password") + ) if "cipher" in sslp: ctx.set_ciphers(sslp["cipher"]) ctx.options |= ssl.OP_NO_SSLv2 diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 0803efc92..ccfc4a320 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -574,7 +574,11 @@ def test_ssl_connect(self): assert create_default_context.called assert dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) dummy_ssl_context.set_ciphers.assert_called_with("cipher") dummy_ssl_context = mock.Mock(options=0) @@ -592,7 +596,34 @@ def test_ssl_connect(self): assert create_default_context.called assert dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: + pymysql.connect( + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + "password": "password", + }, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password="password", + ) dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) @@ -622,7 +653,11 @@ def test_ssl_connect(self): assert create_default_context.called assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) dummy_ssl_context.set_ciphers.assert_not_called for ssl_verify_cert in (True, "1", "yes", "true"): @@ -640,7 +675,9 @@ def test_ssl_connect(self): assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED dummy_ssl_context.load_cert_chain.assert_called_with( - "cert", keyfile="key" + "cert", + keyfile="key", + password=None, ) dummy_ssl_context.set_ciphers.assert_not_called @@ -659,7 +696,9 @@ def test_ssl_connect(self): assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE dummy_ssl_context.load_cert_chain.assert_called_with( - "cert", keyfile="key" + "cert", + keyfile="key", + password=None, ) dummy_ssl_context.set_ciphers.assert_not_called @@ -682,7 +721,9 @@ def test_ssl_connect(self): ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE ), (ssl_ca, ssl_verify_cert) dummy_ssl_context.load_cert_chain.assert_called_with( - "cert", keyfile="key" + "cert", + keyfile="key", + password=None, ) dummy_ssl_context.set_ciphers.assert_not_called @@ -700,7 +741,33 @@ def test_ssl_connect(self): assert create_default_context.called assert dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ssl_key_password="password", + ssl_verify_identity=True, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password="password", + ) dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) From f476773eca5480c75e6d418abd7e73ae6c51ac22 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 13:31:03 +0900 Subject: [PATCH 217/227] chore(deps): update dependency sphinx-rtd-theme to v2 (#1147) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 8d45d0b68..014066235 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ sphinx~=7.2 -sphinx-rtd-theme~=1.3.0 +sphinx-rtd-theme~=2.0.0 From f13f054abcc18b39855a760a84be0a517f0da658 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 22:34:08 +0900 Subject: [PATCH 218/227] chore(deps): update actions/setup-python action to v5 (#1152) --- .github/workflows/django.yaml | 2 +- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/django.yaml b/.github/workflows/django.yaml index 395c64fda..5c4609543 100644 --- a/.github/workflows/django.yaml +++ b/.github/workflows/django.yaml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dcd1abea2..bfe8fff1b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -65,7 +65,7 @@ jobs: /usr/bin/docker ps --all --filter status=exited --no-trunc --format "{{.ID}}" | xargs -r /usr/bin/docker start - name: Set up Python ${{ matrix.py }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} allow-prereleases: true From 1e28be81c24dde66f8acbf4c5e24f60d6b5e72e7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:04:49 +0900 Subject: [PATCH 219/227] chore(deps): update github/codeql-action action to v3 (#1154) --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 13519f183..df49979ea 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: "python" # If you wish to specify custom queries, you can do so here or in a config file. @@ -45,7 +45,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -59,4 +59,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 1f0b7856de4008e7e4c1e8c1b215d5d4dfaecd1a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 12:55:12 +0900 Subject: [PATCH 220/227] chore(deps): update codecov/codecov-action action to v4 (#1158) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bfe8fff1b..6d59d8c4b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,4 +106,4 @@ jobs: - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 From 9694747ae619e88b792a8e0b4c08036572452584 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Feb 2024 15:42:24 +0900 Subject: [PATCH 221/227] pyupgrade --- docs/source/conf.py | 2 -- pymysql/connections.py | 23 +++++++++-------------- pymysql/protocol.py | 6 ++---- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 78dc55ca8..158d0d12f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # PyMySQL documentation build configuration file, created by # sphinx-quickstart on Tue May 17 12:01:11 2016. # diff --git a/pymysql/connections.py b/pymysql/connections.py index 7e12e1696..dc121e1bb 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -84,8 +84,7 @@ def _lenenc_int(i): return b"\xfe" + struct.pack(" len(self._data): raise Exception( - "Invalid advance amount (%s) for cursor. Position=%s" - % (length, new_position) + f"Invalid advance amount ({length}) for cursor. Position={new_position}" ) self._position = new_position From bbd049f40db9c696574ce6f31669880042c56d79 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Feb 2024 17:16:41 +0900 Subject: [PATCH 222/227] Support error packet without sqlstate (#1160) Fix #1156 --- pymysql/connections.py | 2 -- pymysql/err.py | 9 ++++++++- pymysql/tests/test_err.py | 22 ++++++++++++---------- pyproject.toml | 2 ++ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index dc121e1bb..3a04ddd68 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -765,8 +765,6 @@ def _read_packet(self, packet_type=MysqlPacket): dump_packet(recv_data) buff += recv_data # https://dev.mysql.com/doc/internals/en/sending-more-than-16mbyte.html - if bytes_to_read == 0xFFFFFF: - continue if bytes_to_read < MAX_PACKET_LEN: break diff --git a/pymysql/err.py b/pymysql/err.py index 3da5b166f..dac65d3be 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -136,7 +136,14 @@ def _map_error(exc, *errors): def raise_mysql_exception(data): errno = struct.unpack(" Date: Tue, 26 Mar 2024 18:02:41 +1100 Subject: [PATCH 223/227] test json - mariadb without JSON type (#1165) MariaDB-11.0.1 removed the 5.5.5 version hack (MDEV-28910). MariaDB still doesn't support JSON as a type. Use get_mysql_vendor() == mysql for the final part of test_json. --- pymysql/tests/test_basic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index c60b0ccaa..0fe13b59d 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -323,9 +323,10 @@ def test_json(self): res = cur.fetchone()[0] self.assertEqual(json.loads(res), json.loads(json_str)) - cur.execute("SELECT CAST(%s AS JSON) AS x", (json_str,)) - res = cur.fetchone()[0] - self.assertEqual(json.loads(res), json.loads(json_str)) + if self.get_mysql_vendor(conn) == "mysql": + cur.execute("SELECT CAST(%s AS JSON) AS x", (json_str,)) + res = cur.fetchone()[0] + self.assertEqual(json.loads(res), json.loads(json_str)) class TestBulkInserts(base.PyMySQLTestCase): From 69f6c7439bee14784e0ea70ae107af6446cc0c67 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 15:55:22 +0900 Subject: [PATCH 224/227] ruff format --- pymysql/__init__.py | 1 + pymysql/_auth.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 53625d375..373955514 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -21,6 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + import sys from .constants import FIELD_TYPE diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 99987b770..8ce744fb5 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -1,6 +1,7 @@ """ Implements auth methods """ + from .err import OperationalError From 7f032a699d55340f05101deb4d7d4f63db4adc11 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 20 May 2024 13:25:18 +0900 Subject: [PATCH 225/227] remove coveralls from requirements --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 13d7f7fb4..140d37067 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,3 @@ cryptography PyNaCl>=1.4.0 pytest pytest-cov -coveralls From 521e40050cb386a499f68f483fefd144c493053c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 May 2024 11:33:30 +0900 Subject: [PATCH 226/227] forbid dict parameter --- pymysql/converters.py | 6 +----- pymysql/tests/test_connection.py | 7 +++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 1adac7528..dbf97ca75 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -27,11 +27,7 @@ def escape_item(val, charset, mapping=None): def escape_dict(val, charset, mapping=None): - n = {} - for k, v in val.items(): - quoted = escape_item(v, charset, mapping) - n[k] = quoted - return n + raise TypeError("dict can not be used as parameter") def escape_sequence(val, charset, mapping=None): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index ccfc4a320..dcf3394c1 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -848,12 +848,15 @@ def test_escape_no_default(self): self.assertRaises(TypeError, con.escape, 42, {}) - def test_escape_dict_value(self): + def test_escape_dict_raise_typeerror(self): + """con.escape(dict) should raise TypeError""" con = self.connect() mapping = con.encoders.copy() mapping[Foo] = escape_foo - self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) + #self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) + with self.assertRaises(TypeError): + con.escape({"foo": Foo()}) def test_escape_list_item(self): con = self.connect() From 2cab9ecc641e962565c6254a5091f90c47f59b35 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 21 May 2024 20:01:22 +0900 Subject: [PATCH 227/227] v1.1.1 --- CHANGELOG.md | 15 +++++++++++++++ pymysql/__init__.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f371ef324..825dc47c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ * `db` and `passwd` will emit DeprecationWarning in v1.2. See #933. * `Connection.ping(reconnect)` change the default to not reconnect. +## v1.1.1 + +Release date: 2024-05-21 + +> [!WARNING] +> This release fixes a vulnerability (CVE-2024-36039). +> All users are recommended to update to this version. +> +> If you can not update soon, check the input value from +> untrusted source has an expected type. Only dict input +> from untrusted source can be an attack vector. + +* Prohibit dict parameter for `Cursor.execute()`. It didn't produce valid SQL + and might cause SQL injection. (CVE-2024-36039) + ## v1.1.0 Release date: 2023-06-26 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 373955514..bbf9023ef 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -49,8 +49,8 @@ # PyMySQL version. # Used by setuptools and connection_attrs -VERSION = (1, 1, 0, "final", 1) -VERSION_STRING = "1.1.0" +VERSION = (1, 1, 1, "final", 1) +VERSION_STRING = "1.1.1" ### for mysqlclient compatibility ### Django checks mysqlclient version.