From d9f4cec2ca2b768c8d3c1c6e6e505092ebbd051e Mon Sep 17 00:00:00 2001 From: Vilnis Termanis Date: Sat, 20 Jan 2018 17:15:31 +0000 Subject: [PATCH 001/292] Reduce callproc roundtrip time - Make only one single call to SET variables used as procedure parameters --- pymysql/cursors.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index baf0972e..105b0b4f 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -259,9 +259,10 @@ def callproc(self, procname, args=()): disconnected. """ conn = self._get_db() - for index, arg in enumerate(args): - q = "SET @_%s_%d=%s" % (procname, index, conn.escape(arg)) - self._query(q) + if args: + argFmt = '@_{0}_%d=%s'.format(procname) + self._query('SET %s' % ','.join(argFmt % (index, conn.escape(arg)) + for index, arg in enumerate(args))) self.nextset() q = "CALL %s(%s)" % (procname, From 9dde50a2f5fe555fa6ef79f0c8bb2c2f750d5468 Mon Sep 17 00:00:00 2001 From: Phlosioneer Date: Wed, 28 Feb 2018 12:35:07 -0500 Subject: [PATCH 002/292] Improve documentation with errors and links (#642) --- pymysql/connections.py | 62 +++++++++++++++++++++++++++++++++++++----- pymysql/cursors.py | 3 ++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 6d4a0df6..7c4926d3 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -560,6 +560,9 @@ class Connection(object): :param db: Alias for database. (for compatibility to MySQLdb) :param passwd: Alias for password. (for compatibility to MySQLdb) :param binary_prefix: Add _binary prefix on bytes and bytearray. (default: False) + + See `Connection `_ in the + specification. """ _sock = None @@ -716,7 +719,14 @@ def _create_ssl_ctx(self, sslp): return ctx def close(self): - """Send the quit message and close the socket""" + """ + Send the quit message and close the socket. + + See `Connection.close() `_ + in the specification. + + :raise Error: If the connection is already closed. + """ if self._closed: raise err.Error("Already closed") self._closed = True @@ -732,6 +742,7 @@ def close(self): @property def open(self): + """Return True if the connection is open""" return self._sock is not None def _force_close(self): @@ -776,24 +787,38 @@ def begin(self): self._read_ok_packet() def commit(self): - """Commit changes to stable storage""" + """ + Commit changes to stable storage. + + See `Connection.commit() `_ + in the specification. + """ self._execute_command(COMMAND.COM_QUERY, "COMMIT") self._read_ok_packet() def rollback(self): - """Roll back the current transaction""" + """ + Roll back the current transaction. + + See `Connection.rollback() `_ + in the specification. + """ self._execute_command(COMMAND.COM_QUERY, "ROLLBACK") self._read_ok_packet() def show_warnings(self): - """SHOW WARNINGS""" + """Send the "SHOW WARNINGS" SQL command.""" self._execute_command(COMMAND.COM_QUERY, "SHOW WARNINGS") result = MySQLResult(self) result.read() return result.rows def select_db(self, db): - """Set current db""" + """ + Set current db. + + :param db: The name of the db. + """ self._execute_command(COMMAND.COM_INIT_DB, db) self._read_ok_packet() @@ -831,7 +856,13 @@ def _quote_bytes(self, s): return converters.escape_bytes(s) def cursor(self, cursor=None): - """Create a new cursor to execute queries with""" + """ + Create a new cursor to execute queries with. + + :param cursor: The type of cursor to create; one of :py:class:`Cursor`, + :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. + None means use Cursor. + """ if cursor: return cursor(self) return self.cursorclass(self) @@ -873,7 +904,12 @@ def kill(self, thread_id): return self._read_ok_packet() def ping(self, reconnect=True): - """Check if the server is alive""" + """ + Check if the server is alive. + + :param reconnect: If the connection is closed, reconnect. + :raise Error: If the connection is closed and reconnect=False. + """ if self._sock is None: if reconnect: self.connect() @@ -985,6 +1021,9 @@ def write_packet(self, payload): def _read_packet(self, packet_type=MysqlPacket): """Read an entire "mysql packet" in its entirety from the network and return a MysqlPacket type that represents the results. + + :raise OperationalError: If the connection to the MySQL server is lost. + :raise InternalError: If the packet sequence number is wrong. """ buff = b'' while True: @@ -1071,6 +1110,11 @@ def insert_id(self): return 0 def _execute_command(self, command, sql): + """ + :raise InterfaceError: If the connection is closed. + :raise ValueError: If no username was specified. + """ + if not self._sock: raise err.InterfaceError("(0, '')") @@ -1358,6 +1402,10 @@ def read(self): self.connection = None def init_unbuffered_query(self): + """ + :raise OperationalError: If the connection to the MySQL server is lost. + :raise InternalError: + """ self.unbuffered_active = True first_packet = self.connection._read_packet() diff --git a/pymysql/cursors.py b/pymysql/cursors.py index baf0972e..d3b1d610 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -24,6 +24,9 @@ class Cursor(object): Do not create an instance of a Cursor yourself. Call connections.Connection.cursor(). + + See `Cursor `_ in + the specification. """ #: Max statement size which :meth:`executemany` generates. From 9f29c16c872e7c0ccb7854ab0097c523fb46fce6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 10 Mar 2018 23:21:15 +0900 Subject: [PATCH 003/292] Fix failing tests (#643) --- pymysql/tests/base.py | 32 ++++++++++++++---- pymysql/tests/test_SSCursor.py | 10 +++--- pymysql/tests/test_connection.py | 57 ++++++++++++++++---------------- pymysql/tests/test_nextset.py | 24 ++++++++------ 4 files changed, 74 insertions(+), 49 deletions(-) diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index 740157b1..e54afee5 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -40,15 +40,33 @@ def mysql_server_is(self, conn, version_tuple): ) return server_version_tuple >= version_tuple - def setUp(self): - self.connections = [] - for params in self.databases: - self.connections.append(pymysql.connect(**params)) - self.addCleanup(self._teardown_connections) + _connections = None + + @property + def connections(self): + if self._connections is None: + self._connections = [] + for params in self.databases: + self._connections.append(pymysql.connect(**params)) + self.addCleanup(self._teardown_connections) + return self._connections + + def connect(self, **params): + p = self.databases[0].copy() + p.update(params) + conn = pymysql.connect(**p) + @self.addCleanup + def teardown(): + if conn.open: + conn.close() + return conn def _teardown_connections(self): - for connection in self.connections: - connection.close() + if self._connections: + for connection in self._connections: + if connection.open: + connection.close() + self._connections = None def safe_create_table(self, connection, tablename, ddl, cleanup=True): """create a table. diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index e6d6cf53..77eeefa6 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -3,17 +3,19 @@ try: from pymysql.tests import base import pymysql.cursors + from pymysql.constants import CLIENT except Exception: # For local testing from top-level directory, without installing sys.path.append('../pymysql') from pymysql.tests import base import pymysql.cursors + from pymysql.constants import CLIENT class TestSSCursor(base.PyMySQLTestCase): def test_SSCursor(self): affected_rows = 18446744073709551615 - conn = self.connections[0] + conn = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) data = [ ('America', '', 'America/Jamaica'), ('America', '', 'America/Los_Angeles'), @@ -30,10 +32,10 @@ def test_SSCursor(self): cursor = conn.cursor(pymysql.cursors.SSCursor) # Create table - cursor.execute(('CREATE TABLE tz_data (' + cursor.execute('CREATE TABLE tz_data (' 'region VARCHAR(64),' 'zone VARCHAR(64),' - 'name VARCHAR(64))')) + 'name VARCHAR(64))') conn.begin() # Test INSERT @@ -100,7 +102,7 @@ def test_SSCursor(self): self.assertFalse(cursor.nextset()) finally: - cursor.execute('DROP TABLE tz_data') + cursor.execute('DROP TABLE IF EXISTS tz_data') cursor.close() __all__ = ["TestSSCursor"] diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 518b6fe7..1fe908ce 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -5,6 +5,7 @@ import pymysql from pymysql.tests import base from pymysql._compat import text_type +from pymysql.constants import CLIENT class TempUser: @@ -411,7 +412,7 @@ def test_connection_gone_away(self): http://dev.mysql.com/doc/refman/5.0/en/gone-away.html http://dev.mysql.com/doc/refman/5.0/en/error-messages-client.html#error_cr_server_gone_error """ - con = self.connections[0] + con = self.connect() cur = con.cursor() cur.execute("SET wait_timeout=1") time.sleep(2) @@ -422,10 +423,9 @@ def test_connection_gone_away(self): self.assertIn(cm.exception.args[0], (2006, 2013)) def test_init_command(self): - conn = pymysql.connect( + conn = self.connect( init_command='SELECT "bar"; SELECT "baz"', - **self.databases[0] - ) + client_flag=CLIENT.MULTI_STATEMENTS) c = conn.cursor() c.execute('select "foobar";') self.assertEqual(('foobar',), c.fetchone()) @@ -434,22 +434,21 @@ def test_init_command(self): conn.ping(reconnect=False) def test_read_default_group(self): - conn = pymysql.connect( + conn = self.connect( read_default_group='client', - **self.databases[0] ) self.assertTrue(conn.open) def test_context(self): with self.assertRaises(ValueError): - c = pymysql.connect(**self.databases[0]) + c = self.connect() with c as cur: cur.execute('create table test ( a int )') c.begin() cur.execute('insert into test values ((1))') raise ValueError('pseudo abort') c.commit() - c = pymysql.connect(**self.databases[0]) + c = self.connect() with c as cur: cur.execute('select count(*) from test') self.assertEqual(0, cur.fetchone()[0]) @@ -460,31 +459,31 @@ def test_context(self): cur.execute('drop table test') def test_set_charset(self): - c = pymysql.connect(**self.databases[0]) + c = self.connect() c.set_charset('utf8') # TODO validate setting here def test_defer_connect(self): import socket - for db in self.databases: - d = db.copy() + + d = self.databases[0].copy() + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(d['unix_socket']) + except KeyError: + sock = socket.create_connection( + (d.get('host', 'localhost'), d.get('port', 3306))) + for k in ['unix_socket', 'host', 'port']: try: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(d['unix_socket']) + del d[k] except KeyError: - sock = socket.create_connection( - (d.get('host', 'localhost'), d.get('port', 3306))) - for k in ['unix_socket', 'host', 'port']: - try: - del d[k] - except KeyError: - pass - - c = pymysql.connect(defer_connect=True, **d) - self.assertFalse(c.open) - c.connect(sock) - c.close() - sock.close() + pass + + c = pymysql.connect(defer_connect=True, **d) + self.assertFalse(c.open) + c.connect(sock) + c.close() + sock.close() @unittest2.skipUnless(sys.version_info[0:2] >= (3,2), "required py-3.2") def test_no_delay_warning(self): @@ -560,7 +559,9 @@ def test_escape_list_item(self): self.assertEqual(con.escape([Foo()], mapping), "(bar)") def test_previous_cursor_not_closed(self): - con = self.connections[0] + con = self.connect( + init_command='SELECT "bar"; SELECT "baz"', + client_flag=CLIENT.MULTI_STATEMENTS) cur1 = con.cursor() cur1.execute("SELECT 1; SELECT 2") cur2 = con.cursor() @@ -568,7 +569,7 @@ def test_previous_cursor_not_closed(self): self.assertEqual(cur2.fetchone()[0], 3) def test_commit_during_multi_result(self): - con = self.connections[0] + con = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) cur = con.cursor() cur.execute("SELECT 1; SELECT 2") con.commit() diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index cdb6754f..593243e4 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -2,16 +2,16 @@ from pymysql.tests import base from pymysql import util +from pymysql.constants import CLIENT class TestNextset(base.PyMySQLTestCase): - def setUp(self): - super(TestNextset, self).setUp() - self.con = self.connections[0] - def test_nextset(self): - cur = self.con.cursor() + con = self.connect( + init_command='SELECT "bar"; SELECT "baz"', + client_flag=CLIENT.MULTI_STATEMENTS) + cur = con.cursor() cur.execute("SELECT 1; SELECT 2;") self.assertEqual([(1,)], list(cur)) @@ -22,7 +22,7 @@ def test_nextset(self): self.assertIsNone(cur.nextset()) def test_skip_nextset(self): - cur = self.con.cursor() + cur = self.connect(client_flag=CLIENT.MULTI_STATEMENTS).cursor() cur.execute("SELECT 1; SELECT 2;") self.assertEqual([(1,)], list(cur)) @@ -30,7 +30,7 @@ def test_skip_nextset(self): self.assertEqual([(42,)], list(cur)) def test_ok_and_next(self): - cur = self.con.cursor() + cur = self.connect(client_flag=CLIENT.MULTI_STATEMENTS).cursor() cur.execute("SELECT 1; commit; SELECT 2;") self.assertEqual([(1,)], list(cur)) self.assertTrue(cur.nextset()) @@ -40,8 +40,9 @@ def test_ok_and_next(self): @unittest2.expectedFailure def test_multi_cursor(self): - cur1 = self.con.cursor() - cur2 = self.con.cursor() + con = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) + cur1 = con.cursor() + cur2 = con.cursor() cur1.execute("SELECT 1; SELECT 2;") cur2.execute("SELECT 42") @@ -56,7 +57,10 @@ def test_multi_cursor(self): self.assertIsNone(cur1.nextset()) def test_multi_statement_warnings(self): - cursor = self.con.cursor() + con = self.connect( + init_command='SELECT "bar"; SELECT "baz"', + client_flag=CLIENT.MULTI_STATEMENTS) + cursor = con.cursor() try: cursor.execute('DROP TABLE IF EXISTS a; ' From 3dda4a5807174ea2fc428484d116e9e2a12f2983 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Mar 2018 01:57:48 +0900 Subject: [PATCH 004/292] Update cursors.py --- pymysql/cursors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 105b0b4f..bbcc2e25 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -260,8 +260,8 @@ def callproc(self, procname, args=()): """ conn = self._get_db() if args: - argFmt = '@_{0}_%d=%s'.format(procname) - self._query('SET %s' % ','.join(argFmt % (index, conn.escape(arg)) + fmt = '@_{0}_%d=%s'.format(procname) + self._query('SET %s' % ','.join(fmt % (index, conn.escape(arg)) for index, arg in enumerate(args))) self.nextset() From 8293c530e3e7d43537e31b90189c9c8ea9065705 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 14 Mar 2018 18:46:18 +0900 Subject: [PATCH 005/292] Use docker on Travis --- .travis.yml | 48 ++++++++------------------ .travis/database.json | 4 +-- .travis/docker.json | 4 +++ .travis/initializedb.sh | 75 +++++++++++++++++++---------------------- 4 files changed, 55 insertions(+), 76 deletions(-) create mode 100644 .travis/docker.json diff --git a/.travis.yml b/.travis.yml index 132369dc..bd283bf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +# vim: sw=2 ts=2 sts=2 expandtab + sudo: required language: python python: @@ -21,51 +23,29 @@ matrix: python: "2.7" - env: - - DB=5.6.37 - python: "3.3" - addons: - apt: - packages: - - libaio-dev - - libnuma-dev - - - env: - - DB=5.7.19 + - DB=5.7 python: "3.4" - addons: - apt: - packages: - - libaio-dev - - libnuma-dev - + services: + - docker # different py version from 5.6 and 5.7 as cache seems to be based on py version # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version # really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't install: - - export PASSWORD=travis; - - pip install -U coveralls unittest2 coverage + - pip install -U coveralls unittest2 coverage before_script: - - ./.travis/initializedb.sh - - mysql -e 'create database test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' - - mysql -e 'create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' - - mysql -u root -e "create user travis_pymysql2 identified by 'some password'; grant all on test_pymysql2.* to travis_pymysql2;" - - mysql -u root -e "create user travis_pymysql2@localhost identified by 'some password'; grant all on test_pymysql2.* to travis_pymysql2@localhost;" - - mysql -e 'select VERSION()' - - python -VV - - rm -f ~/.my.cnf # set in .travis.initialize.db.sh for the above commands - we should be using database.json however - - export COVERALLS_PARALLEL=true + - ./.travis/initializedb.sh + - python -VV + - rm -f ~/.my.cnf # set in .travis.initialize.db.sh for the above commands - we should be using database.json however + - export COVERALLS_PARALLEL=true script: - coverage run ./runtests.py + - if [ ! -z "${DB}" ]; + then docker logs mysqld; + fi after_success: - - coveralls - - cat /tmp/mysql.err - -after_failure: - - cat /tmp/mysql.err - -# vim: sw=2 ts=2 sts=2 expandtab + - coveralls diff --git a/.travis/database.json b/.travis/database.json index 7acd20c1..ab1f60a3 100644 --- a/.travis/database.json +++ b/.travis/database.json @@ -1,4 +1,4 @@ [ - {"host": "localhost", "unix_socket": "/var/run/mysqld/mysqld.sock", "user": "root", "passwd": "", "db": "test_pymysql", "use_unicode": true, "local_infile": true}, - {"host": "127.0.0.1", "port": 3306, "user": "travis_pymysql2", "password": "some password", "db": "test_pymysql2" } + {"host": "localhost", "unix_socket": "/var/run/mysqld/mysqld.sock", "user": "root", "passwd": "", "db": "test1", "use_unicode": true, "local_infile": true}, + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "db": "test2" } ] diff --git a/.travis/docker.json b/.travis/docker.json new file mode 100644 index 00000000..b851fb6d --- /dev/null +++ b/.travis/docker.json @@ -0,0 +1,4 @@ +[ + {"host": "127.0.0.1", "port": 3306, "user": "root", "passwd": "", "db": "test1", "use_unicode": true, "local_infile": true}, + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "db": "test2" } +] diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 2ff38534..cb21d67a 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -7,47 +7,42 @@ set -v if [ ! -z "${DB}" ]; then # disable existing database server in case of accidential connection - mysql -u root -e 'drop user travis@localhost; drop user root@localhost; drop user travis; create user super@localhost; grant all on *.* to super@localhost with grant option' - mysql -u super -e 'drop user root' - - F=mysql-${DB}-linux-glibc2.12-x86_64 - mkdir -p ${HOME}/mysql - P=${HOME}/mysql/${F} - if [ ! -d "${P}" ]; then - wget https://cdn.mysql.com//Downloads/MySQL-${DB%.*}/${F}.tar.gz -O - | tar -zxf - --directory=${HOME}/mysql - fi - if [ -f "${P}"/my.cnf ]; then - O="--defaults-file=${P}/my.cnf" - fi - if [ -x "${P}"/scripts/mysql_install_db ]; then - I=${P}/scripts/mysql_install_db - O="--defaults-file=${P}/my.cnf" - else - I=${P}/bin/mysqld - IO=" --initialize " - O="--no-defaults " - fi - ${I} ${O} ${IO} --basedir=${P} --datadir=${HOME}/db-"${DB}" --log-error=/tmp/mysql.err - PWLINE=$(grep 'A temporary password is generated for root@localhost:' /tmp/mysql.err) - PASSWD=${PWLINE##* } - if [ -x ${P}/bin/mysql_ssl_rsa_setup ]; then - ${P}/bin/mysql_ssl_rsa_setup --datadir=${HOME}/db-"${DB}" - fi - # sha256 password auth keys: - openssl genrsa -out "${P}"/private_key.pem 2048 - openssl rsa -in "${P}"/private_key.pem -pubout -out "${P}"/public_key.pem - ${P}/bin/mysqld_safe ${O} --ledir=/ --mysqld=${P}/bin/mysqld --datadir=${HOME}/db-${DB} --socket=/tmp/mysql.sock --port 3307 --innodb-buffer-pool-size=200M --lc-messages-dir=${P}/share --plugin-dir=${P}/lib/plugin/ --log-error=/tmp/mysql.err & - while [ ! -S /tmp/mysql.sock ]; do - sleep 3 - tail /tmp/mysql.err + sudo service mysql stop + + docker pull mysql:${DB} + docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 mysql:${DB} + sleep 10 + + while : + do + sleep 5 + mysql -uroot -h 127.0.0.1 -P 3306 -e 'select version()' + if [ $? = 0 ]; then + break + fi + echo "server logs" + docker logs --tail 5 mysqld done - tail /tmp/mysql.err - if [ ! -z "${PASSWD}" ]; then - ${P}/bin/mysql -S /tmp/mysql.sock -u root -p"${PASSWD}" --connect-expired-password -e "SET PASSWORD = PASSWORD('')" - fi - mysql -S /tmp/mysql.sock -u root -e "create user ${USER}@localhost; create user ${USER}@'%'; grant all on *.* to ${USER}@localhost WITH GRANT OPTION;grant all on *.* to ${USER}@'%' WITH GRANT OPTION;" - sed -e 's/3306/3307/g' -e 's:/var/run/mysqld/mysqld.sock:/tmp/mysql.sock:g' .travis/database.json > pymysql/tests/databases.json - echo -e "[client]\nsocket = /tmp/mysql.sock\n" > "${HOME}"/.my.cnf + + echo -e "[client]\nhost = 127.0.0.1\n" > "${HOME}"/.my.cnf + + mysql -e 'select VERSION()' + mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' + mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' + + mysql -u root -e "create user test2 identified by 'some password'; grant all on test2.* to test2;" + mysql -u root -e "create user test2@localhost identified by 'some password'; grant all on test2.* to test2@localhost;" + + cp .travis/docker.json pymysql/tests/databases.json else + cat ~/.my.cnf + + mysql -e 'select VERSION()' + mysql -e 'create database test1 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' + mysql -e 'create database test2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' + + mysql -u root -e "create user test2 identified by 'some password'; grant all on test2.* to test2;" + mysql -u root -e "create user test2@localhost identified by 'some password'; grant all on test2.* to test2@localhost;" + cp .travis/database.json pymysql/tests/databases.json fi From f15dd428fd7fb7089fe0e869edbe7ad893064b1d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 15 Mar 2018 02:35:32 +0900 Subject: [PATCH 006/292] Investigate travis error --- pymysql/tests/base.py | 4 +- pymysql/tests/test_SSCursor.py | 150 ++++++++++++++++----------------- 2 files changed, 76 insertions(+), 78 deletions(-) diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index e54afee5..091cccfa 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -20,8 +20,8 @@ class PyMySQLTestCase(unittest2.TestCase): else: databases = [ {"host":"localhost","user":"root", - "passwd":"","db":"test_pymysql", "use_unicode": True, 'local_infile': True}, - {"host":"localhost","user":"root","passwd":"","db":"test_pymysql2"}] + "passwd":"","db":"test1", "use_unicode": True, 'local_infile': True}, + {"host":"localhost","user":"root","passwd":"","db":"test2"}] def mysql_server_is(self, conn, version_tuple): """Return True if the given connection is on the version given or diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index 77eeefa6..3bbfcfa4 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -28,82 +28,80 @@ def test_SSCursor(self): ('America', '', 'America/Denver'), ('America', '', 'America/Detroit'),] - try: - cursor = conn.cursor(pymysql.cursors.SSCursor) - - # Create table - cursor.execute('CREATE TABLE tz_data (' - 'region VARCHAR(64),' - 'zone VARCHAR(64),' - 'name VARCHAR(64))') - - conn.begin() - # Test INSERT - for i in data: - cursor.execute('INSERT INTO tz_data VALUES (%s, %s, %s)', i) - self.assertEqual(conn.affected_rows(), 1, 'affected_rows does not match') - conn.commit() - - # Test fetchone() - iter = 0 - cursor.execute('SELECT * FROM tz_data') - while True: - row = cursor.fetchone() - if row is None: - break - iter += 1 - - # Test cursor.rowcount - self.assertEqual(cursor.rowcount, affected_rows, - 'cursor.rowcount != %s' % (str(affected_rows))) - - # Test cursor.rownumber - self.assertEqual(cursor.rownumber, iter, - 'cursor.rowcount != %s' % (str(iter))) - - # Test row came out the same as it went in - self.assertEqual((row in data), True, - 'Row not found in source data') - - # Test fetchall - cursor.execute('SELECT * FROM tz_data') - self.assertEqual(len(cursor.fetchall()), len(data), - 'fetchall failed. Number of rows does not match') - - # Test fetchmany - cursor.execute('SELECT * FROM tz_data') - self.assertEqual(len(cursor.fetchmany(2)), 2, - 'fetchmany failed. Number of rows does not match') - - # So MySQLdb won't throw "Commands out of sync" - while True: - res = cursor.fetchone() - if res is None: - break - - # Test update, affected_rows() - cursor.execute('UPDATE tz_data SET zone = %s', ['Foo']) - conn.commit() - self.assertEqual(cursor.rowcount, len(data), - 'Update failed. affected_rows != %s' % (str(len(data)))) - - # Test executemany - cursor.executemany('INSERT INTO tz_data VALUES (%s, %s, %s)', data) - self.assertEqual(cursor.rowcount, len(data), - 'executemany failed. cursor.rowcount != %s' % (str(len(data)))) - - # Test multiple datasets - cursor.execute('SELECT 1; SELECT 2; SELECT 3') - self.assertListEqual(list(cursor), [(1, )]) - self.assertTrue(cursor.nextset()) - self.assertListEqual(list(cursor), [(2, )]) - self.assertTrue(cursor.nextset()) - self.assertListEqual(list(cursor), [(3, )]) - self.assertFalse(cursor.nextset()) - - finally: - cursor.execute('DROP TABLE IF EXISTS tz_data') - cursor.close() + cursor = conn.cursor(pymysql.cursors.SSCursor) + + # Create table + cursor.execute('CREATE TABLE tz_data (' + 'region VARCHAR(64),' + 'zone VARCHAR(64),' + 'name VARCHAR(64))') + + conn.begin() + # Test INSERT + for i in data: + cursor.execute('INSERT INTO tz_data VALUES (%s, %s, %s)', i) + self.assertEqual(conn.affected_rows(), 1, 'affected_rows does not match') + conn.commit() + + # Test fetchone() + iter = 0 + cursor.execute('SELECT * FROM tz_data') + while True: + row = cursor.fetchone() + if row is None: + break + iter += 1 + + # Test cursor.rowcount + self.assertEqual(cursor.rowcount, affected_rows, + 'cursor.rowcount != %s' % (str(affected_rows))) + + # Test cursor.rownumber + self.assertEqual(cursor.rownumber, iter, + 'cursor.rowcount != %s' % (str(iter))) + + # Test row came out the same as it went in + self.assertEqual((row in data), True, + 'Row not found in source data') + + # Test fetchall + cursor.execute('SELECT * FROM tz_data') + self.assertEqual(len(cursor.fetchall()), len(data), + 'fetchall failed. Number of rows does not match') + + # Test fetchmany + cursor.execute('SELECT * FROM tz_data') + self.assertEqual(len(cursor.fetchmany(2)), 2, + 'fetchmany failed. Number of rows does not match') + + # So MySQLdb won't throw "Commands out of sync" + while True: + res = cursor.fetchone() + if res is None: + break + + # Test update, affected_rows() + cursor.execute('UPDATE tz_data SET zone = %s', ['Foo']) + conn.commit() + self.assertEqual(cursor.rowcount, len(data), + 'Update failed. affected_rows != %s' % (str(len(data)))) + + # Test executemany + cursor.executemany('INSERT INTO tz_data VALUES (%s, %s, %s)', data) + self.assertEqual(cursor.rowcount, len(data), + 'executemany failed. cursor.rowcount != %s' % (str(len(data)))) + + # Test multiple datasets + cursor.execute('SELECT 1; SELECT 2; SELECT 3') + self.assertListEqual(list(cursor), [(1, )]) + self.assertTrue(cursor.nextset()) + self.assertListEqual(list(cursor), [(2, )]) + self.assertTrue(cursor.nextset()) + self.assertListEqual(list(cursor), [(3, )]) + self.assertFalse(cursor.nextset()) + + cursor.execute('DROP TABLE IF EXISTS tz_data') + cursor.close() __all__ = ["TestSSCursor"] From 0516250f163d4c414ef016b2f06a2f309a308e41 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 17 Apr 2018 18:51:26 +0900 Subject: [PATCH 007/292] Add Link to MySQL Community Slack --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index b8f9c89d..f6a6f363 100644 --- a/README.rst +++ b/README.rst @@ -129,6 +129,8 @@ MySQL Reference Manuals: http://dev.mysql.com/doc/ MySQL client/server protocol: http://dev.mysql.com/doc/internals/en/client-server-protocol.html +"Connector" channel in MySQL Community Slack: http://lefred.be/mysql-community-on-slack/ + PyMySQL mailing list: https://groups.google.com/forum/#!forum/pymysql-users License From 4478e3aeb7e92d6a35c1cfc927a21a21fa57b953 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 18 Apr 2018 02:15:59 +0900 Subject: [PATCH 008/292] Clear cursor attributes before calling nextset() (#649) fixes #647 Signed-off-by: INADA Naoki --- pymysql/connections.py | 1 + pymysql/cursors.py | 14 +++++++++++++- pymysql/tests/test_nextset.py | 14 +++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 7c4926d3..293efb26 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1087,6 +1087,7 @@ def _write_bytes(self, data): "MySQL server has gone away (%r)" % (e,)) def _read_query_result(self, unbuffered=False): + self._result = None if unbuffered: try: result = MySQLResult(self) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 093c9fbd..816de37f 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -97,6 +97,8 @@ def _nextset(self, unbuffered=False): return None if not current_result.has_next: return None + self._result = None + self._clear_result() conn.next_result(unbuffered=unbuffered) self._do_get_result() return True @@ -322,14 +324,23 @@ def scroll(self, value, mode='relative'): def _query(self, q): conn = self._get_db() self._last_executed = q + self._clear_result() conn.query(q) self._do_get_result() return self.rowcount + def _clear_result(self): + self.rownumber = 0 + self._result = result = None + + self.rowcount = 0 + self.description = None + self.lastrowid = None + self._rows = None + def _do_get_result(self): conn = self._get_db() - self.rownumber = 0 self._result = result = conn._result self.rowcount = result.affected_rows @@ -438,6 +449,7 @@ def close(self): def _query(self, q): conn = self._get_db() self._last_executed = q + self._clear_result() conn.query(q, unbuffered=True) self._do_get_result() return self.rowcount diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index 593243e4..99844107 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -1,7 +1,8 @@ import unittest2 -from pymysql.tests import base +import pymysql from pymysql import util +from pymysql.tests import base from pymysql.constants import CLIENT @@ -29,6 +30,17 @@ def test_skip_nextset(self): cur.execute("SELECT 42") self.assertEqual([(42,)], list(cur)) + def test_nextset_error(self): + con = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) + cur = con.cursor() + + for i in range(3): + cur.execute("SELECT %s; xyzzy;", (i,)) + self.assertEqual([(i,)], list(cur)) + with self.assertRaises(pymysql.ProgrammingError): + cur.nextset() + self.assertEqual((), cur.fetchall()) + def test_ok_and_next(self): cur = self.connect(client_flag=CLIENT.MULTI_STATEMENTS).cursor() cur.execute("SELECT 1; commit; SELECT 2;") From ed3eacd591e1600a4af766269bd3ad824181a226 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 28 Apr 2018 22:06:58 +1000 Subject: [PATCH 009/292] testfix: MySQL-5.1, test_context requires transactional engine (#653) --- pymysql/tests/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 1fe908ce..388856e2 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -443,7 +443,7 @@ def test_context(self): with self.assertRaises(ValueError): c = self.connect() with c as cur: - cur.execute('create table test ( a int )') + cur.execute('create table test ( a int ) ENGINE=InnoDB') c.begin() cur.execute('insert into test values ((1))') raise ValueError('pseudo abort') From 24ecee6bd19a473528d6f8552dabc655de912130 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 30 Apr 2018 15:17:11 +1000 Subject: [PATCH 010/292] test_defer_connect: don't leak file descriptor (#660) --- pymysql/tests/test_connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 388856e2..b361c45c 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -471,6 +471,7 @@ def test_defer_connect(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.connect(d['unix_socket']) except KeyError: + sock.close() sock = socket.create_connection( (d.get('host', 'localhost'), d.get('port', 3306))) for k in ['unix_socket', 'host', 'port']: From ccef546213b0d9e994af28c0a8ef2e579cde5831 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 30 Apr 2018 16:40:04 +1000 Subject: [PATCH 011/292] SSCursor: failed to clear unused results on deletion (#655) A cursor may contain unretrieved results. These should be consumed before a new cursor is created on the same connection. --- pymysql/cursors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 816de37f..705f9e26 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -446,6 +446,8 @@ def close(self): finally: self.connection = None + __del__ = close + def _query(self, q): conn = self._get_db() self._last_executed = q From 279e2587b2a85e0aa258094521e498c225e8da12 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 30 Apr 2018 16:48:46 +1000 Subject: [PATCH 012/292] test: issue3 MySQL-8.0 now returns NULL(None) for timestamp (#658) Like all of the other temporal times. --- pymysql/tests/test_issues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 7cc29be1..6b034515 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -36,7 +36,7 @@ def test_issue_3(self): c.execute("select dt from issue3") self.assertEqual(None, c.fetchone()[0]) c.execute("select ts from issue3") - self.assertTrue(isinstance(c.fetchone()[0], datetime.datetime)) + self.assertIn(type(c.fetchone()[0]), (type(None), datetime.datetime), 'expected Python type None or datetime from SQL timestamp') finally: c.execute("drop table issue3") From 57466b4fc21350011c5248e33fdbb4fe29dc893b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 May 2018 00:29:06 +1000 Subject: [PATCH 013/292] fix test issue363 for MySQL-8.0 (#659) Removed explict utf8 usage (wasn't used) to avoid warning: pymysql.err.Warning: (3719, "'utf8' is currently an alias for the character set UTF8MB3, which will be replaced by UTF8MB4 in a future release. Please consider using UTF8MB4 in order to be unambiguous.") Added SRID to avoid warning: pymysql.err.Warning: (3674, "The spatial index on column 'geom' will not be used by the query optimizer since the column does not have an SRID attribute. Consider adding an SRID attribute to the column.") --- pymysql/tests/test_issues.py | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 6b034515..1d680644 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -443,37 +443,33 @@ def test_issue_363(self): self.safe_create_table( conn, "issue363", "CREATE TABLE issue363 ( " - "id INTEGER PRIMARY KEY, geom LINESTRING NOT NULL, " + "id INTEGER PRIMARY KEY, geom LINESTRING NOT NULL /*!80003 SRID 0 */, " "SPATIAL KEY geom (geom)) " - "ENGINE=MyISAM default charset=utf8") + "ENGINE=MyISAM") cur = conn.cursor() - query = ("INSERT INTO issue363 (id, geom) VALUES" - "(1998, GeomFromText('LINESTRING(1.1 1.1,2.2 2.2)'))") # From MySQL 5.7, ST_GeomFromText is added and GeomFromText is deprecated. if self.mysql_server_is(conn, (5, 7, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - cur.execute(query) + geom_from_text = "ST_GeomFromText" + geom_as_text = "ST_AsText" + geom_as_bin = "ST_AsBinary" else: - cur.execute(query) + geom_from_text = "GeomFromText" + geom_as_text = "AsText" + geom_as_bin = "AsBinary" + query = ("INSERT INTO issue363 (id, geom) VALUES" + "(1998, %s('LINESTRING(1.1 1.1,2.2 2.2)'))" % geom_from_text) + cur.execute(query) # select WKT - query = "SELECT AsText(geom) FROM issue363" - if self.mysql_server_is(conn, (5, 7, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - cur.execute(query) - else: - cur.execute(query) + query = "SELECT %s(geom) FROM issue363" % geom_as_text + cur.execute(query) row = cur.fetchone() self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)", )) # select WKB - query = "SELECT AsBinary(geom) FROM issue363" - if self.mysql_server_is(conn, (5, 7, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - cur.execute(query) - else: - cur.execute(query) + query = "SELECT %s(geom) FROM issue363" % geom_as_bin + cur.execute(query) row = cur.fetchone() self.assertEqual(row, (b"\x01\x02\x00\x00\x00\x02\x00\x00\x00" From b51d5dae391035faa8d8115e0c6f06e25afcc25c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 May 2018 00:29:29 +1000 Subject: [PATCH 014/292] test_set_charset: use utf8mb4 (#664) Works on all supported MySQL versions now that 5.1 support is dropped. --- pymysql/tests/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index b361c45c..8a94ca5d 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -460,7 +460,7 @@ def test_context(self): def test_set_charset(self): c = self.connect() - c.set_charset('utf8') + c.set_charset('utf8mb4') # TODO validate setting here def test_defer_connect(self): From 217666d3d6ad4d63783f53403a26912a647bc1a7 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 May 2018 00:30:07 +1000 Subject: [PATCH 015/292] Travis: use docker for mysql-5.5, mysql-5.6, mariadb-5.5, 10.0, 10.1, 10.2, 10.3 (#662) --- .travis.yml | 37 +++++++++++++++++++++---------------- .travis/initializedb.sh | 4 ++-- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd283bf2..a93608d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,32 +2,37 @@ sudo: required language: python -python: - - "3.7-dev" - - "3.6" +services: + - docker cache: pip matrix: include: - - addons: - mariadb: 5.5 + - env: + - DB=mariadb:5.5 python: "3.5" - - - addons: - mariadb: 10.1 + - env: + - DB=mariadb:10.0 + python: "3.6" + - env: + - DB=mariadb:10.1 python: "pypy" - - - addons: - mariadb: 10.2 + - env: + - DB=mariadb:10.2 python: "2.7" - - env: - - DB=5.7 + - DB=mariadb:10.3 + python: "3.7-dev" + - env: + - DB=mysql:5.5 + python: "3.5" + - env: + - DB=mysql:5.6 + python: "3.6" + - env: + - DB=mysql:5.7 python: "3.4" - services: - - docker - # different py version from 5.6 and 5.7 as cache seems to be based on py version # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index cb21d67a..73189d50 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -9,8 +9,8 @@ if [ ! -z "${DB}" ]; then # disable existing database server in case of accidential connection sudo service mysql stop - docker pull mysql:${DB} - docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 mysql:${DB} + docker pull ${DB} + docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} sleep 10 while : From 023cc6da05d6793e528781b3dc0345dd87df4b36 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 May 2018 00:35:23 +1000 Subject: [PATCH 016/292] Test mysql db capabilities / test_issue_364 (#661) * tests: thirdparty MySQLdb, use utf8mb4 Prevent warnings like the following on MySQL-8.0: pymysql.err.Warning: (3719, "'utf8' is currently an alias for the character set UTF8MB3, which will be replaced by UTF8MB4 in a future release. Please consider using UTF8MB4 in order to be unambiguous.") * test_issue_364: utf8mb4 --- pymysql/tests/test_issues.py | 4 ++-- pymysql/tests/thirdparty/test_MySQLdb/capabilities.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 1d680644..cedd0925 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -409,11 +409,11 @@ def test_issue_321(self): def test_issue_364(self): """ Test mixed unicode/binary arguments in executemany. """ - conn = pymysql.connect(charset="utf8", **self.databases[0]) + conn = pymysql.connect(charset="utf8mb4", **self.databases[0]) self.safe_create_table( conn, "issue364", "create table issue364 (value_1 binary(3), value_2 varchar(3)) " - "engine=InnoDB default charset=utf8") + "engine=InnoDB default charset=utf8mb4") sql = "insert into issue364 (value_1, value_2) values (_binary %s, %s)" usql = u"insert into issue364 (value_1, value_2) values (_binary %s, %s)" diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index 14725bc4..bcf9eecb 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -17,8 +17,8 @@ class DatabaseTest(unittest.TestCase): db_module = None connect_args = () - connect_kwargs = dict(use_unicode=True, charset="utf8", binary_prefix=True) - create_table_extra = "ENGINE=INNODB CHARACTER SET UTF8" + connect_kwargs = dict(use_unicode=True, charset="utf8mb4", binary_prefix=True) + create_table_extra = "ENGINE=INNODB CHARACTER SET UTF8MB4" rows = 10 debug = False From 6a55ce0f5f717fdcb8ca0af24a320ac020f756e8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 May 2018 00:46:31 +1000 Subject: [PATCH 017/292] tests: mysql-8.0, sql_mode NO_AUTO_CREATE_USER removed (#657) --- pymysql/tests/test_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 8a94ca5d..681bc260 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -510,7 +510,11 @@ def test_escape_string(self): self.assertEqual(con.escape("foo'bar"), "'foo\\'bar'") # added NO_AUTO_CREATE_USER as not including it in 5.7 generates warnings - cur.execute("SET sql_mode='NO_BACKSLASH_ESCAPES,NO_AUTO_CREATE_USER'") + # mysql-8.0 removes the option however + if self.mysql_server_is(con, (8, 0, 0)): + cur.execute("SET sql_mode='NO_BACKSLASH_ESCAPES'") + else: + cur.execute("SET sql_mode='NO_BACKSLASH_ESCAPES,NO_AUTO_CREATE_USER'") self.assertEqual(con.escape("foo'bar"), "'foo''bar'") def test_escape_builtin_encoders(self): From d4f5929197e875e0e2f861c714f45327e8fb971b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 May 2018 00:54:30 +1000 Subject: [PATCH 018/292] test_plugin: examine mysql.user (#652) In MySQL-8.0 the plugin is caching_sha2_password. Test is fixed here by examining the plugin column from mysql.user and comparing that to what was discovered during authentication. MariaDB and MySQL-5.5 have a blank plugin column but still report mysql_native_password in authentication. --- pymysql/tests/test_connection.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 681bc260..c626a0d3 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -96,8 +96,12 @@ class TestAuthentication(base.PyMySQLTestCase): # print("plugin: %r" % r[0]) def test_plugin(self): - # Bit of an assumption that the current user is a native password - self.assertEqual('mysql_native_password', self.connections[0]._auth_plugin_name) + if not self.mysql_server_is(self.connections[0], (5, 5, 0)): + raise unittest2.SkipTest("MySQL-5.5 required for plugins") + cur = self.connections[0].cursor() + cur.execute("select plugin from mysql.user where concat(user, '@', host)=current_user()") + for r in cur: + self.assertIn(self.connections[0]._auth_plugin_name, (r[0], 'mysql_native_password')) @unittest2.skipUnless(socket_auth, "connection to unix_socket required") @unittest2.skipIf(socket_found, "socket plugin already installed") From db20bf2aa98e35fdad20f03a9a4d63554963c761 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 2 May 2018 15:36:07 +1000 Subject: [PATCH 019/292] add docs for read_timeout/write_timeout (#665) --- pymysql/connections.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymysql/connections.py b/pymysql/connections.py index 293efb26..967d0c59 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -524,6 +524,8 @@ class Connection(object): the interface from which to connect to the host. Argument can be a hostname or an IP address. :param unix_socket: Optionally, you can use a unix socket rather than TCP/IP. + :param read_timeout: The timeout for reading from the connection in seconds (default: None - no timeout) + :param write_timeout: The timeout for writing to the connection in seconds (default: None - no timeout) :param charset: Charset you want to use. :param sql_mode: Default SQL_MODE to use. :param read_default_file: From 49a64f3ffd321627694139dfdd4ae0999a8b9a1a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 7 May 2018 18:58:28 +1000 Subject: [PATCH 020/292] travis: Add MySQL 8.0 (#663) --- .travis.yml | 3 ++ .travis/initializedb.sh | 28 ++++++++++++++----- .../test_MySQLdb/test_MySQLdb_capabilities.py | 3 +- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index a93608d5..2822cd05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,9 @@ matrix: - env: - DB=mysql:5.7 python: "3.4" + - env: + - DB=mysql:8.0 + python: "3.7-dev" # different py version from 5.6 and 5.7 as cache seems to be based on py version # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 73189d50..18c00eca 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -13,10 +13,13 @@ if [ ! -z "${DB}" ]; then docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} sleep 10 + mysql() { + docker exec mysqld mysql "${@}" + } while : do sleep 5 - mysql -uroot -h 127.0.0.1 -P 3306 -e 'select version()' + mysql -e 'select version()' if [ $? = 0 ]; then break fi @@ -24,14 +27,25 @@ if [ ! -z "${DB}" ]; then docker logs --tail 5 mysqld done - echo -e "[client]\nhost = 127.0.0.1\n" > "${HOME}"/.my.cnf - mysql -e 'select VERSION()' - mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' - mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' - mysql -u root -e "create user test2 identified by 'some password'; grant all on test2.* to test2;" - mysql -u root -e "create user test2@localhost identified by 'some password'; grant all on test2.* to test2@localhost;" + if [ $DB == 'mysql:8.0' ]; then + WITH_PLUGIN='with mysql_native_password' + mysql -e 'SET GLOBAL local_infile=on' + docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" + else + WITH_PLUGIN='' + fi + + mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' + mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' + + mysql -u root -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" + mysql -u root -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" cp .travis/docker.json pymysql/tests/databases.json else diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 657425e9..0fc5e831 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -16,9 +16,8 @@ class test_MySQLdb(capabilities.DatabaseTest): connect_kwargs = base.PyMySQLTestCase.databases[0].copy() connect_kwargs.update(dict(read_default_file='~/.my.cnf', use_unicode=True, binary_prefix=True, - charset='utf8', sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL")) + charset='utf8mb4', sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL")) - create_table_extra = "ENGINE=INNODB CHARACTER SET UTF8" leak_test = False def quote_identifier(self, ident): From 85c61c09c68ba9dca31b2864bd84a323d16a6829 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 7 May 2018 18:45:10 +0900 Subject: [PATCH 021/292] Add WRONG_DB_NAME and WRONG_COLUMN_NAME to ProgrammingError (#667) fixes #629 --- pymysql/err.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymysql/err.py b/pymysql/err.py index e4208ab3..f3513ae8 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -78,7 +78,9 @@ def _map_error(exc, *errors): ER.PARSE_ERROR, ER.NO_SUCH_TABLE, ER.WRONG_DB_NAME, ER.WRONG_TABLE_NAME, ER.FIELD_SPECIFIED_TWICE, ER.INVALID_GROUP_FUNC_USE, ER.UNSUPPORTED_EXTENSION, - ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION) + ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION, + ER.WRONG_DB_NAME, ER.WRONG_COLUMN_NAME, + ) _map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL, ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL, ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW) From e2979e7f559907e2b6f82b42ed19a3abca0ea1fc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 7 May 2018 19:11:27 +0900 Subject: [PATCH 022/292] Update README --- README.rst | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/README.rst b/README.rst index f6a6f363..ac515715 100644 --- a/README.rst +++ b/README.rst @@ -33,37 +33,37 @@ Requirements * Python -- one of the following: - - CPython_ >= 2.6 or >= 3.3 - - PyPy_ >= 4.0 - - IronPython_ 2.7 + - CPython_ : 2.7 and >= 3.4 + - PyPy_ : Latest version * MySQL Server -- one of the following: - - MySQL_ >= 4.1 (tested with only 5.5~) - - MariaDB_ >= 5.1 + - MySQL_ >= 5.5 + - MariaDB_ >= 5.5 -.. _CPython: http://www.python.org/ -.. _PyPy: http://pypy.org/ -.. _IronPython: http://ironpython.net/ -.. _MySQL: http://www.mysql.com/ +.. _CPython: https://www.python.org/ +.. _PyPy: https://pypy.org/ +.. _MySQL: https://www.mysql.com/ .. _MariaDB: https://mariadb.org/ Installation ------------ -The last stable release is available on PyPI and can be installed with ``pip``:: +Package is uploaded on `PyPI `_. - $ pip install PyMySQL +You can install it with pip:: + + $ pip3 install PyMySQL Documentation ------------- -Documentation is available online: http://pymysql.readthedocs.io/ +Documentation is available online: https://pymysql.readthedocs.io/ For support, please refer to the `StackOverflow -`_. +`_. Example ------- @@ -122,16 +122,17 @@ This example will print: Resources --------- -DB-API 2.0: http://www.python.org/dev/peps/pep-0249 +* DB-API 2.0: http://www.python.org/dev/peps/pep-0249 -MySQL Reference Manuals: http://dev.mysql.com/doc/ +* MySQL Reference Manuals: http://dev.mysql.com/doc/ -MySQL client/server protocol: -http://dev.mysql.com/doc/internals/en/client-server-protocol.html +* MySQL client/server protocol: + http://dev.mysql.com/doc/internals/en/client-server-protocol.html -"Connector" channel in MySQL Community Slack: http://lefred.be/mysql-community-on-slack/ +* "Connector" channel in MySQL Community Slack: + http://lefred.be/mysql-community-on-slack/ -PyMySQL mailing list: https://groups.google.com/forum/#!forum/pymysql-users +* PyMySQL mailing list: https://groups.google.com/forum/#!forum/pymysql-users License ------- From ba330574fbf0e60cc4742ddfc883c84f029c7fd7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 7 May 2018 19:22:05 +0900 Subject: [PATCH 023/292] Update CHANGELOG --- CHANGELOG | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 448c1513..fce9fb30 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,21 @@ # Changes +## 0.8.1 + +Release date: 2018-05-07 + +* Reduce `cursor.callproc()` roundtrip time. (#636) + +* Fixed `cursor.query()` is hunged after multi statement failed. (#647) + +* WRONG_DB_NAME and WRONG_COLUMN_NAME is ProgrammingError for now. (#629) + +* Many test suite improvements, especially adding MySQL 8.0 and using Docker. + Thanks to Daniel Black. + +* Droppped support for old Python and MySQL whih is not tested long time. + + ## 0.8 Release date: 2017-12-20 From 68e5800c30ed1ea3c5bd5e9f91f6913034f69ec2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 7 May 2018 19:27:03 +0900 Subject: [PATCH 024/292] 0.8.1 --- pymysql/__init__.py | 2 +- setup.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 6025732a..a881ebed 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 8, 0, None) +VERSION = (0, 8, 1, None) threadsafety = 1 apilevel = "2.0" paramstyle = "pyformat" diff --git a/setup.py b/setup.py index d7fdf3aa..37342d4b 100755 --- a/setup.py +++ b/setup.py @@ -16,6 +16,9 @@ name="PyMySQL", version=version, url='https://github.com/PyMySQL/PyMySQL/', + project_urls={ + "Documentation": "https://pymysql.readthedocs.io/", + }, author='yutaka.matsubara', author_email='yutaka.matsubara@gmail.com', maintainer='INADA Naoki', @@ -32,10 +35,12 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Topic :: Database', ], + keywords="MySQL", ) From 8c566602e9446acc8910dde920531d016d796b64 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 8 May 2018 17:49:36 +0900 Subject: [PATCH 025/292] "drop-in replacement" is not goal anymore. --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ac515715..ccdd8578 100644 --- a/README.rst +++ b/README.rst @@ -18,8 +18,9 @@ PyMySQL .. contents:: Table of Contents :local: -This package contains a pure-Python MySQL client library. The goal of PyMySQL -is to be a drop-in replacement for MySQLdb and work on CPython, PyPy and IronPython. +This package contains a pure-Python MySQL client library, based on `PEP 249`_. + +Most public APIs are compatible with mysqlclient and MySQLdb. NOTE: PyMySQL doesn't support low level APIs `_mysql` provides like `data_seek`, `store_result`, and `use_result`. You should use high level APIs defined in `PEP 249`_. @@ -28,6 +29,7 @@ their usecase. .. _`PEP 249`: https://www.python.org/dev/peps/pep-0249/ + Requirements ------------- From 9105a9ebc98e280a3a4731bec928277714ae6a93 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 9 May 2018 19:59:36 +0900 Subject: [PATCH 026/292] Remove unused variable --- pymysql/cursors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 705f9e26..cc169987 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -331,7 +331,7 @@ def _query(self, q): def _clear_result(self): self.rownumber = 0 - self._result = result = None + self._result = None self.rowcount = 0 self.description = None From 14e4c25f0221900eadd155121997856186438fd9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 9 May 2018 20:04:21 +0900 Subject: [PATCH 027/292] Split connections module to protocol (#670) --- pymysql/connections.py | 329 +--------------------------------------- pymysql/protocol.py | 336 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 341 insertions(+), 324 deletions(-) create mode 100644 pymysql/protocol.py diff --git a/pymysql/connections.py b/pymysql/connections.py index 967d0c59..53e18e3c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -16,11 +16,15 @@ import traceback import warnings -from .charset import MBLENGTH, charset_by_name, charset_by_id +from .charset import charset_by_name, charset_by_id from .constants import CLIENT, COMMAND, CR, FIELD_TYPE, SERVER_STATUS from . import converters from .cursors import Cursor from .optionfile import Parser +from .protocol import ( + dump_packet, MysqlPacket, FieldDescriptorPacket, OKPacketWrapper, + EOFPacketWrapper, LoadLocalPacketWrapper +) from .util import byte2int, int2byte from . import err @@ -85,42 +89,10 @@ def _makefile(sock, mode): sha_new = partial(hashlib.new, 'sha1') -NULL_COLUMN = 251 -UNSIGNED_CHAR_COLUMN = 251 -UNSIGNED_SHORT_COLUMN = 252 -UNSIGNED_INT24_COLUMN = 253 -UNSIGNED_INT64_COLUMN = 254 - DEFAULT_CHARSET = 'latin1' MAX_PACKET_LEN = 2**24-1 - -def dump_packet(data): # pragma: no cover - def is_ascii(data): - if 65 <= byte2int(data) <= 122: - if isinstance(data, int): - return chr(data) - return data - return '.' - - try: - print("packet length:", len(data)) - for i in range(1, 6): - f = sys._getframe(i) - print("call[%d]: %s (line %d)" % (i, f.f_code.co_name, f.f_lineno)) - print("-" * 66) - except ValueError: - pass - dump_data = [data[i:i+16] for i in range_type(0, min(len(data), 256), 16)] - for d in dump_data: - print(' '.join(map(lambda x: "{:02X}".format(byte2int(x)), d)) + - ' ' * (16 - len(d)) + ' ' * 2 + - ''.join(map(lambda x: "{}".format(is_ascii(x)), d))) - print("-" * 66) - print() - - SCRAMBLE_LENGTH = 20 def _scramble(password, message): @@ -214,297 +186,6 @@ def lenenc_int(i): else: raise ValueError("Encoding %x is larger than %x - no representation in LengthEncodedInteger" % (i, (1 << 64))) -class MysqlPacket(object): - """Representation of a MySQL response packet. - - Provides an interface for reading/parsing the packet results. - """ - __slots__ = ('_position', '_data') - - def __init__(self, data, encoding): - self._position = 0 - self._data = data - - def get_all_data(self): - return self._data - - def read(self, size): - """Read the first 'size' bytes in packet and advance cursor past them.""" - result = self._data[self._position:(self._position+size)] - if len(result) != size: - error = ('Result length not requested length:\n' - 'Expected=%s. Actual=%s. Position: %s. Data Length: %s' - % (size, len(result), self._position, len(self._data))) - if DEBUG: - print(error) - self.dump() - raise AssertionError(error) - self._position += size - return result - - def read_all(self): - """Read all remaining data in the packet. - - (Subsequent read() will return errors.) - """ - result = self._data[self._position:] - self._position = None # ensure no subsequent read() - return result - - def advance(self, length): - """Advance the cursor in data buffer 'length' bytes.""" - new_position = self._position + length - if new_position < 0 or new_position > len(self._data): - raise Exception('Invalid advance amount (%s) for cursor. ' - 'Position=%s' % (length, new_position)) - self._position = new_position - - def rewind(self, position=0): - """Set the position of the data buffer cursor to 'position'.""" - if position < 0 or position > len(self._data): - raise Exception("Invalid position to rewind cursor to: %s." % position) - self._position = position - - def get_bytes(self, position, length=1): - """Get 'length' bytes starting at 'position'. - - Position is start of payload (first four packet header bytes are not - included) starting at index '0'. - - No error checking is done. If requesting outside end of buffer - an empty string (or string shorter than 'length') may be returned! - """ - return self._data[position:(position+length)] - - if PY2: - def read_uint8(self): - result = ord(self._data[self._position]) - self._position += 1 - return result - else: - def read_uint8(self): - result = self._data[self._position] - self._position += 1 - return result - - def read_uint16(self): - result = struct.unpack_from('= 7 - - def is_eof_packet(self): - # http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet - # Caution: \xFE may be LengthEncodedInteger. - # If \xFE is LengthEncodedInteger header, 8bytes followed. - return self._data[0:1] == b'\xfe' and len(self._data) < 9 - - def is_auth_switch_request(self): - # http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest - return self._data[0:1] == b'\xfe' - - def is_resultset_packet(self): - field_count = ord(self._data[0:1]) - return 1 <= field_count <= 250 - - def is_load_local_packet(self): - return self._data[0:1] == b'\xfb' - - def is_error_packet(self): - return self._data[0:1] == b'\xff' - - def check_error(self): - if self.is_error_packet(): - self.rewind() - self.advance(1) # field_count == error (we already know that) - errno = self.read_uint16() - if DEBUG: print("errno =", errno) - err.raise_mysql_exception(self._data) - - def dump(self): - dump_packet(self._data) - - -class FieldDescriptorPacket(MysqlPacket): - """A MysqlPacket that represents a specific column's metadata in the result. - - Parsing is automatically done and the results are exported via public - attributes on the class such as: db, table_name, name, length, type_code. - """ - - def __init__(self, data, encoding): - MysqlPacket.__init__(self, data, encoding) - self._parse_field_descriptor(encoding) - - def _parse_field_descriptor(self, encoding): - """Parse the 'Field Descriptor' (Metadata) packet. - - This is compatible with MySQL 4.1+ (not compatible with MySQL 4.0). - """ - self.catalog = self.read_length_coded_string() - self.db = self.read_length_coded_string() - self.table_name = self.read_length_coded_string().decode(encoding) - self.org_table = self.read_length_coded_string().decode(encoding) - self.name = self.read_length_coded_string().decode(encoding) - self.org_name = self.read_length_coded_string().decode(encoding) - self.charsetnr, self.length, self.type_code, self.flags, self.scale = ( - self.read_struct(' len(self._data): + raise Exception('Invalid advance amount (%s) for cursor. ' + 'Position=%s' % (length, new_position)) + self._position = new_position + + def rewind(self, position=0): + """Set the position of the data buffer cursor to 'position'.""" + if position < 0 or position > len(self._data): + raise Exception("Invalid position to rewind cursor to: %s." % position) + self._position = position + + def get_bytes(self, position, length=1): + """Get 'length' bytes starting at 'position'. + + Position is start of payload (first four packet header bytes are not + included) starting at index '0'. + + No error checking is done. If requesting outside end of buffer + an empty string (or string shorter than 'length') may be returned! + """ + return self._data[position:(position+length)] + + if PY2: + def read_uint8(self): + result = ord(self._data[self._position]) + self._position += 1 + return result + else: + def read_uint8(self): + result = self._data[self._position] + self._position += 1 + return result + + def read_uint16(self): + result = struct.unpack_from('= 7 + + def is_eof_packet(self): + # http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet + # Caution: \xFE may be LengthEncodedInteger. + # If \xFE is LengthEncodedInteger header, 8bytes followed. + return self._data[0:1] == b'\xfe' and len(self._data) < 9 + + def is_auth_switch_request(self): + # http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest + return self._data[0:1] == b'\xfe' + + def is_resultset_packet(self): + field_count = ord(self._data[0:1]) + return 1 <= field_count <= 250 + + def is_load_local_packet(self): + return self._data[0:1] == b'\xfb' + + def is_error_packet(self): + return self._data[0:1] == b'\xff' + + def check_error(self): + if self.is_error_packet(): + self.rewind() + self.advance(1) # field_count == error (we already know that) + errno = self.read_uint16() + if DEBUG: print("errno =", errno) + err.raise_mysql_exception(self._data) + + def dump(self): + dump_packet(self._data) + + +class FieldDescriptorPacket(MysqlPacket): + """A MysqlPacket that represents a specific column's metadata in the result. + + Parsing is automatically done and the results are exported via public + attributes on the class such as: db, table_name, name, length, type_code. + """ + + def __init__(self, data, encoding): + MysqlPacket.__init__(self, data, encoding) + self._parse_field_descriptor(encoding) + + def _parse_field_descriptor(self, encoding): + """Parse the 'Field Descriptor' (Metadata) packet. + + This is compatible with MySQL 4.1+ (not compatible with MySQL 4.0). + """ + self.catalog = self.read_length_coded_string() + self.db = self.read_length_coded_string() + self.table_name = self.read_length_coded_string().decode(encoding) + self.org_table = self.read_length_coded_string().decode(encoding) + self.name = self.read_length_coded_string().decode(encoding) + self.org_name = self.read_length_coded_string().decode(encoding) + self.charsetnr, self.length, self.type_code, self.flags, self.scale = ( + self.read_struct(' Date: Fri, 22 Jun 2018 01:35:15 +0200 Subject: [PATCH 029/292] Implement connect attributes (#679) --- pymysql/__init__.py | 4 ++++ pymysql/connections.py | 26 ++++++++++++++++++++++++-- pymysql/constants/CLIENT.py | 4 ++-- setup.py | 7 +------ 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index a881ebed..66559e92 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -36,6 +36,10 @@ VERSION = (0, 8, 1, None) +if VERSION[3] is not None: + VERSION_STRING = "%d.%d.%d_%s" % VERSION +else: + VERSION_STRING = "%d.%d.%d" % VERSION[:3] threadsafety = 1 apilevel = "2.0" paramstyle = "pyformat" diff --git a/pymysql/connections.py b/pymysql/connections.py index 53e18e3c..d5dc0aff 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -26,7 +26,7 @@ EOFPacketWrapper, LoadLocalPacketWrapper ) from .util import byte2int, int2byte -from . import err +from . import err, VERSION_STRING try: import ssl @@ -262,7 +262,7 @@ def __init__(self, host=None, user=None, password="", autocommit=False, db=None, passwd=None, local_infile=False, max_allowed_packet=16*1024*1024, defer_connect=False, auth_plugin_map={}, read_timeout=None, write_timeout=None, - bind_address=None, binary_prefix=False): + bind_address=None, binary_prefix=False, program_name=None): if no_delay is not None: warnings.warn("no_delay option is deprecated", DeprecationWarning) @@ -357,6 +357,7 @@ def _config(key, arg): client_flag |= CLIENT.CAPABILITIES if self.db: client_flag |= CLIENT.CONNECT_WITH_DB + self.client_flag = client_flag self.cursorclass = cursorclass @@ -379,6 +380,18 @@ def _config(key, arg): self.max_allowed_packet = max_allowed_packet self._auth_plugin_map = auth_plugin_map self._binary_prefix = binary_prefix + + self._connect_attrs = { + '_client_name': 'pymysql', + '_pid': str(os.getpid()), + '_client_version': VERSION_STRING, + } + + if program_name: + self._connect_attrs["program_name"] = program_name + elif sys.argv: + self._connect_attrs["program_name"] = sys.argv[0] + if defer_connect: self._sock = None else: @@ -880,6 +893,15 @@ def _request_authentication(self): name = name.encode('ascii') data += name + b'\0' + if self.server_capabilities & CLIENT.CONNECT_ATTRS: + connect_attrs = b'' + for k, v in self._connect_attrs.items(): + k = k.encode('utf8') + connect_attrs += struct.pack('B', len(k)) + k + v = v.encode('utf8') + connect_attrs += struct.pack('B', len(v)) + v + data += struct.pack('B', len(connect_attrs)) + connect_attrs + self.write_packet(data) auth_packet = self._read_packet() diff --git a/pymysql/constants/CLIENT.py b/pymysql/constants/CLIENT.py index e5e6180c..b42f1523 100644 --- a/pymysql/constants/CLIENT.py +++ b/pymysql/constants/CLIENT.py @@ -18,14 +18,14 @@ MULTI_RESULTS = 1 << 17 PS_MULTI_RESULTS = 1 << 18 PLUGIN_AUTH = 1 << 19 +CONNECT_ATTRS = 1 << 20 PLUGIN_AUTH_LENENC_CLIENT_DATA = 1 << 21 CAPABILITIES = ( LONG_PASSWORD | LONG_FLAG | PROTOCOL_41 | TRANSACTIONS | SECURE_CONNECTION | MULTI_RESULTS - | PLUGIN_AUTH | PLUGIN_AUTH_LENENC_CLIENT_DATA) + | PLUGIN_AUTH | PLUGIN_AUTH_LENENC_CLIENT_DATA | CONNECT_ATTRS) # Not done yet -CONNECT_ATTRS = 1 << 20 HANDLE_EXPIRED_PASSWORDS = 1 << 22 SESSION_TRACK = 1 << 23 DEPRECATE_EOF = 1 << 24 diff --git a/setup.py b/setup.py index 37342d4b..f9903258 100755 --- a/setup.py +++ b/setup.py @@ -2,12 +2,7 @@ import io from setuptools import setup, find_packages -version_tuple = __import__('pymysql').VERSION - -if version_tuple[3] is not None: - version = "%d.%d.%d_%s" % version_tuple -else: - version = "%d.%d.%d" % version_tuple[:3] +version = __import__('pymysql').VERSION_STRING with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() From 83a8c9247d11297ede7fa5c89c98d079f10112de Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 26 Jun 2018 21:59:45 +0900 Subject: [PATCH 030/292] Add sha256 and chaching_sha2 auth support (#682) --- .gitignore | 16 +- .travis.yml | 6 +- .travis/initializedb.sh | 10 ++ pymysql/_auth.py | 252 +++++++++++++++++++++++++++++++ pymysql/connections.py | 214 +++++++++++--------------- pymysql/protocol.py | 15 +- pymysql/tests/test_connection.py | 5 +- runtests.py | 4 + setup.py | 3 + tests/__init__.py | 0 tests/test_auth.py | 63 ++++++++ 11 files changed, 449 insertions(+), 139 deletions(-) create mode 100644 pymysql/_auth.py create mode 100644 tests/__init__.py create mode 100644 tests/test_auth.py diff --git a/.gitignore b/.gitignore index cd93a4b0..0b2c85be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,14 @@ *.pyc *.pyo -__pycache__ -.coverage -/dist -/PyMySQL.egg-info +/.cache +/.coverage +/.idea /.tox +/.venv +/.vscode +/PyMySQL.egg-info /build +/dist +/docs/build /pymysql/tests/databases.json - -/.idea -docs/build +__pycache__ diff --git a/.travis.yml b/.travis.yml index 2822cd05..8d960249 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,13 +35,14 @@ matrix: python: "3.4" - env: - DB=mysql:8.0 + - TEST_AUTH=yes python: "3.7-dev" # different py version from 5.6 and 5.7 as cache seems to be based on py version # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version # really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't install: - - pip install -U coveralls unittest2 coverage + - pip install -U coveralls unittest2 coverage cryptography pytest before_script: - ./.travis/initializedb.sh @@ -51,6 +52,9 @@ before_script: script: - coverage run ./runtests.py + - if [ "${TEST_AUTH}" = "yes" ]; + then pytest -v tests; + fi - if [ ! -z "${DB}" ]; then docker logs mysqld; fi diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 18c00eca..d9897e49 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -37,6 +37,16 @@ if [ ! -z "${DB}" ]; then docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" + + # Test user for auth test + mysql -e ' + CREATE USER + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", + nopass_sha256 IDENTIFIED WITH "sha256_password", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER;' + mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' else WITH_PLUGIN='' fi diff --git a/pymysql/_auth.py b/pymysql/_auth.py new file mode 100644 index 00000000..ddf6e4e5 --- /dev/null +++ b/pymysql/_auth.py @@ -0,0 +1,252 @@ +""" +Implements auth methods +""" +from ._compat import text_type +from .constants import CLIENT +from .err import OperationalError + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization, hashes +from cryptography.hazmat.primitives.asymmetric import padding + +from functools import partial +import hashlib +import struct + + +DEBUG = True +SCRAMBLE_LENGTH = 20 +sha1_new = partial(hashlib.new, 'sha1') + + +# mysql_native_password +# https://dev.mysql.com/doc/internals/en/secure-password-authentication.html#packet-Authentication::Native41 + + +def scramble_native_password(password, message): + """Scramble used for mysql_native_password""" + if not password: + return b'' + + stage1 = sha1_new(password).digest() + stage2 = sha1_new(stage1).digest() + s = sha1_new() + s.update(message[:SCRAMBLE_LENGTH]) + s.update(stage2) + result = s.digest() + return _my_crypt(result, stage1) + + +def _my_crypt(message1, message2): + length = len(message1) + result = b'' + for i in range(length): + x = ( + struct.unpack('B', message1[i:i + 1])[0] ^ + struct.unpack('B', message2[i:i + 1])[0] + ) + result += struct.pack('B', x) + return result + + +# old_passwords support ported from libmysql/password.c +# https://dev.mysql.com/doc/internals/en/old-password-authentication.html + +SCRAMBLE_LENGTH_323 = 8 + + +class RandStruct_323(object): + + def __init__(self, seed1, seed2): + self.max_value = 0x3FFFFFFF + self.seed1 = seed1 % self.max_value + self.seed2 = seed2 % self.max_value + + def my_rnd(self): + self.seed1 = (self.seed1 * 3 + self.seed2) % self.max_value + self.seed2 = (self.seed1 + self.seed2 + 33) % self.max_value + return float(self.seed1) / float(self.max_value) + + +def scramble_old_password(password, message): + """Scramble for old_password""" + hash_pass = _hash_password_323(password) + hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) + hash_pass_n = struct.unpack(">LL", hash_pass) + hash_message_n = struct.unpack(">LL", hash_message) + + rand_st = RandStruct_323( + hash_pass_n[0] ^ hash_message_n[0], hash_pass_n[1] ^ hash_message_n[1] + ) + outbuf = io.BytesIO() + for _ in range(min(SCRAMBLE_LENGTH_323, len(message))): + outbuf.write(int2byte(int(rand_st.my_rnd() * 31) + 64)) + extra = int2byte(int(rand_st.my_rnd() * 31)) + out = outbuf.getvalue() + outbuf = io.BytesIO() + for c in out: + outbuf.write(int2byte(byte2int(c) ^ byte2int(extra))) + return outbuf.getvalue() + + +def _hash_password_323(password): + nr = 1345345333 + add = 7 + nr2 = 0x12345671 + + # x in py3 is numbers, p27 is chars + for c in [byte2int(x) for x in password if x not in (' ', '\t', 32, 9)]: + nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF + nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF + add = (add + c) & 0xFFFFFFFF + + r1 = nr & ((1 << 31) - 1) # kill sign bits + r2 = nr2 & ((1 << 31) - 1) + return struct.pack(">LL", r1, r2) + + +# sha256_password + + +def _roundtrip(conn, send_data): + conn.write_packet(send_data) + pkt = conn._read_packet() + pkt.check_error() + return pkt + + +def _xor_password(password, salt): + password_bytes = bytearray(password) + salt = bytearray(salt) # for PY2 compat. + salt_len = len(salt) + for i in range(len(password_bytes)): + password_bytes[i] ^= salt[i % salt_len] + return bytes(password_bytes) + + +def sha2_rsa_encrypt(password, salt, public_key): + """Encrypt password with salt and public_key. + + Used for sha256_password and caching_sha2_password. + """ + message = _xor_password(password + b'\0', salt) + rsa_key = serialization.load_pem_public_key(public_key, default_backend()) + return rsa_key.encrypt( + message, + padding.OAEP( + mgf=padding.MGF1(algorithm=hashes.SHA1()), + algorithm=hashes.SHA1(), + label=None, + ), + ) + + +def sha256_password_auth(conn, pkt): + if conn.ssl and conn.server_capabilities & CLIENT.SSL: + if DEBUG: + print("sha256: Sending plain password") + data = conn.password + b'\0' + return _roundtrip(conn, data) + + if pkt.is_auth_switch_request(): + conn.salt = pkt.read_all() + if not conn.server_public_key and conn.password: + # Request server public key + if DEBUG: + print("sha256: Requesting server public key") + pkt = _roundtrip(conn, b'\1') + + if pkt.is_extra_auth_data(): + conn.server_public_key = pkt._data[1:] + if DEBUG: + print("Received public key:\n", conn.server_public_key.decode('ascii')) + + if conn.password: + if not conn.server_public_key: + raise OperationalError("Couldn't receive server's public key") + + data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) + else: + data = b'' + + return _roundtrip(conn, data) + + +def scramble_caching_sha2(password, nonce): + # (bytes, bytes) -> bytes + """Scramble algorithm used in cached_sha2_password fast path. + + XOR(SHA256(password), SHA256(SHA256(SHA256(password)), nonce)) + """ + if not password: + return b'' + + p1 = hashlib.sha256(password).digest() + p2 = hashlib.sha256(p1).digest() + p3 = hashlib.sha256(p2 + nonce).digest() + + res = bytearray(p1) + for i in range(len(p3)): + res[i] ^= p3[i] + + return bytes(res) + + +def caching_sha2_password_auth(conn, pkt): + # No password fast path + if not conn.password: + return _roundtrip(conn, b'') + + if pkt.is_auth_switch_request(): + # Try from fast auth + if DEBUG: + print("caching sha2: Trying fast path") + conn.salt = pkt.read_all() + scrambled = scramble_caching_sha2(conn.password, conn.salt) + pkt = _roundtrip(conn, scrambled) + # else: fast auth is tried in initial handshake + + if not pkt.is_extra_auth_data(): + raise OperationalError( + "caching sha2: Unknown packet for fast auth: %s" % pkt._data[:1] + ) + + # magic numbers: + # 2 - request public key + # 3 - fast auth succeeded + # 4 - need full auth + + pkt.advance(1) + n = pkt.read_uint8() + + if n == 3: + if DEBUG: + print("caching sha2: succeeded by fast path.") + pkt = conn._read_packet() + pkt.check_error() # pkt must be OK packet + return pkt + + if n != 4: + raise OperationalError("caching sha2: Unknwon result for fast auth: %s" % n) + + if DEBUG: + print("caching sha2: Trying full auth...") + + if conn.ssl and conn.server_capabilities & CLIENT.SSL: + if DEBUG: + print("caching sha2: Sending plain password via SSL") + return _roundtrip(conn, conn.password + b'\0') + + if not conn.server_public_key: + pkt = _roundtrip(conn, b'\x02') # Request public key + if not pkt.is_extra_auth_data(): + raise OperationalError( + "caching sha2: Unknown packet for public key: %s" % pkt._data[:1] + ) + + conn.server_public_key = pkt._data[1:] + if DEBUG: + print(conn.server_public_key.decode('ascii')) + + data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) + pkt = _roundtrip(conn, data) diff --git a/pymysql/connections.py b/pymysql/connections.py index d5dc0aff..14ae76d9 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -6,8 +6,6 @@ from ._compat import PY2, range_type, text_type, str_type, JYTHON, IRONPYTHON import errno -from functools import partial -import hashlib import io import os import socket @@ -16,6 +14,8 @@ import traceback import warnings +from . import _auth + from .charset import charset_by_name, charset_by_id from .constants import CLIENT, COMMAND, CR, FIELD_TYPE, SERVER_STATUS from . import converters @@ -43,7 +43,6 @@ # KeyError occurs when there's no entry in OS database for a current user. DEFAULT_USER = None - DEBUG = False _py_version = sys.version_info[:2] @@ -87,90 +86,16 @@ def _makefile(sock, mode): FIELD_TYPE.VARCHAR, FIELD_TYPE.GEOMETRY]) -sha_new = partial(hashlib.new, 'sha1') -DEFAULT_CHARSET = 'latin1' +DEFAULT_CHARSET = 'latin1' # TODO: change to utf8mb4 MAX_PACKET_LEN = 2**24-1 -SCRAMBLE_LENGTH = 20 - -def _scramble(password, message): - if not password: - return b'' - if DEBUG: print('password=' + str(password)) - stage1 = sha_new(password).digest() - stage2 = sha_new(stage1).digest() - s = sha_new() - s.update(message[:SCRAMBLE_LENGTH]) - s.update(stage2) - result = s.digest() - return _my_crypt(result, stage1) - - -def _my_crypt(message1, message2): - length = len(message1) - result = b'' - for i in range_type(length): - x = (struct.unpack('B', message1[i:i+1])[0] ^ - struct.unpack('B', message2[i:i+1])[0]) - result += struct.pack('B', x) - return result - -# old_passwords support ported from libmysql/password.c -SCRAMBLE_LENGTH_323 = 8 - - -class RandStruct_323(object): - def __init__(self, seed1, seed2): - self.max_value = 0x3FFFFFFF - self.seed1 = seed1 % self.max_value - self.seed2 = seed2 % self.max_value - - def my_rnd(self): - self.seed1 = (self.seed1 * 3 + self.seed2) % self.max_value - self.seed2 = (self.seed1 + self.seed2 + 33) % self.max_value - return float(self.seed1) / float(self.max_value) - - -def _scramble_323(password, message): - hash_pass = _hash_password_323(password) - hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) - hash_pass_n = struct.unpack(">LL", hash_pass) - hash_message_n = struct.unpack(">LL", hash_message) - - rand_st = RandStruct_323(hash_pass_n[0] ^ hash_message_n[0], - hash_pass_n[1] ^ hash_message_n[1]) - outbuf = io.BytesIO() - for _ in range_type(min(SCRAMBLE_LENGTH_323, len(message))): - outbuf.write(int2byte(int(rand_st.my_rnd() * 31) + 64)) - extra = int2byte(int(rand_st.my_rnd() * 31)) - out = outbuf.getvalue() - outbuf = io.BytesIO() - for c in out: - outbuf.write(int2byte(byte2int(c) ^ byte2int(extra))) - return outbuf.getvalue() - - -def _hash_password_323(password): - nr = 1345345333 - add = 7 - nr2 = 0x12345671 - - # x in py3 is numbers, p27 is chars - for c in [byte2int(x) for x in password if x not in (' ', '\t', 32, 9)]: - nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF - nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF - add = (add + c) & 0xFFFFFFFF - - r1 = nr & ((1 << 31) - 1) # kill sign bits - r2 = nr2 & ((1 << 31) - 1) - return struct.pack(">LL", r1, r2) - def pack_int24(n): return struct.pack('`_ in the specification. - + :raise Error: If the connection is already closed. """ if self._closed: @@ -446,7 +376,7 @@ def _force_close(self): if self._sock: try: self._sock.close() - except: + except: # noqa pass self._sock = None self._rfile = None @@ -485,7 +415,7 @@ def begin(self): def commit(self): """ Commit changes to stable storage. - + See `Connection.commit() `_ in the specification. """ @@ -495,7 +425,7 @@ def commit(self): def rollback(self): """ Roll back the current transaction. - + See `Connection.rollback() `_ in the specification. """ @@ -512,7 +442,7 @@ def show_warnings(self): def select_db(self, db): """ Set current db. - + :param db: The name of the db. """ self._execute_command(COMMAND.COM_INIT_DB, db) @@ -520,7 +450,7 @@ def select_db(self, db): def escape(self, obj, mapping=None): """Escape whatever value you pass to it. - + Non-standard, for internal use; do not use this in your applications. """ if isinstance(obj, str_type): @@ -534,7 +464,7 @@ def escape(self, obj, mapping=None): def literal(self, obj): """Alias for escape() - + Non-standard, for internal use; do not use this in your applications. """ return self.escape(obj, self.encoders) @@ -554,7 +484,7 @@ def _quote_bytes(self, s): def cursor(self, cursor=None): """ Create a new cursor to execute queries with. - + :param cursor: The type of cursor to create; one of :py:class:`Cursor`, :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. None means use Cursor. @@ -602,7 +532,7 @@ def kill(self, thread_id): def ping(self, reconnect=True): """ Check if the server is alive. - + :param reconnect: If the connection is closed, reconnect. :raise Error: If the connection is closed and reconnect=False. """ @@ -684,7 +614,7 @@ def connect(self, sock=None): if sock is not None: try: sock.close() - except: + except: # noqa pass if isinstance(e, (OSError, IOError, socket.error)): @@ -811,7 +741,6 @@ def _execute_command(self, command, sql): :raise InterfaceError: If the connection is closed. :raise ValueError: If no username was specified. """ - if not self._sock: raise err.InterfaceError("(0, '')") @@ -861,7 +790,7 @@ def _request_authentication(self): if isinstance(self.user, text_type): self.user = self.user.encode(self.encoding) - data_init = struct.pack(' Date: Wed, 27 Jun 2018 17:01:23 +0900 Subject: [PATCH 031/292] Map LOCK_DEADLOCK error to OperationalError (#693) --- pymysql/err.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/err.py b/pymysql/err.py index f3513ae8..fbc60558 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -91,7 +91,7 @@ def _map_error(exc, *errors): ER.NOT_SUPPORTED_YET, ER.FEATURE_DISABLED, ER.UNKNOWN_STORAGE_ENGINE) _map_error(OperationalError, ER.DBACCESS_DENIED_ERROR, ER.ACCESS_DENIED_ERROR, ER.CON_COUNT_ERROR, ER.TABLEACCESS_DENIED_ERROR, - ER.COLUMNACCESS_DENIED_ERROR, ER.CONSTRAINT_FAILED) + ER.COLUMNACCESS_DENIED_ERROR, ER.CONSTRAINT_FAILED, ER.LOCK_DEADLOCK) del _map_error, ER From c3fee50c755f196055b8cf664ba3c84e3f7b1199 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2018 17:02:03 +0900 Subject: [PATCH 032/292] Use pytest-cov for running test_auth (#691) --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d960249..e439db83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ matrix: # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version # really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't install: - - pip install -U coveralls unittest2 coverage cryptography pytest + - pip install -U coveralls unittest2 coverage cryptography pytest pytest-cov before_script: - ./.travis/initializedb.sh @@ -53,7 +53,7 @@ before_script: script: - coverage run ./runtests.py - if [ "${TEST_AUTH}" = "yes" ]; - then pytest -v tests; + then pytest -v --cov-config .coveragerc tests; fi - if [ ! -z "${DB}" ]; then docker logs mysqld; From e9ee7a823670f7033c1d0a99aae1de5a2a9db90a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2018 17:15:42 +0900 Subject: [PATCH 033/292] Use utf8mb4 for default encoding (#692) --- pymysql/connections.py | 7 ++++--- pymysql/tests/test_basic.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 14ae76d9..a34f70eb 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -75,7 +75,7 @@ def _makefile(sock, mode): return sock.makefile(mode) -TEXT_TYPES = set([ +TEXT_TYPES = { FIELD_TYPE.BIT, FIELD_TYPE.BLOB, FIELD_TYPE.LONG_BLOB, @@ -84,10 +84,11 @@ def _makefile(sock, mode): FIELD_TYPE.TINY_BLOB, FIELD_TYPE.VAR_STRING, FIELD_TYPE.VARCHAR, - FIELD_TYPE.GEOMETRY]) + FIELD_TYPE.GEOMETRY, +} -DEFAULT_CHARSET = 'latin1' # TODO: change to utf8mb4 +DEFAULT_CHARSET = 'utf8mb4' # TODO: change to utf8mb4 MAX_PACKET_LEN = 2**24-1 diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index cabb9e56..a5337322 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -24,7 +24,7 @@ def test_datatypes(self): try: # insert values - v = (True, -3, 123456789012, 5.7, "hello'\" world", u"Espa\xc3\xb1ol", "binary\x00data".encode(conn.charset), datetime.date(1988,2,2), datetime.datetime(2014, 5, 15, 7, 45, 57), datetime.timedelta(5,6), datetime.time(16,32), time.localtime()) + v = (True, -3, 123456789012, 5.7, "hello'\" world", u"Espa\xc3\xb1ol", "binary\x00data".encode(conn.encoding), datetime.date(1988,2,2), datetime.datetime(2014, 5, 15, 7, 45, 57), datetime.timedelta(5,6), datetime.time(16,32), time.localtime()) c.execute("insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", v) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") r = c.fetchone() @@ -99,7 +99,7 @@ def test_binary(self): conn, "test_binary", "create table test_binary (b binary(255))") with conn.cursor() as c: - c.execute("insert into test_binary (b) values (%s)", (data,)) + c.execute("insert into test_binary (b) values (_binary %s)", (data,)) c.execute("select b from test_binary") self.assertEqual(data, c.fetchone()[0]) @@ -111,7 +111,7 @@ def test_blob(self): conn, "test_blob", "create table test_blob (b blob)") with conn.cursor() as c: - c.execute("insert into test_blob (b) values (%s)", (data,)) + c.execute("insert into test_blob (b) values (_binary %s)", (data,)) c.execute("select b from test_blob") self.assertEqual(data, c.fetchone()[0]) From 8b31b8902da9bd66c3fb69733cad9ed5d9b1a3d0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2018 17:24:13 +0900 Subject: [PATCH 034/292] Remove deprecated no_delay option (#694) --- pymysql/connections.py | 5 +---- pymysql/tests/test_connection.py | 7 ------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index a34f70eb..615c6146 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -186,15 +186,12 @@ def __init__(self, host=None, user=None, password="", read_default_file=None, conv=None, use_unicode=None, client_flag=0, cursorclass=Cursor, init_command=None, connect_timeout=10, ssl=None, read_default_group=None, - compress=None, named_pipe=None, no_delay=None, + compress=None, named_pipe=None, autocommit=False, db=None, passwd=None, local_infile=False, max_allowed_packet=16*1024*1024, defer_connect=False, auth_plugin_map=None, read_timeout=None, write_timeout=None, bind_address=None, binary_prefix=False, program_name=None, server_public_key=None): - if no_delay is not None: - warnings.warn("no_delay option is deprecated", DeprecationWarning) - if use_unicode is None and sys.version_info[0] > 2: use_unicode = True diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 28091be2..5e95b1c8 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -491,13 +491,6 @@ def test_defer_connect(self): c.close() sock.close() - @unittest2.skipUnless(sys.version_info[0:2] >= (3,2), "required py-3.2") - def test_no_delay_warning(self): - current_db = self.databases[0].copy() - current_db['no_delay'] = True - with self.assertWarns(DeprecationWarning) as cm: - conn = pymysql.connect(**current_db) - # A custom type and function to escape it class Foo(object): From 2e57bb3debe4b16eaf301075d7da2a3ddab0a0cf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2018 17:32:12 +0900 Subject: [PATCH 035/292] Update CHANGELOG --- CHANGELOG | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index fce9fb30..4d2aa37c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,13 @@ # Changes +## 0.9.0 + +* Change default charset from latin1 to utf8mb4. (because MySQL 8 changed) (#692) +* Remove deprecated `no_delay` option (#694) +* Support connection attributes (#679) +* Support sha256_password and caching_sha2_password auth method (#682) +* Map LOCK_DEADLOCK to OperationalError (#693) + ## 0.8.1 Release date: 2018-05-07 From 2fca94f24302591ba54e473d7a5cacba4111ae84 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 27 Jun 2018 18:56:26 +0900 Subject: [PATCH 036/292] Add unix socket shortcut for new auth methods (#696) --- CHANGELOG | 5 ++++- pymysql/__init__.py | 2 +- pymysql/_auth.py | 6 +++--- pymysql/connections.py | 5 ++++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 4d2aa37c..c076b3de 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,10 +2,13 @@ ## 0.9.0 +Release date: 2018-06-27 + * Change default charset from latin1 to utf8mb4. (because MySQL 8 changed) (#692) +* Support sha256_password and caching_sha2_password auth method (#682) +* Add cryptography dependency, because it's needed for new auth methods. * Remove deprecated `no_delay` option (#694) * Support connection attributes (#679) -* Support sha256_password and caching_sha2_password auth method (#682) * Map LOCK_DEADLOCK to OperationalError (#693) ## 0.8.1 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 1a6ecc09..9fcfcd30 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 8, 1, None) +VERSION = (0, 9, 0, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/pymysql/_auth.py b/pymysql/_auth.py index ddf6e4e5..8c3e6cd9 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -142,7 +142,7 @@ def sha2_rsa_encrypt(password, salt, public_key): def sha256_password_auth(conn, pkt): - if conn.ssl and conn.server_capabilities & CLIENT.SSL: + if conn._secure: if DEBUG: print("sha256: Sending plain password") data = conn.password + b'\0' @@ -232,9 +232,9 @@ def caching_sha2_password_auth(conn, pkt): if DEBUG: print("caching sha2: Trying full auth...") - if conn.ssl and conn.server_capabilities & CLIENT.SSL: + if conn._secure: if DEBUG: - print("caching sha2: Sending plain password via SSL") + print("caching sha2: Sending plain password via secure connection") return _roundtrip(conn, conn.password + b'\0') if not conn.server_public_key: diff --git a/pymysql/connections.py b/pymysql/connections.py index 615c6146..1e580d21 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -179,6 +179,7 @@ class Connection(object): _sock = None _auth_plugin_name = '' _closed = False + _secure = False def __init__(self, host=None, user=None, password="", database=None, port=0, unix_socket=None, @@ -563,11 +564,12 @@ def connect(self, sock=None): self._closed = False try: if sock is None: - if self.unix_socket and self.host in ('localhost', '127.0.0.1'): + if self.unix_socket: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.settimeout(self.connect_timeout) sock.connect(self.unix_socket) self.host_info = "Localhost via UNIX socket" + self._secure = True if DEBUG: print('connected using unix_socket') else: kwargs = {} @@ -795,6 +797,7 @@ def _request_authentication(self): self._sock = self.ctx.wrap_socket(self._sock, server_hostname=self.host) self._rfile = _makefile(self._sock, 'rb') + self._secure = True data = data_init + self.user + b'\0' From 2fb8b1b0b291e50273c0e509e9779f3ee9ce8daf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 Jul 2018 00:52:59 +0900 Subject: [PATCH 037/292] travis: Run auth test with Python 2.7 (#702) To find bugs like #700 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index e439db83..f4a7cc74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,10 @@ matrix: - DB=mysql:8.0 - TEST_AUTH=yes python: "3.7-dev" + - env: + - DB=mysql:8.0 + - TEST_AUTH=yes + python: "2.7" # different py version from 5.6 and 5.7 as cache seems to be based on py version # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version From 24f093a33fdbe92bb5674f3ba482922ac9b43447 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 Jul 2018 01:10:44 +0900 Subject: [PATCH 038/292] Fix caching_sha2_password didn't work with PY2 (#701) Fixes #700 --- pymysql/_auth.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 8c3e6cd9..eb76e4ba 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -1,7 +1,7 @@ """ Implements auth methods """ -from ._compat import text_type +from ._compat import text_type, PY2 from .constants import CLIENT from .err import OperationalError @@ -38,15 +38,14 @@ def scramble_native_password(password, message): def _my_crypt(message1, message2): - length = len(message1) - result = b'' - for i in range(length): - x = ( - struct.unpack('B', message1[i:i + 1])[0] ^ - struct.unpack('B', message2[i:i + 1])[0] - ) - result += struct.pack('B', x) - return result + result = bytearray(message1) + if PY2: + message2 = bytearray(message2) + + for i in range(len(result)): + result[i] ^= message2[i] + + return bytes(result) # old_passwords support ported from libmysql/password.c @@ -186,6 +185,8 @@ def scramble_caching_sha2(password, nonce): p3 = hashlib.sha256(p2 + nonce).digest() res = bytearray(p1) + if PY2: + p3 = bytearray(p3) for i in range(len(p3)): res[i] ^= p3[i] From 2864ae87c901cd0ccf78e06caf26f726d372040b Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 Jul 2018 01:21:33 +0900 Subject: [PATCH 039/292] 0.9.1 --- CHANGELOG | 8 ++++++++ pymysql/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index c076b3de..b4372fed 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,13 @@ # Changes +## 0.9.1 + +Release date: 2018-07-03 + +* Fixed caching_sha2_password and sha256_password raise TypeError on PY2 + (#700, #702) + + ## 0.9.0 Release date: 2018-06-27 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 9fcfcd30..af1b3425 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 9, 0, None) +VERSION = (0, 9, 1, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: From 66d624b46f67e00bccf0c393eb2c32ebd199f0e8 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 Jul 2018 01:30:04 +0900 Subject: [PATCH 040/292] Add requirements file for readthedocs --- Pipfile | 12 ++++++++++++ requirements.txt | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 Pipfile create mode 100644 requirements.txt diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..0e142ba3 --- /dev/null +++ b/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +cryptography = "*" + +[dev-packages] +pytest = "*" +unittest2 = "*" +twine = "*" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..70f05161 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +cryptography + From baeb4aae9fc81b5465cc2d75185519acb500b7b1 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 Jul 2018 01:42:40 +0900 Subject: [PATCH 041/292] README: Update badges --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ccdd8578..1c7fba54 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,10 @@ .. image:: https://readthedocs.org/projects/pymysql/badge/?version=latest - :target: http://pymysql.readthedocs.io/en/latest/?badge=latest + :target: https://pymysql.readthedocs.io/ :alt: Documentation Status +.. image:: https://badge.fury.io/py/PyMySQL.svg + :target: https://badge.fury.io/py/PyMySQL + .. image:: https://travis-ci.org/PyMySQL/PyMySQL.svg?branch=master :target: https://travis-ci.org/PyMySQL/PyMySQL From f0bbe54ef7f977c8cc8a55f583e85d06bea44e7c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 3 Jul 2018 12:20:05 +0900 Subject: [PATCH 042/292] Disable debug logging in _auth (#704) Fixes #703 --- pymysql/_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index eb76e4ba..bbb742d3 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -14,7 +14,7 @@ import struct -DEBUG = True +DEBUG = False SCRAMBLE_LENGTH = 20 sha1_new = partial(hashlib.new, 'sha1') From 44c422d9d099a72c80b3ac9e007e1d838ceaa9ba Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 4 Jul 2018 14:43:54 +0900 Subject: [PATCH 043/292] Add Pipfile.lock to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0b2c85be..98f4d45c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ /docs/build /pymysql/tests/databases.json __pycache__ +Pipfile.lock From fa43bf7d7d967283a668137c3cdb45c1da8c186a Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 4 Jul 2018 15:33:10 +0900 Subject: [PATCH 044/292] Don't install test files (#706) --- MANIFEST.in | 2 -- setup.cfg | 10 ++++++++++ setup.py | 9 ++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0e4c15a0..0a520792 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1 @@ include README.rst LICENSE CHANGELOG -include runtests.py tox.ini -include example.py diff --git a/setup.cfg b/setup.cfg index 2b4fe304..a26a846b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,13 @@ max-line-length = 119 [bdist_wheel] universal = 1 + +[metadata] +license = "MIT" +license_file = LICENSE + +author=yutaka.matsubara +author_email=yutaka.matsubara@gmail.com + +maintainer=INADA Naoki +maintainer_email=songofacandy@gmail.com diff --git a/setup.py b/setup.py index 0c0357cc..a64cf170 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = __import__('pymysql').VERSION_STRING +version = "0.9.1" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() @@ -14,14 +14,9 @@ project_urls={ "Documentation": "https://pymysql.readthedocs.io/", }, - author='yutaka.matsubara', - author_email='yutaka.matsubara@gmail.com', - maintainer='INADA Naoki', - maintainer_email='songofacandy@gmail.com', description='Pure Python MySQL Driver', long_description=readme, - license="MIT", - packages=find_packages(), + packages=find_packages(exclude=['tests*', 'pymysql.tests*']), install_requires=[ "cryptography", ], From 99f4720f446463721d68c29f52d97cb01dbf0c3f Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 4 Jul 2018 15:36:57 +0900 Subject: [PATCH 045/292] 0.9.2 --- pymysql/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index af1b3425..b79b4b83 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 9, 1, None) +VERSION = (0, 9, 2, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index a64cf170..14650d1c 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = "0.9.1" +version = "0.9.2" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() From c3becee5e94223fac50d3fcfe8a3b157d76bbea7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 5 Jul 2018 11:47:08 +0900 Subject: [PATCH 046/292] Update documents --- README.rst | 2 +- docs/source/user/installation.rst | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 1c7fba54..163c4f2a 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ Package is uploaded on `PyPI `_. You can install it with pip:: - $ pip3 install PyMySQL + $ python3 -m pip install PyMySQL Documentation diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index e3bfe84d..8a81fddb 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -6,24 +6,22 @@ Installation The last stable release is available on PyPI and can be installed with ``pip``:: - $ pip install PyMySQL + $ python3 -m pip install PyMySQL Requirements ------------- * Python -- one of the following: - - CPython_ >= 2.6 or >= 3.3 - - PyPy_ >= 4.0 - - IronPython_ 2.7 + - CPython_ >= 2.7 or >= 3.4 + - Latest PyPy_ * MySQL Server -- one of the following: - - MySQL_ >= 4.1 (tested with only 5.5~) - - MariaDB_ >= 5.1 + - MySQL_ >= 5.5 + - MariaDB_ >= 5.5 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ -.. _IronPython: http://ironpython.net/ .. _MySQL: http://www.mysql.com/ .. _MariaDB: https://mariadb.org/ From 7ea89711cf4ade1970a8cf359dc074b19c272184 Mon Sep 17 00:00:00 2001 From: aaron jheng Date: Mon, 16 Jul 2018 19:42:04 +0800 Subject: [PATCH 047/292] Update Pipfile (#712) --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 0e142ba3..a18fa51a 100644 --- a/Pipfile +++ b/Pipfile @@ -1,5 +1,5 @@ [[source]] -url = "https://pypi.python.org/simple" +url = "https://pypi.org/simple" verify_ssl = true name = "pypi" From 3571e904f4f598a9010456ba8e56542846a68b60 Mon Sep 17 00:00:00 2001 From: Alex Lee Date: Mon, 16 Jul 2018 21:56:17 -0700 Subject: [PATCH 048/292] Return "mysql_native_password" in auth response (#709) --- pymysql/connections.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 1e580d21..e9dd4c99 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -804,7 +804,11 @@ def _request_authentication(self): authresp = b'' plugin_name = None - if self._auth_plugin_name in ('', 'mysql_native_password'): + if self._auth_plugin_name == '': + plugin_name = b'' + authresp = _auth.scramble_native_password(self.password, self.salt) + elif self._auth_plugin_name == 'mysql_native_password': + plugin_name = b'mysql_native_password' authresp = _auth.scramble_native_password(self.password, self.salt) elif self._auth_plugin_name == 'caching_sha2_password': plugin_name = b'caching_sha2_password' From 1c8ee8fa71757f1a8be2b0c041a9174ae35963f6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 25 Jul 2018 11:56:34 +0900 Subject: [PATCH 049/292] Update CHANGES --- CHANGELOG | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index b4372fed..d73ddd79 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,13 @@ # Changes +## 0.9.2 + +Release date: 2018-07-04 + +* Disalbled unintentinally enabled debug log +* Removed unintentionally installed tests + + ## 0.9.1 Release date: 2018-07-03 From 55e195cdf2f19c3f36007b72c1a9093dd9b0ebdb Mon Sep 17 00:00:00 2001 From: pick2510 Date: Sun, 12 Aug 2018 05:10:49 +0200 Subject: [PATCH 050/292] Fix old password support (#713) --- pymysql/_auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index bbb742d3..7a7377bf 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -4,6 +4,8 @@ from ._compat import text_type, PY2 from .constants import CLIENT from .err import OperationalError +from .util import byte2int, int2byte + from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization, hashes @@ -11,6 +13,7 @@ from functools import partial import hashlib +import io import struct From 104e84f5c9c84e5e75bfcad3d23306871c838a8b Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 10 Sep 2018 16:56:32 +0100 Subject: [PATCH 051/292] Cleanup after Python 2.6 support removed (#725) --- pymysql/connections.py | 4 ++-- pymysql/cursors.py | 6 +++--- tox.ini | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index e9dd4c99..f1ae621d 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -302,8 +302,8 @@ def _config(key, arg): conv = converters.conversions # Need for MySQLdb compatibility. - self.encoders = dict([(k, v) for (k, v) in conv.items() if type(k) is not int]) - self.decoders = dict([(k, v) for (k, v) in conv.items() if type(k) is int]) + self.encoders = {k: v for (k, v) in conv.items() if type(k) is not int} + self.decoders = {k: v for (k, v) in conv.items() if type(k) is int} self.sql_mode = sql_mode self.init_command = init_command self.max_allowed_packet = max_allowed_packet diff --git a/pymysql/cursors.py b/pymysql/cursors.py index cc169987..a6d645d4 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -122,9 +122,9 @@ def _escape_args(self, args, conn): return tuple(conn.literal(arg) for arg in args) elif isinstance(args, dict): if PY2: - args = dict((ensure_bytes(key), ensure_bytes(val)) for - (key, val) in args.items()) - return dict((key, conn.literal(val)) for (key, val) in args.items()) + args = {ensure_bytes(key): ensure_bytes(val) for + (key, val) in args.items()} + return {key: conn.literal(val) for (key, val) in args.items()} else: # If it's not a dictionary let's try escaping it anyways. # Worst case it will throw a Value error diff --git a/tox.ini b/tox.ini index a50364c9..e2f2917c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34,pypy,pypy3 +envlist = py27,py34,py35,py36,py37,pypy,pypy3 [testenv] commands = coverage run ./runtests.py From 09040abb0a8153b68b76113539632e7eaa53ce69 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 14 Sep 2018 12:45:31 +1000 Subject: [PATCH 052/292] Fix docstring (#727) capath and cipher are supported. --- pymysql/connections.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index f1ae621d..4dae75d3 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -152,7 +152,6 @@ class Connection(object): (default: 10, min: 1, max: 31536000) :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. - For now the capath and cipher arguments are not supported. :param read_default_group: Group to read from in the configuration file. :param compress: Not supported :param named_pipe: Not supported From 3ab3b275e3d60be733f2c3f1bf6cfd644863466c Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 29 Oct 2018 16:41:14 +0900 Subject: [PATCH 053/292] sys.argv is not always available (#739) fixes #736 --- pymysql/connections.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 4dae75d3..92ba3861 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -315,10 +315,12 @@ def _config(key, arg): '_pid': str(os.getpid()), '_client_version': VERSION_STRING, } + if program_name is None: + argv = getattr(sys, "argv") + if argv: + program_name = argv[0] if program_name: self._connect_attrs["program_name"] = program_name - elif sys.argv: - self._connect_attrs["program_name"] = sys.argv[0] if defer_connect: self._sock = None From 27546ef14fc33f058b1003492d4ba72b7a1b58da Mon Sep 17 00:00:00 2001 From: IWAMOTO Toshihiro Date: Wed, 28 Nov 2018 10:38:25 +0900 Subject: [PATCH 054/292] Hide Connection.autocommit_mode docstring (#746) It is an internal variable and the rendered document can be misread as if pymysql defaults to server default autocommit mode. --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 92ba3861..32a07169 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -294,7 +294,7 @@ def _config(key, arg): self._affected_rows = 0 self.host_info = "Not connected" - #: specified autocommit mode. None means use server default. + # specified autocommit mode. None means use server default. self.autocommit_mode = autocommit if conv is None: From c7154e6c6b091611fa105b124cc6975e07f37441 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 6 Dec 2018 20:56:24 +0900 Subject: [PATCH 055/292] Support non-ascii program_name on Python 2 (#748) Fixes #747 --- pymysql/connections.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 32a07169..c01d9993 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -319,6 +319,9 @@ def _config(key, arg): argv = getattr(sys, "argv") if argv: program_name = argv[0] + if PY2: + program_name = program_name.decode('utf-8', 'replace') + if program_name: self._connect_attrs["program_name"] = program_name @@ -847,9 +850,9 @@ def _request_authentication(self): if self.server_capabilities & CLIENT.CONNECT_ATTRS: connect_attrs = b'' for k, v in self._connect_attrs.items(): - k = k.encode('utf8') + k = k.encode('utf-8') connect_attrs += struct.pack('B', len(k)) + k - v = v.encode('utf8') + v = v.encode('utf-8') connect_attrs += struct.pack('B', len(v)) + v data += struct.pack('B', len(connect_attrs)) + connect_attrs From 9fa5daf8a8157fb80725e6bca78e29cb80833618 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 15 Dec 2018 11:10:20 +0900 Subject: [PATCH 056/292] Remove auto program_name from sys.argv (#755) --- pymysql/connections.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index c01d9993..7f5646e8 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -315,12 +315,6 @@ def _config(key, arg): '_pid': str(os.getpid()), '_client_version': VERSION_STRING, } - if program_name is None: - argv = getattr(sys, "argv") - if argv: - program_name = argv[0] - if PY2: - program_name = program_name.decode('utf-8', 'replace') if program_name: self._connect_attrs["program_name"] = program_name From c42ddff9412740ad785c633e31967174eec931e0 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sun, 16 Dec 2018 13:45:44 +0900 Subject: [PATCH 057/292] Warn when old password is used (#756) --- pymysql/_auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 7a7377bf..e0a48f74 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -15,6 +15,7 @@ import hashlib import io import struct +import warnings DEBUG = False @@ -72,6 +73,8 @@ def my_rnd(self): def scramble_old_password(password, message): """Scramble for old_password""" + warnings.warn("old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" + "old password support will be removed in future PyMySQL version") hash_pass = _hash_password_323(password) hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) hash_pass_n = struct.unpack(">LL", hash_pass) From 7b18bb6588903bce502308099d6f007fd165e8fc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Dec 2018 21:19:36 +0900 Subject: [PATCH 058/292] test: self.connections[0] -> self.connect() (#758) --- pymysql/tests/test_DictCursor.py | 2 +- pymysql/tests/test_basic.py | 40 ++++++------ pymysql/tests/test_connection.py | 108 ++++++++++++++++--------------- pymysql/tests/test_cursor.py | 2 +- pymysql/tests/test_issues.py | 32 ++++----- pymysql/tests/test_load_local.py | 8 +-- 6 files changed, 98 insertions(+), 94 deletions(-) diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index 9a0d638b..122882e6 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -14,7 +14,7 @@ class TestDictCursor(base.PyMySQLTestCase): def setUp(self): super(TestDictCursor, self).setUp() - self.conn = conn = self.connections[0] + self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) # create a table ane some data to query diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index a5337322..940661f7 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -18,7 +18,7 @@ class TestConversion(base.PyMySQLTestCase): def test_datatypes(self): """ test every data type """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)") try: @@ -57,7 +57,7 @@ def test_datatypes(self): def test_dict(self): """ test dict escaping """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a integer, b integer, c integer)") try: @@ -68,7 +68,7 @@ def test_dict(self): c.execute("drop table test_dict") def test_string(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a text)") test_value = "I am a test string" @@ -80,7 +80,7 @@ def test_string(self): c.execute("drop table test_dict") def test_integer(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a integer)") test_value = 12345 @@ -94,7 +94,7 @@ def test_integer(self): def test_binary(self): """test binary data""" data = bytes(bytearray(range(255))) - conn = self.connections[0] + conn = self.connect() self.safe_create_table( conn, "test_binary", "create table test_binary (b binary(255))") @@ -106,7 +106,7 @@ def test_binary(self): def test_blob(self): """test blob data""" data = bytes(bytearray(range(256)) * 4) - conn = self.connections[0] + conn = self.connect() self.safe_create_table( conn, "test_blob", "create table test_blob (b blob)") @@ -117,7 +117,7 @@ def test_blob(self): def test_untyped(self): """ test conversion of null, empty string """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("select null,''") self.assertEqual((None,u''), c.fetchone()) @@ -126,7 +126,7 @@ def test_untyped(self): def test_timedelta(self): """ test timedelta conversion """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')") self.assertEqual((datetime.timedelta(0, 45000), @@ -141,7 +141,7 @@ def test_timedelta(self): def test_datetime_microseconds(self): """ test datetime conversion w microseconds""" - conn = self.connections[0] + conn = self.connect() if not self.mysql_server_is(conn, (5, 6, 4)): raise SkipTest("target backend does not support microseconds") c = conn.cursor() @@ -206,7 +206,7 @@ class TestCursor(base.PyMySQLTestCase): # ('max_updates', 3, 1, 11, 11, 0, 0), # ('max_connections', 3, 1, 11, 11, 0, 0), # ('max_user_connections', 3, 1, 11, 11, 0, 0)) - # conn = self.connections[0] + # conn = self.connect() # c = conn.cursor() # c.execute("select * from mysql.user") # @@ -214,7 +214,7 @@ class TestCursor(base.PyMySQLTestCase): def test_fetch_no_result(self): """ test a fetchone() with no rows """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("create table test_nr (b varchar(32))") try: @@ -226,7 +226,7 @@ def test_fetch_no_result(self): def test_aggregates(self): """ test aggregate functions """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() try: c.execute('create table test_aggregates (i integer)') @@ -240,7 +240,7 @@ def test_aggregates(self): def test_single_tuple(self): """ test a single tuple """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() self.safe_create_table( conn, 'mystuff', @@ -283,7 +283,7 @@ class TestBulkInserts(base.PyMySQLTestCase): def setUp(self): super(TestBulkInserts, self).setUp() - self.conn = conn = self.connections[0] + self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) # create a table ane some data to query @@ -299,14 +299,14 @@ def setUp(self): """) def _verify_records(self, data): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() cursor.execute("SELECT id, name, age, height from bulkinsert") result = cursor.fetchall() self.assertEqual(sorted(data), sorted(result)) def test_bulk_insert(self): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] @@ -320,7 +320,7 @@ def test_bulk_insert(self): self._verify_records(data) def test_bulk_insert_multiline_statement(self): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] cursor.executemany("""insert @@ -344,7 +344,7 @@ def test_bulk_insert_multiline_statement(self): self._verify_records(data) def test_bulk_insert_single_record(self): - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123)] cursor.executemany("insert into bulkinsert (id, name, age, height) " @@ -354,7 +354,7 @@ def test_bulk_insert_single_record(self): def test_issue_288(self): """executemany should work with "insert ... on update" """ - conn = self.connections[0] + conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] cursor.executemany("""insert @@ -380,7 +380,7 @@ def test_issue_288(self): self._verify_records(data) def test_warnings(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 5e95b1c8..3f162780 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -96,18 +96,19 @@ class TestAuthentication(base.PyMySQLTestCase): # print("plugin: %r" % r[0]) def test_plugin(self): - if not self.mysql_server_is(self.connections[0], (5, 5, 0)): + conn = self.connect() + if not self.mysql_server_is(conn, (5, 5, 0)): raise unittest2.SkipTest("MySQL-5.5 required for plugins") - cur = self.connections[0].cursor() + cur = conn.cursor() cur.execute("select plugin from mysql.user where concat(user, '@', host)=current_user()") for r in cur: - self.assertIn(self.connections[0]._auth_plugin_name, (r[0], 'mysql_native_password')) + self.assertIn(conn._auth_plugin_name, (r[0], 'mysql_native_password')) @unittest2.skipUnless(socket_auth, "connection to unix_socket required") @unittest2.skipIf(socket_found, "socket plugin already installed") def testSocketAuthInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin auth_socket soname 'auth_socket.so'") TestAuthentication.socket_found = True @@ -132,7 +133,7 @@ def testSocketAuth(self): self.realtestSocketAuth() def realtestSocketAuth(self): - with TempUser(self.connections[0].cursor(), TestAuthentication.osuser + '@localhost', + with TempUser(self.connect().cursor(), TestAuthentication.osuser + '@localhost', self.databases[0]['db'], self.socket_plugin_name) as u: c = pymysql.connect(user=TestAuthentication.osuser, **self.db) @@ -180,7 +181,7 @@ def __init__(self, con): @unittest2.skipIf(two_questions_found, "two_questions plugin already installed") def testDialogAuthTwoQuestionsInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin two_questions soname 'dialog_examples.so'") TestAuthentication.two_questions_found = True @@ -200,7 +201,7 @@ def realTestDialogAuthTwoQuestions(self): TestAuthentication.Dialog.fail=False TestAuthentication.Dialog.m = {b'Password, please:': b'notverysecret', b'Are you sure ?': b'yes, of course'} - with TempUser(self.connections[0].cursor(), 'pymysql_2q@localhost', + with TempUser(self.connect().cursor(), 'pymysql_2q@localhost', self.databases[0]['db'], 'two_questions', 'notverysecret') as u: with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user='pymysql_2q', **self.db) @@ -210,7 +211,7 @@ def realTestDialogAuthTwoQuestions(self): @unittest2.skipIf(three_attempts_found, "three_attempts plugin already installed") def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin three_attempts soname 'dialog_examples.so'") TestAuthentication.three_attempts_found = True @@ -229,7 +230,7 @@ def testDialogAuthThreeAttempts(self): def realTestDialogAuthThreeAttempts(self): TestAuthentication.Dialog.m = {b'Password, please:': b'stillnotverysecret'} TestAuthentication.Dialog.fail=True # fail just once. We've got three attempts after all - with TempUser(self.connections[0].cursor(), 'pymysql_3a@localhost', + with TempUser(self.connect().cursor(), 'pymysql_3a@localhost', self.databases[0]['db'], 'three_attempts', 'stillnotverysecret') as u: pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.DialogHandler}, **self.db) @@ -253,7 +254,7 @@ def realTestDialogAuthThreeAttempts(self): @unittest2.skipIf(os.environ.get('PAMSERVICE') is None, "PAMSERVICE env var required") def testPamAuthInstallPlugin(self): # needs plugin. lets install it. - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute("install plugin pam soname 'auth_pam.so'") TestAuthentication.pam_found = True @@ -276,7 +277,7 @@ def realTestPamAuth(self): db = self.db.copy() import os db['password'] = os.environ.get('PASSWORD') - cur = self.connections[0].cursor() + cur = self.connect().cursor() try: cur.execute('show grants for ' + TestAuthentication.osuser + '@localhost') grants = cur.fetchone()[0] @@ -311,51 +312,54 @@ def realTestPamAuth(self): @unittest2.skipUnless(socket_auth, "connection to unix_socket required") @unittest2.skipUnless(mysql_old_password_found, "no mysql_old_password plugin") def testMySQLOldPasswordAuth(self): - if self.mysql_server_is(self.connections[0], (5, 7, 0)): + conn = self.connect() + if self.mysql_server_is(conn, (5, 7, 0)): raise unittest2.SkipTest('Old passwords aren\'t supported in 5.7') # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") # from login in MySQL-5.6 - if self.mysql_server_is(self.connections[0], (5, 6, 0)): + if self.mysql_server_is(conn, (5, 6, 0)): raise unittest2.SkipTest('Old passwords don\'t authenticate in 5.6') db = self.db.copy() db['password'] = "crummy p\tassword" - with self.connections[0] as c: - # deprecated in 5.6 - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(self.connections[0], (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) - else: + c = conn.cursor() + + # deprecated in 5.6 + if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + with self.assertWarns(pymysql.err.Warning) as cm: c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) - v = c.fetchone()[0] - self.assertEqual(v, '2a01785203b08770') - # only works in MariaDB and MySQL-5.6 - can't separate out by version - #if self.mysql_server_is(self.connections[0], (5, 5, 0)): - # with TempUser(c, 'old_pass_user@localhost', - # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: - # cur = pymysql.connect(user='old_pass_user', **db).cursor() - # cur.execute("SELECT VERSION()") - c.execute("SELECT @@secure_auth") - secure_auth_setting = c.fetchone()[0] - c.execute('set old_passwords=1') - # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(self.connections[0], (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute('set global secure_auth=0') - else: + else: + c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) + v = c.fetchone()[0] + self.assertEqual(v, '2a01785203b08770') + # only works in MariaDB and MySQL-5.6 - can't separate out by version + #if self.mysql_server_is(self.connect(), (5, 5, 0)): + # with TempUser(c, 'old_pass_user@localhost', + # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: + # cur = pymysql.connect(user='old_pass_user', **db).cursor() + # cur.execute("SELECT VERSION()") + c.execute("SELECT @@secure_auth") + secure_auth_setting = c.fetchone()[0] + c.execute('set old_passwords=1') + # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead + if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + with self.assertWarns(pymysql.err.Warning) as cm: c.execute('set global secure_auth=0') - with TempUser(c, 'old_pass_user@localhost', - self.databases[0]['db'], password=db['password']) as u: - cur = pymysql.connect(user='old_pass_user', **db).cursor() - cur.execute("SELECT VERSION()") - c.execute('set global secure_auth=%r' % secure_auth_setting) + else: + c.execute('set global secure_auth=0') + with TempUser(c, 'old_pass_user@localhost', + self.databases[0]['db'], password=db['password']) as u: + cur = pymysql.connect(user='old_pass_user', **db).cursor() + cur.execute("SELECT VERSION()") + c.execute('set global secure_auth=%r' % secure_auth_setting) @unittest2.skipUnless(socket_auth, "connection to unix_socket required") @unittest2.skipUnless(sha256_password_found, "no sha256 password authentication plugin found") def testAuthSHA256(self): - c = self.connections[0].cursor() + conn = self.connect() + c = conn.cursor() with TempUser(c, 'pymysql_sha256@localhost', self.databases[0]['db'], 'sha256_password') as u: - if self.mysql_server_is(self.connections[0], (5, 7, 0)): + if self.mysql_server_is(conn, (5, 7, 0)): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") else: c.execute('SET old_passwords = 2') @@ -377,7 +381,7 @@ def test_utf8mb4(self): def test_largedata(self): """Large query and response (>=16MB)""" - cur = self.connections[0].cursor() + cur = self.connect().cursor() cur.execute("SELECT @@max_allowed_packet") if cur.fetchone()[0] < 16*1024*1024 + 10: print("Set max_allowed_packet to bigger than 17MB") @@ -387,7 +391,7 @@ def test_largedata(self): assert cur.fetchone()[0] == t def test_autocommit(self): - con = self.connections[0] + con = self.connect() self.assertFalse(con.get_autocommit()) cur = con.cursor() @@ -400,7 +404,7 @@ def test_autocommit(self): self.assertEqual(cur.fetchone()[0], 0) def test_select_db(self): - con = self.connections[0] + con = self.connect() current_db = self.databases[0]['db'] other_db = self.databases[1]['db'] @@ -503,7 +507,7 @@ def escape_foo(x, d): class TestEscape(base.PyMySQLTestCase): def test_escape_string(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() self.assertEqual(con.escape("foo'bar"), "'foo\\'bar'") @@ -516,21 +520,21 @@ def test_escape_string(self): self.assertEqual(con.escape("foo'bar"), "'foo''bar'") def test_escape_builtin_encoders(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() val = datetime.datetime(2012, 3, 4, 5, 6) self.assertEqual(con.escape(val, con.encoders), "'2012-03-04 05:06:00'") def test_escape_custom_object(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() mapping = {Foo: escape_foo} self.assertEqual(con.escape(Foo(), mapping), "bar") def test_escape_fallback_encoder(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() class Custom(str): @@ -540,13 +544,13 @@ class Custom(str): self.assertEqual(con.escape(Custom('foobar'), mapping), "'foobar'") def test_escape_no_default(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() self.assertRaises(TypeError, con.escape, 42, {}) def test_escape_dict_value(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() mapping = con.encoders.copy() @@ -554,7 +558,7 @@ def test_escape_dict_value(self): self.assertEqual(con.escape({'foo': Foo()}, mapping), {'foo': "bar"}) def test_escape_list_item(self): - con = self.connections[0] + con = self.connect() cur = con.cursor() mapping = con.encoders.copy() diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index add04755..fb3e8bed 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -7,7 +7,7 @@ class CursorTest(base.PyMySQLTestCase): def setUp(self): super(CursorTest, self).setUp() - conn = self.connections[0] + conn = self.connect() self.safe_create_table( conn, "test", "create table test (data varchar(10))", diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index cedd0925..8dca31b7 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -21,7 +21,7 @@ class TestOldIssues(base.PyMySQLTestCase): def test_issue_3(self): """ undefined methods datetime_or_None, date_or_None """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -42,7 +42,7 @@ def test_issue_3(self): def test_issue_4(self): """ can't retrieve TIMESTAMP fields """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -57,7 +57,7 @@ def test_issue_4(self): def test_issue_5(self): """ query on information_schema.tables fails """ - con = self.connections[0] + con = self.connect() cur = con.cursor() cur.execute("select * from information_schema.tables") @@ -73,7 +73,7 @@ def test_issue_6(self): def test_issue_8(self): """ Primary Key and Index error when selecting data """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -98,7 +98,7 @@ def test_issue_9(self): def test_issue_13(self): """ can't handle large result fields """ - conn = self.connections[0] + conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -117,7 +117,7 @@ def test_issue_13(self): def test_issue_15(self): """ query should be expanded before perform character encoding """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -132,7 +132,7 @@ def test_issue_15(self): def test_issue_16(self): """ Patch for string and tuple escaping """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -148,7 +148,7 @@ def test_issue_16(self): @unittest2.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") def test_issue_17(self): """could not connect mysql use passwod""" - conn = self.connections[0] + conn = self.connect() host = self.databases[0]["host"] db = self.databases[0]["db"] c = conn.cursor() @@ -191,7 +191,7 @@ def test_issue_33(self): @unittest2.skip("This test requires manual intervention") def test_issue_35(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() print("sudo killall -9 mysqld within the next 10 seconds") try: @@ -237,7 +237,7 @@ def test_issue_36(self): del self.connections[1] def test_issue_37(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() self.assertEqual(1, c.execute("SELECT @foo")) self.assertEqual((None,), c.fetchone()) @@ -245,7 +245,7 @@ def test_issue_37(self): c.execute("set @foo = 'bar'") def test_issue_38(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() datum = "a" * 1024 * 1023 # reduced size for most default mysql installs @@ -259,7 +259,7 @@ def test_issue_38(self): c.execute("drop table issue38") def disabled_test_issue_54(self): - conn = self.connections[0] + conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -278,7 +278,7 @@ def disabled_test_issue_54(self): class TestGitHubIssues(base.PyMySQLTestCase): def test_issue_66(self): """ 'Connection' object has no attribute 'insert_id' """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor() self.assertEqual(0, conn.insert_id()) try: @@ -294,7 +294,7 @@ def test_issue_66(self): def test_issue_79(self): """ Duplicate field overwrites the previous one in the result of DictCursor """ - conn = self.connections[0] + conn = self.connect() c = conn.cursor(pymysql.cursors.DictCursor) with warnings.catch_warnings(): @@ -321,7 +321,7 @@ def test_issue_79(self): def test_issue_95(self): """ Leftover trailing OK packet for "CALL my_sp" queries """ - conn = self.connections[0] + conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): warnings.filterwarnings("ignore") @@ -366,7 +366,7 @@ def test_issue_114(self): def test_issue_175(self): """ The number of fields returned by server is read in wrong way """ - conn = self.connections[0] + conn = self.connect() cur = conn.cursor() for length in (200, 300): columns = ', '.join('c{0} integer'.format(i) for i in range(length)) diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index 85fd94ea..eafa6e19 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -10,7 +10,7 @@ class TestLoadLocal(base.PyMySQLTestCase): def test_no_file(self): """Test load local infile when the file does not exist""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") try: @@ -26,7 +26,7 @@ def test_no_file(self): def test_load_file(self): """Test load local infile with a valid file""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -44,7 +44,7 @@ def test_load_file(self): def test_unbuffered_load_file(self): """Test unbuffered load local infile with a valid file""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor(cursors.SSCursor) c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), @@ -66,7 +66,7 @@ def test_unbuffered_load_file(self): def test_load_warnings(self): """Test load local infile produces the appropriate warnings""" - conn = self.connections[0] + conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), From 15eaee5f0a40125275de2c53671fff888b8c7439 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Dec 2018 21:27:40 +0900 Subject: [PATCH 059/292] Close connection on unknown error (#759) Fixes #275 --- pymysql/connections.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pymysql/connections.py b/pymysql/connections.py index 7f5646e8..64246114 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -694,6 +694,10 @@ def _read_bytes(self, num_bytes): raise err.OperationalError( CR.CR_SERVER_LOST, "Lost connection to MySQL server during query (%s)" % (e,)) + except BaseException: + # Don't convert unknown exception to MySQLError. + self._force_close() + raise if len(data) < num_bytes: self._force_close() raise err.OperationalError( From 1ef6c587337bd6ff3272c2c4771948676fd2a9e6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Mon, 17 Dec 2018 21:58:01 +0900 Subject: [PATCH 060/292] Deprecate context manager interface of Connection (#742) --- pymysql/connections.py | 3 +++ pymysql/tests/test_connection.py | 30 +++++++++++++++++------------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 64246114..c8ed12a3 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -492,6 +492,9 @@ def cursor(self, cursor=None): def __enter__(self): """Context manager that returns a Cursor""" + warnings.warn( + "Context manager API of Connection object is deprecated; Use conn.begin()", + DeprecationWarning) return self.cursor() def __exit__(self, exc, value, traceback): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 3f162780..7f31f6c2 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -7,6 +7,8 @@ from pymysql._compat import text_type from pymysql.constants import CLIENT +import pytest + class TempUser: def __init__(self, c, user, db, auth=None, authdata=None, password=None): @@ -451,21 +453,23 @@ def test_read_default_group(self): def test_context(self): with self.assertRaises(ValueError): c = self.connect() + with pytest.warns(DeprecationWarning): + with c as cur: + cur.execute('create table test ( a int ) ENGINE=InnoDB') + c.begin() + cur.execute('insert into test values ((1))') + raise ValueError('pseudo abort') + c = self.connect() + with pytest.warns(DeprecationWarning): with c as cur: - cur.execute('create table test ( a int ) ENGINE=InnoDB') - c.begin() + cur.execute('select count(*) from test') + self.assertEqual(0, cur.fetchone()[0]) cur.execute('insert into test values ((1))') - raise ValueError('pseudo abort') - c.commit() - c = self.connect() - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(0, cur.fetchone()[0]) - cur.execute('insert into test values ((1))') - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(1,cur.fetchone()[0]) - cur.execute('drop table test') + with pytest.warns(DeprecationWarning): + with c as cur: + cur.execute('select count(*) from test') + self.assertEqual(1,cur.fetchone()[0]) + cur.execute('drop table test') def test_set_charset(self): c = self.connect() From 4e6d5f3d1db5241c47fe8742fe951c49819fd44c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 18 Dec 2018 17:17:22 +0900 Subject: [PATCH 061/292] Update URLs in README --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 163c4f2a..cee5053d 100644 --- a/README.rst +++ b/README.rst @@ -127,15 +127,15 @@ This example will print: Resources --------- -* DB-API 2.0: http://www.python.org/dev/peps/pep-0249 +* DB-API 2.0: https://www.python.org/dev/peps/pep-0249/ -* MySQL Reference Manuals: http://dev.mysql.com/doc/ +* MySQL Reference Manuals: https://dev.mysql.com/doc/ * MySQL client/server protocol: - http://dev.mysql.com/doc/internals/en/client-server-protocol.html + https://dev.mysql.com/doc/internals/en/client-server-protocol.html * "Connector" channel in MySQL Community Slack: - http://lefred.be/mysql-community-on-slack/ + https://lefred.be/mysql-community-on-slack/ * PyMySQL mailing list: https://groups.google.com/forum/#!forum/pymysql-users From 91954b1ca84b454ec995c65a2c4b08fc67934067 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 18 Dec 2018 18:32:49 +0900 Subject: [PATCH 062/292] Update charsets based on MySQL 8.0.12 (#733) --- pymysql/charset.py | 90 ++++++++---------------------------------- pymysql/connections.py | 2 +- pymysql/converters.py | 15 ------- 3 files changed, 17 insertions(+), 90 deletions(-) diff --git a/pymysql/charset.py b/pymysql/charset.py index 968376cf..07d80638 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -18,7 +18,7 @@ def __repr__(self): @property def encoding(self): name = self.name - if name == 'utf8mb4': + if name in ('utf8mb4', 'utf8mb3'): return 'utf8' return name @@ -30,18 +30,18 @@ def is_binary(self): class Charsets: def __init__(self): self._by_id = {} + self._by_name = {} def add(self, c): self._by_id[c.id] = c + if c.is_default: + self._by_name[c.name] = c def by_id(self, id): return self._by_id[id] def by_name(self, name): - name = name.lower() - for c in self._by_id.values(): - if c.name == name and c.is_default: - return c + return self._by_name.get(name.lower()) _charsets = Charsets() """ @@ -89,7 +89,6 @@ def by_name(self, name): _charsets.add(Charset(32, 'armscii8', 'armscii8_general_ci', 'Yes')) _charsets.add(Charset(33, 'utf8', 'utf8_general_ci', 'Yes')) _charsets.add(Charset(34, 'cp1250', 'cp1250_czech_cs', '')) -_charsets.add(Charset(35, 'ucs2', 'ucs2_general_ci', 'Yes')) _charsets.add(Charset(36, 'cp866', 'cp866_general_ci', 'Yes')) _charsets.add(Charset(37, 'keybcs2', 'keybcs2_general_ci', 'Yes')) _charsets.add(Charset(38, 'macce', 'macce_general_ci', 'Yes')) @@ -108,13 +107,9 @@ def by_name(self, name): _charsets.add(Charset(51, 'cp1251', 'cp1251_general_ci', 'Yes')) _charsets.add(Charset(52, 'cp1251', 'cp1251_general_cs', '')) _charsets.add(Charset(53, 'macroman', 'macroman_bin', '')) -_charsets.add(Charset(54, 'utf16', 'utf16_general_ci', 'Yes')) -_charsets.add(Charset(55, 'utf16', 'utf16_bin', '')) _charsets.add(Charset(57, 'cp1256', 'cp1256_general_ci', 'Yes')) _charsets.add(Charset(58, 'cp1257', 'cp1257_bin', '')) _charsets.add(Charset(59, 'cp1257', 'cp1257_general_ci', 'Yes')) -_charsets.add(Charset(60, 'utf32', 'utf32_general_ci', 'Yes')) -_charsets.add(Charset(61, 'utf32', 'utf32_bin', '')) _charsets.add(Charset(63, 'binary', 'binary', 'Yes')) _charsets.add(Charset(64, 'armscii8', 'armscii8_bin', '')) _charsets.add(Charset(65, 'ascii', 'ascii_bin', '')) @@ -128,6 +123,7 @@ def by_name(self, name): _charsets.add(Charset(73, 'keybcs2', 'keybcs2_bin', '')) _charsets.add(Charset(74, 'koi8r', 'koi8r_bin', '')) _charsets.add(Charset(75, 'koi8u', 'koi8u_bin', '')) +_charsets.add(Charset(76, 'utf8', 'utf8_tolower_ci', '')) _charsets.add(Charset(77, 'latin2', 'latin2_bin', '')) _charsets.add(Charset(78, 'latin5', 'latin5_bin', '')) _charsets.add(Charset(79, 'latin7', 'latin7_bin', '')) @@ -141,7 +137,6 @@ def by_name(self, name): _charsets.add(Charset(87, 'gbk', 'gbk_bin', '')) _charsets.add(Charset(88, 'sjis', 'sjis_bin', '')) _charsets.add(Charset(89, 'tis620', 'tis620_bin', '')) -_charsets.add(Charset(90, 'ucs2', 'ucs2_bin', '')) _charsets.add(Charset(91, 'ujis', 'ujis_bin', '')) _charsets.add(Charset(92, 'geostd8', 'geostd8_general_ci', 'Yes')) _charsets.add(Charset(93, 'geostd8', 'geostd8_bin', '')) @@ -151,67 +146,6 @@ def by_name(self, name): _charsets.add(Charset(97, 'eucjpms', 'eucjpms_japanese_ci', 'Yes')) _charsets.add(Charset(98, 'eucjpms', 'eucjpms_bin', '')) _charsets.add(Charset(99, 'cp1250', 'cp1250_polish_ci', '')) -_charsets.add(Charset(101, 'utf16', 'utf16_unicode_ci', '')) -_charsets.add(Charset(102, 'utf16', 'utf16_icelandic_ci', '')) -_charsets.add(Charset(103, 'utf16', 'utf16_latvian_ci', '')) -_charsets.add(Charset(104, 'utf16', 'utf16_romanian_ci', '')) -_charsets.add(Charset(105, 'utf16', 'utf16_slovenian_ci', '')) -_charsets.add(Charset(106, 'utf16', 'utf16_polish_ci', '')) -_charsets.add(Charset(107, 'utf16', 'utf16_estonian_ci', '')) -_charsets.add(Charset(108, 'utf16', 'utf16_spanish_ci', '')) -_charsets.add(Charset(109, 'utf16', 'utf16_swedish_ci', '')) -_charsets.add(Charset(110, 'utf16', 'utf16_turkish_ci', '')) -_charsets.add(Charset(111, 'utf16', 'utf16_czech_ci', '')) -_charsets.add(Charset(112, 'utf16', 'utf16_danish_ci', '')) -_charsets.add(Charset(113, 'utf16', 'utf16_lithuanian_ci', '')) -_charsets.add(Charset(114, 'utf16', 'utf16_slovak_ci', '')) -_charsets.add(Charset(115, 'utf16', 'utf16_spanish2_ci', '')) -_charsets.add(Charset(116, 'utf16', 'utf16_roman_ci', '')) -_charsets.add(Charset(117, 'utf16', 'utf16_persian_ci', '')) -_charsets.add(Charset(118, 'utf16', 'utf16_esperanto_ci', '')) -_charsets.add(Charset(119, 'utf16', 'utf16_hungarian_ci', '')) -_charsets.add(Charset(120, 'utf16', 'utf16_sinhala_ci', '')) -_charsets.add(Charset(128, 'ucs2', 'ucs2_unicode_ci', '')) -_charsets.add(Charset(129, 'ucs2', 'ucs2_icelandic_ci', '')) -_charsets.add(Charset(130, 'ucs2', 'ucs2_latvian_ci', '')) -_charsets.add(Charset(131, 'ucs2', 'ucs2_romanian_ci', '')) -_charsets.add(Charset(132, 'ucs2', 'ucs2_slovenian_ci', '')) -_charsets.add(Charset(133, 'ucs2', 'ucs2_polish_ci', '')) -_charsets.add(Charset(134, 'ucs2', 'ucs2_estonian_ci', '')) -_charsets.add(Charset(135, 'ucs2', 'ucs2_spanish_ci', '')) -_charsets.add(Charset(136, 'ucs2', 'ucs2_swedish_ci', '')) -_charsets.add(Charset(137, 'ucs2', 'ucs2_turkish_ci', '')) -_charsets.add(Charset(138, 'ucs2', 'ucs2_czech_ci', '')) -_charsets.add(Charset(139, 'ucs2', 'ucs2_danish_ci', '')) -_charsets.add(Charset(140, 'ucs2', 'ucs2_lithuanian_ci', '')) -_charsets.add(Charset(141, 'ucs2', 'ucs2_slovak_ci', '')) -_charsets.add(Charset(142, 'ucs2', 'ucs2_spanish2_ci', '')) -_charsets.add(Charset(143, 'ucs2', 'ucs2_roman_ci', '')) -_charsets.add(Charset(144, 'ucs2', 'ucs2_persian_ci', '')) -_charsets.add(Charset(145, 'ucs2', 'ucs2_esperanto_ci', '')) -_charsets.add(Charset(146, 'ucs2', 'ucs2_hungarian_ci', '')) -_charsets.add(Charset(147, 'ucs2', 'ucs2_sinhala_ci', '')) -_charsets.add(Charset(159, 'ucs2', 'ucs2_general_mysql500_ci', '')) -_charsets.add(Charset(160, 'utf32', 'utf32_unicode_ci', '')) -_charsets.add(Charset(161, 'utf32', 'utf32_icelandic_ci', '')) -_charsets.add(Charset(162, 'utf32', 'utf32_latvian_ci', '')) -_charsets.add(Charset(163, 'utf32', 'utf32_romanian_ci', '')) -_charsets.add(Charset(164, 'utf32', 'utf32_slovenian_ci', '')) -_charsets.add(Charset(165, 'utf32', 'utf32_polish_ci', '')) -_charsets.add(Charset(166, 'utf32', 'utf32_estonian_ci', '')) -_charsets.add(Charset(167, 'utf32', 'utf32_spanish_ci', '')) -_charsets.add(Charset(168, 'utf32', 'utf32_swedish_ci', '')) -_charsets.add(Charset(169, 'utf32', 'utf32_turkish_ci', '')) -_charsets.add(Charset(170, 'utf32', 'utf32_czech_ci', '')) -_charsets.add(Charset(171, 'utf32', 'utf32_danish_ci', '')) -_charsets.add(Charset(172, 'utf32', 'utf32_lithuanian_ci', '')) -_charsets.add(Charset(173, 'utf32', 'utf32_slovak_ci', '')) -_charsets.add(Charset(174, 'utf32', 'utf32_spanish2_ci', '')) -_charsets.add(Charset(175, 'utf32', 'utf32_roman_ci', '')) -_charsets.add(Charset(176, 'utf32', 'utf32_persian_ci', '')) -_charsets.add(Charset(177, 'utf32', 'utf32_esperanto_ci', '')) -_charsets.add(Charset(178, 'utf32', 'utf32_hungarian_ci', '')) -_charsets.add(Charset(179, 'utf32', 'utf32_sinhala_ci', '')) _charsets.add(Charset(192, 'utf8', 'utf8_unicode_ci', '')) _charsets.add(Charset(193, 'utf8', 'utf8_icelandic_ci', '')) _charsets.add(Charset(194, 'utf8', 'utf8_latvian_ci', '')) @@ -232,6 +166,10 @@ def by_name(self, name): _charsets.add(Charset(209, 'utf8', 'utf8_esperanto_ci', '')) _charsets.add(Charset(210, 'utf8', 'utf8_hungarian_ci', '')) _charsets.add(Charset(211, 'utf8', 'utf8_sinhala_ci', '')) +_charsets.add(Charset(212, 'utf8', 'utf8_german2_ci', '')) +_charsets.add(Charset(213, 'utf8', 'utf8_croatian_ci', '')) +_charsets.add(Charset(214, 'utf8', 'utf8_unicode_520_ci', '')) +_charsets.add(Charset(215, 'utf8', 'utf8_vietnamese_ci', '')) _charsets.add(Charset(223, 'utf8', 'utf8_general_mysql500_ci', '')) _charsets.add(Charset(224, 'utf8mb4', 'utf8mb4_unicode_ci', '')) _charsets.add(Charset(225, 'utf8mb4', 'utf8mb4_icelandic_ci', '')) @@ -257,14 +195,18 @@ def by_name(self, name): _charsets.add(Charset(245, 'utf8mb4', 'utf8mb4_croatian_ci', '')) _charsets.add(Charset(246, 'utf8mb4', 'utf8mb4_unicode_520_ci', '')) _charsets.add(Charset(247, 'utf8mb4', 'utf8mb4_vietnamese_ci', '')) - +_charsets.add(Charset(248, 'gb18030', 'gb18030_chinese_ci', 'Yes')) +_charsets.add(Charset(249, 'gb18030', 'gb18030_bin', '')) +_charsets.add(Charset(250, 'gb18030', 'gb18030_unicode_520_ci', '')) +_charsets.add(Charset(255, 'utf8mb4', 'utf8mb4_0900_ai_ci', '')) charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id +#TODO: remove this def charset_to_encoding(name): """Convert MySQL's charset name to Python's codec name""" - if name == 'utf8mb4': + if name in ('utf8mb4', 'utf8mb3'): return 'utf8' return name diff --git a/pymysql/connections.py b/pymysql/connections.py index c8ed12a3..2e4122b4 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -88,7 +88,7 @@ def _makefile(sock, mode): } -DEFAULT_CHARSET = 'utf8mb4' # TODO: change to utf8mb4 +DEFAULT_CHARSET = 'utf8mb4' MAX_PACKET_LEN = 2**24-1 diff --git a/pymysql/converters.py b/pymysql/converters.py index bf1db9d7..ce2be062 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -354,21 +354,6 @@ def through(x): convert_bit = through -def convert_characters(connection, field, data): - field_charset = charset_by_id(field.charsetnr).name - encoding = charset_to_encoding(field_charset) - if field.flags & FLAG.SET: - return convert_set(data.decode(encoding)) - if field.flags & FLAG.BINARY: - return data - - if connection.use_unicode: - data = data.decode(encoding) - elif connection.charset != field_charset: - data = data.decode(encoding) - data = data.encode(connection.encoding) - return data - encoders = { bool: escape_bool, int: escape_int, From 4bf04205359251ca35015a01359d6687d0b4bbcf Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 18 Dec 2018 19:25:12 +0900 Subject: [PATCH 063/292] Optional cryptography (#760) --- Pipfile | 1 + README.rst | 5 +++++ docs/source/user/installation.rst | 5 +++++ pymysql/_auth.py | 12 +++++++++--- pymysql/util.py | 9 --------- setup.py | 6 +++--- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Pipfile b/Pipfile index a18fa51a..07939550 100644 --- a/Pipfile +++ b/Pipfile @@ -10,3 +10,4 @@ cryptography = "*" pytest = "*" unittest2 = "*" twine = "*" +flake8 = "*" diff --git a/README.rst b/README.rst index cee5053d..cd1e3bd9 100644 --- a/README.rst +++ b/README.rst @@ -61,6 +61,11 @@ You can install it with pip:: $ python3 -m pip install PyMySQL +To use "sha256_password" or "caching_sha2_password" for authenticate, +you need to install additional dependency:: + + $ python3 -m pip install PyMySQL[rsa] + Documentation ------------- diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index 8a81fddb..656e3c7a 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -8,6 +8,11 @@ The last stable release is available on PyPI and can be installed with ``pip``:: $ python3 -m pip install PyMySQL +To use "sha256_password" or "caching_sha2_password" for authenticate, +you need to install additional dependency:: + + $ python3 -m pip install PyMySQL[rsa] + Requirements ------------- diff --git a/pymysql/_auth.py b/pymysql/_auth.py index e0a48f74..199f36c7 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -7,9 +7,13 @@ from .util import byte2int, int2byte -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization, hashes -from cryptography.hazmat.primitives.asymmetric import padding +try: + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import serialization, hashes + from cryptography.hazmat.primitives.asymmetric import padding + _have_cryptography = True +except ImportError: + _have_cryptography = False from functools import partial import hashlib @@ -134,6 +138,8 @@ def sha2_rsa_encrypt(password, salt, public_key): Used for sha256_password and caching_sha2_password. """ + if not _have_cryptography: + raise RuntimeError("cryptography is required for sha256_password or caching_sha2_password") message = _xor_password(password + b'\0', salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) return rsa_key.encrypt( diff --git a/pymysql/util.py b/pymysql/util.py index 3e82ac7b..04683f83 100644 --- a/pymysql/util.py +++ b/pymysql/util.py @@ -11,12 +11,3 @@ def byte2int(b): def int2byte(i): return struct.pack("!B", i) - -def join_bytes(bs): - if len(bs) == 0: - return "" - else: - rv = bs[0] - for b in bs[1:]: - rv += b - return rv diff --git a/setup.py b/setup.py index 14650d1c..71bc09b8 100755 --- a/setup.py +++ b/setup.py @@ -17,9 +17,9 @@ description='Pure Python MySQL Driver', long_description=readme, packages=find_packages(exclude=['tests*', 'pymysql.tests*']), - install_requires=[ - "cryptography", - ], + extras_require={ + "rsa": ["cryptography"], + }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python :: 2', From 8eee26692726e7c923186322a68e15e4d98c138e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 18 Dec 2018 20:35:06 +0900 Subject: [PATCH 064/292] v0.9.3 --- CHANGELOG | 11 +++++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d73ddd79..9ddb8f0b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,16 @@ # Changes +## 0.9.3 + +Release date: 2018-12-18 + +* cryptography dependency is optional now. +* Fix old_password (used before MySQL 4.1) support. +* Deprecate old_password. +* Stop sending ``sys.argv[0]`` for connection attribute "program_name". +* Close connection when unknown error is happened. +* Deprecate context manager API of Connection object. + ## 0.9.2 Release date: 2018-07-04 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index b79b4b83..0cb5006c 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 9, 2, None) +VERSION = (0, 9, 3, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 71bc09b8..6157243a 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = "0.9.2" +version = "0.9.3" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() From fe0cd60e6f9e3bc0fb81f76f7e4fa30a1e1b34cc Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 18 Dec 2018 21:41:06 +0900 Subject: [PATCH 065/292] Drop Python 3.4 support (#762) --- .travis.yml | 10 +++---- README.rst | 2 +- docs/source/user/installation.rst | 2 +- pymysql/converters.py | 43 +------------------------------ pymysql/err.py | 7 +---- pymysql/tests/test_err.py | 6 ----- setup.py | 1 - 7 files changed, 9 insertions(+), 62 deletions(-) diff --git a/.travis.yml b/.travis.yml index f4a7cc74..32e0e4b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,12 @@ # vim: sw=2 ts=2 sts=2 expandtab -sudo: required +dist: xenial language: python +cache: pip + services: - docker -cache: pip - matrix: include: - env: @@ -17,7 +17,7 @@ matrix: python: "3.6" - env: - DB=mariadb:10.1 - python: "pypy" + python: "pypy3.5" - env: - DB=mariadb:10.2 python: "2.7" @@ -32,7 +32,7 @@ matrix: python: "3.6" - env: - DB=mysql:5.7 - python: "3.4" + python: "3.7" - env: - DB=mysql:8.0 - TEST_AUTH=yes diff --git a/README.rst b/README.rst index cd1e3bd9..175bf43e 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ Requirements * Python -- one of the following: - - CPython_ : 2.7 and >= 3.4 + - CPython_ : 2.7 and >= 3.5 - PyPy_ : Latest version * MySQL Server -- one of the following: diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index 656e3c7a..d95961c6 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -18,7 +18,7 @@ Requirements * Python -- one of the following: - - CPython_ >= 2.7 or >= 3.4 + - CPython_ >= 2.7 or >= 3.5 - Latest PyPy_ * MySQL Server -- one of the following: diff --git a/pymysql/converters.py b/pymysql/converters.py index ce2be062..be2e697c 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -301,46 +301,6 @@ def convert_date(obj): return obj -def convert_mysql_timestamp(timestamp): - """Convert a MySQL TIMESTAMP to a Timestamp object. - - MySQL >= 4.1 returns TIMESTAMP in the same format as DATETIME: - - >>> mysql_timestamp_converter('2007-02-25 22:32:17') - datetime.datetime(2007, 2, 25, 22, 32, 17) - - MySQL < 4.1 uses a big string of numbers: - - >>> mysql_timestamp_converter('20070225223217') - datetime.datetime(2007, 2, 25, 22, 32, 17) - - Illegal values are returned as None: - - >>> mysql_timestamp_converter('2007-02-31 22:32:17') is None - True - >>> mysql_timestamp_converter('00000000000000') is None - True - - """ - if not PY2 and isinstance(timestamp, (bytes, bytearray)): - timestamp = timestamp.decode('ascii') - if timestamp[4] == '-': - return convert_datetime(timestamp) - timestamp += "0"*(14-len(timestamp)) # padding - year, month, day, hour, minute, second = \ - int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]), \ - int(timestamp[8:10]), int(timestamp[10:12]), int(timestamp[12:14]) - try: - return datetime.datetime(year, month, day, hour, minute, second) - except ValueError: - return timestamp - -def convert_set(s): - if isinstance(s, (bytes, bytearray)): - return set(s.split(b",")) - return set(s.split(",")) - - def through(x): return x @@ -388,11 +348,10 @@ def through(x): FIELD_TYPE.LONGLONG: int, FIELD_TYPE.INT24: int, FIELD_TYPE.YEAR: int, - FIELD_TYPE.TIMESTAMP: convert_mysql_timestamp, + FIELD_TYPE.TIMESTAMP: convert_datetime, FIELD_TYPE.DATETIME: convert_datetime, FIELD_TYPE.TIME: convert_timedelta, FIELD_TYPE.DATE: convert_date, - FIELD_TYPE.SET: convert_set, FIELD_TYPE.BLOB: through, FIELD_TYPE.TINY_BLOB: through, FIELD_TYPE.MEDIUM_BLOB: through, diff --git a/pymysql/err.py b/pymysql/err.py index fbc60558..e93ba9be 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -99,11 +99,6 @@ def _map_error(exc, *errors): def raise_mysql_exception(data): errno = struct.unpack(' Date: Wed, 19 Dec 2018 17:45:43 +0900 Subject: [PATCH 066/292] Remove context manager (#763) --- pymysql/connections.py | 14 -------------- pymysql/tests/test_connection.py | 21 --------------------- 2 files changed, 35 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 2e4122b4..af074e21 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -490,20 +490,6 @@ def cursor(self, cursor=None): return cursor(self) return self.cursorclass(self) - def __enter__(self): - """Context manager that returns a Cursor""" - warnings.warn( - "Context manager API of Connection object is deprecated; Use conn.begin()", - DeprecationWarning) - return self.cursor() - - def __exit__(self, exc, value, traceback): - """On successful exit, commit. On exception, rollback""" - if exc: - self.rollback() - else: - self.commit() - # The following methods are INTERNAL USE ONLY (called from Cursor) def query(self, sql, unbuffered=False): # if DEBUG: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 7f31f6c2..7c258df8 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -450,27 +450,6 @@ def test_read_default_group(self): ) self.assertTrue(conn.open) - def test_context(self): - with self.assertRaises(ValueError): - c = self.connect() - with pytest.warns(DeprecationWarning): - with c as cur: - cur.execute('create table test ( a int ) ENGINE=InnoDB') - c.begin() - cur.execute('insert into test values ((1))') - raise ValueError('pseudo abort') - c = self.connect() - with pytest.warns(DeprecationWarning): - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(0, cur.fetchone()[0]) - cur.execute('insert into test values ((1))') - with pytest.warns(DeprecationWarning): - with c as cur: - cur.execute('select count(*) from test') - self.assertEqual(1,cur.fetchone()[0]) - cur.execute('drop table test') - def test_set_charset(self): c = self.connect() c.set_charset('utf8mb4') From a500fcd64d4500417540a2a2ff7b16a88d1872ad Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 19 Dec 2018 21:39:58 +0900 Subject: [PATCH 067/292] Remove unittest2 dependency (#764) --- .travis.yml | 2 +- Pipfile | 13 ---- pymysql/tests/__init__.py | 4 +- pymysql/tests/base.py | 26 ++++++-- pymysql/tests/test_basic.py | 6 +- pymysql/tests/test_connection.py | 64 +++++++++---------- pymysql/tests/test_err.py | 4 +- pymysql/tests/test_issues.py | 7 +- pymysql/tests/test_nextset.py | 4 +- pymysql/tests/thirdparty/__init__.py | 5 +- .../thirdparty/test_MySQLdb/capabilities.py | 5 +- .../tests/thirdparty/test_MySQLdb/dbapi20.py | 6 +- .../test_MySQLdb/test_MySQLdb_capabilities.py | 5 +- .../test_MySQLdb/test_MySQLdb_dbapi20.py | 5 +- .../test_MySQLdb/test_MySQLdb_nonstandard.py | 5 +- runtests.py | 31 --------- tox.ini | 7 +- 17 files changed, 77 insertions(+), 122 deletions(-) delete mode 100644 Pipfile delete mode 100755 runtests.py diff --git a/.travis.yml b/.travis.yml index 32e0e4b5..b8b07b90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ before_script: - export COVERALLS_PARALLEL=true script: - - coverage run ./runtests.py + - pytest -v --cov-config .coveragerc pymysql - if [ "${TEST_AUTH}" = "yes" ]; then pytest -v --cov-config .coveragerc tests; fi diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 07939550..00000000 --- a/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -cryptography = "*" - -[dev-packages] -pytest = "*" -unittest2 = "*" -twine = "*" -flake8 = "*" diff --git a/pymysql/tests/__init__.py b/pymysql/tests/__init__.py index a9f5a4bf..91ad5763 100644 --- a/pymysql/tests/__init__.py +++ b/pymysql/tests/__init__.py @@ -14,5 +14,5 @@ from pymysql.tests.thirdparty import * if __name__ == "__main__": - import unittest2 - unittest2.main() + import unittest + unittest.main() diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index 091cccfa..22bed9d8 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -3,14 +3,33 @@ import os import re import warnings - -import unittest2 +import unittest import pymysql from .._compat import CPYTHON -class PyMySQLTestCase(unittest2.TestCase): +if CPYTHON: + import atexit + gc.set_debug(gc.DEBUG_UNCOLLECTABLE) + + @atexit.register + def report_uncollectable(): + import gc + if not gc.garbage: + print("No garbages!") + return + print('uncollectable objects') + for obj in gc.garbage: + print(obj) + if hasattr(obj, '__dict__'): + print(obj.__dict__) + for ref in gc.get_referrers(obj): + print("referrer:", ref) + print('---') + + +class PyMySQLTestCase(unittest.TestCase): # You can specify your test environment creating a file named # "databases.json" or editing the `databases` variable below. fname = os.path.join(os.path.dirname(__file__), "databases.json") @@ -97,7 +116,6 @@ def safe_gc_collect(self): """Ensure cycles are collected via gc. Runs additional times on non-CPython platforms. - """ gc.collect() if not CPYTHON: diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 940661f7..c2d53904 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -4,7 +4,7 @@ import time import warnings -from unittest2 import SkipTest +import pytest from pymysql import util import pymysql.cursors @@ -143,7 +143,7 @@ def test_datetime_microseconds(self): conn = self.connect() if not self.mysql_server_is(conn, (5, 6, 4)): - raise SkipTest("target backend does not support microseconds") + pytest.skip("target backend does not support microseconds") c = conn.cursor() dt = datetime.datetime(2013, 11, 12, 9, 9, 9, 123450) c.execute("create table test_datetime (id int, ts datetime(6))") @@ -256,7 +256,7 @@ def test_json(self): args["charset"] = "utf8mb4" conn = pymysql.connect(**args) if not self.mysql_server_is(conn, (5, 7, 0)): - raise SkipTest("JSON type is not supported on MySQL <= 5.6") + pytest.skip("JSON type is not supported on MySQL <= 5.6") self.safe_create_table(conn, "test_json", """\ create table test_json ( diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 7c258df8..e4d24c44 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,7 +1,7 @@ import datetime import sys import time -import unittest2 +import pytest import pymysql from pymysql.tests import base from pymysql._compat import text_type @@ -100,14 +100,14 @@ class TestAuthentication(base.PyMySQLTestCase): def test_plugin(self): conn = self.connect() if not self.mysql_server_is(conn, (5, 5, 0)): - raise unittest2.SkipTest("MySQL-5.5 required for plugins") + pytest.skip("MySQL-5.5 required for plugins") cur = conn.cursor() cur.execute("select plugin from mysql.user where concat(user, '@', host)=current_user()") for r in cur: self.assertIn(conn._auth_plugin_name, (r[0], 'mysql_native_password')) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(socket_found, "socket plugin already installed") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(socket_found, reason="socket plugin already installed") def testSocketAuthInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -124,13 +124,13 @@ def testSocketAuthInstallPlugin(self): self.realtestSocketAuth() except pymysql.err.InternalError: TestAuthentication.socket_found = False - raise unittest2.SkipTest('we couldn\'t install the socket plugin') + pytest.skip('we couldn\'t install the socket plugin') finally: if TestAuthentication.socket_found: cur.execute("uninstall plugin %s" % self.socket_plugin_name) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(socket_found, "no socket plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not socket_found, reason="no socket plugin") def testSocketAuth(self): self.realtestSocketAuth() @@ -179,8 +179,8 @@ def __init__(self, con): self.con=con - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(two_questions_found, "two_questions plugin already installed") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(two_questions_found, reason="two_questions plugin already installed") def testDialogAuthTwoQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -189,13 +189,13 @@ def testDialogAuthTwoQuestionsInstallPlugin(self): TestAuthentication.two_questions_found = True self.realTestDialogAuthTwoQuestions() except pymysql.err.InternalError: - raise unittest2.SkipTest('we couldn\'t install the two_questions plugin') + pytest.skip('we couldn\'t install the two_questions plugin') finally: if TestAuthentication.two_questions_found: cur.execute("uninstall plugin two_questions") - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(two_questions_found, "no two questions auth plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not two_questions_found, reason="no two questions auth plugin") def testDialogAuthTwoQuestions(self): self.realTestDialogAuthTwoQuestions() @@ -209,8 +209,8 @@ def realTestDialogAuthTwoQuestions(self): pymysql.connect(user='pymysql_2q', **self.db) pymysql.connect(user='pymysql_2q', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(three_attempts_found, "three_attempts plugin already installed") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(three_attempts_found, reason="three_attempts plugin already installed") def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -219,13 +219,13 @@ def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): TestAuthentication.three_attempts_found = True self.realTestDialogAuthThreeAttempts() except pymysql.err.InternalError: - raise unittest2.SkipTest('we couldn\'t install the three_attempts plugin') + pytest.skip('we couldn\'t install the three_attempts plugin') finally: if TestAuthentication.three_attempts_found: cur.execute("uninstall plugin three_attempts") - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(three_attempts_found, "no three attempts plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not three_attempts_found, reason="no three attempts plugin") def testDialogAuthThreeAttempts(self): self.realTestDialogAuthThreeAttempts() @@ -250,10 +250,10 @@ def realTestDialogAuthThreeAttempts(self): with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipIf(pam_found, "pam plugin already installed") - @unittest2.skipIf(os.environ.get('PASSWORD') is None, "PASSWORD env var required") - @unittest2.skipIf(os.environ.get('PAMSERVICE') is None, "PAMSERVICE env var required") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(pam_found, reason="pam plugin already installed") + @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") + @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") def testPamAuthInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -262,16 +262,16 @@ def testPamAuthInstallPlugin(self): TestAuthentication.pam_found = True self.realTestPamAuth() except pymysql.err.InternalError: - raise unittest2.SkipTest('we couldn\'t install the auth_pam plugin') + pytest.skip('we couldn\'t install the auth_pam plugin') finally: if TestAuthentication.pam_found: cur.execute("uninstall plugin pam") - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(pam_found, "no pam plugin") - @unittest2.skipIf(os.environ.get('PASSWORD') is None, "PASSWORD env var required") - @unittest2.skipIf(os.environ.get('PAMSERVICE') is None, "PAMSERVICE env var required") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not pam_found, reason="no pam plugin") + @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") + @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") def testPamAuth(self): self.realTestPamAuth() @@ -311,16 +311,16 @@ def realTestPamAuth(self): # select old_password("crummy p\tassword"); #| old_password("crummy p\tassword") | #| 2a01785203b08770 | - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(mysql_old_password_found, "no mysql_old_password plugin") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not mysql_old_password_found, reason="no mysql_old_password plugin") def testMySQLOldPasswordAuth(self): conn = self.connect() if self.mysql_server_is(conn, (5, 7, 0)): - raise unittest2.SkipTest('Old passwords aren\'t supported in 5.7') + pytest.skip('Old passwords aren\'t supported in 5.7') # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") # from login in MySQL-5.6 if self.mysql_server_is(conn, (5, 6, 0)): - raise unittest2.SkipTest('Old passwords don\'t authenticate in 5.6') + pytest.skip('Old passwords don\'t authenticate in 5.6') db = self.db.copy() db['password'] = "crummy p\tassword" c = conn.cursor() @@ -354,8 +354,8 @@ def testMySQLOldPasswordAuth(self): cur.execute("SELECT VERSION()") c.execute('set global secure_auth=%r' % secure_auth_setting) - @unittest2.skipUnless(socket_auth, "connection to unix_socket required") - @unittest2.skipUnless(sha256_password_found, "no sha256 password authentication plugin found") + @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") + @pytest.mark.skipif(not sha256_password_found, reason="no sha256 password authentication plugin found") def testAuthSHA256(self): conn = self.connect() c = conn.cursor() diff --git a/pymysql/tests/test_err.py b/pymysql/tests/test_err.py index 895c2afb..bb6a5c49 100644 --- a/pymysql/tests/test_err.py +++ b/pymysql/tests/test_err.py @@ -1,4 +1,4 @@ -import unittest2 +import unittest from pymysql import err @@ -6,7 +6,7 @@ __all__ = ["TestRaiseException"] -class TestRaiseException(unittest2.TestCase): +class TestRaiseException(unittest.TestCase): def test_raise_mysql_exception(self): data = b"\xff\x15\x04#28000Access denied" diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 8dca31b7..05ecf286 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -3,11 +3,12 @@ import warnings import sys +import pytest + import pymysql from pymysql import cursors from pymysql._compat import text_type from pymysql.tests import base -import unittest2 try: import imp @@ -145,7 +146,7 @@ def test_issue_16(self): finally: c.execute("drop table issue16") - @unittest2.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") + @pytest.mark.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") def test_issue_17(self): """could not connect mysql use passwod""" conn = self.connect() @@ -189,7 +190,7 @@ def test_issue_33(self): c.execute(u"select name from hei\xdfe") self.assertEqual(u"Pi\xdfata", c.fetchone()[0]) - @unittest2.skip("This test requires manual intervention") + @pytest.mark.skip("This test requires manual intervention") def test_issue_35(self): conn = self.connect() c = conn.cursor() diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index 99844107..d5467b11 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -1,4 +1,4 @@ -import unittest2 +import pytest import pymysql from pymysql import util @@ -50,7 +50,7 @@ def test_ok_and_next(self): self.assertEqual([(2,)], list(cur)) self.assertFalse(bool(cur.nextset())) - @unittest2.expectedFailure + @pytest.mark.xfail def test_multi_cursor(self): con = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) cur1 = con.cursor() diff --git a/pymysql/tests/thirdparty/__init__.py b/pymysql/tests/thirdparty/__init__.py index 6d59e112..7a613478 100644 --- a/pymysql/tests/thirdparty/__init__.py +++ b/pymysql/tests/thirdparty/__init__.py @@ -1,8 +1,5 @@ from .test_MySQLdb import * if __name__ == "__main__": - try: - import unittest2 as unittest - except ImportError: - import unittest + import unittest unittest.main() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index bcf9eecb..6be9d1ba 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -6,10 +6,7 @@ """ import sys from time import time -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest PY2 = sys.version_info[0] == 2 diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 3cbf2263..1cc202e2 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -14,12 +14,8 @@ __version__ = '$Revision$'[11:-2] __author__ = 'Stuart Bishop ' -try: - import unittest2 as unittest -except ImportError: - import unittest - import time +import unittest # $Log$ # Revision 1.1.2.1 2006/02/25 03:44:32 adustman diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 0fc5e831..13b43d3f 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -1,8 +1,5 @@ from . import capabilities -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import pymysql from pymysql.tests import base import warnings diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index a2669162..2c9a0600 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -2,10 +2,7 @@ import pymysql from pymysql.tests import base -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest class test_MySQLdb(dbapi20.DatabaseAPI20Test): diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py index 17fc2cde..5c739a42 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py @@ -1,8 +1,5 @@ import sys -try: - import unittest2 as unittest -except ImportError: - import unittest +import unittest import pymysql _mysql = pymysql diff --git a/runtests.py b/runtests.py deleted file mode 100755 index ea3d9e8d..00000000 --- a/runtests.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -import unittest2 - -from pymysql._compat import PYPY, JYTHON, IRONPYTHON - -#import pymysql -#pymysql.connections.DEBUG = True -#pymysql._auth.DEBUG = True - -if not (PYPY or JYTHON or IRONPYTHON): - import atexit - import gc - gc.set_debug(gc.DEBUG_UNCOLLECTABLE) - - @atexit.register - def report_uncollectable(): - import gc - if not gc.garbage: - print("No garbages!") - return - print('uncollectable objects') - for obj in gc.garbage: - print(obj) - if hasattr(obj, '__dict__'): - print(obj.__dict__) - for ref in gc.get_referrers(obj): - print("referrer:", ref) - print('---') - -import pymysql.tests -unittest2.main(pymysql.tests, verbosity=2) diff --git a/tox.ini b/tox.ini index e2f2917c..d13e49f4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py27,py34,py35,py36,py37,pypy,pypy3 +envlist = py27,py35,py36,py37,pypy,pypy3 [testenv] -commands = coverage run ./runtests.py -deps = unittest2 - coverage +commands = pytest -v pymysql/tests/ +deps = coverage pytest passenv = USER PASSWORD PAMSERVICE From a4d6630d8c9f5b098d4bcb0f2dce3eec39190aad Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 19 Dec 2018 22:09:59 +0900 Subject: [PATCH 068/292] travis: Remove unittest2 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b8b07b90..b2f91aab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ matrix: # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version # really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't install: - - pip install -U coveralls unittest2 coverage cryptography pytest pytest-cov + - pip install -U coveralls coverage cryptography pytest pytest-cov before_script: - ./.travis/initializedb.sh @@ -55,9 +55,9 @@ before_script: - export COVERALLS_PARALLEL=true script: - - pytest -v --cov-config .coveragerc pymysql + - pytest -v --cov --cov-config .coveragerc pymysql - if [ "${TEST_AUTH}" = "yes" ]; - then pytest -v --cov-config .coveragerc tests; + then pytest -v --cov --cov-config .coveragerc tests; fi - if [ ! -z "${DB}" ]; then docker logs mysqld; From 6d85d0ad1133e419b53b6517cf87b1ccdebf7ab1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 13:42:45 +0900 Subject: [PATCH 069/292] requirements->requirements-dev --- requirements.txt => requirements-dev.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename requirements.txt => requirements-dev.txt (100%) diff --git a/requirements.txt b/requirements-dev.txt similarity index 100% rename from requirements.txt rename to requirements-dev.txt From 0b91c36715c2755cb593d2f0fc616e38c22d97c4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 13:43:04 +0900 Subject: [PATCH 070/292] Add pytest to requirements-dev --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 70f05161..5e85e522 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ cryptography - +pytest From 3b51a620ce825c826bb217e9326d52a849298798 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 13:59:02 +0900 Subject: [PATCH 071/292] Use set -exv in initializedb.sh --- .travis/initializedb.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index d9897e49..251c1a71 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -1,9 +1,7 @@ #!/bin/bash -#debug -set -x -#verbose -set -v +#error,debug,verbose, +set -exv if [ ! -z "${DB}" ]; then # disable existing database server in case of accidential connection From 60b6dfe9a3c844aa666e54d9486f2c9af72c6362 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 20 Dec 2018 14:21:27 +0900 Subject: [PATCH 072/292] fix travis --- .travis/initializedb.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 251c1a71..3f71a549 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -4,12 +4,7 @@ set -exv if [ ! -z "${DB}" ]; then - # disable existing database server in case of accidential connection - sudo service mysql stop - - docker pull ${DB} docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} - sleep 10 mysql() { docker exec mysqld mysql "${@}" @@ -17,16 +12,11 @@ if [ ! -z "${DB}" ]; then while : do sleep 5 - mysql -e 'select version()' - if [ $? = 0 ]; then - break - fi + mysql -e 'select version()' && break echo "server logs" docker logs --tail 5 mysqld done - mysql -e 'select VERSION()' - if [ $DB == 'mysql:8.0' ]; then WITH_PLUGIN='with mysql_native_password' mysql -e 'SET GLOBAL local_infile=on' From a3c06319791009436315eb9387ca3fa00bfd4b17 Mon Sep 17 00:00:00 2001 From: Scott Cole Date: Mon, 7 Jan 2019 00:23:43 -0800 Subject: [PATCH 073/292] Update error message for cryptography package (#769) --- pymysql/_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 199f36c7..aa082dfe 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -139,7 +139,7 @@ def sha2_rsa_encrypt(password, salt, public_key): Used for sha256_password and caching_sha2_password. """ if not _have_cryptography: - raise RuntimeError("cryptography is required for sha256_password or caching_sha2_password") + raise RuntimeError("'cryptography' package is required for sha256_password or caching_sha2_password auth methods") message = _xor_password(password + b'\0', salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) return rsa_key.encrypt( From 501abf0d33b23e95f1a2ec3b40d4b33b513c1c33 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 16 Jan 2019 21:37:10 +0900 Subject: [PATCH 074/292] travis: Sleep more (#772) --- .travis/initializedb.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 3f71a549..9ec35d31 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -4,7 +4,9 @@ set -exv if [ ! -z "${DB}" ]; then + docker pull ${DB} docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} + sleep 15 mysql() { docker exec mysqld mysql "${@}" From 34adc28316e2e92de7c46e81fa24435c5a562914 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 17 Jan 2019 20:08:18 +0900 Subject: [PATCH 075/292] travis: faster DB startup wait (#773) --- .travis/initializedb.sh | 96 +++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 9ec35d31..17d06100 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -1,62 +1,46 @@ #!/bin/bash -#error,debug,verbose, -set -exv - -if [ ! -z "${DB}" ]; then - docker pull ${DB} - docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} - sleep 15 - - mysql() { - docker exec mysqld mysql "${@}" - } - while : - do - sleep 5 - mysql -e 'select version()' && break - echo "server logs" - docker logs --tail 5 mysqld - done - - if [ $DB == 'mysql:8.0' ]; then - WITH_PLUGIN='with mysql_native_password' - mysql -e 'SET GLOBAL local_infile=on' - docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - - # Test user for auth test - mysql -e ' - CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", - nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", - nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" - PASSWORD EXPIRE NEVER;' - mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' - else - WITH_PLUGIN='' - fi - - mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' - mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - - mysql -u root -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" - mysql -u root -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - - cp .travis/docker.json pymysql/tests/databases.json +set -ex + +docker pull ${DB} +docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} + +mysql() { + docker exec mysqld mysql "${@}" +} +while : +do + sleep 3 + mysql --protocol=tcp -e 'select version()' && break +done +docker logs mysqld + +if [ $DB == 'mysql:8.0' ]; then + WITH_PLUGIN='with mysql_native_password' + mysql -e 'SET GLOBAL local_infile=on' + docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" + + # Test user for auth test + mysql -e ' + CREATE USER + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", + nopass_sha256 IDENTIFIED WITH "sha256_password", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER;' + mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' else - cat ~/.my.cnf + WITH_PLUGIN='' +fi - mysql -e 'select VERSION()' - mysql -e 'create database test1 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' - mysql -e 'create database test2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' +mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' +mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - mysql -u root -e "create user test2 identified by 'some password'; grant all on test2.* to test2;" - mysql -u root -e "create user test2@localhost identified by 'some password'; grant all on test2.* to test2@localhost;" +mysql -u root -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" +mysql -u root -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - cp .travis/database.json pymysql/tests/databases.json -fi +cp .travis/docker.json pymysql/tests/databases.json From d063f68e890739f0e582f1b8049e1af170c94d77 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 17 Jan 2019 21:39:18 +0900 Subject: [PATCH 076/292] Remove auto show warnings (#774) --- pymysql/cursors.py | 31 ------------------------------- pymysql/tests/test_basic.py | 12 ------------ pymysql/tests/test_issues.py | 25 ------------------------- pymysql/tests/test_load_local.py | 24 ------------------------ 4 files changed, 92 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index a6d645d4..b3a690e6 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -2,7 +2,6 @@ from __future__ import print_function, absolute_import from functools import partial import re -import warnings from ._compat import range_type, text_type, PY2 from . import err @@ -35,8 +34,6 @@ class Cursor(object): #: Default value of max_allowed_packet is 1048576. max_stmt_length = 1024000 - _defer_warnings = False - def __init__(self, connection): self.connection = connection self.description = None @@ -46,7 +43,6 @@ def __init__(self, connection): self._executed = None self._result = None self._rows = None - self._warnings_handled = False def close(self): """ @@ -90,9 +86,6 @@ def _nextset(self, unbuffered=False): """Get the next query set""" conn = self._get_db() current_result = self._result - # for unbuffered queries warnings are only available once whole result has been read - if unbuffered: - self._show_warnings() if current_result is None or current_result is not conn._result: return None if not current_result.has_next: @@ -347,26 +340,6 @@ def _do_get_result(self): self.description = result.description self.lastrowid = result.insert_id self._rows = result.rows - self._warnings_handled = False - - if not self._defer_warnings: - self._show_warnings() - - def _show_warnings(self): - if self._warnings_handled: - return - self._warnings_handled = True - if self._result and (self._result.has_next or not self._result.warning_count): - return - ws = self._get_db().show_warnings() - if ws is None: - return - for w in ws: - msg = w[-1] - if PY2: - if isinstance(msg, unicode): - msg = msg.encode('utf-8', 'replace') - warnings.warn(err.Warning(*w[1:3]), stacklevel=4) def __iter__(self): return iter(self.fetchone, None) @@ -427,8 +400,6 @@ class SSCursor(Cursor): possible to scroll backwards, as only the current row is held in memory. """ - _defer_warnings = True - def _conv_row(self, row): return row @@ -468,7 +439,6 @@ def fetchone(self): self._check_executed() row = self.read_next() if row is None: - self._show_warnings() return None self.rownumber += 1 return row @@ -502,7 +472,6 @@ def fetchmany(self, size=None): for i in range_type(size): row = self.read_next() if row is None: - self._show_warnings() break rows.append(row) self.rownumber += 1 diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index c2d53904..38c8cb64 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -2,7 +2,6 @@ import datetime import json import time -import warnings import pytest @@ -378,14 +377,3 @@ def test_issue_288(self): age = values(age)""")) cursor.execute('commit') self._verify_records(data) - - def test_warnings(self): - con = self.connect() - cur = con.cursor() - with warnings.catch_warnings(record=True) as ws: - warnings.simplefilter("always") - cur.execute("drop table if exists no_exists_table") - self.assertEqual(len(ws), 1) - self.assertEqual(ws[0].category, pymysql.Warning) - if u"no_exists_table" not in str(ws[0].message): - self.fail("'no_exists_table' not in %s" % (str(ws[0].message),)) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 05ecf286..3775f314 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -485,28 +485,3 @@ def test_issue_363(self): # don't assert the exact internal binary value, as it could # vary across implementations self.assertTrue(isinstance(row[0], bytes)) - - def test_issue_491(self): - """ Test warning propagation """ - conn = pymysql.connect(charset="utf8", **self.databases[0]) - - with warnings.catch_warnings(): - # Ignore all warnings other than pymysql generated ones - warnings.simplefilter("ignore") - warnings.simplefilter("error", category=pymysql.Warning) - - # verify for both buffered and unbuffered cursor types - for cursor_class in (cursors.Cursor, cursors.SSCursor): - c = conn.cursor(cursor_class) - try: - c.execute("SELECT CAST('124b' AS SIGNED)") - c.fetchall() - except pymysql.Warning as e: - # Warnings should have errorcode and string message, just like exceptions - self.assertEqual(len(e.args), 2) - self.assertEqual(e.args[0], 1292) - self.assertTrue(isinstance(e.args[1], text_type)) - else: - self.fail("Should raise Warning") - finally: - c.close() diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index eafa6e19..30186e3a 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -2,7 +2,6 @@ from pymysql.tests import base import os -import warnings __all__ = ["TestLoadLocal"] @@ -64,29 +63,6 @@ def test_unbuffered_load_file(self): c = conn.cursor() c.execute("DROP TABLE test_load_local") - def test_load_warnings(self): - """Test load local infile produces the appropriate warnings""" - conn = self.connect() - c = conn.cursor() - c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") - filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'data', - 'load_local_warn_data.txt') - try: - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - c.execute( - ("LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + - "test_load_local FIELDS TERMINATED BY ','").format(filename) - ) - self.assertEqual(w[0].category, Warning) - expected_message = "Incorrect integer value" - if expected_message not in str(w[-1].message): - self.fail("%r not in %r" % (expected_message, w[-1].message)) - finally: - c.execute("DROP TABLE test_load_local") - c.close() - if __name__ == "__main__": import unittest From 3539f87ed59b443916f195967ece4ba5b32cd8e3 Mon Sep 17 00:00:00 2001 From: Deneby67 Date: Tue, 5 Feb 2019 10:55:38 +0300 Subject: [PATCH 077/292] Optimize reading huge packet (#779) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index af074e21..7cf3fef6 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -638,7 +638,7 @@ def _read_packet(self, packet_type=MysqlPacket): :raise OperationalError: If the connection to the MySQL server is lost. :raise InternalError: If the packet sequence number is wrong. """ - buff = b'' + buff = bytearray() while True: packet_header = self._read_bytes(4) #if DEBUG: dump_packet(packet_header) @@ -666,7 +666,7 @@ def _read_packet(self, packet_type=MysqlPacket): if bytes_to_read < MAX_PACKET_LEN: break - packet = packet_type(buff, self.encoding) + packet = packet_type(bytes(buff), self.encoding) packet.check_error() return packet From 3f04fcaa8fc9d96f717b76e58553e87f890c6ba4 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Fri, 1 Mar 2019 16:03:51 +0800 Subject: [PATCH 078/292] Fix typo in CHANGELOG (#783) --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 9ddb8f0b..a7272aa9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -51,7 +51,7 @@ Release date: 2018-05-07 * Many test suite improvements, especially adding MySQL 8.0 and using Docker. Thanks to Daniel Black. -* Droppped support for old Python and MySQL whih is not tested long time. +* Droppped support for old Python and MySQL which is not tested long time. ## 0.8 From 188efc56ef0e3c5e6745e52907e8677bf86f7487 Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Mon, 4 Mar 2019 13:56:15 +0800 Subject: [PATCH 079/292] Fix minor typos in comments (#784) --- pymysql/connections.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 7cf3fef6..f54344ed 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -140,7 +140,7 @@ class Connection(object): Specifies my.cnf file to read these parameters from under the [client] section. :param conv: Conversion dictionary to use instead of the default one. - This is used to provide custom marshalling and unmarshaling of types. + This is used to provide custom marshalling and unmarshalling of types. See converters. :param use_unicode: Whether or not to default to unicode strings. @@ -159,14 +159,14 @@ class Connection(object): :param local_infile: Boolean to enable the use of LOAD DATA LOCAL command. (default: False) :param max_allowed_packet: Max size of packet sent to server in bytes. (default: 16MB) Only used to limit size of "LOAD LOCAL INFILE" data packet smaller than default (16KB). - :param defer_connect: Don't explicitly connect on contruction - wait for connect call. + :param defer_connect: Don't explicitly connect on construction - wait for connect call. (default: False) :param auth_plugin_map: A dict of plugin names to a class that processes that plugin. The class will take the Connection object as the argument to the constructor. The class needs an authenticate method taking an authentication packet as an argument. For the dialog plugin, a prompt(echo, prompt) method can be used (if no authenticate method) for returning a string from the user. (experimental) - :param server_public_key: SHA256 authenticaiton plugin public key value. (default: None) + :param server_public_key: SHA256 authentication plugin public key value. (default: None) :param db: Alias for database. (for compatibility to MySQLdb) :param passwd: Alias for password. (for compatibility to MySQLdb) :param binary_prefix: Add _binary prefix on bytes and bytearray. (default: False) @@ -622,9 +622,9 @@ def connect(self, sock=None): def write_packet(self, payload): """Writes an entire "mysql packet" in its entirety to the network - addings its length and sequence number. + adding its length and sequence number. """ - # Internal note: when you build packet manualy and calls _write_bytes() + # Internal note: when you build packet manually and calls _write_bytes() # directly, you should set self._next_seq_id properly. data = pack_int24(len(payload)) + int2byte(self._next_seq_id) + payload if DEBUG: dump_packet(data) From 3674bc6fd064bf88524e839c07690e8c35223709 Mon Sep 17 00:00:00 2001 From: Ludger Heide Date: Wed, 13 Mar 2019 18:05:11 +0100 Subject: [PATCH 080/292] Setting SO_KEEPALIVE only for TCP (#785) --- pymysql/connections.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index f54344ed..d9ade9a2 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -575,8 +575,9 @@ def connect(self, sock=None): self.host_info = "socket %s:%d" % (self.host, self.port) if DEBUG: print('connected using socket') sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.settimeout(None) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + self._sock = sock self._rfile = _makefile(sock, 'rb') self._next_seq_id = 0 From 383c0438fe74464ad65b9850bd13f310de7a878e Mon Sep 17 00:00:00 2001 From: parthgandhi Date: Fri, 30 Aug 2019 12:00:19 +0530 Subject: [PATCH 081/292] fix spelling mistakes in changelog (#808) --- CHANGELOG | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a7272aa9..503b043a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -15,7 +15,7 @@ Release date: 2018-12-18 Release date: 2018-07-04 -* Disalbled unintentinally enabled debug log +* Disabled unintentinally enabled debug log * Removed unintentionally installed tests @@ -51,7 +51,7 @@ Release date: 2018-05-07 * Many test suite improvements, especially adding MySQL 8.0 and using Docker. Thanks to Daniel Black. -* Droppped support for old Python and MySQL which is not tested long time. +* Dropped support for old Python and MySQL which is not tested long time. ## 0.8 From f8c31d40c5abda9e03de5df34ea692b428fb6677 Mon Sep 17 00:00:00 2001 From: ppd0705 Date: Fri, 13 Sep 2019 13:16:40 +0800 Subject: [PATCH 082/292] Fix error packet handling for SSCursor (#810) --- pymysql/connections.py | 5 ++++- pymysql/protocol.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index d9ade9a2..93efd9be 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -668,7 +668,10 @@ def _read_packet(self, packet_type=MysqlPacket): break packet = packet_type(bytes(buff), self.encoding) - packet.check_error() + if packet.is_error_packet(): + if self._result is not None and self._result.unbuffered_active is True: + self._result.unbuffered_active = False + packet.raise_for_error() return packet def _read_bytes(self, num_bytes): diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 8ccf7c4d..e302edab 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -213,11 +213,14 @@ def is_error_packet(self): def check_error(self): if self.is_error_packet(): - self.rewind() - self.advance(1) # field_count == error (we already know that) - errno = self.read_uint16() - if DEBUG: print("errno =", errno) - err.raise_mysql_exception(self._data) + self.raise_for_error() + + def raise_for_error(self): + self.rewind() + self.advance(1) # field_count == error (we already know that) + errno = self.read_uint16() + if DEBUG: print("errno =", errno) + err.raise_mysql_exception(self._data) def dump(self): dump_packet(self._data) From 18b0bcb9bf0561fa2d191ff946e97d99a244b211 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 21 Sep 2019 18:16:35 +0900 Subject: [PATCH 083/292] use better format for float (#806) --- pymysql/converters.py | 7 ++++++- .../thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index be2e697c..889cd7a2 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -54,7 +54,12 @@ def escape_int(value, mapping=None): return str(value) def escape_float(value, mapping=None): - return ('%.15g' % value) + s = repr(value) + if s in ('inf', 'nan'): + raise ProgrammingError("%s can not be used with MySQL" % s) + if 'e' not in s: + s += 'e0' + return s _escape_table = [unichr(x) for x in range(128)] _escape_table[0] = u'\\0' diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 13b43d3f..8c1dd535 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -90,7 +90,7 @@ def test_literal_int(self): self.assertTrue("2" == self.connection.literal(2)) def test_literal_float(self): - self.assertTrue("3.1415" == self.connection.literal(3.1415)) + self.assertEqual("3.1415e0", self.connection.literal(3.1415)) def test_literal_string(self): self.assertTrue("'foo'" == self.connection.literal("foo")) From ec8306b2331881bedc3aa19c13ec1400aa939ec3 Mon Sep 17 00:00:00 2001 From: brettl-sprint <57368682+brettl-sprint@users.noreply.github.com> Date: Thu, 7 Nov 2019 00:33:14 -0500 Subject: [PATCH 084/292] Updates link to error handling documentation (#821) --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 93efd9be..22738606 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1,7 +1,7 @@ # Python implementation of the MySQL client-server protocol # http://dev.mysql.com/doc/internals/en/client-server-protocol.html # Error codes: -# http://dev.mysql.com/doc/refman/5.5/en/error-messages-client.html +# https://dev.mysql.com/doc/refman/5.5/en/error-handling.html from __future__ import print_function from ._compat import PY2, range_type, text_type, str_type, JYTHON, IRONPYTHON From 9dcefe9814bb053b1718a4407bb06790cb5de955 Mon Sep 17 00:00:00 2001 From: Bastien Vallet Date: Thu, 7 Nov 2019 16:55:03 +0100 Subject: [PATCH 085/292] Add Python 3.8 support (#822) --- .travis.yml | 6 +++--- setup.py | 1 + tox.ini | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index b2f91aab..69ca5317 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,13 +17,13 @@ matrix: python: "3.6" - env: - DB=mariadb:10.1 - python: "pypy3.5" + python: "pypy3" - env: - DB=mariadb:10.2 python: "2.7" - env: - DB=mariadb:10.3 - python: "3.7-dev" + python: "3.7" - env: - DB=mysql:5.5 python: "3.5" @@ -36,7 +36,7 @@ matrix: - env: - DB=mysql:8.0 - TEST_AUTH=yes - python: "3.7-dev" + python: "3.8" - env: - DB=mysql:8.0 - TEST_AUTH=yes diff --git a/setup.py b/setup.py index b888c01f..6a9b2d80 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', diff --git a/tox.ini b/tox.ini index d13e49f4..95430ae8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py35,py36,py37,pypy,pypy3 +envlist = py{27,35,36,37,38,py,py3} [testenv] commands = pytest -v pymysql/tests/ From c3e5a63514c57d1f4c9d5e7bf4b7e10b0608b0e1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 13 Nov 2019 14:14:58 +0900 Subject: [PATCH 086/292] Use OperationalError for unknown error with code>1000. (#823) Fixes #816. --- pymysql/err.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymysql/err.py b/pymysql/err.py index e93ba9be..8ca23655 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -100,5 +100,7 @@ def _map_error(exc, *errors): def raise_mysql_exception(data): errno = struct.unpack(' Date: Thu, 21 Nov 2019 15:51:19 +0900 Subject: [PATCH 087/292] Use cp1252 encoding for latin1 charset (#824) --- pymysql/charset.py | 14 ++++++-------- pymysql/converters.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pymysql/charset.py b/pymysql/charset.py index 07d80638..d3ced67c 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -20,6 +20,12 @@ def encoding(self): name = self.name if name in ('utf8mb4', 'utf8mb3'): return 'utf8' + if name == 'latin1': + return 'cp1252' + if name == 'koi8r': + return 'koi8_r' + if name == 'koi8u': + return 'koi8_u' return name @property @@ -202,11 +208,3 @@ def by_name(self, name): charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id - - -#TODO: remove this -def charset_to_encoding(name): - """Convert MySQL's charset name to Python's codec name""" - if name in ('utf8mb4', 'utf8mb3'): - return 'utf8' - return name diff --git a/pymysql/converters.py b/pymysql/converters.py index 889cd7a2..2793a2ae 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -6,7 +6,7 @@ import time from .constants import FIELD_TYPE, FLAG -from .charset import charset_by_id, charset_to_encoding +from .charset import charset_by_id def escape_item(val, charset, mapping=None): From 2330bf798894b35f3fcc796e9c5df5bac44105ab Mon Sep 17 00:00:00 2001 From: brettl-sprint <57368682+brettl-sprint@users.noreply.github.com> Date: Thu, 21 Nov 2019 11:33:03 -0500 Subject: [PATCH 088/292] Raise more graceful error when port is not int (#820) --- pymysql/connections.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymysql/connections.py b/pymysql/connections.py index 22738606..d74af4fa 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -254,6 +254,8 @@ def _config(key, arg): self.host = host or "localhost" self.port = port or 3306 + if type(self.port) is not int: + raise ValueError("port should be of type int") self.user = user or DEFAULT_USER self.password = password or b"" if isinstance(self.password, text_type): From 0f4d45e5a20b47959ba7d16f130cbc0c7ce8506c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 26 Nov 2019 20:56:41 +0900 Subject: [PATCH 089/292] Fix decimal literal. (#828) Fixes #818. --- pymysql/converters.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 2793a2ae..efb0e4d4 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -159,6 +159,11 @@ def escape_date(obj, mapping=None): def escape_struct_time(obj, mapping=None): return escape_datetime(datetime.datetime(*obj[:6])) + +def Decimal2Literal(o, d): + return format(o, "f") + + def _convert_second_fraction(s): if not s: return 0 @@ -337,7 +342,7 @@ def through(x): datetime.timedelta: escape_timedelta, datetime.time: escape_time, time.struct_time: escape_struct_time, - Decimal: escape_object, + Decimal: Decimal2Literal, } if not PY2 or JYTHON or IRONPYTHON: From c3c87a7e773dbb09def0b081c70dd55fe83b9633 Mon Sep 17 00:00:00 2001 From: Sebastien Volle Date: Wed, 4 Dec 2019 11:31:11 +0100 Subject: [PATCH 090/292] Fix connection timeout error messages (#830) Fix inconsistency between connection read/write timeout error messages and actual value checks. --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index d74af4fa..a1cd8c25 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -267,10 +267,10 @@ def _config(key, arg): raise ValueError("connect_timeout should be >0 and <=31536000") self.connect_timeout = connect_timeout or None if read_timeout is not None and read_timeout <= 0: - raise ValueError("read_timeout should be >= 0") + raise ValueError("read_timeout should be > 0") self._read_timeout = read_timeout if write_timeout is not None and write_timeout <= 0: - raise ValueError("write_timeout should be >= 0") + raise ValueError("write_timeout should be > 0") self._write_timeout = write_timeout if charset: self.charset = charset From 577276a952499fdc4c6786e164dfb3f12dad7272 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 8 Dec 2019 01:06:34 +1100 Subject: [PATCH 091/292] Fix typo. (#833) Closes #832 --- pymysql/cursors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index b3a690e6..033b5e7f 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -8,7 +8,7 @@ #: Regular expression for :meth:`Cursor.executemany`. -#: executemany only suports simple bulk insert. +#: executemany only supports simple bulk insert. #: You can use it to load large dataset. RE_INSERT_VALUES = re.compile( r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" + From 6faa8b679df6ca97a83f3028228eaa2803278171 Mon Sep 17 00:00:00 2001 From: Vilhelm Prytz Date: Wed, 11 Dec 2019 23:24:33 +0100 Subject: [PATCH 092/292] Remove unused imports (#835) --- pymysql/_auth.py | 3 +-- pymysql/converters.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index aa082dfe..a7fdaa48 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -1,8 +1,7 @@ """ Implements auth methods """ -from ._compat import text_type, PY2 -from .constants import CLIENT +from ._compat import PY2 from .err import OperationalError from .util import byte2int, int2byte diff --git a/pymysql/converters.py b/pymysql/converters.py index efb0e4d4..b084ed2f 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -5,8 +5,7 @@ import re import time -from .constants import FIELD_TYPE, FLAG -from .charset import charset_by_id +from .constants import FIELD_TYPE def escape_item(val, charset, mapping=None): From 9f1b8569032ec7eaff36fe9ef5e40f82c47260b2 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 14 Feb 2020 10:18:40 +0000 Subject: [PATCH 093/292] Fix test suite compatibility with MySQL 8 (#840) MySQL 8 deprecates the use of display format for int columns: https://dev.mysql.com/doc/refman/8.0/en/numeric-type-syntax.html This results in warnings being generated during test suite execution which results in test failures. Drop use of display widths - they don't materially change the tests so this should be safe across all MySQL versions and variants. --- pymysql/tests/test_basic.py | 2 +- pymysql/tests/test_issues.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 38c8cb64..aa23e065 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -289,7 +289,7 @@ def setUp(self): self.safe_create_table(conn, 'bulkinsert', """\ CREATE TABLE bulkinsert ( -id int(11), +id int, name char(20), age int, height int, diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 3775f314..604aeaff 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -79,8 +79,8 @@ def test_issue_8(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists test") - c.execute("""CREATE TABLE `test` (`station` int(10) NOT NULL DEFAULT '0', `dh` -datetime NOT NULL DEFAULT '2015-01-01 00:00:00', `echeance` int(1) NOT NULL + c.execute("""CREATE TABLE `test` (`station` int NOT NULL DEFAULT '0', `dh` +datetime NOT NULL DEFAULT '2015-01-01 00:00:00', `echeance` int NOT NULL DEFAULT '0', `me` double DEFAULT NULL, `mo` double DEFAULT NULL, PRIMARY KEY (`station`,`dh`,`echeance`)) ENGINE=MyISAM DEFAULT CHARSET=latin1;""") try: From 8f9060042f0987656039d0588a54b6df30d3ba57 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 Mar 2020 18:15:10 +0900 Subject: [PATCH 094/292] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..f2bd4d30 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,39 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Complete steps to reproduce the behavior: + +Schema: + +``` +CREATE DATABASE ... +CREATE TABLE ... +``` + +Code: + +```py +import pymysql +con = pymysql.connect(...) +``` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment** + - OS: [e.g. Windows, Linux] + - Server and version: [e.g. MySQL 8.0.19, MariaDB] + - PyMySQL version: + +**Additional context** +Add any other context about the problem here. From 33bb6b6640bd7004054b105de3da62f489f0df03 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 25 Mar 2020 18:19:49 +0900 Subject: [PATCH 095/292] Remove old ISSUE_TEMPLATE --- .github/ISSUE_TEMPLATE.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 3e0fbe82..00000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,11 +0,0 @@ -This project is maintained one busy person with a frail wife and an infant daughter. -My time and energy is a very limited resource. I'm not a teacher or free tech support. -Don't ask a question here. Don't file an issue until you believe it's a not a problem with your code. -Search for friendly volunteers who can teach you or review your code on ML or Q&A sites. - -See also: https://medium.com/@methane/why-you-must-not-ask-questions-on-github-issues-51d741d83fde - - -If you're sure it's PyMySQL's issue, report the complete steps to reproduce, from creating database. - -I don't have time to investigate your issue from an incomplete code snippet. From d895719372d00378b17a42d60109d10b0d1a10ed Mon Sep 17 00:00:00 2001 From: Uri Date: Wed, 13 May 2020 07:45:43 +0300 Subject: [PATCH 096/292] updated doctored version info for MySQLdb compatibility (#858) Fixes #790 --- pymysql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 0cb5006c..6ffb2ae6 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -108,7 +108,7 @@ def get_client_info(): # for MySQLdb compatibility connect = Connection = Connect # we include a doctored version_info here for MySQLdb compatibility -version_info = (1, 3, 12, "final", 0) +version_info = (1, 3, 13, "final", 0) NULL = "NULL" From 466ecfe61eab666658b6f2141b0dfb457c4c72a5 Mon Sep 17 00:00:00 2001 From: Justin Chang Date: Mon, 13 Jul 2020 00:10:36 -0400 Subject: [PATCH 097/292] Fix InterfaceError response when connection lost (#872) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index a1cd8c25..fe7a2abd 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -739,7 +739,7 @@ def _execute_command(self, command, sql): :raise ValueError: If no username was specified. """ if not self._sock: - raise err.InterfaceError("(0, '')") + raise err.InterfaceError(0, '') # If the last query was unbuffered, make sure it finishes before # sending new commands @@ -1253,7 +1253,7 @@ def __init__(self, filename, connection): def send_data(self): """Send data packets from the local file to the server""" if not self.connection._sock: - raise err.InterfaceError("(0, '')") + raise err.InterfaceError(0, '') conn = self.connection try: From 221d411cb2acfae34d95908aa841f7bb5a1d6e74 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 09:16:05 +0900 Subject: [PATCH 098/292] travis: Use Python 3.9-dev --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 69ca5317..bff6a0ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ matrix: python: "3.5" - env: - DB=mysql:5.6 - python: "3.6" + python: "3.9-dev" - env: - DB=mysql:5.7 python: "3.7" From f75c0024c6bd89a165b559b2bacd7afdb8858cce Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 10:03:05 +0900 Subject: [PATCH 099/292] Update error mapping (#873) --- pymysql/constants/ER.py | 1 - pymysql/err.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/constants/ER.py b/pymysql/constants/ER.py index 79b88afb..ddcc4e90 100644 --- a/pymysql/constants/ER.py +++ b/pymysql/constants/ER.py @@ -1,4 +1,3 @@ - ERROR_FIRST = 1000 HASHCHK = 1000 NISAMCHK = 1001 diff --git a/pymysql/err.py b/pymysql/err.py index 8ca23655..94100cfe 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -83,7 +83,8 @@ def _map_error(exc, *errors): ) _map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL, ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL, - ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW) + ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW, ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, + ER.ILLEGAL_VALUE_FOR_TYPE) _map_error(IntegrityError, ER.DUP_ENTRY, ER.NO_REFERENCED_ROW, ER.NO_REFERENCED_ROW_2, ER.ROW_IS_REFERENCED, ER.ROW_IS_REFERENCED_2, ER.CANNOT_ADD_FOREIGN, ER.BAD_NULL_ERROR) From 73f977029e2c076719a7ea8d0c3df84cb44ebe7c Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Fri, 17 Jul 2020 03:06:23 +0200 Subject: [PATCH 100/292] Support for MariaDB's auth_ed25519 authentication plugin (#786) (#791) --- .travis.yml | 8 +++-- .travis/initializedb.sh | 10 ++++++- README.rst | 5 ++++ pymysql/_auth.py | 60 ++++++++++++++++++++++++++++++++++++++ pymysql/connections.py | 2 ++ requirements-dev.txt | 1 + setup.py | 1 + tests/test_mariadb_auth.py | 23 +++++++++++++++ 8 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 tests/test_mariadb_auth.py diff --git a/.travis.yml b/.travis.yml index bff6a0ee..553d9cd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ matrix: python: "2.7" - env: - DB=mariadb:10.3 + - TEST_MARIADB_AUTH=yes python: "3.7" - env: - DB=mysql:5.5 @@ -46,7 +47,7 @@ matrix: # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version # really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't install: - - pip install -U coveralls coverage cryptography pytest pytest-cov + - pip install -U coveralls coverage cryptography PyNaCl pytest pytest-cov before_script: - ./.travis/initializedb.sh @@ -57,7 +58,10 @@ before_script: script: - pytest -v --cov --cov-config .coveragerc pymysql - if [ "${TEST_AUTH}" = "yes" ]; - then pytest -v --cov --cov-config .coveragerc tests; + then pytest -v --cov --cov-config .coveragerc tests/test_auth.py; + fi + - if [ "${TEST_MARIADB_AUTH}" = "yes" ]; + then pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py; fi - if [ ! -z "${DB}" ]; then docker logs mysqld; diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 17d06100..98c1cd3b 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -6,7 +6,7 @@ docker pull ${DB} docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} mysql() { - docker exec mysqld mysql "${@}" + docker exec -i mysqld mysql "${@}" } while : do @@ -33,6 +33,14 @@ if [ $DB == 'mysql:8.0' ]; then nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" PASSWORD EXPIRE NEVER;' mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' +elif [[ $DB == mariadb:10.* ]] && [ ${DB#mariadb:10.} -ge 3 ]; then + mysql -e ' + INSTALL SONAME "auth_ed25519"; + CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' + # we need to pass the hashed password manually until 10.4, so hide it here + mysql -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql + mysql -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql + WITH_PLUGIN='' else WITH_PLUGIN='' fi diff --git a/README.rst b/README.rst index 175bf43e..7bed7f7e 100644 --- a/README.rst +++ b/README.rst @@ -66,6 +66,11 @@ you need to install additional dependency:: $ python3 -m pip install PyMySQL[rsa] +To use MariaDB's "ed25519" authentication method, you need to install +additional dependency:: + + $ python3 -m pip install PyMySQL[ed25519] + Documentation ------------- diff --git a/pymysql/_auth.py b/pymysql/_auth.py index a7fdaa48..72e9579b 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -113,6 +113,66 @@ def _hash_password_323(password): return struct.pack(">LL", r1, r2) +# MariaDB's client_ed25519-plugin +# https://mariadb.com/kb/en/library/connection/#client_ed25519-plugin + +_nacl_bindings = False + + +def _init_nacl(): + global _nacl_bindings + try: + from nacl import bindings + _nacl_bindings = bindings + except ImportError: + raise RuntimeError("'pynacl' package is required for ed25519_password auth method") + + +def _scalar_clamp(s32): + ba = bytearray(s32) + ba0 = bytes(bytearray([ba[0] & 248])) + ba31 = bytes(bytearray([(ba[31] & 127) | 64])) + return ba0 + bytes(s32[1:31]) + ba31 + + +def ed25519_password(password, scramble): + """Sign a random scramble with elliptic curve Ed25519. + + Secret and public key are derived from password. + """ + # variable names based on rfc8032 section-5.1.6 + # + if not _nacl_bindings: + _init_nacl() + + # h = SHA512(password) + h = hashlib.sha512(password).digest() + + # s = prune(first_half(h)) + s = _scalar_clamp(h[:32]) + + # r = SHA512(second_half(h) || M) + r = hashlib.sha512(h[32:] + scramble).digest() + + # R = encoded point [r]B + r = _nacl_bindings.crypto_core_ed25519_scalar_reduce(r) + R = _nacl_bindings.crypto_scalarmult_ed25519_base_noclamp(r) + + # A = encoded point [s]B + A = _nacl_bindings.crypto_scalarmult_ed25519_base_noclamp(s) + + # k = SHA512(R || A || M) + k = hashlib.sha512(R + A + scramble).digest() + + # S = (k * s + r) mod L + k = _nacl_bindings.crypto_core_ed25519_scalar_reduce(k) + ks = _nacl_bindings.crypto_core_ed25519_scalar_mul(k, s) + S = _nacl_bindings.crypto_core_ed25519_scalar_add(ks, r) + + # signature = R || S + return R + S + + # sha256_password diff --git a/pymysql/connections.py b/pymysql/connections.py index fe7a2abd..75e07f34 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -894,6 +894,8 @@ def _process_auth(self, plugin_name, auth_packet): return _auth.sha256_password_auth(self, auth_packet) elif plugin_name == b"mysql_native_password": data = _auth.scramble_native_password(self.password, auth_packet.read_all()) + elif plugin_name == b'client_ed25519': + data = _auth.ed25519_password(self.password, auth_packet.read_all()) elif plugin_name == b"mysql_old_password": data = _auth.scramble_old_password(self.password, auth_packet.read_all()) + b'\0' elif plugin_name == b"mysql_clear_password": diff --git a/requirements-dev.txt b/requirements-dev.txt index 5e85e522..d65512fb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ cryptography +PyNaCl>=1.4.0 pytest diff --git a/setup.py b/setup.py index 6a9b2d80..3dbdca2d 100755 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ packages=find_packages(exclude=['tests*', 'pymysql.tests*']), extras_require={ "rsa": ["cryptography"], + "ed25519": ["PyNaCl>=1.4.0"], }, classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/test_mariadb_auth.py b/tests/test_mariadb_auth.py new file mode 100644 index 00000000..2f336fec --- /dev/null +++ b/tests/test_mariadb_auth.py @@ -0,0 +1,23 @@ +"""Test for auth methods supported by MariaDB 10.3+""" + +import pymysql + +# pymysql.connections.DEBUG = True +# pymysql._auth.DEBUG = True + +host = "127.0.0.1" +port = 3306 + + +def test_ed25519_no_password(): + con = pymysql.connect(user="nopass_ed25519", host=host, port=port, ssl=None) + con.close() + + +def test_ed25519_password(): # nosec + con = pymysql.connect(user="user_ed25519", password="pass_ed25519", + host=host, port=port, ssl=None) + con.close() + + +# default mariadb docker images aren't configured with SSL From e929f94b2e26bd71eeb8253d16ab5e537f27ae91 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 11:34:04 +0900 Subject: [PATCH 101/292] Update changelog --- CHANGELOG | 18 ++++++++++++++++++ setup.cfg | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 503b043a..8f13b9ff 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,23 @@ # Changes +## 0.10 + +Release date: 2020-07-17 + +* MariaDB ed25519 auth is supported. +* Python 3.4 support is dropped. +* Context manager interface is removed from `Connection`. It will be added + with different meaning. +* MySQL warnings are not shown by default because many user report issue to + PyMySQL issue tracker when they see warning. You need to call "SHOW WARNINGS" + explicitly when you want to see warnings. +* Formatting of float object is changed from "3.14" to "3.14e0". +* Use cp1252 codec for latin1 charset. +* Fix decimal literal. +* TRUNCATED_WRONG_VALUE_FOR_FIELD, and ILLEGAL_VALUE_FOR_TYPE are now + DataError instead of InternalError. + + ## 0.9.3 Release date: 2018-12-18 diff --git a/setup.cfg b/setup.cfg index a26a846b..ca7a9ae3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,5 +13,5 @@ license_file = LICENSE author=yutaka.matsubara author_email=yutaka.matsubara@gmail.com -maintainer=INADA Naoki +maintainer=Inada Naoki maintainer_email=songofacandy@gmail.com From d78581ec246a22758fc397242b74ccaebf07cb62 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 17 Jul 2020 17:34:31 +0900 Subject: [PATCH 102/292] v0.10.0 --- CHANGELOG | 2 +- pymysql/__init__.py | 2 +- setup.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 8f13b9ff..d2e3bd86 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,6 @@ # Changes -## 0.10 +## v0.10.0 Release date: 2020-07-17 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 6ffb2ae6..9c4e8f57 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 9, 3, None) +VERSION = (0, 10, 0, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 3dbdca2d..8c72060f 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = "0.9.3" +version = "0.10.0" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() @@ -30,6 +30,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Intended Audience :: Developers', From a262df2d5f0bf0f39864521f9efcc37dbee5005d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 Jul 2020 16:25:02 +0900 Subject: [PATCH 103/292] Update Changelog --- CHANGELOG | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index d2e3bd86..186c75ea 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,7 +2,9 @@ ## v0.10.0 -Release date: 2020-07-17 +Release date: 2020-07-18 + +This version is the last version supporting Python 2.7. * MariaDB ed25519 auth is supported. * Python 3.4 support is dropped. From 95e313acdbd522827fb2eaea5520c3b280b08195 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 Jul 2020 16:25:58 +0900 Subject: [PATCH 104/292] CHANGELOG -> CHANGELOG.md --- CHANGELOG => CHANGELOG.md | 0 MANIFEST.in | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename CHANGELOG => CHANGELOG.md (100%) diff --git a/CHANGELOG b/CHANGELOG.md similarity index 100% rename from CHANGELOG rename to CHANGELOG.md diff --git a/MANIFEST.in b/MANIFEST.in index 0a520792..e9e1eebc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst LICENSE CHANGELOG +include README.rst LICENSE CHANGELOG.md From a27cbcb9be99b5b0038855eb6313083fe7feed3b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 Jul 2020 16:30:58 +0900 Subject: [PATCH 105/292] fix warning --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ca7a9ae3..db1af545 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ universal = 1 [metadata] license = "MIT" -license_file = LICENSE +license_files = LICENSE author=yutaka.matsubara author_email=yutaka.matsubara@gmail.com From 3e71dd32e8ce868b090c282759eebdeabc960f58 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 28 Jul 2020 13:06:07 +0900 Subject: [PATCH 106/292] Add missing import (#879) Fixes #878 --- pymysql/converters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymysql/converters.py b/pymysql/converters.py index b084ed2f..1b582904 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -5,6 +5,7 @@ import re import time +from .err import ProgrammingError from .constants import FIELD_TYPE From 2f6bb5d720286ef4efb84749877980c3157f15d5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 9 Sep 2020 18:03:05 +0900 Subject: [PATCH 107/292] Fix sha256 and caching_sha2 auth (#892) --- .travis/initializedb.sh | 4 ++-- pymysql/_auth.py | 3 +++ tests/test_auth.py | 17 ++++++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh index 98c1cd3b..6991cfe6 100755 --- a/.travis/initializedb.sh +++ b/.travis/initializedb.sh @@ -27,9 +27,9 @@ if [ $DB == 'mysql:8.0' ]; then # Test user for auth test mysql -e ' CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256", + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" PASSWORD EXPIRE NEVER;' mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 72e9579b..57f9abb1 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -184,6 +184,9 @@ def _roundtrip(conn, send_data): def _xor_password(password, salt): + # Trailing NUL character will be added in Auth Switch Request. + # See https://github.com/mysql/mysql-server/blob/7d10c82196c8e45554f27c00681474a9fb86d137/sql/auth/sha2_password.cc#L939-L945 + salt = salt[:SCRAMBLE_LENGTH] password_bytes = bytearray(password) salt = bytearray(salt) # for PY2 compat. salt_len = len(salt) diff --git a/tests/test_auth.py b/tests/test_auth.py index 7d857344..61957655 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -12,6 +12,9 @@ ca = os.path.expanduser("~/ca.pem") ssl = {'ca': ca, 'check_hostname': False} +pass_sha256 = "pass_sha256_01234567890123456789" +pass_caching_sha2 = "pass_caching_sha2_01234567890123456789" + def test_sha256_no_password(): con = pymysql.connect(user="nopass_sha256", host=host, port=port, ssl=None) @@ -24,12 +27,12 @@ def test_sha256_no_passowrd_ssl(): def test_sha256_password(): - con = pymysql.connect(user="user_sha256", password="pass_sha256", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=None) con.close() def test_sha256_password_ssl(): - con = pymysql.connect(user="user_sha256", password="pass_sha256", host=host, port=port, ssl=ssl) + con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=ssl) con.close() @@ -38,26 +41,26 @@ def test_caching_sha2_no_password(): con.close() -def test_caching_sha2_no_password(): +def test_caching_sha2_no_password_ssl(): con = pymysql.connect(user="nopass_caching_sha2", host=host, port=port, ssl=ssl) con.close() def test_caching_sha2_password(): - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) con.query("FLUSH PRIVILEGES") con.close() def test_caching_sha2_password_ssl(): - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=ssl) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=ssl) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password="pass_caching_sha2", host=host, port=port, ssl=None) + con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) con.query("FLUSH PRIVILEGES") con.close() From 37fd1e1b0126d75d80eef59c053f80634b09bd75 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 10 Sep 2020 16:29:31 +0900 Subject: [PATCH 108/292] v0.10.1 --- CHANGELOG.md | 8 ++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 186c75ea..0d1313aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changes +## v0.10.1 + +Release date: 2020-09-10 + +* Fix missing import of ProgrammingError. (#878) +* Fix auth switch request handling. (#890) + + ## v0.10.0 Release date: 2020-07-18 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 9c4e8f57..5148fa77 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -35,7 +35,7 @@ DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 10, 0, None) +VERSION = (0, 10, 1, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 8c72060f..e35e7b29 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import io from setuptools import setup, find_packages -version = "0.10.0" +version = "0.10.1" with io.open('./README.rst', encoding='utf-8') as f: readme = f.read() From 99b703cccb8011692c398caf0c0fbd97b1355e90 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 10 Dec 2020 13:51:06 +1100 Subject: [PATCH 109/292] Fix test unix_socket for MariaDB-10.4 (#907) --- .travis.yml | 12 +++++++----- pymysql/tests/test_connection.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 553d9cd1..e1398170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # vim: sw=2 ts=2 sts=2 expandtab -dist: xenial +dist: bionic language: python cache: pip @@ -13,16 +13,18 @@ matrix: - DB=mariadb:5.5 python: "3.5" - env: - - DB=mariadb:10.0 + - DB=mariadb:10.2 python: "3.6" - env: - - DB=mariadb:10.1 + - DB=mariadb:10.3 + - TEST_MARIADB_AUTH=yes python: "pypy3" - env: - - DB=mariadb:10.2 + - DB=mariadb:10.4 + - TEST_MARIADB_AUTH=yes python: "2.7" - env: - - DB=mariadb:10.3 + - DB=mariadb:10.5 - TEST_MARIADB_AUTH=yes python: "3.7" - env: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index e4d24c44..51b9f3a5 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -70,7 +70,7 @@ class TestAuthentication(base.PyMySQLTestCase): for r in cur: if (r[1], r[2]) != (u'ACTIVE', u'AUTHENTICATION'): continue - if r[3] == u'auth_socket.so': + if r[3] == u'auth_socket.so' or r[0] == u'unix_socket': socket_plugin_name = r[0] socket_found = True elif r[3] == u'dialog_examples.so': From 907b45374ec8d09f1b83f4afca00b291d09e5d16 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 16:16:13 +0900 Subject: [PATCH 110/292] travis: Remove Python 2.7, 3.5, MySQL 5.5, MariaDB 5.5. (#913) --- .travis.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1398170..aa1f0f34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,6 @@ services: matrix: include: - - env: - - DB=mariadb:5.5 - python: "3.5" - env: - DB=mariadb:10.2 python: "3.6" @@ -19,20 +16,13 @@ matrix: - DB=mariadb:10.3 - TEST_MARIADB_AUTH=yes python: "pypy3" - - env: - - DB=mariadb:10.4 - - TEST_MARIADB_AUTH=yes - python: "2.7" - env: - DB=mariadb:10.5 - TEST_MARIADB_AUTH=yes python: "3.7" - - env: - - DB=mysql:5.5 - python: "3.5" - env: - DB=mysql:5.6 - python: "3.9-dev" + python: "3.9" - env: - DB=mysql:5.7 python: "3.7" @@ -40,10 +30,6 @@ matrix: - DB=mysql:8.0 - TEST_AUTH=yes python: "3.8" - - env: - - DB=mysql:8.0 - - TEST_AUTH=yes - python: "2.7" # different py version from 5.6 and 5.7 as cache seems to be based on py version # http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version From 4e481fa52262e35498cd7ee187ebe4903f9a1771 Mon Sep 17 00:00:00 2001 From: CJ Mauro <57578688+cmauro1@users.noreply.github.com> Date: Sat, 2 Jan 2021 02:18:18 -0500 Subject: [PATCH 111/292] Add context manager support to Connection (#886) --- pymysql/connections.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 75e07f34..9e87e0b0 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -325,7 +325,14 @@ def _config(key, arg): self._sock = None else: self.connect() - + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + del exc_info + self.close() + def _create_ssl_ctx(self, sslp): if isinstance(sslp, ssl.SSLContext): return sslp From b2e580f6edfe4198efe03bff07847580599df649 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 16:18:50 +0900 Subject: [PATCH 112/292] Create FUNDING.yml (#914) --- .github/FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..89fc5cf8 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: [methane] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 2d440dfcbeadb26d13c1779c02872f840ec455f5 Mon Sep 17 00:00:00 2001 From: Uri Date: Sat, 2 Jan 2021 09:33:07 +0200 Subject: [PATCH 113/292] Updated mysqlclient version to 1.4.0 (#885) --- pymysql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 5148fa77..29e6b87c 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -108,7 +108,7 @@ def get_client_info(): # for MySQLdb compatibility connect = Connection = Connect # we include a doctored version_info here for MySQLdb compatibility -version_info = (1, 3, 13, "final", 0) +version_info = (1, 4, 0, "final", 0) NULL = "NULL" From 1489819a47cdeae830002435ac2fc4d43c6c949d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 16:35:24 +0900 Subject: [PATCH 114/292] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7bed7f7e..0a09f892 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ .. image:: https://badge.fury.io/py/PyMySQL.svg :target: https://badge.fury.io/py/PyMySQL -.. image:: https://travis-ci.org/PyMySQL/PyMySQL.svg?branch=master - :target: https://travis-ci.org/PyMySQL/PyMySQL +.. image:: https://travis-ci.com/PyMySQL/PyMySQL.svg?branch=master + :target: https://travis-ci.com/PyMySQL/PyMySQL .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master From aefbdbe1dc6dc022f2b02d2f4c4564d4ec929175 Mon Sep 17 00:00:00 2001 From: Moriyoshi Koizumi Date: Sat, 2 Jan 2021 17:11:19 +0900 Subject: [PATCH 115/292] Add MySQL Connector/Python compatible SSL options. (#903) Add connector-python compatible options. Also fixes #842. https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html --- pymysql/connections.py | 50 ++++++++-- pymysql/tests/test_connection.py | 160 ++++++++++++++++++++++++++++++- requirements-dev.txt | 1 + 3 files changed, 201 insertions(+), 10 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 9e87e0b0..7ecfb616 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -152,6 +152,12 @@ class Connection(object): (default: 10, min: 1, max: 31536000) :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. + :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate + :param ssl_cert: Path to the file that contains a PEM-formatted client certificate + :param ssl_disabled: A boolean value that disables usage of TLS + :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate + :param ssl_verify_cert: Set to true to check the validity of server certificates + :param ssl_verify_identity: Set to true to check the server's identity :param read_default_group: Group to read from in the configuration file. :param compress: Not supported :param named_pipe: Not supported @@ -191,7 +197,9 @@ def __init__(self, host=None, user=None, password="", max_allowed_packet=16*1024*1024, defer_connect=False, auth_plugin_map=None, read_timeout=None, write_timeout=None, bind_address=None, binary_prefix=False, program_name=None, - server_public_key=None): + server_public_key=None, ssl_ca=None, ssl_cert=None, + ssl_disabled=None, ssl_key=None, ssl_verify_cert=None, + ssl_verify_identity=None): if use_unicode is None and sys.version_info[0] > 2: use_unicode = True @@ -245,12 +253,23 @@ def _config(key, arg): ssl[key] = value self.ssl = False - if ssl: - if not SSL_ENABLED: - raise NotImplementedError("ssl module not found") - self.ssl = True - client_flag |= CLIENT.SSL - self.ctx = self._create_ssl_ctx(ssl) + if not ssl_disabled: + if ssl_ca or ssl_cert or ssl_key or ssl_verify_cert or ssl_verify_identity: + ssl = { + "ca": ssl_ca, + "check_hostname": bool(ssl_verify_identity), + "verify_mode": ssl_verify_cert if ssl_verify_cert is not None else False, + } + if ssl_cert is not None: + ssl["cert"] = ssl_cert + if ssl_key is not None: + ssl["key" ] = ssl_key + if ssl: + if not SSL_ENABLED: + raise NotImplementedError("ssl module not found") + self.ssl = True + client_flag |= CLIENT.SSL + self.ctx = self._create_ssl_ctx(ssl) self.host = host or "localhost" self.port = port or 3306 @@ -341,7 +360,22 @@ def _create_ssl_ctx(self, sslp): hasnoca = ca is None and capath is None ctx = ssl.create_default_context(cafile=ca, capath=capath) ctx.check_hostname = not hasnoca and sslp.get('check_hostname', True) - ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED + verify_mode_value = sslp.get('verify_mode') + if verify_mode_value is None: + ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED + elif isinstance(verify_mode_value, bool): + ctx.verify_mode = ssl.CERT_REQUIRED if verify_mode_value else ssl.CERT_NONE + else: + if isinstance(verify_mode_value, (text_type, str_type)): + verify_mode_value = verify_mode_value.lower() + if verify_mode_value in ("none", "0", "false", "no"): + ctx.verify_mode = ssl.CERT_NONE + elif verify_mode_value == "optional": + ctx.verify_mode = ssl.CERT_OPTIONAL + elif verify_mode_value in ("required", "1", "true", "yes"): + ctx.verify_mode = ssl.CERT_REQUIRED + else: + ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED if 'cert' in sslp: ctx.load_cert_chain(sslp['cert'], keyfile=sslp.get('key')) if 'cipher' in sslp: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 51b9f3a5..d04cdd48 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,14 +1,14 @@ import datetime +import ssl import sys import time +import mock import pytest import pymysql from pymysql.tests import base from pymysql._compat import text_type from pymysql.constants import CLIENT -import pytest - class TempUser: def __init__(self, c, user, db, auth=None, authdata=None, password=None): @@ -478,6 +478,162 @@ def test_defer_connect(self): c.close() sock.close() + def test_ssl_connect(self): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + "cipher": "cipher", + }, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_called_with("cipher") + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + }, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_not_called + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + for ssl_verify_cert in (True, "1", "yes", "true"): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_cert="cert", + ssl_key="key", + ssl_verify_cert=ssl_verify_cert, + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + for ssl_verify_cert in (None, False, "0", "no", "false"): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_cert="cert", + ssl_key="key", + ssl_verify_cert=ssl_verify_cert, + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + for ssl_ca in ("ca", None): + for ssl_verify_cert in ("foo", "bar", ""): + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca=ssl_ca, + ssl_cert="cert", + ssl_key="key", + ssl_verify_cert=ssl_verify_cert, + ) + assert create_default_context.called + assert not dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == (ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE), (ssl_ca, ssl_verify_cert) + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ssl_verify_identity=True, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_disabled=True, + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + }, + ) + assert not create_default_context.called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect") as connect, \ + mock.patch("pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + pymysql.connect( + ssl_disabled=True, + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ) + assert not create_default_context.called + # A custom type and function to escape it class Foo(object): diff --git a/requirements-dev.txt b/requirements-dev.txt index d65512fb..69d3f68a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ cryptography PyNaCl>=1.4.0 pytest +mock From 66947bf8ccba9986a8503d4a7d5b77b1b21be54e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 2 Jan 2021 18:22:37 +0900 Subject: [PATCH 116/292] Remove Python 2.7 and 3.5 support. (#915) --- pymysql/__init__.py | 6 +- pymysql/_auth.py | 9 +-- pymysql/_compat.py | 21 ------ pymysql/charset.py | 2 +- pymysql/connections.py | 60 +++++----------- pymysql/converters.py | 68 ++++--------------- pymysql/cursors.py | 49 ++++--------- pymysql/optionfile.py | 7 +- pymysql/protocol.py | 26 +++---- pymysql/tests/base.py | 30 -------- pymysql/tests/test_SSCursor.py | 1 + pymysql/tests/test_basic.py | 1 - pymysql/tests/test_connection.py | 11 ++- pymysql/tests/test_converters.py | 9 --- pymysql/tests/test_issues.py | 15 ---- pymysql/tests/test_optionfile.py | 14 +--- .../test_MySQLdb/test_MySQLdb_nonstandard.py | 22 ++---- 17 files changed, 71 insertions(+), 280 deletions(-) delete mode 100644 pymysql/_compat.py diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 29e6b87c..1e126dcd 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -23,7 +23,6 @@ """ import sys -from ._compat import PY2 from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string from .err import ( @@ -79,10 +78,7 @@ def __hash__(self): def Binary(x): """Return x as a binary type.""" - if PY2: - return bytearray(x) - else: - return bytes(x) + return bytes(x) def Connect(*args, **kwargs): diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 57f9abb1..77caeafd 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -1,7 +1,6 @@ """ Implements auth methods """ -from ._compat import PY2 from .err import OperationalError from .util import byte2int, int2byte @@ -46,8 +45,6 @@ def scramble_native_password(password, message): def _my_crypt(message1, message2): result = bytearray(message1) - if PY2: - message2 = bytearray(message2) for i in range(len(result)): result[i] ^= message2[i] @@ -61,7 +58,7 @@ def _my_crypt(message1, message2): SCRAMBLE_LENGTH_323 = 8 -class RandStruct_323(object): +class RandStruct_323: def __init__(self, seed1, seed2): self.max_value = 0x3FFFFFFF @@ -188,7 +185,7 @@ def _xor_password(password, salt): # See https://github.com/mysql/mysql-server/blob/7d10c82196c8e45554f27c00681474a9fb86d137/sql/auth/sha2_password.cc#L939-L945 salt = salt[:SCRAMBLE_LENGTH] password_bytes = bytearray(password) - salt = bytearray(salt) # for PY2 compat. + #salt = bytearray(salt) # for PY2 compat. salt_len = len(salt) for i in range(len(password_bytes)): password_bytes[i] ^= salt[i % salt_len] @@ -259,8 +256,6 @@ def scramble_caching_sha2(password, nonce): p3 = hashlib.sha256(p2 + nonce).digest() res = bytearray(p1) - if PY2: - p3 = bytearray(p3) for i in range(len(p3)): res[i] ^= p3[i] diff --git a/pymysql/_compat.py b/pymysql/_compat.py deleted file mode 100644 index 252789ec..00000000 --- a/pymysql/_compat.py +++ /dev/null @@ -1,21 +0,0 @@ -import sys - -PY2 = sys.version_info[0] == 2 -PYPY = hasattr(sys, 'pypy_translation_info') -JYTHON = sys.platform.startswith('java') -IRONPYTHON = sys.platform == 'cli' -CPYTHON = not PYPY and not JYTHON and not IRONPYTHON - -if PY2: - import __builtin__ - range_type = xrange - text_type = unicode - long_type = long - str_type = basestring - unichr = __builtin__.unichr -else: - range_type = range - text_type = str - long_type = int - str_type = str - unichr = chr diff --git a/pymysql/charset.py b/pymysql/charset.py index d3ced67c..3ef3ea46 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -6,7 +6,7 @@ } -class Charset(object): +class Charset: def __init__(self, id, name, collation, is_default): self.id, self.name, self.collation = id, name, collation self.is_default = is_default == 'Yes' diff --git a/pymysql/connections.py b/pymysql/connections.py index 7ecfb616..e426d151 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -2,11 +2,7 @@ # http://dev.mysql.com/doc/internals/en/client-server-protocol.html # Error codes: # https://dev.mysql.com/doc/refman/5.5/en/error-handling.html -from __future__ import print_function -from ._compat import PY2, range_type, text_type, str_type, JYTHON, IRONPYTHON - import errno -import io import os import socket import struct @@ -47,32 +43,11 @@ _py_version = sys.version_info[:2] -if PY2: - pass -elif _py_version < (3, 6): - # See http://bugs.python.org/issue24870 - _surrogateescape_table = [chr(i) if i < 0x80 else chr(i + 0xdc00) for i in range(256)] - - def _fast_surrogateescape(s): - return s.decode('latin1').translate(_surrogateescape_table) -else: - def _fast_surrogateescape(s): - return s.decode('ascii', 'surrogateescape') - -# socket.makefile() in Python 2 is not usable because very inefficient and -# bad behavior about timeout. -# XXX: ._socketio doesn't work under IronPython. -if PY2 and not IRONPYTHON: - # read method of file-like returned by sock.makefile() is very slow. - # So we copy io-based one from Python 3. - from ._socketio import SocketIO - - def _makefile(sock, mode): - return io.BufferedReader(SocketIO(sock, mode)) -else: - # socket.makefile in Python 3 is nice. - def _makefile(sock, mode): - return sock.makefile(mode) +def _fast_surrogateescape(s): + return s.decode('ascii', 'surrogateescape') + +def _makefile(sock, mode): + return sock.makefile(mode) TEXT_TYPES = { @@ -113,7 +88,7 @@ def lenenc_int(i): raise ValueError("Encoding %x is larger than %x - no representation in LengthEncodedInteger" % (i, (1 << 64))) -class Connection(object): +class Connection: """ Representation of a socket with a mysql server. @@ -277,7 +252,7 @@ def _config(key, arg): raise ValueError("port should be of type int") self.user = user or DEFAULT_USER self.password = password or b"" - if isinstance(self.password, text_type): + if isinstance(self.password, str): self.password = self.password.encode('latin1') self.db = database self.unix_socket = unix_socket @@ -493,7 +468,7 @@ def escape(self, obj, mapping=None): Non-standard, for internal use; do not use this in your applications. """ - if isinstance(obj, str_type): + if isinstance(obj, str): return "'" + self.escape_string(obj) + "'" if isinstance(obj, (bytes, bytearray)): ret = self._quote_bytes(obj) @@ -537,11 +512,8 @@ def cursor(self, cursor=None): def query(self, sql, unbuffered=False): # if DEBUG: # print("DEBUG: sending query:", sql) - if isinstance(sql, text_type) and not (JYTHON or IRONPYTHON): - if PY2: - sql = sql.encode(self.encoding) - else: - sql = sql.encode(self.encoding, 'surrogateescape') + if isinstance(sql, str): + sql = sql.encode(self.encoding, 'surrogateescape') self._execute_command(COMMAND.COM_QUERY, sql) self._affected_rows = self._read_query_result(unbuffered=unbuffered) return self._affected_rows @@ -792,7 +764,7 @@ def _execute_command(self, command, sql): self.next_result() self._result = None - if isinstance(sql, text_type): + if isinstance(sql, str): sql = sql.encode(self.encoding) packet_size = min(MAX_PACKET_LEN, len(sql) + 1) # +1 is for command @@ -825,7 +797,7 @@ def _request_authentication(self): raise ValueError("Did not specify a username") charset_id = charset_by_name(self.charset).id - if isinstance(self.user, text_type): + if isinstance(self.user, str): self.user = self.user.encode(self.encoding) data_init = struct.pack(' max_stmt_length: rows += self.execute(sql + postfix) sql = bytearray(prefix) @@ -265,7 +242,7 @@ def callproc(self, procname, args=()): q = "CALL %s(%s)" % (procname, ','.join(['@_%s_%d' % (procname, i) - for i in range_type(len(args))])) + for i in range(len(args))])) self._query(q) self._executed = q return args @@ -356,7 +333,7 @@ def __iter__(self): NotSupportedError = err.NotSupportedError -class DictCursorMixin(object): +class DictCursorMixin: # You can override this to use OrderedDict or other dict-like types. dict_type = dict @@ -469,7 +446,7 @@ def fetchmany(self, size=None): size = self.arraysize rows = [] - for i in range_type(size): + for i in range(size): row = self.read_next() if row is None: break @@ -485,7 +462,7 @@ def scroll(self, value, mode='relative'): raise err.NotSupportedError( "Backwards scrolling not supported by this cursor") - for _ in range_type(value): + for _ in range(value): self.read_next() self.rownumber += value elif mode == 'absolute': @@ -494,7 +471,7 @@ def scroll(self, value, mode='relative'): "Backwards scrolling not supported by this cursor") end = value - self.rownumber - for _ in range_type(end): + for _ in range(end): self.read_next() self.rownumber = value else: diff --git a/pymysql/optionfile.py b/pymysql/optionfile.py index 91e2dfe3..79810ef3 100644 --- a/pymysql/optionfile.py +++ b/pymysql/optionfile.py @@ -1,9 +1,4 @@ -from ._compat import PY2 - -if PY2: - import ConfigParser as configparser -else: - import configparser +import configparser class Parser(configparser.RawConfigParser): diff --git a/pymysql/protocol.py b/pymysql/protocol.py index e302edab..541475ad 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -1,9 +1,7 @@ # Python implementation of low level MySQL client-server protocol # http://dev.mysql.com/doc/internals/en/client-server-protocol.html -from __future__ import print_function from .charset import MBLENGTH -from ._compat import PY2, range_type from .constants import FIELD_TYPE, SERVER_STATUS from . import err from .util import byte2int @@ -37,7 +35,7 @@ def printable(data): print("-" * 66) except ValueError: pass - dump_data = [data[i:i+16] for i in range_type(0, min(len(data), 256), 16)] + dump_data = [data[i:i+16] for i in range(0, min(len(data), 256), 16)] for d in dump_data: print(' '.join("{:02X}".format(byte2int(x)) for x in d) + ' ' * (16 - len(d)) + ' ' * 2 + @@ -46,7 +44,7 @@ def printable(data): print() -class MysqlPacket(object): +class MysqlPacket: """Representation of a MySQL response packet. Provides an interface for reading/parsing the packet results. @@ -108,16 +106,10 @@ def get_bytes(self, position, length=1): """ return self._data[position:(position+length)] - if PY2: - def read_uint8(self): - result = ord(self._data[self._position]) - self._position += 1 - return result - else: - def read_uint8(self): - result = self._data[self._position] - self._position += 1 - return result + def read_uint8(self): + result = self._data[self._position] + self._position += 1 + return result def read_uint16(self): result = struct.unpack_from(' Date: Sun, 3 Jan 2021 10:11:53 +0900 Subject: [PATCH 117/292] Use GitHub Actions (#917) --- .github/workflows/test.yaml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000..369b5067 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,54 @@ +name: Test + +on: + push: + pull_request: + +jobs: + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - db: "mariadb:10.2" + py: "3.9" + - db: "mariadb:10.3" + py: "3.8" + - db: "mariadb:10.5" + py: "3.7" + - db: "mysql:5.6" + py: "3.6" + - db: "mysql:5.7" + py: "pypy-3.6" + - db: "mysql:8.0" + py: "3.9" + + services: + mysql: + image: "${{ matrix.db }}" + ports: + - 3306:3306 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + options: "--name=mysqld" + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.py }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.py }} + - name: Set up MySQL + run: | + sleep 10 + mysql -h 127.0.0.1 -uroot -e "select version()" + mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" + mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' + mysql -h 127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' + mysql -h 127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" + mysql -h 127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" + cp .travis/docker.json pymysql/tests/databases.json + - name: Run test + run: | + pip install -U cryptography PyNaCl pytest pytest-cov mock + pytest -v --cov --cov-config .coveragerc pymysql From 8d0c6c20f608f40726ee94d3b56be71481e55c59 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 10:16:53 +0900 Subject: [PATCH 118/292] Update README.rst --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 0a09f892..269928b8 100644 --- a/README.rst +++ b/README.rst @@ -5,9 +5,6 @@ .. image:: https://badge.fury.io/py/PyMySQL.svg :target: https://badge.fury.io/py/PyMySQL -.. image:: https://travis-ci.com/PyMySQL/PyMySQL.svg?branch=master - :target: https://travis-ci.com/PyMySQL/PyMySQL - .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master From 6ec449aa068922405350813df1001f635871d437 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 11:32:08 +0900 Subject: [PATCH 119/292] Fix regression, enable coveralls (#918) --- .github/workflows/test.yaml | 23 +++++++++++++++++-- pymysql/connections.py | 2 +- pymysql/converters.py | 4 ++-- pymysql/cursors.py | 1 - pymysql/tests/test_cursor.py | 3 --- .../thirdparty/test_MySQLdb/capabilities.py | 18 +++++---------- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 369b5067..c68f7239 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,7 +5,7 @@ on: pull_request: jobs: - build: + test: runs-on: ubuntu-20.04 strategy: matrix: @@ -50,5 +50,24 @@ jobs: cp .travis/docker.json pymysql/tests/databases.json - name: Run test run: | - pip install -U cryptography PyNaCl pytest pytest-cov mock + pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls pytest -v --cov --cov-config .coveragerc pymysql + - name: Report coverage + run: coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: ${{ matrix.test-name }} + COVERALLS_PARALLEL: true + + coveralls: + name: Finish coveralls + runs-on: ubuntu-20.04 + needs: test + container: python:3-slim + steps: + - name: Finished + run: | + pip3 install --upgrade coveralls + coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/pymysql/connections.py b/pymysql/connections.py index e426d151..6fd15e13 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -341,7 +341,7 @@ def _create_ssl_ctx(self, sslp): elif isinstance(verify_mode_value, bool): ctx.verify_mode = ssl.CERT_REQUIRED if verify_mode_value else ssl.CERT_NONE else: - if isinstance(verify_mode_value, (text_type, str_type)): + if isinstance(verify_mode_value, str): verify_mode_value = verify_mode_value.lower() if verify_mode_value in ("none", "0", "false", "no"): ctx.verify_mode = ssl.CERT_NONE diff --git a/pymysql/converters.py b/pymysql/converters.py index 0e40eab7..6d1fc9ee 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -74,11 +74,11 @@ def escape_string(value, mapping=None): def escape_bytes_prefixed(value, mapping=None): - return "_binary'%s'" % value.decode('ascii', 'surrogateescape') + return "_binary'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) def escape_bytes(value, mapping=None): - return "'%s'" % value.decode('ascii', 'surrogateescape') + return "'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) def escape_str(value, mapping=None): diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 6f72ba35..a8c52836 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -1,5 +1,4 @@ import re - from . import err diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index fb3e8bed..4c9174f5 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -30,7 +30,6 @@ def test_cleanup_rows_unbuffered(self): break del cursor - self.safe_gc_collect() c2 = conn.cursor() @@ -48,10 +47,8 @@ def test_cleanup_rows_buffered(self): break del cursor - self.safe_gc_collect() c2 = conn.cursor() - c2.execute("select 1") self.assertEqual( diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index 6be9d1ba..e261a78e 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -8,7 +8,6 @@ from time import time import unittest -PY2 = sys.version_info[0] == 2 class DatabaseTest(unittest.TestCase): @@ -24,10 +23,7 @@ def setUp(self): self.connection = db self.cursor = db.cursor() self.BLOBText = ''.join([chr(i) for i in range(256)] * 100); - if PY2: - self.BLOBUText = unicode().join(unichr(i) for i in range(16834)) - else: - self.BLOBUText = "".join(chr(i) for i in range(16834)) + self.BLOBUText = "".join(chr(i) for i in range(16834)) data = bytearray(range(256)) * 16 self.BLOBBinary = self.db_module.Binary(data) @@ -64,14 +60,12 @@ def new_table_name(self): i = i + 1 def create_table(self, columndefs): + """ + Create a table using a list of column definitions given in columndefs. - """ Create a table using a list of column definitions given in - columndefs. - - generator must be a function taking arguments (row_number, - col_number) returning a suitable data object for insertion - into the table. - + generator must be a function taking arguments (row_number, + col_number) returning a suitable data object for insertion + into the table. """ self.table = self.new_table_name() self.cursor.execute('CREATE TABLE %s (%s) %s' % From b93a87a25ea22c1563cbbcaf943799b3f7e40887 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 11:55:37 +0900 Subject: [PATCH 120/292] Actions: Run auth tests (#919) --- .github/workflows/test.yaml | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c68f7239..71cc4e82 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,16 +12,24 @@ jobs: include: - db: "mariadb:10.2" py: "3.9" + - db: "mariadb:10.3" py: "3.8" + mariadb_auth: true + - db: "mariadb:10.5" py: "3.7" + mariadb_auth: true + - db: "mysql:5.6" py: "3.6" + - db: "mysql:5.7" py: "pypy-3.6" + - db: "mysql:8.0" py: "3.9" + mysql_auth: true services: mysql: @@ -48,10 +56,41 @@ jobs: mysql -h 127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" mysql -h 127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" cp .travis/docker.json pymysql/tests/databases.json + - name: Run test run: | pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls pytest -v --cov --cov-config .coveragerc pymysql + + - name: Run MySQL8 auth test + if: ${{ matrix.mysql_auth }} + run: | + docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" + docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" + mysql -uroot -h127.0.0.1 -e ' + CREATE USER + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", + nopass_sha256 IDENTIFIED WITH "sha256_password", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER; + GRANT RELOAD ON *.* TO user_caching_sha2;' + pytest -v --cov --cov-config .coveragerc tests/test_auth.py; + + - name: Run MariaDB auth test + if: ${{ matrix.mariadb_auth }} + run: | + mysql -uroot -h127.0.0.1 -e ' + INSTALL SONAME "auth_ed25519"; + CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' + # we need to pass the hashed password manually until 10.4, so hide it here + mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql -uroot -h127.0.0.1 + mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql -uroot -h127.0.0.1 + pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py + - name: Report coverage run: coveralls env: From f889038f1b6b134806fb158d34cfb59f31905da2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:05:46 +0900 Subject: [PATCH 121/292] Reformat with black (#920) --- pymysql/__init__.py | 125 ++- pymysql/_auth.py | 45 +- pymysql/_socketio.py | 16 +- pymysql/charset.py | 317 ++++--- pymysql/connections.py | 423 ++++++---- pymysql/constants/CLIENT.py | 13 +- pymysql/constants/COMMAND.py | 25 +- pymysql/constants/CR.py | 100 +-- pymysql/constants/FIELD_TYPE.py | 2 - pymysql/constants/SERVER_STATUS.py | 1 - pymysql/converters.py | 86 +- pymysql/cursors.py | 75 +- pymysql/err.py | 78 +- pymysql/optionfile.py | 4 +- pymysql/protocol.py | 111 ++- pymysql/tests/__init__.py | 1 + pymysql/tests/base.py | 18 +- pymysql/tests/test_DictCursor.py | 52 +- pymysql/tests/test_SSCursor.py | 102 ++- pymysql/tests/test_basic.py | 186 ++-- pymysql/tests/test_connection.py | 445 ++++++---- pymysql/tests/test_converters.py | 24 +- pymysql/tests/test_cursor.py | 74 +- pymysql/tests/test_err.py | 3 +- pymysql/tests/test_issues.py | 140 +-- pymysql/tests/test_load_local.py | 31 +- pymysql/tests/test_nextset.py | 12 +- pymysql/tests/test_optionfile.py | 7 +- pymysql/tests/thirdparty/__init__.py | 1 + .../tests/thirdparty/test_MySQLdb/__init__.py | 1 + .../thirdparty/test_MySQLdb/capabilities.py | 243 +++--- .../tests/thirdparty/test_MySQLdb/dbapi20.py | 794 +++++++++--------- .../test_MySQLdb/test_MySQLdb_capabilities.py | 73 +- .../test_MySQLdb/test_MySQLdb_dbapi20.py | 200 +++-- .../test_MySQLdb/test_MySQLdb_nonstandard.py | 46 +- pymysql/util.py | 1 - tests/test_auth.py | 42 +- tests/test_mariadb_auth.py | 5 +- 38 files changed, 2296 insertions(+), 1626 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 1e126dcd..5b49262e 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -26,12 +26,26 @@ from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string from .err import ( - Warning, Error, InterfaceError, DataError, - DatabaseError, OperationalError, IntegrityError, InternalError, - NotSupportedError, ProgrammingError, MySQLError) + Warning, + Error, + InterfaceError, + DataError, + DatabaseError, + OperationalError, + IntegrityError, + InternalError, + NotSupportedError, + ProgrammingError, + MySQLError, +) from .times import ( - Date, Time, Timestamp, - DateFromTicks, TimeFromTicks, TimestampFromTicks) + Date, + Time, + Timestamp, + DateFromTicks, + TimeFromTicks, + TimestampFromTicks, +) VERSION = (0, 10, 1, None) @@ -45,7 +59,6 @@ class DBAPISet(frozenset): - def __ne__(self, other): if isinstance(other, set): return frozenset.__ne__(self, other) @@ -62,18 +75,32 @@ def __hash__(self): return frozenset.__hash__(self) -STRING = DBAPISet([FIELD_TYPE.ENUM, FIELD_TYPE.STRING, - FIELD_TYPE.VAR_STRING]) -BINARY = DBAPISet([FIELD_TYPE.BLOB, FIELD_TYPE.LONG_BLOB, - FIELD_TYPE.MEDIUM_BLOB, FIELD_TYPE.TINY_BLOB]) -NUMBER = DBAPISet([FIELD_TYPE.DECIMAL, FIELD_TYPE.DOUBLE, FIELD_TYPE.FLOAT, - FIELD_TYPE.INT24, FIELD_TYPE.LONG, FIELD_TYPE.LONGLONG, - FIELD_TYPE.TINY, FIELD_TYPE.YEAR]) -DATE = DBAPISet([FIELD_TYPE.DATE, FIELD_TYPE.NEWDATE]) -TIME = DBAPISet([FIELD_TYPE.TIME]) +STRING = DBAPISet([FIELD_TYPE.ENUM, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING]) +BINARY = DBAPISet( + [ + FIELD_TYPE.BLOB, + FIELD_TYPE.LONG_BLOB, + FIELD_TYPE.MEDIUM_BLOB, + FIELD_TYPE.TINY_BLOB, + ] +) +NUMBER = DBAPISet( + [ + FIELD_TYPE.DECIMAL, + FIELD_TYPE.DOUBLE, + FIELD_TYPE.FLOAT, + FIELD_TYPE.INT24, + FIELD_TYPE.LONG, + FIELD_TYPE.LONGLONG, + FIELD_TYPE.TINY, + FIELD_TYPE.YEAR, + ] +) +DATE = DBAPISet([FIELD_TYPE.DATE, FIELD_TYPE.NEWDATE]) +TIME = DBAPISet([FIELD_TYPE.TIME]) TIMESTAMP = DBAPISet([FIELD_TYPE.TIMESTAMP, FIELD_TYPE.DATETIME]) -DATETIME = TIMESTAMP -ROWID = DBAPISet() +DATETIME = TIMESTAMP +ROWID = DBAPISet() def Binary(x): @@ -87,9 +114,12 @@ def Connect(*args, **kwargs): more information. """ from .connections import Connection + return Connection(*args, **kwargs) + from . import connections as _orig_conn + if _orig_conn.Connection.__init__.__doc__ is not None: Connect.__doc__ = _orig_conn.Connection.__init__.__doc__ del _orig_conn @@ -99,7 +129,8 @@ def get_client_info(): # for MySQLdb compatibility version = VERSION if VERSION[3] is None: version = VERSION[:3] - return '.'.join(map(str, version)) + return ".".join(map(str, version)) + connect = Connection = Connect @@ -110,9 +141,11 @@ def get_client_info(): # for MySQLdb compatibility __version__ = get_client_info() + def thread_safe(): return True # match MySQLdb.thread_safe() + def install_as_MySQLdb(): """ After this function is called, any application that imports MySQLdb or @@ -122,16 +155,50 @@ def install_as_MySQLdb(): __all__ = [ - 'BINARY', 'Binary', 'Connect', 'Connection', 'DATE', 'Date', - 'Time', 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks', - 'DataError', 'DatabaseError', 'Error', 'FIELD_TYPE', 'IntegrityError', - 'InterfaceError', 'InternalError', 'MySQLError', 'NULL', 'NUMBER', - 'NotSupportedError', 'DBAPISet', 'OperationalError', 'ProgrammingError', - 'ROWID', 'STRING', 'TIME', 'TIMESTAMP', 'Warning', 'apilevel', 'connect', - 'connections', 'constants', 'converters', 'cursors', - 'escape_dict', 'escape_sequence', 'escape_string', 'get_client_info', - 'paramstyle', 'threadsafety', 'version_info', - + "BINARY", + "Binary", + "Connect", + "Connection", + "DATE", + "Date", + "Time", + "Timestamp", + "DateFromTicks", + "TimeFromTicks", + "TimestampFromTicks", + "DataError", + "DatabaseError", + "Error", + "FIELD_TYPE", + "IntegrityError", + "InterfaceError", + "InternalError", + "MySQLError", + "NULL", + "NUMBER", + "NotSupportedError", + "DBAPISet", + "OperationalError", + "ProgrammingError", + "ROWID", + "STRING", + "TIME", + "TIMESTAMP", + "Warning", + "apilevel", + "connect", + "connections", + "constants", + "converters", + "cursors", + "escape_dict", + "escape_sequence", + "escape_string", + "get_client_info", + "paramstyle", + "threadsafety", + "version_info", "install_as_MySQLdb", - "NULL", "__version__", + "NULL", + "__version__", ] diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 77caeafd..d16a0895 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -9,6 +9,7 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization, hashes from cryptography.hazmat.primitives.asymmetric import padding + _have_cryptography = True except ImportError: _have_cryptography = False @@ -22,7 +23,7 @@ DEBUG = False SCRAMBLE_LENGTH = 20 -sha1_new = partial(hashlib.new, 'sha1') +sha1_new = partial(hashlib.new, "sha1") # mysql_native_password @@ -32,7 +33,7 @@ def scramble_native_password(password, message): """Scramble used for mysql_native_password""" if not password: - return b'' + return b"" stage1 = sha1_new(password).digest() stage2 = sha1_new(stage1).digest() @@ -59,7 +60,6 @@ def _my_crypt(message1, message2): class RandStruct_323: - def __init__(self, seed1, seed2): self.max_value = 0x3FFFFFFF self.seed1 = seed1 % self.max_value @@ -73,8 +73,10 @@ def my_rnd(self): def scramble_old_password(password, message): """Scramble for old_password""" - warnings.warn("old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" - "old password support will be removed in future PyMySQL version") + warnings.warn( + "old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" + "old password support will be removed in future PyMySQL version" + ) hash_pass = _hash_password_323(password) hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) hash_pass_n = struct.unpack(">LL", hash_pass) @@ -100,7 +102,7 @@ def _hash_password_323(password): nr2 = 0x12345671 # x in py3 is numbers, p27 is chars - for c in [byte2int(x) for x in password if x not in (' ', '\t', 32, 9)]: + for c in [byte2int(x) for x in password if x not in (" ", "\t", 32, 9)]: nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF add = (add + c) & 0xFFFFFFFF @@ -120,9 +122,12 @@ def _init_nacl(): global _nacl_bindings try: from nacl import bindings + _nacl_bindings = bindings except ImportError: - raise RuntimeError("'pynacl' package is required for ed25519_password auth method") + raise RuntimeError( + "'pynacl' package is required for ed25519_password auth method" + ) def _scalar_clamp(s32): @@ -185,7 +190,7 @@ def _xor_password(password, salt): # See https://github.com/mysql/mysql-server/blob/7d10c82196c8e45554f27c00681474a9fb86d137/sql/auth/sha2_password.cc#L939-L945 salt = salt[:SCRAMBLE_LENGTH] password_bytes = bytearray(password) - #salt = bytearray(salt) # for PY2 compat. + # salt = bytearray(salt) # for PY2 compat. salt_len = len(salt) for i in range(len(password_bytes)): password_bytes[i] ^= salt[i % salt_len] @@ -198,8 +203,10 @@ def sha2_rsa_encrypt(password, salt, public_key): Used for sha256_password and caching_sha2_password. """ if not _have_cryptography: - raise RuntimeError("'cryptography' package is required for sha256_password or caching_sha2_password auth methods") - message = _xor_password(password + b'\0', salt) + raise RuntimeError( + "'cryptography' package is required for sha256_password or caching_sha2_password auth methods" + ) + message = _xor_password(password + b"\0", salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) return rsa_key.encrypt( message, @@ -215,7 +222,7 @@ def sha256_password_auth(conn, pkt): if conn._secure: if DEBUG: print("sha256: Sending plain password") - data = conn.password + b'\0' + data = conn.password + b"\0" return _roundtrip(conn, data) if pkt.is_auth_switch_request(): @@ -224,12 +231,12 @@ def sha256_password_auth(conn, pkt): # Request server public key if DEBUG: print("sha256: Requesting server public key") - pkt = _roundtrip(conn, b'\1') + pkt = _roundtrip(conn, b"\1") if pkt.is_extra_auth_data(): conn.server_public_key = pkt._data[1:] if DEBUG: - print("Received public key:\n", conn.server_public_key.decode('ascii')) + print("Received public key:\n", conn.server_public_key.decode("ascii")) if conn.password: if not conn.server_public_key: @@ -237,7 +244,7 @@ def sha256_password_auth(conn, pkt): data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) else: - data = b'' + data = b"" return _roundtrip(conn, data) @@ -249,7 +256,7 @@ def scramble_caching_sha2(password, nonce): XOR(SHA256(password), SHA256(SHA256(SHA256(password)), nonce)) """ if not password: - return b'' + return b"" p1 = hashlib.sha256(password).digest() p2 = hashlib.sha256(p1).digest() @@ -265,7 +272,7 @@ def scramble_caching_sha2(password, nonce): def caching_sha2_password_auth(conn, pkt): # No password fast path if not conn.password: - return _roundtrip(conn, b'') + return _roundtrip(conn, b"") if pkt.is_auth_switch_request(): # Try from fast auth @@ -305,10 +312,10 @@ def caching_sha2_password_auth(conn, pkt): if conn._secure: if DEBUG: print("caching sha2: Sending plain password via secure connection") - return _roundtrip(conn, conn.password + b'\0') + return _roundtrip(conn, conn.password + b"\0") if not conn.server_public_key: - pkt = _roundtrip(conn, b'\x02') # Request public key + pkt = _roundtrip(conn, b"\x02") # Request public key if not pkt.is_extra_auth_data(): raise OperationalError( "caching sha2: Unknown packet for public key: %s" % pkt._data[:1] @@ -316,7 +323,7 @@ def caching_sha2_password_auth(conn, pkt): conn.server_public_key = pkt._data[1:] if DEBUG: - print(conn.server_public_key.decode('ascii')) + print(conn.server_public_key.decode("ascii")) data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key) pkt = _roundtrip(conn, data) diff --git a/pymysql/_socketio.py b/pymysql/_socketio.py index 6a11d42e..6b2d65a3 100644 --- a/pymysql/_socketio.py +++ b/pymysql/_socketio.py @@ -8,11 +8,12 @@ import io import errno -__all__ = ['SocketIO'] +__all__ = ["SocketIO"] EINTR = errno.EINTR _blocking_errnos = (errno.EAGAIN, errno.EWOULDBLOCK) + class SocketIO(io.RawIOBase): """Raw I/O implementation for stream sockets. @@ -85,29 +86,25 @@ def write(self, b): raise def readable(self): - """True if the SocketIO is open for reading. - """ + """True if the SocketIO is open for reading.""" if self.closed: raise ValueError("I/O operation on closed socket.") return self._reading def writable(self): - """True if the SocketIO is open for writing. - """ + """True if the SocketIO is open for writing.""" if self.closed: raise ValueError("I/O operation on closed socket.") return self._writing def seekable(self): - """True if the SocketIO is open for seeking. - """ + """True if the SocketIO is open for seeking.""" if self.closed: raise ValueError("I/O operation on closed socket.") return super().seekable() def fileno(self): - """Return the file descriptor of the underlying socket. - """ + """Return the file descriptor of the underlying socket.""" self._checkClosed() return self._sock.fileno() @@ -131,4 +128,3 @@ def close(self): io.RawIOBase.close(self) self._sock._decref_socketios() self._sock = None - diff --git a/pymysql/charset.py b/pymysql/charset.py index 3ef3ea46..ac87c53d 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -1,31 +1,29 @@ -MBLENGTH = { - 8:1, - 33:3, - 88:2, - 91:2 - } +MBLENGTH = {8: 1, 33: 3, 88: 2, 91: 2} class Charset: def __init__(self, id, name, collation, is_default): self.id, self.name, self.collation = id, name, collation - self.is_default = is_default == 'Yes' + self.is_default = is_default == "Yes" def __repr__(self): return "Charset(id=%s, name=%r, collation=%r)" % ( - self.id, self.name, self.collation) + self.id, + self.name, + self.collation, + ) @property def encoding(self): name = self.name - if name in ('utf8mb4', 'utf8mb3'): - return 'utf8' - if name == 'latin1': - return 'cp1252' - if name == 'koi8r': - return 'koi8_r' - if name == 'koi8u': - return 'koi8_u' + if name in ("utf8mb4", "utf8mb3"): + return "utf8" + if name == "latin1": + return "cp1252" + if name == "koi8r": + return "koi8_r" + if name == "koi8u": + return "koi8_u" return name @property @@ -49,6 +47,7 @@ def by_id(self, id): def by_name(self, name): return self._by_name.get(name.lower()) + _charsets = Charsets() """ Generated with: @@ -62,149 +61,149 @@ def by_name(self, name): " """ -_charsets.add(Charset(1, 'big5', 'big5_chinese_ci', 'Yes')) -_charsets.add(Charset(2, 'latin2', 'latin2_czech_cs', '')) -_charsets.add(Charset(3, 'dec8', 'dec8_swedish_ci', 'Yes')) -_charsets.add(Charset(4, 'cp850', 'cp850_general_ci', 'Yes')) -_charsets.add(Charset(5, 'latin1', 'latin1_german1_ci', '')) -_charsets.add(Charset(6, 'hp8', 'hp8_english_ci', 'Yes')) -_charsets.add(Charset(7, 'koi8r', 'koi8r_general_ci', 'Yes')) -_charsets.add(Charset(8, 'latin1', 'latin1_swedish_ci', 'Yes')) -_charsets.add(Charset(9, 'latin2', 'latin2_general_ci', 'Yes')) -_charsets.add(Charset(10, 'swe7', 'swe7_swedish_ci', 'Yes')) -_charsets.add(Charset(11, 'ascii', 'ascii_general_ci', 'Yes')) -_charsets.add(Charset(12, 'ujis', 'ujis_japanese_ci', 'Yes')) -_charsets.add(Charset(13, 'sjis', 'sjis_japanese_ci', 'Yes')) -_charsets.add(Charset(14, 'cp1251', 'cp1251_bulgarian_ci', '')) -_charsets.add(Charset(15, 'latin1', 'latin1_danish_ci', '')) -_charsets.add(Charset(16, 'hebrew', 'hebrew_general_ci', 'Yes')) -_charsets.add(Charset(18, 'tis620', 'tis620_thai_ci', 'Yes')) -_charsets.add(Charset(19, 'euckr', 'euckr_korean_ci', 'Yes')) -_charsets.add(Charset(20, 'latin7', 'latin7_estonian_cs', '')) -_charsets.add(Charset(21, 'latin2', 'latin2_hungarian_ci', '')) -_charsets.add(Charset(22, 'koi8u', 'koi8u_general_ci', 'Yes')) -_charsets.add(Charset(23, 'cp1251', 'cp1251_ukrainian_ci', '')) -_charsets.add(Charset(24, 'gb2312', 'gb2312_chinese_ci', 'Yes')) -_charsets.add(Charset(25, 'greek', 'greek_general_ci', 'Yes')) -_charsets.add(Charset(26, 'cp1250', 'cp1250_general_ci', 'Yes')) -_charsets.add(Charset(27, 'latin2', 'latin2_croatian_ci', '')) -_charsets.add(Charset(28, 'gbk', 'gbk_chinese_ci', 'Yes')) -_charsets.add(Charset(29, 'cp1257', 'cp1257_lithuanian_ci', '')) -_charsets.add(Charset(30, 'latin5', 'latin5_turkish_ci', 'Yes')) -_charsets.add(Charset(31, 'latin1', 'latin1_german2_ci', '')) -_charsets.add(Charset(32, 'armscii8', 'armscii8_general_ci', 'Yes')) -_charsets.add(Charset(33, 'utf8', 'utf8_general_ci', 'Yes')) -_charsets.add(Charset(34, 'cp1250', 'cp1250_czech_cs', '')) -_charsets.add(Charset(36, 'cp866', 'cp866_general_ci', 'Yes')) -_charsets.add(Charset(37, 'keybcs2', 'keybcs2_general_ci', 'Yes')) -_charsets.add(Charset(38, 'macce', 'macce_general_ci', 'Yes')) -_charsets.add(Charset(39, 'macroman', 'macroman_general_ci', 'Yes')) -_charsets.add(Charset(40, 'cp852', 'cp852_general_ci', 'Yes')) -_charsets.add(Charset(41, 'latin7', 'latin7_general_ci', 'Yes')) -_charsets.add(Charset(42, 'latin7', 'latin7_general_cs', '')) -_charsets.add(Charset(43, 'macce', 'macce_bin', '')) -_charsets.add(Charset(44, 'cp1250', 'cp1250_croatian_ci', '')) -_charsets.add(Charset(45, 'utf8mb4', 'utf8mb4_general_ci', 'Yes')) -_charsets.add(Charset(46, 'utf8mb4', 'utf8mb4_bin', '')) -_charsets.add(Charset(47, 'latin1', 'latin1_bin', '')) -_charsets.add(Charset(48, 'latin1', 'latin1_general_ci', '')) -_charsets.add(Charset(49, 'latin1', 'latin1_general_cs', '')) -_charsets.add(Charset(50, 'cp1251', 'cp1251_bin', '')) -_charsets.add(Charset(51, 'cp1251', 'cp1251_general_ci', 'Yes')) -_charsets.add(Charset(52, 'cp1251', 'cp1251_general_cs', '')) -_charsets.add(Charset(53, 'macroman', 'macroman_bin', '')) -_charsets.add(Charset(57, 'cp1256', 'cp1256_general_ci', 'Yes')) -_charsets.add(Charset(58, 'cp1257', 'cp1257_bin', '')) -_charsets.add(Charset(59, 'cp1257', 'cp1257_general_ci', 'Yes')) -_charsets.add(Charset(63, 'binary', 'binary', 'Yes')) -_charsets.add(Charset(64, 'armscii8', 'armscii8_bin', '')) -_charsets.add(Charset(65, 'ascii', 'ascii_bin', '')) -_charsets.add(Charset(66, 'cp1250', 'cp1250_bin', '')) -_charsets.add(Charset(67, 'cp1256', 'cp1256_bin', '')) -_charsets.add(Charset(68, 'cp866', 'cp866_bin', '')) -_charsets.add(Charset(69, 'dec8', 'dec8_bin', '')) -_charsets.add(Charset(70, 'greek', 'greek_bin', '')) -_charsets.add(Charset(71, 'hebrew', 'hebrew_bin', '')) -_charsets.add(Charset(72, 'hp8', 'hp8_bin', '')) -_charsets.add(Charset(73, 'keybcs2', 'keybcs2_bin', '')) -_charsets.add(Charset(74, 'koi8r', 'koi8r_bin', '')) -_charsets.add(Charset(75, 'koi8u', 'koi8u_bin', '')) -_charsets.add(Charset(76, 'utf8', 'utf8_tolower_ci', '')) -_charsets.add(Charset(77, 'latin2', 'latin2_bin', '')) -_charsets.add(Charset(78, 'latin5', 'latin5_bin', '')) -_charsets.add(Charset(79, 'latin7', 'latin7_bin', '')) -_charsets.add(Charset(80, 'cp850', 'cp850_bin', '')) -_charsets.add(Charset(81, 'cp852', 'cp852_bin', '')) -_charsets.add(Charset(82, 'swe7', 'swe7_bin', '')) -_charsets.add(Charset(83, 'utf8', 'utf8_bin', '')) -_charsets.add(Charset(84, 'big5', 'big5_bin', '')) -_charsets.add(Charset(85, 'euckr', 'euckr_bin', '')) -_charsets.add(Charset(86, 'gb2312', 'gb2312_bin', '')) -_charsets.add(Charset(87, 'gbk', 'gbk_bin', '')) -_charsets.add(Charset(88, 'sjis', 'sjis_bin', '')) -_charsets.add(Charset(89, 'tis620', 'tis620_bin', '')) -_charsets.add(Charset(91, 'ujis', 'ujis_bin', '')) -_charsets.add(Charset(92, 'geostd8', 'geostd8_general_ci', 'Yes')) -_charsets.add(Charset(93, 'geostd8', 'geostd8_bin', '')) -_charsets.add(Charset(94, 'latin1', 'latin1_spanish_ci', '')) -_charsets.add(Charset(95, 'cp932', 'cp932_japanese_ci', 'Yes')) -_charsets.add(Charset(96, 'cp932', 'cp932_bin', '')) -_charsets.add(Charset(97, 'eucjpms', 'eucjpms_japanese_ci', 'Yes')) -_charsets.add(Charset(98, 'eucjpms', 'eucjpms_bin', '')) -_charsets.add(Charset(99, 'cp1250', 'cp1250_polish_ci', '')) -_charsets.add(Charset(192, 'utf8', 'utf8_unicode_ci', '')) -_charsets.add(Charset(193, 'utf8', 'utf8_icelandic_ci', '')) -_charsets.add(Charset(194, 'utf8', 'utf8_latvian_ci', '')) -_charsets.add(Charset(195, 'utf8', 'utf8_romanian_ci', '')) -_charsets.add(Charset(196, 'utf8', 'utf8_slovenian_ci', '')) -_charsets.add(Charset(197, 'utf8', 'utf8_polish_ci', '')) -_charsets.add(Charset(198, 'utf8', 'utf8_estonian_ci', '')) -_charsets.add(Charset(199, 'utf8', 'utf8_spanish_ci', '')) -_charsets.add(Charset(200, 'utf8', 'utf8_swedish_ci', '')) -_charsets.add(Charset(201, 'utf8', 'utf8_turkish_ci', '')) -_charsets.add(Charset(202, 'utf8', 'utf8_czech_ci', '')) -_charsets.add(Charset(203, 'utf8', 'utf8_danish_ci', '')) -_charsets.add(Charset(204, 'utf8', 'utf8_lithuanian_ci', '')) -_charsets.add(Charset(205, 'utf8', 'utf8_slovak_ci', '')) -_charsets.add(Charset(206, 'utf8', 'utf8_spanish2_ci', '')) -_charsets.add(Charset(207, 'utf8', 'utf8_roman_ci', '')) -_charsets.add(Charset(208, 'utf8', 'utf8_persian_ci', '')) -_charsets.add(Charset(209, 'utf8', 'utf8_esperanto_ci', '')) -_charsets.add(Charset(210, 'utf8', 'utf8_hungarian_ci', '')) -_charsets.add(Charset(211, 'utf8', 'utf8_sinhala_ci', '')) -_charsets.add(Charset(212, 'utf8', 'utf8_german2_ci', '')) -_charsets.add(Charset(213, 'utf8', 'utf8_croatian_ci', '')) -_charsets.add(Charset(214, 'utf8', 'utf8_unicode_520_ci', '')) -_charsets.add(Charset(215, 'utf8', 'utf8_vietnamese_ci', '')) -_charsets.add(Charset(223, 'utf8', 'utf8_general_mysql500_ci', '')) -_charsets.add(Charset(224, 'utf8mb4', 'utf8mb4_unicode_ci', '')) -_charsets.add(Charset(225, 'utf8mb4', 'utf8mb4_icelandic_ci', '')) -_charsets.add(Charset(226, 'utf8mb4', 'utf8mb4_latvian_ci', '')) -_charsets.add(Charset(227, 'utf8mb4', 'utf8mb4_romanian_ci', '')) -_charsets.add(Charset(228, 'utf8mb4', 'utf8mb4_slovenian_ci', '')) -_charsets.add(Charset(229, 'utf8mb4', 'utf8mb4_polish_ci', '')) -_charsets.add(Charset(230, 'utf8mb4', 'utf8mb4_estonian_ci', '')) -_charsets.add(Charset(231, 'utf8mb4', 'utf8mb4_spanish_ci', '')) -_charsets.add(Charset(232, 'utf8mb4', 'utf8mb4_swedish_ci', '')) -_charsets.add(Charset(233, 'utf8mb4', 'utf8mb4_turkish_ci', '')) -_charsets.add(Charset(234, 'utf8mb4', 'utf8mb4_czech_ci', '')) -_charsets.add(Charset(235, 'utf8mb4', 'utf8mb4_danish_ci', '')) -_charsets.add(Charset(236, 'utf8mb4', 'utf8mb4_lithuanian_ci', '')) -_charsets.add(Charset(237, 'utf8mb4', 'utf8mb4_slovak_ci', '')) -_charsets.add(Charset(238, 'utf8mb4', 'utf8mb4_spanish2_ci', '')) -_charsets.add(Charset(239, 'utf8mb4', 'utf8mb4_roman_ci', '')) -_charsets.add(Charset(240, 'utf8mb4', 'utf8mb4_persian_ci', '')) -_charsets.add(Charset(241, 'utf8mb4', 'utf8mb4_esperanto_ci', '')) -_charsets.add(Charset(242, 'utf8mb4', 'utf8mb4_hungarian_ci', '')) -_charsets.add(Charset(243, 'utf8mb4', 'utf8mb4_sinhala_ci', '')) -_charsets.add(Charset(244, 'utf8mb4', 'utf8mb4_german2_ci', '')) -_charsets.add(Charset(245, 'utf8mb4', 'utf8mb4_croatian_ci', '')) -_charsets.add(Charset(246, 'utf8mb4', 'utf8mb4_unicode_520_ci', '')) -_charsets.add(Charset(247, 'utf8mb4', 'utf8mb4_vietnamese_ci', '')) -_charsets.add(Charset(248, 'gb18030', 'gb18030_chinese_ci', 'Yes')) -_charsets.add(Charset(249, 'gb18030', 'gb18030_bin', '')) -_charsets.add(Charset(250, 'gb18030', 'gb18030_unicode_520_ci', '')) -_charsets.add(Charset(255, 'utf8mb4', 'utf8mb4_0900_ai_ci', '')) +_charsets.add(Charset(1, "big5", "big5_chinese_ci", "Yes")) +_charsets.add(Charset(2, "latin2", "latin2_czech_cs", "")) +_charsets.add(Charset(3, "dec8", "dec8_swedish_ci", "Yes")) +_charsets.add(Charset(4, "cp850", "cp850_general_ci", "Yes")) +_charsets.add(Charset(5, "latin1", "latin1_german1_ci", "")) +_charsets.add(Charset(6, "hp8", "hp8_english_ci", "Yes")) +_charsets.add(Charset(7, "koi8r", "koi8r_general_ci", "Yes")) +_charsets.add(Charset(8, "latin1", "latin1_swedish_ci", "Yes")) +_charsets.add(Charset(9, "latin2", "latin2_general_ci", "Yes")) +_charsets.add(Charset(10, "swe7", "swe7_swedish_ci", "Yes")) +_charsets.add(Charset(11, "ascii", "ascii_general_ci", "Yes")) +_charsets.add(Charset(12, "ujis", "ujis_japanese_ci", "Yes")) +_charsets.add(Charset(13, "sjis", "sjis_japanese_ci", "Yes")) +_charsets.add(Charset(14, "cp1251", "cp1251_bulgarian_ci", "")) +_charsets.add(Charset(15, "latin1", "latin1_danish_ci", "")) +_charsets.add(Charset(16, "hebrew", "hebrew_general_ci", "Yes")) +_charsets.add(Charset(18, "tis620", "tis620_thai_ci", "Yes")) +_charsets.add(Charset(19, "euckr", "euckr_korean_ci", "Yes")) +_charsets.add(Charset(20, "latin7", "latin7_estonian_cs", "")) +_charsets.add(Charset(21, "latin2", "latin2_hungarian_ci", "")) +_charsets.add(Charset(22, "koi8u", "koi8u_general_ci", "Yes")) +_charsets.add(Charset(23, "cp1251", "cp1251_ukrainian_ci", "")) +_charsets.add(Charset(24, "gb2312", "gb2312_chinese_ci", "Yes")) +_charsets.add(Charset(25, "greek", "greek_general_ci", "Yes")) +_charsets.add(Charset(26, "cp1250", "cp1250_general_ci", "Yes")) +_charsets.add(Charset(27, "latin2", "latin2_croatian_ci", "")) +_charsets.add(Charset(28, "gbk", "gbk_chinese_ci", "Yes")) +_charsets.add(Charset(29, "cp1257", "cp1257_lithuanian_ci", "")) +_charsets.add(Charset(30, "latin5", "latin5_turkish_ci", "Yes")) +_charsets.add(Charset(31, "latin1", "latin1_german2_ci", "")) +_charsets.add(Charset(32, "armscii8", "armscii8_general_ci", "Yes")) +_charsets.add(Charset(33, "utf8", "utf8_general_ci", "Yes")) +_charsets.add(Charset(34, "cp1250", "cp1250_czech_cs", "")) +_charsets.add(Charset(36, "cp866", "cp866_general_ci", "Yes")) +_charsets.add(Charset(37, "keybcs2", "keybcs2_general_ci", "Yes")) +_charsets.add(Charset(38, "macce", "macce_general_ci", "Yes")) +_charsets.add(Charset(39, "macroman", "macroman_general_ci", "Yes")) +_charsets.add(Charset(40, "cp852", "cp852_general_ci", "Yes")) +_charsets.add(Charset(41, "latin7", "latin7_general_ci", "Yes")) +_charsets.add(Charset(42, "latin7", "latin7_general_cs", "")) +_charsets.add(Charset(43, "macce", "macce_bin", "")) +_charsets.add(Charset(44, "cp1250", "cp1250_croatian_ci", "")) +_charsets.add(Charset(45, "utf8mb4", "utf8mb4_general_ci", "Yes")) +_charsets.add(Charset(46, "utf8mb4", "utf8mb4_bin", "")) +_charsets.add(Charset(47, "latin1", "latin1_bin", "")) +_charsets.add(Charset(48, "latin1", "latin1_general_ci", "")) +_charsets.add(Charset(49, "latin1", "latin1_general_cs", "")) +_charsets.add(Charset(50, "cp1251", "cp1251_bin", "")) +_charsets.add(Charset(51, "cp1251", "cp1251_general_ci", "Yes")) +_charsets.add(Charset(52, "cp1251", "cp1251_general_cs", "")) +_charsets.add(Charset(53, "macroman", "macroman_bin", "")) +_charsets.add(Charset(57, "cp1256", "cp1256_general_ci", "Yes")) +_charsets.add(Charset(58, "cp1257", "cp1257_bin", "")) +_charsets.add(Charset(59, "cp1257", "cp1257_general_ci", "Yes")) +_charsets.add(Charset(63, "binary", "binary", "Yes")) +_charsets.add(Charset(64, "armscii8", "armscii8_bin", "")) +_charsets.add(Charset(65, "ascii", "ascii_bin", "")) +_charsets.add(Charset(66, "cp1250", "cp1250_bin", "")) +_charsets.add(Charset(67, "cp1256", "cp1256_bin", "")) +_charsets.add(Charset(68, "cp866", "cp866_bin", "")) +_charsets.add(Charset(69, "dec8", "dec8_bin", "")) +_charsets.add(Charset(70, "greek", "greek_bin", "")) +_charsets.add(Charset(71, "hebrew", "hebrew_bin", "")) +_charsets.add(Charset(72, "hp8", "hp8_bin", "")) +_charsets.add(Charset(73, "keybcs2", "keybcs2_bin", "")) +_charsets.add(Charset(74, "koi8r", "koi8r_bin", "")) +_charsets.add(Charset(75, "koi8u", "koi8u_bin", "")) +_charsets.add(Charset(76, "utf8", "utf8_tolower_ci", "")) +_charsets.add(Charset(77, "latin2", "latin2_bin", "")) +_charsets.add(Charset(78, "latin5", "latin5_bin", "")) +_charsets.add(Charset(79, "latin7", "latin7_bin", "")) +_charsets.add(Charset(80, "cp850", "cp850_bin", "")) +_charsets.add(Charset(81, "cp852", "cp852_bin", "")) +_charsets.add(Charset(82, "swe7", "swe7_bin", "")) +_charsets.add(Charset(83, "utf8", "utf8_bin", "")) +_charsets.add(Charset(84, "big5", "big5_bin", "")) +_charsets.add(Charset(85, "euckr", "euckr_bin", "")) +_charsets.add(Charset(86, "gb2312", "gb2312_bin", "")) +_charsets.add(Charset(87, "gbk", "gbk_bin", "")) +_charsets.add(Charset(88, "sjis", "sjis_bin", "")) +_charsets.add(Charset(89, "tis620", "tis620_bin", "")) +_charsets.add(Charset(91, "ujis", "ujis_bin", "")) +_charsets.add(Charset(92, "geostd8", "geostd8_general_ci", "Yes")) +_charsets.add(Charset(93, "geostd8", "geostd8_bin", "")) +_charsets.add(Charset(94, "latin1", "latin1_spanish_ci", "")) +_charsets.add(Charset(95, "cp932", "cp932_japanese_ci", "Yes")) +_charsets.add(Charset(96, "cp932", "cp932_bin", "")) +_charsets.add(Charset(97, "eucjpms", "eucjpms_japanese_ci", "Yes")) +_charsets.add(Charset(98, "eucjpms", "eucjpms_bin", "")) +_charsets.add(Charset(99, "cp1250", "cp1250_polish_ci", "")) +_charsets.add(Charset(192, "utf8", "utf8_unicode_ci", "")) +_charsets.add(Charset(193, "utf8", "utf8_icelandic_ci", "")) +_charsets.add(Charset(194, "utf8", "utf8_latvian_ci", "")) +_charsets.add(Charset(195, "utf8", "utf8_romanian_ci", "")) +_charsets.add(Charset(196, "utf8", "utf8_slovenian_ci", "")) +_charsets.add(Charset(197, "utf8", "utf8_polish_ci", "")) +_charsets.add(Charset(198, "utf8", "utf8_estonian_ci", "")) +_charsets.add(Charset(199, "utf8", "utf8_spanish_ci", "")) +_charsets.add(Charset(200, "utf8", "utf8_swedish_ci", "")) +_charsets.add(Charset(201, "utf8", "utf8_turkish_ci", "")) +_charsets.add(Charset(202, "utf8", "utf8_czech_ci", "")) +_charsets.add(Charset(203, "utf8", "utf8_danish_ci", "")) +_charsets.add(Charset(204, "utf8", "utf8_lithuanian_ci", "")) +_charsets.add(Charset(205, "utf8", "utf8_slovak_ci", "")) +_charsets.add(Charset(206, "utf8", "utf8_spanish2_ci", "")) +_charsets.add(Charset(207, "utf8", "utf8_roman_ci", "")) +_charsets.add(Charset(208, "utf8", "utf8_persian_ci", "")) +_charsets.add(Charset(209, "utf8", "utf8_esperanto_ci", "")) +_charsets.add(Charset(210, "utf8", "utf8_hungarian_ci", "")) +_charsets.add(Charset(211, "utf8", "utf8_sinhala_ci", "")) +_charsets.add(Charset(212, "utf8", "utf8_german2_ci", "")) +_charsets.add(Charset(213, "utf8", "utf8_croatian_ci", "")) +_charsets.add(Charset(214, "utf8", "utf8_unicode_520_ci", "")) +_charsets.add(Charset(215, "utf8", "utf8_vietnamese_ci", "")) +_charsets.add(Charset(223, "utf8", "utf8_general_mysql500_ci", "")) +_charsets.add(Charset(224, "utf8mb4", "utf8mb4_unicode_ci", "")) +_charsets.add(Charset(225, "utf8mb4", "utf8mb4_icelandic_ci", "")) +_charsets.add(Charset(226, "utf8mb4", "utf8mb4_latvian_ci", "")) +_charsets.add(Charset(227, "utf8mb4", "utf8mb4_romanian_ci", "")) +_charsets.add(Charset(228, "utf8mb4", "utf8mb4_slovenian_ci", "")) +_charsets.add(Charset(229, "utf8mb4", "utf8mb4_polish_ci", "")) +_charsets.add(Charset(230, "utf8mb4", "utf8mb4_estonian_ci", "")) +_charsets.add(Charset(231, "utf8mb4", "utf8mb4_spanish_ci", "")) +_charsets.add(Charset(232, "utf8mb4", "utf8mb4_swedish_ci", "")) +_charsets.add(Charset(233, "utf8mb4", "utf8mb4_turkish_ci", "")) +_charsets.add(Charset(234, "utf8mb4", "utf8mb4_czech_ci", "")) +_charsets.add(Charset(235, "utf8mb4", "utf8mb4_danish_ci", "")) +_charsets.add(Charset(236, "utf8mb4", "utf8mb4_lithuanian_ci", "")) +_charsets.add(Charset(237, "utf8mb4", "utf8mb4_slovak_ci", "")) +_charsets.add(Charset(238, "utf8mb4", "utf8mb4_spanish2_ci", "")) +_charsets.add(Charset(239, "utf8mb4", "utf8mb4_roman_ci", "")) +_charsets.add(Charset(240, "utf8mb4", "utf8mb4_persian_ci", "")) +_charsets.add(Charset(241, "utf8mb4", "utf8mb4_esperanto_ci", "")) +_charsets.add(Charset(242, "utf8mb4", "utf8mb4_hungarian_ci", "")) +_charsets.add(Charset(243, "utf8mb4", "utf8mb4_sinhala_ci", "")) +_charsets.add(Charset(244, "utf8mb4", "utf8mb4_german2_ci", "")) +_charsets.add(Charset(245, "utf8mb4", "utf8mb4_croatian_ci", "")) +_charsets.add(Charset(246, "utf8mb4", "utf8mb4_unicode_520_ci", "")) +_charsets.add(Charset(247, "utf8mb4", "utf8mb4_vietnamese_ci", "")) +_charsets.add(Charset(248, "gb18030", "gb18030_chinese_ci", "Yes")) +_charsets.add(Charset(249, "gb18030", "gb18030_bin", "")) +_charsets.add(Charset(250, "gb18030", "gb18030_unicode_520_ci", "")) +_charsets.add(Charset(255, "utf8mb4", "utf8mb4_0900_ai_ci", "")) charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id diff --git a/pymysql/connections.py b/pymysql/connections.py index 6fd15e13..dc69868b 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -18,14 +18,19 @@ from .cursors import Cursor from .optionfile import Parser from .protocol import ( - dump_packet, MysqlPacket, FieldDescriptorPacket, OKPacketWrapper, - EOFPacketWrapper, LoadLocalPacketWrapper + dump_packet, + MysqlPacket, + FieldDescriptorPacket, + OKPacketWrapper, + EOFPacketWrapper, + LoadLocalPacketWrapper, ) from .util import byte2int, int2byte from . import err, VERSION_STRING try: import ssl + SSL_ENABLED = True except ImportError: ssl = None @@ -33,6 +38,7 @@ try: import getpass + DEFAULT_USER = getpass.getuser() del getpass except (ImportError, KeyError): @@ -43,8 +49,10 @@ _py_version = sys.version_info[:2] + def _fast_surrogateescape(s): - return s.decode('ascii', 'surrogateescape') + return s.decode("ascii", "surrogateescape") + def _makefile(sock, mode): return sock.makefile(mode) @@ -63,29 +71,34 @@ def _makefile(sock, mode): } -DEFAULT_CHARSET = 'utf8mb4' +DEFAULT_CHARSET = "utf8mb4" -MAX_PACKET_LEN = 2**24-1 +MAX_PACKET_LEN = 2 ** 24 - 1 def pack_int24(n): - return struct.pack(' 2: use_unicode = True @@ -184,7 +224,9 @@ def __init__(self, host=None, user=None, password="", password = passwd if compress or named_pipe: - raise NotImplementedError("compress and named_pipe arguments are not supported") + raise NotImplementedError( + "compress and named_pipe arguments are not supported" + ) self._local_infile = bool(local_infile) if self._local_infile: @@ -233,12 +275,14 @@ def _config(key, arg): ssl = { "ca": ssl_ca, "check_hostname": bool(ssl_verify_identity), - "verify_mode": ssl_verify_cert if ssl_verify_cert is not None else False, + "verify_mode": ssl_verify_cert + if ssl_verify_cert is not None + else False, } if ssl_cert is not None: ssl["cert"] = ssl_cert if ssl_key is not None: - ssl["key" ] = ssl_key + ssl["key"] = ssl_key if ssl: if not SSL_ENABLED: raise NotImplementedError("ssl module not found") @@ -253,7 +297,7 @@ def _config(key, arg): self.user = user or DEFAULT_USER self.password = password or b"" if isinstance(self.password, str): - self.password = self.password.encode('latin1') + self.password = self.password.encode("latin1") self.db = database self.unix_socket = unix_socket self.bind_address = bind_address @@ -307,9 +351,9 @@ def _config(key, arg): self.server_public_key = server_public_key self._connect_attrs = { - '_client_name': 'pymysql', - '_pid': str(os.getpid()), - '_client_version': VERSION_STRING, + "_client_name": "pymysql", + "_pid": str(os.getpid()), + "_client_version": VERSION_STRING, } if program_name: @@ -319,23 +363,23 @@ def _config(key, arg): self._sock = None else: self.connect() - + def __enter__(self): return self - + def __exit__(self, *exc_info): del exc_info self.close() - + def _create_ssl_ctx(self, sslp): if isinstance(sslp, ssl.SSLContext): return sslp - ca = sslp.get('ca') - capath = sslp.get('capath') + ca = sslp.get("ca") + capath = sslp.get("capath") hasnoca = ca is None and capath is None ctx = ssl.create_default_context(cafile=ca, capath=capath) - ctx.check_hostname = not hasnoca and sslp.get('check_hostname', True) - verify_mode_value = sslp.get('verify_mode') + ctx.check_hostname = not hasnoca and sslp.get("check_hostname", True) + verify_mode_value = sslp.get("verify_mode") if verify_mode_value is None: ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED elif isinstance(verify_mode_value, bool): @@ -351,10 +395,10 @@ def _create_ssl_ctx(self, sslp): ctx.verify_mode = ssl.CERT_REQUIRED else: ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED - if 'cert' in sslp: - ctx.load_cert_chain(sslp['cert'], keyfile=sslp.get('key')) - if 'cipher' in sslp: - ctx.set_ciphers(sslp['cipher']) + if "cert" in sslp: + ctx.load_cert_chain(sslp["cert"], keyfile=sslp.get("key")) + if "cipher" in sslp: + ctx.set_ciphers(sslp["cipher"]) ctx.options |= ssl.OP_NO_SSLv2 ctx.options |= ssl.OP_NO_SSLv3 return ctx @@ -373,7 +417,7 @@ def close(self): self._closed = True if self._sock is None: return - send_data = struct.pack('= 5: + if int(self.server_version.split(".", 1)[0]) >= 5: self.client_flag |= CLIENT.MULTI_RESULTS if self.user is None: @@ -800,28 +851,30 @@ def _request_authentication(self): if isinstance(self.user, str): self.user = self.user.encode(self.encoding) - data_init = struct.pack('=5.0) - data += authresp + b'\0' + data += authresp + b"\0" if self.db and self.server_capabilities & CLIENT.CONNECT_WITH_DB: if isinstance(self.db, str): self.db = self.db.encode(self.encoding) - data += self.db + b'\0' + data += self.db + b"\0" if self.server_capabilities & CLIENT.PLUGIN_AUTH: - data += (plugin_name or b'') + b'\0' + data += (plugin_name or b"") + b"\0" if self.server_capabilities & CLIENT.CONNECT_ATTRS: - connect_attrs = b'' + connect_attrs = b"" for k, v in self._connect_attrs.items(): - k = k.encode('utf-8') - connect_attrs += struct.pack('B', len(k)) + k - v = v.encode('utf-8') - connect_attrs += struct.pack('B', len(v)) + v - data += struct.pack('B', len(connect_attrs)) + connect_attrs + k = k.encode("utf-8") + connect_attrs += struct.pack("B", len(k)) + k + v = v.encode("utf-8") + connect_attrs += struct.pack("B", len(v)) + v + data += struct.pack("B", len(connect_attrs)) + connect_attrs self.write_packet(data) auth_packet = self._read_packet() @@ -868,15 +921,19 @@ def _request_authentication(self): # if authentication method isn't accepted the first byte # will have the octet 254 if auth_packet.is_auth_switch_request(): - if DEBUG: print("received auth switch") + if DEBUG: + print("received auth switch") # https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest - auth_packet.read_uint8() # 0xfe packet identifier + auth_packet.read_uint8() # 0xfe packet identifier plugin_name = auth_packet.read_string() - if self.server_capabilities & CLIENT.PLUGIN_AUTH and plugin_name is not None: + if ( + self.server_capabilities & CLIENT.PLUGIN_AUTH + and plugin_name is not None + ): auth_packet = self._process_auth(plugin_name, auth_packet) else: # send legacy handshake - data = _auth.scramble_old_password(self.password, self.salt) + b'\0' + data = _auth.scramble_old_password(self.password, self.salt) + b"\0" self.write_packet(data) auth_packet = self._read_packet() elif auth_packet.is_extra_auth_data(): @@ -888,9 +945,12 @@ def _request_authentication(self): elif self._auth_plugin_name == "sha256_password": auth_packet = _auth.sha256_password_auth(self, auth_packet) else: - raise err.OperationalError("Received extra packet for auth method %r", self._auth_plugin_name) + raise err.OperationalError( + "Received extra packet for auth method %r", self._auth_plugin_name + ) - if DEBUG: print("Succeed to auth") + if DEBUG: + print("Succeed to auth") def _process_auth(self, plugin_name, auth_packet): handler = self._get_auth_plugin_handler(plugin_name) @@ -898,22 +958,29 @@ def _process_auth(self, plugin_name, auth_packet): try: return handler.authenticate(auth_packet) except AttributeError: - if plugin_name != b'dialog': - raise err.OperationalError(2059, "Authentication plugin '%s'" - " not loaded: - %r missing authenticate method" % (plugin_name, type(handler))) + if plugin_name != b"dialog": + raise err.OperationalError( + 2059, + "Authentication plugin '%s'" + " not loaded: - %r missing authenticate method" + % (plugin_name, type(handler)), + ) if plugin_name == b"caching_sha2_password": return _auth.caching_sha2_password_auth(self, auth_packet) elif plugin_name == b"sha256_password": return _auth.sha256_password_auth(self, auth_packet) elif plugin_name == b"mysql_native_password": data = _auth.scramble_native_password(self.password, auth_packet.read_all()) - elif plugin_name == b'client_ed25519': + elif plugin_name == b"client_ed25519": data = _auth.ed25519_password(self.password, auth_packet.read_all()) elif plugin_name == b"mysql_old_password": - data = _auth.scramble_old_password(self.password, auth_packet.read_all()) + b'\0' + data = ( + _auth.scramble_old_password(self.password, auth_packet.read_all()) + + b"\0" + ) elif plugin_name == b"mysql_clear_password": # https://dev.mysql.com/doc/internals/en/clear-text-authentication.html - data = self.password + b'\0' + data = self.password + b"\0" elif plugin_name == b"dialog": pkt = auth_packet while True: @@ -923,27 +990,41 @@ def _process_auth(self, plugin_name, auth_packet): prompt = pkt.read_all() if prompt == b"Password: ": - self.write_packet(self.password + b'\0') + self.write_packet(self.password + b"\0") elif handler: - resp = 'no response - TypeError within plugin.prompt method' + resp = "no response - TypeError within plugin.prompt method" try: resp = handler.prompt(echo, prompt) - self.write_packet(resp + b'\0') + self.write_packet(resp + b"\0") except AttributeError: - raise err.OperationalError(2059, "Authentication plugin '%s'" \ - " not loaded: - %r missing prompt method" % (plugin_name, handler)) + raise err.OperationalError( + 2059, + "Authentication plugin '%s'" + " not loaded: - %r missing prompt method" + % (plugin_name, handler), + ) except TypeError: - raise err.OperationalError(2061, "Authentication plugin '%s'" \ - " %r didn't respond with string. Returned '%r' to prompt %r" % (plugin_name, handler, resp, prompt)) + raise err.OperationalError( + 2061, + "Authentication plugin '%s'" + " %r didn't respond with string. Returned '%r' to prompt %r" + % (plugin_name, handler, resp, prompt), + ) else: - raise err.OperationalError(2059, "Authentication plugin '%s' (%r) not configured" % (plugin_name, handler)) + raise err.OperationalError( + 2059, + "Authentication plugin '%s' (%r) not configured" + % (plugin_name, handler), + ) pkt = self._read_packet() pkt.check_error() if pkt.is_ok_packet() or last: break return pkt else: - raise err.OperationalError(2059, "Authentication plugin '%s' not configured" % plugin_name) + raise err.OperationalError( + 2059, "Authentication plugin '%s' not configured" % plugin_name + ) self.write_packet(data) pkt = self._read_packet() @@ -953,13 +1034,17 @@ def _process_auth(self, plugin_name, auth_packet): def _get_auth_plugin_handler(self, plugin_name): plugin_class = self._auth_plugin_map.get(plugin_name) if not plugin_class and isinstance(plugin_name, bytes): - plugin_class = self._auth_plugin_map.get(plugin_name.decode('ascii')) + plugin_class = self._auth_plugin_map.get(plugin_name.decode("ascii")) if plugin_class: try: handler = plugin_class(self) except TypeError: - raise err.OperationalError(2059, "Authentication plugin '%s'" - " not loaded: - %r cannot be constructed with connection object" % (plugin_name, plugin_class)) + raise err.OperationalError( + 2059, + "Authentication plugin '%s'" + " not loaded: - %r cannot be constructed with connection object" + % (plugin_name, plugin_class), + ) else: handler = None return handler @@ -982,24 +1067,24 @@ def _get_server_information(self): packet = self._read_packet() data = packet.get_all_data() - self.protocol_version = byte2int(data[i:i+1]) + self.protocol_version = byte2int(data[i : i + 1]) i += 1 - server_end = data.find(b'\0', i) - self.server_version = data[i:server_end].decode('latin1') + server_end = data.find(b"\0", i) + self.server_version = data[i:server_end].decode("latin1") i = server_end + 1 - self.server_thread_id = struct.unpack('= i + 6: - lang, stat, cap_h, salt_len = struct.unpack('= i + salt_len: # salt_len includes auth_plugin_data_part_1 and filler - self.salt += data[i:i+salt_len] + self.salt += data[i : i + salt_len] i += salt_len - i+=1 + i += 1 # AUTH PLUGIN NAME may appear here. if self.server_capabilities & CLIENT.PLUGIN_AUTH and len(data) >= i: # Due to Bug#59453 the auth-plugin-name is missing the terminating @@ -1033,12 +1120,12 @@ def _get_server_information(self): # ref: https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake # didn't use version checks as mariadb is corrected and reports # earlier than those two. - server_end = data.find(b'\0', i) - if server_end < 0: # pragma: no cover - very specific upstream bug + server_end = data.find(b"\0", i) + if server_end < 0: # pragma: no cover - very specific upstream bug # not found \0 and last field so take it all - self._auth_plugin_name = data[i:].decode('utf-8') + self._auth_plugin_name = data[i:].decode("utf-8") else: - self._auth_plugin_name = data[i:server_end].decode('utf-8') + self._auth_plugin_name = data[i:server_end].decode("utf-8") def get_server_info(self): return self.server_version @@ -1056,7 +1143,6 @@ def get_server_info(self): class MySQLResult: - def __init__(self, connection): """ :type connection: Connection @@ -1127,7 +1213,8 @@ def _read_ok_packet(self, first_packet): def _read_load_local_packet(self, first_packet): if not self.connection._local_infile: raise RuntimeError( - "**WARN**: Received LOAD_LOCAL packet but local_infile option is false.") + "**WARN**: Received LOAD_LOCAL packet but local_infile option is false." + ) load_packet = LoadLocalPacketWrapper(first_packet) sender = LoadLocalFile(load_packet.filename, self.connection) try: @@ -1137,14 +1224,16 @@ def _read_load_local_packet(self, first_packet): raise ok_packet = self.connection._read_packet() - if not ok_packet.is_ok_packet(): # pragma: no cover - upstream induced protocol error + if ( + not ok_packet.is_ok_packet() + ): # pragma: no cover - upstream induced protocol error raise err.OperationalError(2014, "Commands Out of Sync") self._read_ok_packet(ok_packet) def _check_packet_is_eof(self, packet): if not packet.is_eof_packet(): return False - #TODO: Support CLIENT.DEPRECATE_EOF + # TODO: Support CLIENT.DEPRECATE_EOF # 1) Add DEPRECATE_EOF to CAPABILITIES # 2) Mask CAPABILITIES with server_capabilities # 3) if server_capabilities & CLIENT.DEPRECATE_EOF: use OKPacketWrapper instead of EOFPacketWrapper @@ -1211,7 +1300,8 @@ def _read_row_from_packet(self, packet): if data is not None: if encoding is not None: data = data.decode(encoding) - if DEBUG: print("DEBUG: DATA = ", data) + if DEBUG: + print("DEBUG: DATA = ", data) if converter is not None: data = converter(data) row.append(data) @@ -1246,17 +1336,18 @@ def _get_descriptions(self): encoding = conn_encoding else: # Integers, Dates and Times, and other basic data is encoded in ascii - encoding = 'ascii' + encoding = "ascii" else: encoding = None converter = self.connection.decoders.get(field_type) if converter is converters.through: converter = None - if DEBUG: print("DEBUG: field={}, converter={}".format(field, converter)) + if DEBUG: + print("DEBUG: field={}, converter={}".format(field, converter)) self.converters.append((encoding, converter)) eof_packet = self.connection._read_packet() - assert eof_packet.is_eof_packet(), 'Protocol error, expecting EOF' + assert eof_packet.is_eof_packet(), "Protocol error, expecting EOF" self.description = tuple(description) @@ -1268,19 +1359,23 @@ def __init__(self, filename, connection): def send_data(self): """Send data packets from the local file to the server""" if not self.connection._sock: - raise err.InterfaceError(0, '') + raise err.InterfaceError(0, "") conn = self.connection try: - with open(self.filename, 'rb') as open_file: - packet_size = min(conn.max_allowed_packet, 16*1024) # 16KB is efficient enough + with open(self.filename, "rb") as open_file: + packet_size = min( + conn.max_allowed_packet, 16 * 1024 + ) # 16KB is efficient enough while True: chunk = open_file.read(packet_size) if not chunk: break conn.write_packet(chunk) except IOError: - raise err.OperationalError(1017, "Can't find file '{0}'".format(self.filename)) + raise err.OperationalError( + 1017, "Can't find file '{0}'".format(self.filename) + ) finally: # send the empty packet to signify we are done sending data - conn.write_packet(b'') + conn.write_packet(b"") diff --git a/pymysql/constants/CLIENT.py b/pymysql/constants/CLIENT.py index b42f1523..34fe57a5 100644 --- a/pymysql/constants/CLIENT.py +++ b/pymysql/constants/CLIENT.py @@ -21,9 +21,16 @@ CONNECT_ATTRS = 1 << 20 PLUGIN_AUTH_LENENC_CLIENT_DATA = 1 << 21 CAPABILITIES = ( - LONG_PASSWORD | LONG_FLAG | PROTOCOL_41 | TRANSACTIONS - | SECURE_CONNECTION | MULTI_RESULTS - | PLUGIN_AUTH | PLUGIN_AUTH_LENENC_CLIENT_DATA | CONNECT_ATTRS) + LONG_PASSWORD + | LONG_FLAG + | PROTOCOL_41 + | TRANSACTIONS + | SECURE_CONNECTION + | MULTI_RESULTS + | PLUGIN_AUTH + | PLUGIN_AUTH_LENENC_CLIENT_DATA + | CONNECT_ATTRS +) # Not done yet HANDLE_EXPIRED_PASSWORDS = 1 << 22 diff --git a/pymysql/constants/COMMAND.py b/pymysql/constants/COMMAND.py index 1da27553..2d98850b 100644 --- a/pymysql/constants/COMMAND.py +++ b/pymysql/constants/COMMAND.py @@ -1,4 +1,3 @@ - COM_SLEEP = 0x00 COM_QUIT = 0x01 COM_INIT_DB = 0x02 @@ -9,12 +8,12 @@ COM_REFRESH = 0x07 COM_SHUTDOWN = 0x08 COM_STATISTICS = 0x09 -COM_PROCESS_INFO = 0x0a -COM_CONNECT = 0x0b -COM_PROCESS_KILL = 0x0c -COM_DEBUG = 0x0d -COM_PING = 0x0e -COM_TIME = 0x0f +COM_PROCESS_INFO = 0x0A +COM_CONNECT = 0x0B +COM_PROCESS_KILL = 0x0C +COM_DEBUG = 0x0D +COM_PING = 0x0E +COM_TIME = 0x0F COM_DELAYED_INSERT = 0x10 COM_CHANGE_USER = 0x11 COM_BINLOG_DUMP = 0x12 @@ -25,9 +24,9 @@ COM_STMT_EXECUTE = 0x17 COM_STMT_SEND_LONG_DATA = 0x18 COM_STMT_CLOSE = 0x19 -COM_STMT_RESET = 0x1a -COM_SET_OPTION = 0x1b -COM_STMT_FETCH = 0x1c -COM_DAEMON = 0x1d -COM_BINLOG_DUMP_GTID = 0x1e -COM_END = 0x1f +COM_STMT_RESET = 0x1A +COM_SET_OPTION = 0x1B +COM_STMT_FETCH = 0x1C +COM_DAEMON = 0x1D +COM_BINLOG_DUMP_GTID = 0x1E +COM_END = 0x1F diff --git a/pymysql/constants/CR.py b/pymysql/constants/CR.py index 48ca956e..25579a7c 100644 --- a/pymysql/constants/CR.py +++ b/pymysql/constants/CR.py @@ -1,68 +1,68 @@ # flake8: noqa # errmsg.h -CR_ERROR_FIRST = 2000 -CR_UNKNOWN_ERROR = 2000 -CR_SOCKET_CREATE_ERROR = 2001 -CR_CONNECTION_ERROR = 2002 -CR_CONN_HOST_ERROR = 2003 -CR_IPSOCK_ERROR = 2004 -CR_UNKNOWN_HOST = 2005 -CR_SERVER_GONE_ERROR = 2006 -CR_VERSION_ERROR = 2007 -CR_OUT_OF_MEMORY = 2008 -CR_WRONG_HOST_INFO = 2009 +CR_ERROR_FIRST = 2000 +CR_UNKNOWN_ERROR = 2000 +CR_SOCKET_CREATE_ERROR = 2001 +CR_CONNECTION_ERROR = 2002 +CR_CONN_HOST_ERROR = 2003 +CR_IPSOCK_ERROR = 2004 +CR_UNKNOWN_HOST = 2005 +CR_SERVER_GONE_ERROR = 2006 +CR_VERSION_ERROR = 2007 +CR_OUT_OF_MEMORY = 2008 +CR_WRONG_HOST_INFO = 2009 CR_LOCALHOST_CONNECTION = 2010 -CR_TCP_CONNECTION = 2011 +CR_TCP_CONNECTION = 2011 CR_SERVER_HANDSHAKE_ERR = 2012 -CR_SERVER_LOST = 2013 +CR_SERVER_LOST = 2013 CR_COMMANDS_OUT_OF_SYNC = 2014 CR_NAMEDPIPE_CONNECTION = 2015 -CR_NAMEDPIPEWAIT_ERROR = 2016 -CR_NAMEDPIPEOPEN_ERROR = 2017 +CR_NAMEDPIPEWAIT_ERROR = 2016 +CR_NAMEDPIPEOPEN_ERROR = 2017 CR_NAMEDPIPESETSTATE_ERROR = 2018 -CR_CANT_READ_CHARSET = 2019 +CR_CANT_READ_CHARSET = 2019 CR_NET_PACKET_TOO_LARGE = 2020 -CR_EMBEDDED_CONNECTION = 2021 -CR_PROBE_SLAVE_STATUS = 2022 -CR_PROBE_SLAVE_HOSTS = 2023 -CR_PROBE_SLAVE_CONNECT = 2024 +CR_EMBEDDED_CONNECTION = 2021 +CR_PROBE_SLAVE_STATUS = 2022 +CR_PROBE_SLAVE_HOSTS = 2023 +CR_PROBE_SLAVE_CONNECT = 2024 CR_PROBE_MASTER_CONNECT = 2025 CR_SSL_CONNECTION_ERROR = 2026 -CR_MALFORMED_PACKET = 2027 -CR_WRONG_LICENSE = 2028 +CR_MALFORMED_PACKET = 2027 +CR_WRONG_LICENSE = 2028 -CR_NULL_POINTER = 2029 -CR_NO_PREPARE_STMT = 2030 -CR_PARAMS_NOT_BOUND = 2031 -CR_DATA_TRUNCATED = 2032 +CR_NULL_POINTER = 2029 +CR_NO_PREPARE_STMT = 2030 +CR_PARAMS_NOT_BOUND = 2031 +CR_DATA_TRUNCATED = 2032 CR_NO_PARAMETERS_EXISTS = 2033 CR_INVALID_PARAMETER_NO = 2034 -CR_INVALID_BUFFER_USE = 2035 +CR_INVALID_BUFFER_USE = 2035 CR_UNSUPPORTED_PARAM_TYPE = 2036 -CR_SHARED_MEMORY_CONNECTION = 2037 -CR_SHARED_MEMORY_CONNECT_REQUEST_ERROR = 2038 -CR_SHARED_MEMORY_CONNECT_ANSWER_ERROR = 2039 +CR_SHARED_MEMORY_CONNECTION = 2037 +CR_SHARED_MEMORY_CONNECT_REQUEST_ERROR = 2038 +CR_SHARED_MEMORY_CONNECT_ANSWER_ERROR = 2039 CR_SHARED_MEMORY_CONNECT_FILE_MAP_ERROR = 2040 -CR_SHARED_MEMORY_CONNECT_MAP_ERROR = 2041 -CR_SHARED_MEMORY_FILE_MAP_ERROR = 2042 -CR_SHARED_MEMORY_MAP_ERROR = 2043 -CR_SHARED_MEMORY_EVENT_ERROR = 2044 +CR_SHARED_MEMORY_CONNECT_MAP_ERROR = 2041 +CR_SHARED_MEMORY_FILE_MAP_ERROR = 2042 +CR_SHARED_MEMORY_MAP_ERROR = 2043 +CR_SHARED_MEMORY_EVENT_ERROR = 2044 CR_SHARED_MEMORY_CONNECT_ABANDONED_ERROR = 2045 -CR_SHARED_MEMORY_CONNECT_SET_ERROR = 2046 -CR_CONN_UNKNOW_PROTOCOL = 2047 -CR_INVALID_CONN_HANDLE = 2048 -CR_SECURE_AUTH = 2049 -CR_FETCH_CANCELED = 2050 -CR_NO_DATA = 2051 -CR_NO_STMT_METADATA = 2052 -CR_NO_RESULT_SET = 2053 -CR_NOT_IMPLEMENTED = 2054 -CR_SERVER_LOST_EXTENDED = 2055 -CR_STMT_CLOSED = 2056 -CR_NEW_STMT_METADATA = 2057 -CR_ALREADY_CONNECTED = 2058 -CR_AUTH_PLUGIN_CANNOT_LOAD = 2059 -CR_DUPLICATE_CONNECTION_ATTR = 2060 -CR_AUTH_PLUGIN_ERR = 2061 +CR_SHARED_MEMORY_CONNECT_SET_ERROR = 2046 +CR_CONN_UNKNOW_PROTOCOL = 2047 +CR_INVALID_CONN_HANDLE = 2048 +CR_SECURE_AUTH = 2049 +CR_FETCH_CANCELED = 2050 +CR_NO_DATA = 2051 +CR_NO_STMT_METADATA = 2052 +CR_NO_RESULT_SET = 2053 +CR_NOT_IMPLEMENTED = 2054 +CR_SERVER_LOST_EXTENDED = 2055 +CR_STMT_CLOSED = 2056 +CR_NEW_STMT_METADATA = 2057 +CR_ALREADY_CONNECTED = 2058 +CR_AUTH_PLUGIN_CANNOT_LOAD = 2059 +CR_DUPLICATE_CONNECTION_ATTR = 2060 +CR_AUTH_PLUGIN_ERR = 2061 CR_ERROR_LAST = 2061 diff --git a/pymysql/constants/FIELD_TYPE.py b/pymysql/constants/FIELD_TYPE.py index 51bd5143..b8b44866 100644 --- a/pymysql/constants/FIELD_TYPE.py +++ b/pymysql/constants/FIELD_TYPE.py @@ -1,5 +1,3 @@ - - DECIMAL = 0 TINY = 1 SHORT = 2 diff --git a/pymysql/constants/SERVER_STATUS.py b/pymysql/constants/SERVER_STATUS.py index 6f5d5663..8f8d7768 100644 --- a/pymysql/constants/SERVER_STATUS.py +++ b/pymysql/constants/SERVER_STATUS.py @@ -1,4 +1,3 @@ - SERVER_STATUS_IN_TRANS = 1 SERVER_STATUS_AUTOCOMMIT = 2 SERVER_MORE_RESULTS_EXISTS = 8 diff --git a/pymysql/converters.py b/pymysql/converters.py index 6d1fc9ee..113dd298 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -25,6 +25,7 @@ def escape_item(val, charset, mapping=None): val = encoder(val, mapping) return val + def escape_dict(val, charset, mapping=None): n = {} for k, v in val.items(): @@ -32,6 +33,7 @@ def escape_dict(val, charset, mapping=None): n[k] = quoted return n + def escape_sequence(val, charset, mapping=None): n = [] for item in val: @@ -39,32 +41,38 @@ def escape_sequence(val, charset, mapping=None): n.append(quoted) return "(" + ",".join(n) + ")" + def escape_set(val, charset, mapping=None): - return ','.join([escape_item(x, charset, mapping) for x in val]) + return ",".join([escape_item(x, charset, mapping) for x in val]) + def escape_bool(value, mapping=None): return str(int(value)) + def escape_int(value, mapping=None): return str(value) + def escape_float(value, mapping=None): s = repr(value) - if s in ('inf', 'nan'): + if s in ("inf", "nan"): raise ProgrammingError("%s can not be used with MySQL" % s) - if 'e' not in s: - s += 'e0' + if "e" not in s: + s += "e0" return s + _escape_table = [chr(x) for x in range(128)] -_escape_table[0] = u'\\0' -_escape_table[ord('\\')] = u'\\\\' -_escape_table[ord('\n')] = u'\\n' -_escape_table[ord('\r')] = u'\\r' -_escape_table[ord('\032')] = u'\\Z' +_escape_table[0] = u"\\0" +_escape_table[ord("\\")] = u"\\\\" +_escape_table[ord("\n")] = u"\\n" +_escape_table[ord("\r")] = u"\\r" +_escape_table[ord("\032")] = u"\\Z" _escape_table[ord('"')] = u'\\"' _escape_table[ord("'")] = u"\\'" + def escape_string(value, mapping=None): """escapes *value* without adding quote. @@ -74,18 +82,22 @@ def escape_string(value, mapping=None): def escape_bytes_prefixed(value, mapping=None): - return "_binary'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) + return "_binary'%s'" % value.decode("ascii", "surrogateescape").translate( + _escape_table + ) def escape_bytes(value, mapping=None): - return "'%s'" % value.decode('ascii', 'surrogateescape').translate(_escape_table) + return "'%s'" % value.decode("ascii", "surrogateescape").translate(_escape_table) def escape_str(value, mapping=None): return "'%s'" % escape_string(str(value), mapping) + def escape_None(value, mapping=None): - return 'NULL' + return "NULL" + def escape_timedelta(obj, mapping=None): seconds = int(obj.seconds) % 60 @@ -97,6 +109,7 @@ def escape_timedelta(obj, mapping=None): fmt = "'{0:02d}:{1:02d}:{2:02d}'" return fmt.format(hours, minutes, seconds, obj.microseconds) + def escape_time(obj, mapping=None): if obj.microsecond: fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" @@ -104,6 +117,7 @@ def escape_time(obj, mapping=None): fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}'" return fmt.format(obj) + def escape_datetime(obj, mapping=None): if obj.microsecond: fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" @@ -111,10 +125,12 @@ def escape_datetime(obj, mapping=None): fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}'" return fmt.format(obj) + def escape_date(obj, mapping=None): fmt = "'{0.year:04}-{0.month:02}-{0.day:02}'" return fmt.format(obj) + def escape_struct_time(obj, mapping=None): return escape_datetime(datetime.datetime(*obj[:6])) @@ -127,10 +143,13 @@ def _convert_second_fraction(s): if not s: return 0 # Pad zeros to ensure the fraction length in microseconds - s = s.ljust(6, '0') + s = s.ljust(6, "0") return int(s[:6]) -DATETIME_RE = re.compile(r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") + +DATETIME_RE = re.compile( + r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?" +) def convert_datetime(obj): @@ -150,7 +169,7 @@ def convert_datetime(obj): """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") m = DATETIME_RE.match(obj) if not m: @@ -159,10 +178,11 @@ def convert_datetime(obj): try: groups = list(m.groups()) groups[-1] = _convert_second_fraction(groups[-1]) - return datetime.datetime(*[ int(x) for x in groups ]) + return datetime.datetime(*[int(x) for x in groups]) except ValueError: return convert_date(obj) + TIMEDELTA_RE = re.compile(r"(-)?(\d{1,3}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") @@ -184,7 +204,7 @@ def convert_timedelta(obj): be parsed correctly by this function. """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") m = TIMEDELTA_RE.match(obj) if not m: @@ -196,16 +216,20 @@ def convert_timedelta(obj): negate = -1 if groups[0] else 1 hours, minutes, seconds, microseconds = groups[1:] - tdelta = datetime.timedelta( - hours = int(hours), - minutes = int(minutes), - seconds = int(seconds), - microseconds = int(microseconds) - ) * negate + tdelta = ( + datetime.timedelta( + hours=int(hours), + minutes=int(minutes), + seconds=int(seconds), + microseconds=int(microseconds), + ) + * negate + ) return tdelta except ValueError: return obj + TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") @@ -232,7 +256,7 @@ def convert_time(obj): use set this function as the converter for FIELD_TYPE.TIME. """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") m = TIME_RE.match(obj) if not m: @@ -242,8 +266,12 @@ def convert_time(obj): groups = list(m.groups()) groups[-1] = _convert_second_fraction(groups[-1]) hours, minutes, seconds, microseconds = groups - return datetime.time(hour=int(hours), minute=int(minutes), - second=int(seconds), microsecond=int(microseconds)) + return datetime.time( + hour=int(hours), + minute=int(minutes), + second=int(seconds), + microsecond=int(microseconds), + ) except ValueError: return obj @@ -263,9 +291,9 @@ def convert_date(obj): """ if isinstance(obj, (bytes, bytearray)): - obj = obj.decode('ascii') + obj = obj.decode("ascii") try: - return datetime.date(*[ int(x) for x in obj.split('-', 2) ]) + return datetime.date(*[int(x) for x in obj.split("-", 2)]) except ValueError: return obj @@ -274,7 +302,7 @@ def through(x): return x -#def convert_bit(b): +# def convert_bit(b): # b = "\x00" * (8 - len(b)) + b # pad w/ zeroes # return struct.unpack(">Q", b)[0] # diff --git a/pymysql/cursors.py b/pymysql/cursors.py index a8c52836..68ac78e7 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -6,10 +6,11 @@ #: executemany only supports simple bulk insert. #: You can use it to load large dataset. RE_INSERT_VALUES = re.compile( - r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" + - r"(\(\s*(?:%s|%\(.+\)s)\s*(?:,\s*(?:%s|%\(.+\)s)\s*)*\))" + - r"(\s*(?:ON DUPLICATE.*)?);?\s*\Z", - re.IGNORECASE | re.DOTALL) + r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" + + r"(\(\s*(?:%s|%\(.+\)s)\s*(?:,\s*(?:%s|%\(.+\)s)\s*)*\))" + + r"(\s*(?:ON DUPLICATE.*)?);?\s*\Z", + re.IGNORECASE | re.DOTALL, +) class Cursor: @@ -167,16 +168,23 @@ def executemany(self, query, args): if m: q_prefix = m.group(1) % () q_values = m.group(2).rstrip() - q_postfix = m.group(3) or '' - assert q_values[0] == '(' and q_values[-1] == ')' - return self._do_execute_many(q_prefix, q_values, q_postfix, args, - self.max_stmt_length, - self._get_db().encoding) + q_postfix = m.group(3) or "" + assert q_values[0] == "(" and q_values[-1] == ")" + return self._do_execute_many( + q_prefix, + q_values, + q_postfix, + args, + self.max_stmt_length, + self._get_db().encoding, + ) self.rowcount = sum(self.execute(query, arg) for arg in args) return self.rowcount - def _do_execute_many(self, prefix, values, postfix, args, max_stmt_length, encoding): + def _do_execute_many( + self, prefix, values, postfix, args, max_stmt_length, encoding + ): conn = self._get_db() escape = self._escape_args if isinstance(prefix, str): @@ -187,18 +195,18 @@ def _do_execute_many(self, prefix, values, postfix, args, max_stmt_length, encod args = iter(args) v = values % escape(next(args), conn) if isinstance(v, str): - v = v.encode(encoding, 'surrogateescape') + v = v.encode(encoding, "surrogateescape") sql += v rows = 0 for arg in args: v = values % escape(arg, conn) if isinstance(v, str): - v = v.encode(encoding, 'surrogateescape') + v = v.encode(encoding, "surrogateescape") if len(sql) + len(v) + len(postfix) + 1 > max_stmt_length: rows += self.execute(sql + postfix) sql = bytearray(prefix) else: - sql += b',' + sql += b"," sql += v rows += self.execute(sql + postfix) self.rowcount = rows @@ -234,14 +242,19 @@ def callproc(self, procname, args=()): """ conn = self._get_db() if args: - fmt = '@_{0}_%d=%s'.format(procname) - self._query('SET %s' % ','.join(fmt % (index, conn.escape(arg)) - for index, arg in enumerate(args))) + fmt = "@_{0}_%d=%s".format(procname) + self._query( + "SET %s" + % ",".join( + fmt % (index, conn.escape(arg)) for index, arg in enumerate(args) + ) + ) self.nextset() - q = "CALL %s(%s)" % (procname, - ','.join(['@_%s_%d' % (procname, i) - for i in range(len(args))])) + q = "CALL %s(%s)" % ( + procname, + ",".join(["@_%s_%d" % (procname, i) for i in range(len(args))]), + ) self._query(q) self._executed = q return args @@ -261,7 +274,7 @@ def fetchmany(self, size=None): if self._rows is None: return () end = self.rownumber + (size or self.arraysize) - result = self._rows[self.rownumber:end] + result = self._rows[self.rownumber : end] self.rownumber = min(end, len(self._rows)) return result @@ -271,17 +284,17 @@ def fetchall(self): if self._rows is None: return () if self.rownumber: - result = self._rows[self.rownumber:] + result = self._rows[self.rownumber :] else: result = self._rows self.rownumber = len(self._rows) return result - def scroll(self, value, mode='relative'): + def scroll(self, value, mode="relative"): self._check_executed() - if mode == 'relative': + if mode == "relative": r = self.rownumber + value - elif mode == 'absolute': + elif mode == "absolute": r = value else: raise err.ProgrammingError("unknown scroll mode %s" % mode) @@ -343,7 +356,7 @@ def _do_get_result(self): for f in self._result.fields: name = f.name if name in fields: - name = f.table_name + '.' + name + name = f.table_name + "." + name fields.append(name) self._fields = fields @@ -453,21 +466,23 @@ def fetchmany(self, size=None): self.rownumber += 1 return rows - def scroll(self, value, mode='relative'): + def scroll(self, value, mode="relative"): self._check_executed() - if mode == 'relative': + if mode == "relative": if value < 0: raise err.NotSupportedError( - "Backwards scrolling not supported by this cursor") + "Backwards scrolling not supported by this cursor" + ) for _ in range(value): self.read_next() self.rownumber += value - elif mode == 'absolute': + elif mode == "absolute": if value < self.rownumber: raise err.NotSupportedError( - "Backwards scrolling not supported by this cursor") + "Backwards scrolling not supported by this cursor" + ) end = value - self.rownumber for _ in range(end): diff --git a/pymysql/err.py b/pymysql/err.py index 94100cfe..3da5b166 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -74,33 +74,69 @@ def _map_error(exc, *errors): error_map[error] = exc -_map_error(ProgrammingError, ER.DB_CREATE_EXISTS, ER.SYNTAX_ERROR, - ER.PARSE_ERROR, ER.NO_SUCH_TABLE, ER.WRONG_DB_NAME, - ER.WRONG_TABLE_NAME, ER.FIELD_SPECIFIED_TWICE, - ER.INVALID_GROUP_FUNC_USE, ER.UNSUPPORTED_EXTENSION, - ER.TABLE_MUST_HAVE_COLUMNS, ER.CANT_DO_THIS_DURING_AN_TRANSACTION, - ER.WRONG_DB_NAME, ER.WRONG_COLUMN_NAME, - ) -_map_error(DataError, ER.WARN_DATA_TRUNCATED, ER.WARN_NULL_TO_NOTNULL, - ER.WARN_DATA_OUT_OF_RANGE, ER.NO_DEFAULT, ER.PRIMARY_CANT_HAVE_NULL, - ER.DATA_TOO_LONG, ER.DATETIME_FUNCTION_OVERFLOW, ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, - ER.ILLEGAL_VALUE_FOR_TYPE) -_map_error(IntegrityError, ER.DUP_ENTRY, ER.NO_REFERENCED_ROW, - ER.NO_REFERENCED_ROW_2, ER.ROW_IS_REFERENCED, ER.ROW_IS_REFERENCED_2, - ER.CANNOT_ADD_FOREIGN, ER.BAD_NULL_ERROR) -_map_error(NotSupportedError, ER.WARNING_NOT_COMPLETE_ROLLBACK, - ER.NOT_SUPPORTED_YET, ER.FEATURE_DISABLED, ER.UNKNOWN_STORAGE_ENGINE) -_map_error(OperationalError, ER.DBACCESS_DENIED_ERROR, ER.ACCESS_DENIED_ERROR, - ER.CON_COUNT_ERROR, ER.TABLEACCESS_DENIED_ERROR, - ER.COLUMNACCESS_DENIED_ERROR, ER.CONSTRAINT_FAILED, ER.LOCK_DEADLOCK) +_map_error( + ProgrammingError, + ER.DB_CREATE_EXISTS, + ER.SYNTAX_ERROR, + ER.PARSE_ERROR, + ER.NO_SUCH_TABLE, + ER.WRONG_DB_NAME, + ER.WRONG_TABLE_NAME, + ER.FIELD_SPECIFIED_TWICE, + ER.INVALID_GROUP_FUNC_USE, + ER.UNSUPPORTED_EXTENSION, + ER.TABLE_MUST_HAVE_COLUMNS, + ER.CANT_DO_THIS_DURING_AN_TRANSACTION, + ER.WRONG_DB_NAME, + ER.WRONG_COLUMN_NAME, +) +_map_error( + DataError, + ER.WARN_DATA_TRUNCATED, + ER.WARN_NULL_TO_NOTNULL, + ER.WARN_DATA_OUT_OF_RANGE, + ER.NO_DEFAULT, + ER.PRIMARY_CANT_HAVE_NULL, + ER.DATA_TOO_LONG, + ER.DATETIME_FUNCTION_OVERFLOW, + ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, + ER.ILLEGAL_VALUE_FOR_TYPE, +) +_map_error( + IntegrityError, + ER.DUP_ENTRY, + ER.NO_REFERENCED_ROW, + ER.NO_REFERENCED_ROW_2, + ER.ROW_IS_REFERENCED, + ER.ROW_IS_REFERENCED_2, + ER.CANNOT_ADD_FOREIGN, + ER.BAD_NULL_ERROR, +) +_map_error( + NotSupportedError, + ER.WARNING_NOT_COMPLETE_ROLLBACK, + ER.NOT_SUPPORTED_YET, + ER.FEATURE_DISABLED, + ER.UNKNOWN_STORAGE_ENGINE, +) +_map_error( + OperationalError, + ER.DBACCESS_DENIED_ERROR, + ER.ACCESS_DENIED_ERROR, + ER.CON_COUNT_ERROR, + ER.TABLEACCESS_DENIED_ERROR, + ER.COLUMNACCESS_DENIED_ERROR, + ER.CONSTRAINT_FAILED, + ER.LOCK_DEADLOCK, +) del _map_error, ER def raise_mysql_exception(data): - errno = struct.unpack('= 2 and value[0] == value[-1] == quote: return value[1:-1] diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 541475ad..24b3f23e 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -25,7 +25,7 @@ def printable(data): if isinstance(data, int): return chr(data) return data - return '.' + return "." try: print("packet length:", len(data)) @@ -35,11 +35,14 @@ def printable(data): print("-" * 66) except ValueError: pass - dump_data = [data[i:i+16] for i in range(0, min(len(data), 256), 16)] + dump_data = [data[i : i + 16] for i in range(0, min(len(data), 256), 16)] for d in dump_data: - print(' '.join("{:02X}".format(byte2int(x)) for x in d) + - ' ' * (16 - len(d)) + ' ' * 2 + - ''.join(printable(x) for x in d)) + print( + " ".join("{:02X}".format(byte2int(x)) for x in d) + + " " * (16 - len(d)) + + " " * 2 + + "".join(printable(x) for x in d) + ) print("-" * 66) print() @@ -49,7 +52,8 @@ class MysqlPacket: Provides an interface for reading/parsing the packet results. """ - __slots__ = ('_position', '_data') + + __slots__ = ("_position", "_data") def __init__(self, data, encoding): self._position = 0 @@ -60,11 +64,13 @@ def get_all_data(self): def read(self, size): """Read the first 'size' bytes in packet and advance cursor past them.""" - result = self._data[self._position:(self._position+size)] + result = self._data[self._position : (self._position + size)] if len(result) != size: - error = ('Result length not requested length:\n' - 'Expected=%s. Actual=%s. Position: %s. Data Length: %s' - % (size, len(result), self._position, len(self._data))) + error = ( + "Result length not requested length:\n" + "Expected=%s. Actual=%s. Position: %s. Data Length: %s" + % (size, len(result), self._position, len(self._data)) + ) if DEBUG: print(error) self.dump() @@ -77,7 +83,7 @@ def read_all(self): (Subsequent read() will return errors.) """ - result = self._data[self._position:] + result = self._data[self._position :] self._position = None # ensure no subsequent read() return result @@ -85,8 +91,10 @@ def advance(self, length): """Advance the cursor in data buffer 'length' bytes.""" new_position = self._position + length if new_position < 0 or new_position > len(self._data): - raise Exception('Invalid advance amount (%s) for cursor. ' - 'Position=%s' % (length, new_position)) + raise Exception( + "Invalid advance amount (%s) for cursor. " + "Position=%s" % (length, new_position) + ) self._position = new_position def rewind(self, position=0): @@ -104,7 +112,7 @@ def get_bytes(self, position, length=1): No error checking is done. If requesting outside end of buffer an empty string (or string shorter than 'length') may be returned! """ - return self._data[position:(position+length)] + return self._data[position : (position + length)] def read_uint8(self): result = self._data[self._position] @@ -112,30 +120,30 @@ def read_uint8(self): return result def read_uint16(self): - result = struct.unpack_from('= 7 + return self._data[0:1] == b"\0" and len(self._data) >= 7 def is_eof_packet(self): # http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet # Caution: \xFE may be LengthEncodedInteger. # If \xFE is LengthEncodedInteger header, 8bytes followed. - return self._data[0:1] == b'\xfe' and len(self._data) < 9 + return self._data[0:1] == b"\xfe" and len(self._data) < 9 def is_auth_switch_request(self): # http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest - return self._data[0:1] == b'\xfe' + return self._data[0:1] == b"\xfe" def is_extra_auth_data(self): # https://dev.mysql.com/doc/internals/en/successful-authentication.html - return self._data[0:1] == b'\x01' + return self._data[0:1] == b"\x01" def is_resultset_packet(self): field_count = ord(self._data[0:1]) return 1 <= field_count <= 250 def is_load_local_packet(self): - return self._data[0:1] == b'\xfb' + return self._data[0:1] == b"\xfb" def is_error_packet(self): - return self._data[0:1] == b'\xff' + return self._data[0:1] == b"\xff" def check_error(self): if self.is_error_packet(): @@ -211,7 +219,8 @@ def raise_for_error(self): self.rewind() self.advance(1) # field_count == error (we already know that) errno = self.read_uint16() - if DEBUG: print("errno =", errno) + if DEBUG: + print("errno =", errno) err.raise_mysql_exception(self._data) def dump(self): @@ -240,8 +249,13 @@ def _parse_field_descriptor(self, encoding): self.org_table = self.read_length_coded_string().decode(encoding) self.name = self.read_length_coded_string().decode(encoding) self.org_name = self.read_length_coded_string().decode(encoding) - self.charsetnr, self.length, self.type_code, self.flags, self.scale = ( - self.read_struct('= version_tuple @@ -53,10 +59,12 @@ def connect(self, **params): p = self.databases[0].copy() p.update(params) conn = pymysql.connect(**p) + @self.addCleanup def teardown(): if conn.open: conn.close() + return conn def _teardown_connections(self): diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index 122882e6..581a0c4a 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -6,9 +6,9 @@ class TestDictCursor(base.PyMySQLTestCase): - bob = {'name': 'bob', 'age': 21, 'DOB': datetime.datetime(1990, 2, 6, 23, 4, 56)} - jim = {'name': 'jim', 'age': 56, 'DOB': datetime.datetime(1955, 5, 9, 13, 12, 45)} - fred = {'name': 'fred', 'age': 100, 'DOB': datetime.datetime(1911, 9, 12, 1, 1, 1)} + bob = {"name": "bob", "age": 21, "DOB": datetime.datetime(1990, 2, 6, 23, 4, 56)} + jim = {"name": "jim", "age": 56, "DOB": datetime.datetime(1955, 5, 9, 13, 12, 45)} + fred = {"name": "fred", "age": 100, "DOB": datetime.datetime(1911, 9, 12, 1, 1, 1)} cursor_type = pymysql.cursors.DictCursor @@ -23,10 +23,14 @@ def setUp(self): c.execute("drop table if exists dictcursor") # include in filterwarnings since for unbuffered dict cursor warning for lack of table # will only be propagated at start of next execute() call - c.execute("""CREATE TABLE dictcursor (name char(20), age int , DOB datetime)""") - data = [("bob", 21, "1990-02-06 23:04:56"), - ("jim", 56, "1955-05-09 13:12:45"), - ("fred", 100, "1911-09-12 01:01:01")] + c.execute( + """CREATE TABLE dictcursor (name char(20), age int , DOB datetime)""" + ) + data = [ + ("bob", 21, "1990-02-06 23:04:56"), + ("jim", 56, "1955-05-09 13:12:45"), + ("fred", 100, "1911-09-12 01:01:01"), + ] c.executemany("insert into dictcursor values (%s,%s,%s)", data) def tearDown(self): @@ -39,13 +43,13 @@ def _ensure_cursor_expired(self, cursor): def test_DictCursor(self): bob, jim, fred = self.bob.copy(), self.jim.copy(), self.fred.copy() - #all assert test compare to the structure as would come out from MySQLdb + # all assert test compare to the structure as would come out from MySQLdb conn = self.conn c = conn.cursor(self.cursor_type) # try an update which should return no rows c.execute("update dictcursor set age=20 where name='bob'") - bob['age'] = 20 + bob["age"] = 20 # pull back the single row dict for bob and check c.execute("SELECT * from dictcursor where name='bob'") r = c.fetchone() @@ -55,19 +59,23 @@ def test_DictCursor(self): # same again, but via fetchall => tuple) c.execute("SELECT * from dictcursor where name='bob'") r = c.fetchall() - self.assertEqual([bob], r, "fetch a 1 row result via fetchall failed via DictCursor") + self.assertEqual( + [bob], r, "fetch a 1 row result via fetchall failed via DictCursor" + ) # same test again but iterate over the c.execute("SELECT * from dictcursor where name='bob'") for r in c: - self.assertEqual(bob, r, "fetch a 1 row result via iteration failed via DictCursor") + self.assertEqual( + bob, r, "fetch a 1 row result via iteration failed via DictCursor" + ) # get all 3 row via fetchall c.execute("SELECT * from dictcursor") r = c.fetchall() - self.assertEqual([bob,jim,fred], r, "fetchall failed via DictCursor") - #same test again but do a list comprehension + self.assertEqual([bob, jim, fred], r, "fetchall failed via DictCursor") + # same test again but do a list comprehension c.execute("SELECT * from dictcursor") r = list(c) - self.assertEqual([bob,jim,fred], r, "DictCursor should be iterable") + self.assertEqual([bob, jim, fred], r, "DictCursor should be iterable") # get all 2 row via fetchmany c.execute("SELECT * from dictcursor") r = c.fetchmany(2) @@ -75,12 +83,13 @@ def test_DictCursor(self): self._ensure_cursor_expired(c) def test_custom_dict(self): - class MyDict(dict): pass + class MyDict(dict): + pass class MyDictCursor(self.cursor_type): dict_type = MyDict - keys = ['name', 'age', 'DOB'] + keys = ["name", "age", "DOB"] bob = MyDict([(k, self.bob[k]) for k in keys]) jim = MyDict([(k, self.jim[k]) for k in keys]) fred = MyDict([(k, self.fred[k]) for k in keys]) @@ -93,18 +102,15 @@ class MyDictCursor(self.cursor_type): cur.execute("SELECT * FROM dictcursor") r = cur.fetchall() - self.assertEqual([bob, jim, fred], r, - "fetchall failed via MyDictCursor") + self.assertEqual([bob, jim, fred], r, "fetchall failed via MyDictCursor") cur.execute("SELECT * FROM dictcursor") r = list(cur) - self.assertEqual([bob, jim, fred], r, - "list failed via MyDictCursor") + self.assertEqual([bob, jim, fred], r, "list failed via MyDictCursor") cur.execute("SELECT * FROM dictcursor") r = cur.fetchmany(2) - self.assertEqual([bob, jim], r, - "list failed via MyDictCursor") + self.assertEqual([bob, jim], r, "list failed via MyDictCursor") self._ensure_cursor_expired(cur) @@ -114,6 +120,8 @@ class TestSSDictCursor(TestDictCursor): def _ensure_cursor_expired(self, cursor): list(cursor.fetchall_unbuffered()) + if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index 2b0de78a..a68a7769 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -6,7 +6,7 @@ from pymysql.constants import CLIENT except Exception: # For local testing from top-level directory, without installing - sys.path.append('../pymysql') + sys.path.append("../pymysql") from pymysql.tests import base import pymysql.cursors from pymysql.constants import CLIENT @@ -18,35 +18,38 @@ def test_SSCursor(self): conn = self.connect(client_flag=CLIENT.MULTI_STATEMENTS) data = [ - ('America', '', 'America/Jamaica'), - ('America', '', 'America/Los_Angeles'), - ('America', '', 'America/Lima'), - ('America', '', 'America/New_York'), - ('America', '', 'America/Menominee'), - ('America', '', 'America/Havana'), - ('America', '', 'America/El_Salvador'), - ('America', '', 'America/Costa_Rica'), - ('America', '', 'America/Denver'), - ('America', '', 'America/Detroit'),] + ("America", "", "America/Jamaica"), + ("America", "", "America/Los_Angeles"), + ("America", "", "America/Lima"), + ("America", "", "America/New_York"), + ("America", "", "America/Menominee"), + ("America", "", "America/Havana"), + ("America", "", "America/El_Salvador"), + ("America", "", "America/Costa_Rica"), + ("America", "", "America/Denver"), + ("America", "", "America/Detroit"), + ] cursor = conn.cursor(pymysql.cursors.SSCursor) # Create table - cursor.execute('CREATE TABLE tz_data (' - 'region VARCHAR(64),' - 'zone VARCHAR(64),' - 'name VARCHAR(64))') + cursor.execute( + "CREATE TABLE tz_data (" + "region VARCHAR(64)," + "zone VARCHAR(64)," + "name VARCHAR(64))" + ) conn.begin() # Test INSERT for i in data: - cursor.execute('INSERT INTO tz_data VALUES (%s, %s, %s)', i) - self.assertEqual(conn.affected_rows(), 1, 'affected_rows does not match') + cursor.execute("INSERT INTO tz_data VALUES (%s, %s, %s)", i) + self.assertEqual(conn.affected_rows(), 1, "affected_rows does not match") conn.commit() # Test fetchone() iter = 0 - cursor.execute('SELECT * FROM tz_data') + cursor.execute("SELECT * FROM tz_data") while True: row = cursor.fetchone() if row is None: @@ -54,26 +57,35 @@ def test_SSCursor(self): iter += 1 # Test cursor.rowcount - self.assertEqual(cursor.rowcount, affected_rows, - 'cursor.rowcount != %s' % (str(affected_rows))) + self.assertEqual( + cursor.rowcount, + affected_rows, + "cursor.rowcount != %s" % (str(affected_rows)), + ) # Test cursor.rownumber - self.assertEqual(cursor.rownumber, iter, - 'cursor.rowcount != %s' % (str(iter))) + self.assertEqual( + cursor.rownumber, iter, "cursor.rowcount != %s" % (str(iter)) + ) # Test row came out the same as it went in - self.assertEqual((row in data), True, - 'Row not found in source data') + self.assertEqual((row in data), True, "Row not found in source data") # Test fetchall - cursor.execute('SELECT * FROM tz_data') - self.assertEqual(len(cursor.fetchall()), len(data), - 'fetchall failed. Number of rows does not match') + cursor.execute("SELECT * FROM tz_data") + self.assertEqual( + len(cursor.fetchall()), + len(data), + "fetchall failed. Number of rows does not match", + ) # Test fetchmany - cursor.execute('SELECT * FROM tz_data') - self.assertEqual(len(cursor.fetchmany(2)), 2, - 'fetchmany failed. Number of rows does not match') + cursor.execute("SELECT * FROM tz_data") + self.assertEqual( + len(cursor.fetchmany(2)), + 2, + "fetchmany failed. Number of rows does not match", + ) # So MySQLdb won't throw "Commands out of sync" while True: @@ -82,30 +94,38 @@ def test_SSCursor(self): break # Test update, affected_rows() - cursor.execute('UPDATE tz_data SET zone = %s', ['Foo']) + cursor.execute("UPDATE tz_data SET zone = %s", ["Foo"]) conn.commit() - self.assertEqual(cursor.rowcount, len(data), - 'Update failed. affected_rows != %s' % (str(len(data)))) + self.assertEqual( + cursor.rowcount, + len(data), + "Update failed. affected_rows != %s" % (str(len(data))), + ) # Test executemany - cursor.executemany('INSERT INTO tz_data VALUES (%s, %s, %s)', data) - self.assertEqual(cursor.rowcount, len(data), - 'executemany failed. cursor.rowcount != %s' % (str(len(data)))) + cursor.executemany("INSERT INTO tz_data VALUES (%s, %s, %s)", data) + self.assertEqual( + cursor.rowcount, + len(data), + "executemany failed. cursor.rowcount != %s" % (str(len(data))), + ) # Test multiple datasets - cursor.execute('SELECT 1; SELECT 2; SELECT 3') - self.assertListEqual(list(cursor), [(1, )]) + cursor.execute("SELECT 1; SELECT 2; SELECT 3") + self.assertListEqual(list(cursor), [(1,)]) self.assertTrue(cursor.nextset()) - self.assertListEqual(list(cursor), [(2, )]) + self.assertListEqual(list(cursor), [(2,)]) self.assertTrue(cursor.nextset()) - self.assertListEqual(list(cursor), [(3, )]) + self.assertListEqual(list(cursor), [(3,)]) self.assertFalse(cursor.nextset()) - cursor.execute('DROP TABLE IF EXISTS tz_data') + cursor.execute("DROP TABLE IF EXISTS tz_data") cursor.close() + __all__ = ["TestSSCursor"] if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 840c4860..f8e622e6 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -18,23 +18,46 @@ def test_datatypes(self): """ test every data type """ conn = self.connect() c = conn.cursor() - c.execute("create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)") + c.execute( + "create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)" + ) try: # insert values - v = (True, -3, 123456789012, 5.7, "hello'\" world", u"Espa\xc3\xb1ol", "binary\x00data".encode(conn.encoding), datetime.date(1988,2,2), datetime.datetime(2014, 5, 15, 7, 45, 57), datetime.timedelta(5,6), datetime.time(16,32), time.localtime()) - c.execute("insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", v) + v = ( + True, + -3, + 123456789012, + 5.7, + "hello'\" world", + u"Espa\xc3\xb1ol", + "binary\x00data".encode(conn.encoding), + datetime.date(1988, 2, 2), + datetime.datetime(2014, 5, 15, 7, 45, 57), + datetime.timedelta(5, 6), + datetime.time(16, 32), + time.localtime(), + ) + c.execute( + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + v, + ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") r = c.fetchone() self.assertEqual(util.int2byte(1), r[0]) self.assertEqual(v[1:10], r[1:10]) - self.assertEqual(datetime.timedelta(0, 60 * (v[10].hour * 60 + v[10].minute)), r[10]) + self.assertEqual( + datetime.timedelta(0, 60 * (v[10].hour * 60 + v[10].minute)), r[10] + ) self.assertEqual(datetime.datetime(*v[-1][:6]), r[-1]) c.execute("delete from test_datatypes") # check nulls - c.execute("insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", [None] * 12) + c.execute( + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + [None] * 12, + ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") r = c.fetchone() self.assertEqual(tuple([None] * 12), r) @@ -43,11 +66,15 @@ def test_datatypes(self): # check sequences type for seq_type in (tuple, list, set, frozenset): - c.execute("insert into test_datatypes (i, l) values (2,4), (6,8), (10,12)") - seq = seq_type([2,6]) - c.execute("select l from test_datatypes where i in %s order by i", (seq,)) + c.execute( + "insert into test_datatypes (i, l) values (2,4), (6,8), (10,12)" + ) + seq = seq_type([2, 6]) + c.execute( + "select l from test_datatypes where i in %s order by i", (seq,) + ) r = c.fetchall() - self.assertEqual(((4,),(8,)), r) + self.assertEqual(((4,), (8,)), r) c.execute("delete from test_datatypes") finally: @@ -59,9 +86,12 @@ def test_dict(self): c = conn.cursor() c.execute("create table test_dict (a integer, b integer, c integer)") try: - c.execute("insert into test_dict (a,b,c) values (%(a)s, %(b)s, %(c)s)", {"a":1,"b":2,"c":3}) + c.execute( + "insert into test_dict (a,b,c) values (%(a)s, %(b)s, %(c)s)", + {"a": 1, "b": 2, "c": 3}, + ) c.execute("select a,b,c from test_dict") - self.assertEqual((1,2,3), c.fetchone()) + self.assertEqual((1, 2, 3), c.fetchone()) finally: c.execute("drop table test_dict") @@ -94,7 +124,8 @@ def test_binary(self): data = bytes(bytearray(range(255))) conn = self.connect() self.safe_create_table( - conn, "test_binary", "create table test_binary (b binary(255))") + conn, "test_binary", "create table test_binary (b binary(255))" + ) with conn.cursor() as c: c.execute("insert into test_binary (b) values (_binary %s)", (data,)) @@ -105,8 +136,7 @@ def test_blob(self): """test blob data""" data = bytes(bytearray(range(256)) * 4) conn = self.connect() - self.safe_create_table( - conn, "test_blob", "create table test_blob (b blob)") + self.safe_create_table(conn, "test_blob", "create table test_blob (b blob)") with conn.cursor() as c: c.execute("insert into test_blob (b) values (_binary %s)", (data,)) @@ -118,23 +148,29 @@ def test_untyped(self): conn = self.connect() c = conn.cursor() c.execute("select null,''") - self.assertEqual((None,u''), c.fetchone()) + self.assertEqual((None, u""), c.fetchone()) c.execute("select '',null") - self.assertEqual((u'',None), c.fetchone()) + self.assertEqual((u"", None), c.fetchone()) def test_timedelta(self): """ test timedelta conversion """ conn = self.connect() c = conn.cursor() - c.execute("select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')") - self.assertEqual((datetime.timedelta(0, 45000), - datetime.timedelta(0, 83579), - datetime.timedelta(0, 83579, 51000), - -datetime.timedelta(0, 45000), - -datetime.timedelta(0, 83579), - -datetime.timedelta(0, 83579, 51000), - -datetime.timedelta(0, 1800)), - c.fetchone()) + c.execute( + "select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')" + ) + self.assertEqual( + ( + datetime.timedelta(0, 45000), + datetime.timedelta(0, 83579), + datetime.timedelta(0, 83579, 51000), + -datetime.timedelta(0, 45000), + -datetime.timedelta(0, 83579), + -datetime.timedelta(0, 83579, 51000), + -datetime.timedelta(0, 1800), + ), + c.fetchone(), + ) def test_datetime_microseconds(self): """ test datetime conversion w microseconds""" @@ -146,10 +182,7 @@ def test_datetime_microseconds(self): dt = datetime.datetime(2013, 11, 12, 9, 9, 9, 123450) c.execute("create table test_datetime (id int, ts datetime(6))") try: - c.execute( - "insert into test_datetime values (%s, %s)", - (1, dt) - ) + c.execute("insert into test_datetime values (%s, %s)", (1, dt)) c.execute("select ts from test_datetime") self.assertEqual((dt,), c.fetchone()) finally: @@ -162,7 +195,7 @@ class TestCursor(base.PyMySQLTestCase): # compatible with the DB-API 2.0 spec and has not broken # any unit tests for anything we've tried. - #def test_description(self): + # def test_description(self): # """ test description attribute """ # # result is from MySQLdb module # r = (('Host', 254, 11, 60, 60, 0, 0), @@ -227,22 +260,22 @@ def test_aggregates(self): conn = self.connect() c = conn.cursor() try: - c.execute('create table test_aggregates (i integer)') + c.execute("create table test_aggregates (i integer)") for i in range(0, 10): - c.execute('insert into test_aggregates (i) values (%s)', (i,)) - c.execute('select sum(i) from test_aggregates') - r, = c.fetchone() - self.assertEqual(sum(range(0,10)), r) + c.execute("insert into test_aggregates (i) values (%s)", (i,)) + c.execute("select sum(i) from test_aggregates") + (r,) = c.fetchone() + self.assertEqual(sum(range(0, 10)), r) finally: - c.execute('drop table test_aggregates') + c.execute("drop table test_aggregates") def test_single_tuple(self): """ test a single tuple """ conn = self.connect() c = conn.cursor() self.safe_create_table( - conn, 'mystuff', - "create table mystuff (id integer primary key)") + conn, "mystuff", "create table mystuff (id integer primary key)" + ) c.execute("insert into mystuff (id) values (1)") c.execute("insert into mystuff (id) values (2)") c.execute("select id from mystuff where id in %s", ((1,),)) @@ -256,12 +289,16 @@ def test_json(self): if not self.mysql_server_is(conn, (5, 7, 0)): pytest.skip("JSON type is not supported on MySQL <= 5.6") - self.safe_create_table(conn, "test_json", """\ + self.safe_create_table( + conn, + "test_json", + """\ create table test_json ( id int not null, json JSON not null, primary key (id) -);""") +);""", + ) cur = conn.cursor() json_str = u'{"hello": "こんãĢãĄã¯"}' @@ -285,7 +322,10 @@ def setUp(self): c = conn.cursor(self.cursor_type) # create a table ane some data to query - self.safe_create_table(conn, 'bulkinsert', """\ + self.safe_create_table( + conn, + "bulkinsert", + """\ CREATE TABLE bulkinsert ( id int, @@ -294,7 +334,8 @@ def setUp(self): height int, PRIMARY KEY (id) ) -""") +""", + ) def _verify_records(self, data): conn = self.connect() @@ -308,27 +349,38 @@ def test_bulk_insert(self): cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] - cursor.executemany("insert into bulkinsert (id, name, age, height) " - "values (%s,%s,%s,%s)", data) + cursor.executemany( + "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + data, + ) self.assertEqual( - cursor._last_executed, bytearray( - b"insert into bulkinsert (id, name, age, height) values " - b"(0,'bob',21,123),(1,'jim',56,45),(2,'fred',100,180)")) - cursor.execute('commit') + cursor._last_executed, + bytearray( + b"insert into bulkinsert (id, name, age, height) values " + b"(0,'bob',21,123),(1,'jim',56,45),(2,'fred',100,180)" + ), + ) + cursor.execute("commit") self._verify_records(data) def test_bulk_insert_multiline_statement(self): conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] - cursor.executemany("""insert + cursor.executemany( + """insert into bulkinsert (id, name, age, height) values (%s, %s , %s, %s ) - """, data) - self.assertEqual(cursor._last_executed.strip(), bytearray(b"""insert + """, + data, + ) + self.assertEqual( + cursor._last_executed.strip(), + bytearray( + b"""insert into bulkinsert (id, name, age, height) values (0, @@ -337,17 +389,21 @@ def test_bulk_insert_multiline_statement(self): 'jim' , 56, 45 ),(2, 'fred' , 100, -180 )""")) - cursor.execute('commit') +180 )""" + ), + ) + cursor.execute("commit") self._verify_records(data) def test_bulk_insert_single_record(self): conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123)] - cursor.executemany("insert into bulkinsert (id, name, age, height) " - "values (%s,%s,%s,%s)", data) - cursor.execute('commit') + cursor.executemany( + "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + data, + ) + cursor.execute("commit") self._verify_records(data) def test_issue_288(self): @@ -355,15 +411,21 @@ def test_issue_288(self): conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] - cursor.executemany("""insert + cursor.executemany( + """insert into bulkinsert (id, name, age, height) values (%s, %s , %s, %s ) on duplicate key update age = values(age) - """, data) - self.assertEqual(cursor._last_executed.strip(), bytearray(b"""insert + """, + data, + ) + self.assertEqual( + cursor._last_executed.strip(), + bytearray( + b"""insert into bulkinsert (id, name, age, height) values (0, @@ -373,6 +435,8 @@ def test_issue_288(self): 45 ),(2, 'fred' , 100, 180 ) on duplicate key update -age = values(age)""")) - cursor.execute('commit') +age = values(age)""" + ), + ) + cursor.execute("commit") self._verify_records(data) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index db36c3e6..abd30e0b 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -54,34 +54,37 @@ class TestAuthentication(base.PyMySQLTestCase): sha256_password_found = False import os - osuser = os.environ.get('USER') + + osuser = os.environ.get("USER") # socket auth requires the current user and for the connection to be a socket # rest do grants @localhost due to incomplete logic - TODO change to @% then db = base.PyMySQLTestCase.databases[0].copy() - socket_auth = db.get('unix_socket') is not None \ - and db.get('host') in ('localhost', '127.0.0.1') + socket_auth = db.get("unix_socket") is not None and db.get("host") in ( + "localhost", + "127.0.0.1", + ) cur = pymysql.connect(**db).cursor() - del db['user'] + del db["user"] cur.execute("SHOW PLUGINS") for r in cur: - if (r[1], r[2]) != (u'ACTIVE', u'AUTHENTICATION'): + if (r[1], r[2]) != (u"ACTIVE", u"AUTHENTICATION"): continue - if r[3] == u'auth_socket.so' or r[0] == u'unix_socket': + if r[3] == u"auth_socket.so" or r[0] == u"unix_socket": socket_plugin_name = r[0] socket_found = True - elif r[3] == u'dialog_examples.so': - if r[0] == 'two_questions': - two_questions_found = True - elif r[0] == 'three_attempts': - three_attempts_found = True - elif r[0] == u'pam': + elif r[3] == u"dialog_examples.so": + if r[0] == "two_questions": + two_questions_found = True + elif r[0] == "three_attempts": + three_attempts_found = True + elif r[0] == u"pam": pam_found = True - pam_plugin_name = r[3].split('.')[0] - if pam_plugin_name == 'auth_pam': - pam_plugin_name = 'pam' + pam_plugin_name = r[3].split(".")[0] + if pam_plugin_name == "auth_pam": + pam_plugin_name = "pam" # MySQL: authentication_pam # https://dev.mysql.com/doc/refman/5.5/en/pam-authentication-plugin.html @@ -89,11 +92,11 @@ class TestAuthentication(base.PyMySQLTestCase): # https://mariadb.com/kb/en/mariadb/pam-authentication-plugin/ # Names differ but functionality is close - elif r[0] == u'mysql_old_password': + elif r[0] == u"mysql_old_password": mysql_old_password_found = True - elif r[0] == u'sha256_password': + elif r[0] == u"sha256_password": sha256_password_found = True - #else: + # else: # print("plugin: %r" % r[0]) def test_plugin(self): @@ -101,9 +104,11 @@ def test_plugin(self): if not self.mysql_server_is(conn, (5, 5, 0)): pytest.skip("MySQL-5.5 required for plugins") cur = conn.cursor() - cur.execute("select plugin from mysql.user where concat(user, '@', host)=current_user()") + cur.execute( + "select plugin from mysql.user where concat(user, '@', host)=current_user()" + ) for r in cur: - self.assertIn(conn._auth_plugin_name, (r[0], 'mysql_native_password')) + self.assertIn(conn._auth_plugin_name, (r[0], "mysql_native_password")) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif(socket_found, reason="socket plugin already installed") @@ -113,17 +118,17 @@ def testSocketAuthInstallPlugin(self): try: cur.execute("install plugin auth_socket soname 'auth_socket.so'") TestAuthentication.socket_found = True - self.socket_plugin_name = 'auth_socket' + self.socket_plugin_name = "auth_socket" self.realtestSocketAuth() except pymysql.err.InternalError: try: cur.execute("install soname 'auth_socket'") TestAuthentication.socket_found = True - self.socket_plugin_name = 'unix_socket' + self.socket_plugin_name = "unix_socket" self.realtestSocketAuth() except pymysql.err.InternalError: TestAuthentication.socket_found = False - pytest.skip('we couldn\'t install the socket plugin') + pytest.skip("we couldn't install the socket plugin") finally: if TestAuthentication.socket_found: cur.execute("uninstall plugin %s" % self.socket_plugin_name) @@ -134,27 +139,30 @@ def testSocketAuth(self): self.realtestSocketAuth() def realtestSocketAuth(self): - with TempUser(self.connect().cursor(), TestAuthentication.osuser + '@localhost', - self.databases[0]['db'], self.socket_plugin_name) as u: + with TempUser( + self.connect().cursor(), + TestAuthentication.osuser + "@localhost", + self.databases[0]["db"], + self.socket_plugin_name, + ) as u: c = pymysql.connect(user=TestAuthentication.osuser, **self.db) class Dialog: - fail=False + fail = False def __init__(self, con): - self.fail=TestAuthentication.Dialog.fail + self.fail = TestAuthentication.Dialog.fail pass def prompt(self, echo, prompt): if self.fail: - self.fail=False - return b'bad guess at a password' + self.fail = False + return b"bad guess at a password" return self.m.get(prompt) class DialogHandler: - def __init__(self, con): - self.con=con + self.con = con def authenticate(self, pkt): while True: @@ -163,10 +171,10 @@ def authenticate(self, pkt): last = (flag & 0x01) == 0x01 prompt = pkt.read_all() - if prompt == b'Password, please:': - self.con.write_packet(b'stillnotverysecret\0') + if prompt == b"Password, please:": + self.con.write_packet(b"stillnotverysecret\0") else: - self.con.write_packet(b'no idea what to do with this prompt\0') + self.con.write_packet(b"no idea what to do with this prompt\0") pkt = self.con._read_packet() pkt.check_error() if pkt.is_ok_packet() or last: @@ -175,11 +183,12 @@ def authenticate(self, pkt): class DefectiveHandler: def __init__(self, con): - self.con=con - + self.con = con @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(two_questions_found, reason="two_questions plugin already installed") + @pytest.mark.skipif( + two_questions_found, reason="two_questions plugin already installed" + ) def testDialogAuthTwoQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -188,7 +197,7 @@ def testDialogAuthTwoQuestionsInstallPlugin(self): TestAuthentication.two_questions_found = True self.realTestDialogAuthTwoQuestions() except pymysql.err.InternalError: - pytest.skip('we couldn\'t install the two_questions plugin') + pytest.skip("we couldn't install the two_questions plugin") finally: if TestAuthentication.two_questions_found: cur.execute("uninstall plugin two_questions") @@ -199,17 +208,30 @@ def testDialogAuthTwoQuestions(self): self.realTestDialogAuthTwoQuestions() def realTestDialogAuthTwoQuestions(self): - TestAuthentication.Dialog.fail=False - TestAuthentication.Dialog.m = {b'Password, please:': b'notverysecret', - b'Are you sure ?': b'yes, of course'} - with TempUser(self.connect().cursor(), 'pymysql_2q@localhost', - self.databases[0]['db'], 'two_questions', 'notverysecret') as u: + TestAuthentication.Dialog.fail = False + TestAuthentication.Dialog.m = { + b"Password, please:": b"notverysecret", + b"Are you sure ?": b"yes, of course", + } + with TempUser( + self.connect().cursor(), + "pymysql_2q@localhost", + self.databases[0]["db"], + "two_questions", + "notverysecret", + ) as u: with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_2q', **self.db) - pymysql.connect(user='pymysql_2q', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) + pymysql.connect(user="pymysql_2q", **self.db) + pymysql.connect( + user="pymysql_2q", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(three_attempts_found, reason="three_attempts plugin already installed") + @pytest.mark.skipif( + three_attempts_found, reason="three_attempts plugin already installed" + ) def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -218,7 +240,7 @@ def testDialogAuthThreeAttemptsQuestionsInstallPlugin(self): TestAuthentication.three_attempts_found = True self.realTestDialogAuthThreeAttempts() except pymysql.err.InternalError: - pytest.skip('we couldn\'t install the three_attempts plugin') + pytest.skip("we couldn't install the three_attempts plugin") finally: if TestAuthentication.three_attempts_found: cur.execute("uninstall plugin three_attempts") @@ -229,30 +251,67 @@ def testDialogAuthThreeAttempts(self): self.realTestDialogAuthThreeAttempts() def realTestDialogAuthThreeAttempts(self): - TestAuthentication.Dialog.m = {b'Password, please:': b'stillnotverysecret'} - TestAuthentication.Dialog.fail=True # fail just once. We've got three attempts after all - with TempUser(self.connect().cursor(), 'pymysql_3a@localhost', - self.databases[0]['db'], 'three_attempts', 'stillnotverysecret') as u: - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.DialogHandler}, **self.db) + TestAuthentication.Dialog.m = {b"Password, please:": b"stillnotverysecret"} + TestAuthentication.Dialog.fail = ( + True # fail just once. We've got three attempts after all + ) + with TempUser( + self.connect().cursor(), + "pymysql_3a@localhost", + self.databases[0]["db"], + "three_attempts", + "stillnotverysecret", + ) as u: + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.DialogHandler}, + **self.db + ) with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': object}, **self.db) + pymysql.connect( + user="pymysql_3a", auth_plugin_map={b"dialog": object}, **self.db + ) with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.DefectiveHandler}, **self.db) + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.DefectiveHandler}, + **self.db + ) with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'notdialogplugin': TestAuthentication.Dialog}, **self.db) - TestAuthentication.Dialog.m = {b'Password, please:': b'I do not know'} + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"notdialogplugin": TestAuthentication.Dialog}, + **self.db + ) + TestAuthentication.Dialog.m = {b"Password, please:": b"I do not know"} with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) - TestAuthentication.Dialog.m = {b'Password, please:': None} + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) + TestAuthentication.Dialog.m = {b"Password, please:": None} with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_3a', auth_plugin_map={b'dialog': TestAuthentication.Dialog}, **self.db) + pymysql.connect( + user="pymysql_3a", + auth_plugin_map={b"dialog": TestAuthentication.Dialog}, + **self.db + ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif(pam_found, reason="pam plugin already installed") - @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") - @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") + @pytest.mark.skipif( + os.environ.get("PASSWORD") is None, reason="PASSWORD env var required" + ) + @pytest.mark.skipif( + os.environ.get("PAMSERVICE") is None, reason="PAMSERVICE env var required" + ) def testPamAuthInstallPlugin(self): # needs plugin. lets install it. cur = self.connect().cursor() @@ -261,133 +320,162 @@ def testPamAuthInstallPlugin(self): TestAuthentication.pam_found = True self.realTestPamAuth() except pymysql.err.InternalError: - pytest.skip('we couldn\'t install the auth_pam plugin') + pytest.skip("we couldn't install the auth_pam plugin") finally: if TestAuthentication.pam_found: cur.execute("uninstall plugin pam") - @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif(not pam_found, reason="no pam plugin") - @pytest.mark.skipif(os.environ.get('PASSWORD') is None, reason="PASSWORD env var required") - @pytest.mark.skipif(os.environ.get('PAMSERVICE') is None, reason="PAMSERVICE env var required") + @pytest.mark.skipif( + os.environ.get("PASSWORD") is None, reason="PASSWORD env var required" + ) + @pytest.mark.skipif( + os.environ.get("PAMSERVICE") is None, reason="PAMSERVICE env var required" + ) def testPamAuth(self): self.realTestPamAuth() def realTestPamAuth(self): db = self.db.copy() import os - db['password'] = os.environ.get('PASSWORD') + + db["password"] = os.environ.get("PASSWORD") cur = self.connect().cursor() try: - cur.execute('show grants for ' + TestAuthentication.osuser + '@localhost') + cur.execute("show grants for " + TestAuthentication.osuser + "@localhost") grants = cur.fetchone()[0] - cur.execute('drop user ' + TestAuthentication.osuser + '@localhost') + cur.execute("drop user " + TestAuthentication.osuser + "@localhost") except pymysql.OperationalError as e: # assuming the user doesn't exist which is ok too self.assertEqual(1045, e.args[0]) grants = None - with TempUser(cur, TestAuthentication.osuser + '@localhost', - self.databases[0]['db'], 'pam', os.environ.get('PAMSERVICE')) as u: + with TempUser( + cur, + TestAuthentication.osuser + "@localhost", + self.databases[0]["db"], + "pam", + os.environ.get("PAMSERVICE"), + ) as u: try: c = pymysql.connect(user=TestAuthentication.osuser, **db) - db['password'] = 'very bad guess at password' + db["password"] = "very bad guess at password" with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user=TestAuthentication.osuser, - auth_plugin_map={b'mysql_cleartext_password': TestAuthentication.DefectiveHandler}, - **self.db) + pymysql.connect( + user=TestAuthentication.osuser, + auth_plugin_map={ + b"mysql_cleartext_password": TestAuthentication.DefectiveHandler + }, + **self.db + ) except pymysql.OperationalError as e: self.assertEqual(1045, e.args[0]) # we had 'bad guess at password' work with pam. Well at least we get a permission denied here with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user=TestAuthentication.osuser, - auth_plugin_map={b'mysql_cleartext_password': TestAuthentication.DefectiveHandler}, - **self.db) + pymysql.connect( + user=TestAuthentication.osuser, + auth_plugin_map={ + b"mysql_cleartext_password": TestAuthentication.DefectiveHandler + }, + **self.db + ) if grants: # recreate the user cur.execute(grants) # select old_password("crummy p\tassword"); - #| old_password("crummy p\tassword") | - #| 2a01785203b08770 | + # | old_password("crummy p\tassword") | + # | 2a01785203b08770 | @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(not mysql_old_password_found, reason="no mysql_old_password plugin") + @pytest.mark.skipif( + not mysql_old_password_found, reason="no mysql_old_password plugin" + ) def testMySQLOldPasswordAuth(self): conn = self.connect() if self.mysql_server_is(conn, (5, 7, 0)): - pytest.skip('Old passwords aren\'t supported in 5.7') + pytest.skip("Old passwords aren't supported in 5.7") # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") # from login in MySQL-5.6 if self.mysql_server_is(conn, (5, 6, 0)): - pytest.skip('Old passwords don\'t authenticate in 5.6') + pytest.skip("Old passwords don't authenticate in 5.6") db = self.db.copy() - db['password'] = "crummy p\tassword" + db["password"] = "crummy p\tassword" c = conn.cursor() # deprecated in 5.6 - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) + c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) else: - c.execute("SELECT OLD_PASSWORD('%s')" % db['password']) + c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) v = c.fetchone()[0] - self.assertEqual(v, '2a01785203b08770') + self.assertEqual(v, "2a01785203b08770") # only works in MariaDB and MySQL-5.6 - can't separate out by version - #if self.mysql_server_is(self.connect(), (5, 5, 0)): + # if self.mysql_server_is(self.connect(), (5, 5, 0)): # with TempUser(c, 'old_pass_user@localhost', # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: # cur = pymysql.connect(user='old_pass_user', **db).cursor() # cur.execute("SELECT VERSION()") c.execute("SELECT @@secure_auth") secure_auth_setting = c.fetchone()[0] - c.execute('set old_passwords=1') + c.execute("set old_passwords=1") # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if sys.version_info[0:2] >= (3,2) and self.mysql_server_is(conn, (5, 6, 0)): + if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: - c.execute('set global secure_auth=0') + c.execute("set global secure_auth=0") else: - c.execute('set global secure_auth=0') - with TempUser(c, 'old_pass_user@localhost', - self.databases[0]['db'], password=db['password']) as u: - cur = pymysql.connect(user='old_pass_user', **db).cursor() + c.execute("set global secure_auth=0") + with TempUser( + c, + "old_pass_user@localhost", + self.databases[0]["db"], + password=db["password"], + ) as u: + cur = pymysql.connect(user="old_pass_user", **db).cursor() cur.execute("SELECT VERSION()") - c.execute('set global secure_auth=%r' % secure_auth_setting) + c.execute("set global secure_auth=%r" % secure_auth_setting) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif(not sha256_password_found, reason="no sha256 password authentication plugin found") + @pytest.mark.skipif( + not sha256_password_found, + reason="no sha256 password authentication plugin found", + ) def testAuthSHA256(self): conn = self.connect() c = conn.cursor() - with TempUser(c, 'pymysql_sha256@localhost', - self.databases[0]['db'], 'sha256_password') as u: + with TempUser( + c, "pymysql_sha256@localhost", self.databases[0]["db"], "sha256_password" + ) as u: if self.mysql_server_is(conn, (5, 7, 0)): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") else: - c.execute('SET old_passwords = 2') - c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' = PASSWORD('Sh@256Pa33')") + c.execute("SET old_passwords = 2") + c.execute( + "SET PASSWORD FOR 'pymysql_sha256'@'localhost' = PASSWORD('Sh@256Pa33')" + ) c.execute("FLUSH PRIVILEGES") db = self.db.copy() - db['password'] = "Sh@256Pa33" - # Although SHA256 is supported, need the configuration of public key of the mysql server. Currently will get error by this test. + db["password"] = "Sh@256Pa33" + # Although SHA256 is supported, need the configuration of public key of the mysql server. Currently will get error by this test. with self.assertRaises(pymysql.err.OperationalError): - pymysql.connect(user='pymysql_sha256', **db) + pymysql.connect(user="pymysql_sha256", **db) -class TestConnection(base.PyMySQLTestCase): +class TestConnection(base.PyMySQLTestCase): def test_utf8mb4(self): """This test requires MySQL >= 5.5""" arg = self.databases[0].copy() - arg['charset'] = 'utf8mb4' + arg["charset"] = "utf8mb4" conn = pymysql.connect(**arg) def test_largedata(self): """Large query and response (>=16MB)""" cur = self.connect().cursor() cur.execute("SELECT @@max_allowed_packet") - if cur.fetchone()[0] < 16*1024*1024 + 10: + if cur.fetchone()[0] < 16 * 1024 * 1024 + 10: print("Set max_allowed_packet to bigger than 17MB") return - t = 'a' * (16*1024*1024) + t = "a" * (16 * 1024 * 1024) cur.execute("SELECT '" + t + "'") assert cur.fetchone()[0] == t @@ -406,15 +494,15 @@ def test_autocommit(self): def test_select_db(self): con = self.connect() - current_db = self.databases[0]['db'] - other_db = self.databases[1]['db'] + current_db = self.databases[0]["db"] + other_db = self.databases[1]["db"] cur = con.cursor() - cur.execute('SELECT database()') + cur.execute("SELECT database()") self.assertEqual(cur.fetchone()[0], current_db) con.select_db(other_db) - cur.execute('SELECT database()') + cur.execute("SELECT database()") self.assertEqual(cur.fetchone()[0], other_db) def test_connection_gone_away(self): @@ -429,29 +517,30 @@ def test_connection_gone_away(self): with self.assertRaises(pymysql.OperationalError) as cm: cur.execute("SELECT 1+1") # error occures while reading, not writing because of socket buffer. - #self.assertEqual(cm.exception.args[0], 2006) + # self.assertEqual(cm.exception.args[0], 2006) self.assertIn(cm.exception.args[0], (2006, 2013)) def test_init_command(self): conn = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) c = conn.cursor() c.execute('select "foobar";') - self.assertEqual(('foobar',), c.fetchone()) + self.assertEqual(("foobar",), c.fetchone()) conn.close() with self.assertRaises(pymysql.err.Error): conn.ping(reconnect=False) def test_read_default_group(self): conn = self.connect( - read_default_group='client', + read_default_group="client", ) self.assertTrue(conn.open) def test_set_charset(self): c = self.connect() - c.set_charset('utf8mb4') + c.set_charset("utf8mb4") # TODO validate setting here def test_defer_connect(self): @@ -460,12 +549,13 @@ def test_defer_connect(self): d = self.databases[0].copy() try: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(d['unix_socket']) + sock.connect(d["unix_socket"]) except KeyError: sock.close() sock = socket.create_connection( - (d.get('host', 'localhost'), d.get('port', 3306))) - for k in ['unix_socket', 'host', 'port']: + (d.get("host", "localhost"), d.get("port", 3306)) + ) + for k in ["unix_socket", "host", "port"]: try: del d[k] except KeyError: @@ -479,9 +569,12 @@ def test_defer_connect(self): def test_ssl_connect(self): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl={ "ca": "ca", @@ -497,9 +590,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_called_with("cipher") dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl={ "ca": "ca", @@ -514,9 +610,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca="ca", ) @@ -527,9 +626,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca="ca", ssl_cert="cert", @@ -543,9 +645,12 @@ def test_ssl_connect(self): for ssl_verify_cert in (True, "1", "yes", "true"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_cert="cert", ssl_key="key", @@ -554,14 +659,19 @@ def test_ssl_connect(self): assert create_default_context.called assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", keyfile="key" + ) dummy_ssl_context.set_ciphers.assert_not_called for ssl_verify_cert in (None, False, "0", "no", "false"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_cert="cert", ssl_key="key", @@ -570,15 +680,20 @@ def test_ssl_connect(self): assert create_default_context.called assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", keyfile="key" + ) dummy_ssl_context.set_ciphers.assert_not_called for ssl_ca in ("ca", None): for ssl_verify_cert in ("foo", "bar", ""): dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca=ssl_ca, ssl_cert="cert", @@ -587,14 +702,21 @@ def test_ssl_connect(self): ) assert create_default_context.called assert not dummy_ssl_context.check_hostname - assert dummy_ssl_context.verify_mode == (ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE), (ssl_ca, ssl_verify_cert) - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + assert dummy_ssl_context.verify_mode == ( + ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE + ), (ssl_ca, ssl_verify_cert) + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", keyfile="key" + ) dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_ca="ca", ssl_cert="cert", @@ -608,9 +730,12 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_disabled=True, ssl={ @@ -622,9 +747,12 @@ def test_ssl_connect(self): assert not create_default_context.called dummy_ssl_context = mock.Mock(options=0) - with mock.patch("pymysql.connections.Connection.connect") as connect, \ - mock.patch("pymysql.connections.ssl.create_default_context", - new=mock.Mock(return_value=dummy_ssl_context)) as create_default_context: + with mock.patch( + "pymysql.connections.Connection.connect" + ) as connect, mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: pymysql.connect( ssl_disabled=True, ssl_ca="ca", @@ -679,7 +807,7 @@ class Custom(str): pass mapping = {str: pymysql.escape_string} - self.assertEqual(con.escape(Custom('foobar'), mapping), "'foobar'") + self.assertEqual(con.escape(Custom("foobar"), mapping), "'foobar'") def test_escape_no_default(self): con = self.connect() @@ -693,7 +821,7 @@ def test_escape_dict_value(self): mapping = con.encoders.copy() mapping[Foo] = escape_foo - self.assertEqual(con.escape({'foo': Foo()}, mapping), {'foo': "bar"}) + self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) def test_escape_list_item(self): con = self.connect() @@ -706,7 +834,8 @@ def test_escape_list_item(self): def test_previous_cursor_not_closed(self): con = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) cur1 = con.cursor() cur1.execute("SELECT 1; SELECT 2") cur2 = con.cursor() diff --git a/pymysql/tests/test_converters.py b/pymysql/tests/test_converters.py index c2c9b6bf..dc194a9e 100644 --- a/pymysql/tests/test_converters.py +++ b/pymysql/tests/test_converters.py @@ -7,34 +7,30 @@ class TestConverter(TestCase): - def test_escape_string(self): - self.assertEqual( - converters.escape_string(u"foo\nbar"), - u"foo\\nbar" - ) + self.assertEqual(converters.escape_string(u"foo\nbar"), u"foo\\nbar") def test_convert_datetime(self): expected = datetime.datetime(2007, 2, 24, 23, 6, 20) - dt = converters.convert_datetime('2007-02-24 23:06:20') + dt = converters.convert_datetime("2007-02-24 23:06:20") self.assertEqual(dt, expected) def test_convert_datetime_with_fsp(self): expected = datetime.datetime(2007, 2, 24, 23, 6, 20, 511581) - dt = converters.convert_datetime('2007-02-24 23:06:20.511581') + dt = converters.convert_datetime("2007-02-24 23:06:20.511581") self.assertEqual(dt, expected) def _test_convert_timedelta(self, with_negate=False, with_fsp=False): - d = {'hours': 789, 'minutes': 12, 'seconds': 34} - s = '%(hours)s:%(minutes)s:%(seconds)s' % d + d = {"hours": 789, "minutes": 12, "seconds": 34} + s = "%(hours)s:%(minutes)s:%(seconds)s" % d if with_fsp: - d['microseconds'] = 511581 - s += '.%(microseconds)s' % d + d["microseconds"] = 511581 + s += ".%(microseconds)s" % d expected = datetime.timedelta(**d) if with_negate: expected = -expected - s = '-' + s + s = "-" + s tdelta = converters.convert_timedelta(s) self.assertEqual(tdelta, expected) @@ -49,10 +45,10 @@ def test_convert_timedelta_with_fsp(self): def test_convert_time(self): expected = datetime.time(23, 6, 20) - time_obj = converters.convert_time('23:06:20') + time_obj = converters.convert_time("23:06:20") self.assertEqual(time_obj, expected) def test_convert_time_with_fsp(self): expected = datetime.time(23, 6, 20, 511581) - time_obj = converters.convert_time('23:06:20.511581') + time_obj = converters.convert_time("23:06:20.511581") self.assertEqual(time_obj, expected) diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 4c9174f5..783caf88 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -3,6 +3,7 @@ from pymysql.tests import base import pymysql.cursors + class CursorTest(base.PyMySQLTestCase): def setUp(self): super(CursorTest, self).setUp() @@ -10,12 +11,14 @@ def setUp(self): conn = self.connect() self.safe_create_table( conn, - "test", "create table test (data varchar(10))", + "test", + "create table test (data varchar(10))", ) cursor = conn.cursor() cursor.execute( "insert into test (data) values " - "('row1'), ('row2'), ('row3'), ('row4'), ('row5')") + "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" + ) cursor.close() self.test_connection = pymysql.connect(**self.databases[0]) self.addCleanup(self.test_connection.close) @@ -51,55 +54,78 @@ def test_cleanup_rows_buffered(self): c2 = conn.cursor() c2.execute("select 1") - self.assertEqual( - c2.fetchone(), (1,) - ) + self.assertEqual(c2.fetchone(), (1,)) self.assertIsNone(c2.fetchone()) def test_executemany(self): conn = self.test_connection cursor = conn.cursor(pymysql.cursors.Cursor) - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%s, %s)") - self.assertIsNotNone(m, 'error parse %s') - self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%s, %s)" + ) + self.assertIsNotNone(m, "error parse %s") + self.assertEqual(m.group(3), "", "group 3 not blank, bug in RE_INSERT_VALUES?") - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id)s, %(name)s)") - self.assertIsNotNone(m, 'error parse %(name)s') - self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%(id)s, %(name)s)" + ) + self.assertIsNotNone(m, "error parse %(name)s") + self.assertEqual(m.group(3), "", "group 3 not blank, bug in RE_INSERT_VALUES?") - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s)") - self.assertIsNotNone(m, 'error parse %(id_name)s') - self.assertEqual(m.group(3), '', 'group 3 not blank, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s)" + ) + self.assertIsNotNone(m, "error parse %(id_name)s") + self.assertEqual(m.group(3), "", "group 3 not blank, bug in RE_INSERT_VALUES?") - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s) ON duplicate update") - self.assertIsNotNone(m, 'error parse %(id_name)s') - self.assertEqual(m.group(3), ' ON duplicate update', 'group 3 not ON duplicate update, bug in RE_INSERT_VALUES?') + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO TEST (ID, NAME) VALUES (%(id_name)s, %(name)s) ON duplicate update" + ) + self.assertIsNotNone(m, "error parse %(id_name)s") + self.assertEqual( + m.group(3), + " ON duplicate update", + "group 3 not ON duplicate update, bug in RE_INSERT_VALUES?", + ) # https://github.com/PyMySQL/PyMySQL/pull/597 - m = pymysql.cursors.RE_INSERT_VALUES.match("INSERT INTO bloup(foo, bar)VALUES(%s, %s)") + m = pymysql.cursors.RE_INSERT_VALUES.match( + "INSERT INTO bloup(foo, bar)VALUES(%s, %s)" + ) assert m is not None # cursor._executed must bee "insert into test (data) values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)" # list args data = range(10) cursor.executemany("insert into test (data) values (%s)", data) - self.assertTrue(cursor._executed.endswith(b",(7),(8),(9)"), 'execute many with %s not in one query') + self.assertTrue( + cursor._executed.endswith(b",(7),(8),(9)"), + "execute many with %s not in one query", + ) # dict args - data_dict = [{'data': i} for i in range(10)] + data_dict = [{"data": i} for i in range(10)] cursor.executemany("insert into test (data) values (%(data)s)", data_dict) - self.assertTrue(cursor._executed.endswith(b",(7),(8),(9)"), 'execute many with %(data)s not in one query') + self.assertTrue( + cursor._executed.endswith(b",(7),(8),(9)"), + "execute many with %(data)s not in one query", + ) # %% in column set - cursor.execute("""\ + cursor.execute( + """\ CREATE TABLE percent_test ( `A%` INTEGER, - `B%` INTEGER)""") + `B%` INTEGER)""" + ) try: q = "INSERT INTO percent_test (`A%%`, `B%%`) VALUES (%s, %s)" self.assertIsNotNone(pymysql.cursors.RE_INSERT_VALUES.match(q)) cursor.executemany(q, [(3, 4), (5, 6)]) - self.assertTrue(cursor._executed.endswith(b"(3, 4),(5, 6)"), "executemany with %% not in one query") + self.assertTrue( + cursor._executed.endswith(b"(3, 4),(5, 6)"), + "executemany with %% not in one query", + ) finally: cursor.execute("DROP TABLE IF EXISTS percent_test") diff --git a/pymysql/tests/test_err.py b/pymysql/tests/test_err.py index bb6a5c49..6b54c6d0 100644 --- a/pymysql/tests/test_err.py +++ b/pymysql/tests/test_err.py @@ -7,9 +7,8 @@ class TestRaiseException(unittest.TestCase): - def test_raise_mysql_exception(self): data = b"\xff\x15\x04#28000Access denied" with self.assertRaises(err.OperationalError) as cm: err.raise_mysql_exception(data) - self.assertEqual(cm.exception.args, (1045, 'Access denied')) + self.assertEqual(cm.exception.args, (1045, "Access denied")) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 2e11ddb5..95765e54 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -11,6 +11,7 @@ __all__ = ["TestOldIssues", "TestNewIssues", "TestGitHubIssues"] + class TestOldIssues(base.PyMySQLTestCase): def test_issue_3(self): """ undefined methods datetime_or_None, date_or_None """ @@ -21,7 +22,10 @@ def test_issue_3(self): c.execute("drop table if exists issue3") c.execute("create table issue3 (d date, t time, dt datetime, ts timestamp)") try: - c.execute("insert into issue3 (d, t, dt, ts) values (%s,%s,%s,%s)", (None, None, None, None)) + c.execute( + "insert into issue3 (d, t, dt, ts) values (%s,%s,%s,%s)", + (None, None, None, None), + ) c.execute("select d from issue3") self.assertEqual(None, c.fetchone()[0]) c.execute("select t from issue3") @@ -29,7 +33,11 @@ def test_issue_3(self): c.execute("select dt from issue3") self.assertEqual(None, c.fetchone()[0]) c.execute("select ts from issue3") - self.assertIn(type(c.fetchone()[0]), (type(None), datetime.datetime), 'expected Python type None or datetime from SQL timestamp') + self.assertIn( + type(c.fetchone()[0]), + (type(None), datetime.datetime), + "expected Python type None or datetime from SQL timestamp", + ) finally: c.execute("drop table issue3") @@ -58,7 +66,7 @@ def test_issue_6(self): """ exception: TypeError: ord() expected a character, but string of length 0 found """ # ToDo: this test requires access to db 'mysql'. kwargs = self.databases[0].copy() - kwargs['db'] = "mysql" + kwargs["db"] = "mysql" conn = pymysql.connect(**kwargs) c = conn.cursor() c.execute("select * from user") @@ -71,10 +79,12 @@ def test_issue_8(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists test") - c.execute("""CREATE TABLE `test` (`station` int NOT NULL DEFAULT '0', `dh` + c.execute( + """CREATE TABLE `test` (`station` int NOT NULL DEFAULT '0', `dh` datetime NOT NULL DEFAULT '2015-01-01 00:00:00', `echeance` int NOT NULL DEFAULT '0', `me` double DEFAULT NULL, `mo` double DEFAULT NULL, PRIMARY -KEY (`station`,`dh`,`echeance`)) ENGINE=MyISAM DEFAULT CHARSET=latin1;""") +KEY (`station`,`dh`,`echeance`)) ENGINE=MyISAM DEFAULT CHARSET=latin1;""" + ) try: self.assertEqual(0, c.execute("SELECT * FROM test")) c.execute("ALTER TABLE `test` ADD INDEX `idx_station` (`station`)") @@ -92,7 +102,7 @@ def test_issue_13(self): try: cur.execute("create table issue13 (t text)") # ticket says 18k - size = 18*1024 + size = 18 * 1024 cur.execute("insert into issue13 (t) values (%s)", ("x" * size,)) cur.execute("select t from issue13") # use assertTrue so that obscenely huge error messages don't print @@ -110,9 +120,9 @@ def test_issue_15(self): c.execute("drop table if exists issue15") c.execute("create table issue15 (t varchar(32))") try: - c.execute("insert into issue15 (t) values (%s)", (u'\xe4\xf6\xfc',)) + c.execute("insert into issue15 (t) values (%s)", (u"\xe4\xf6\xfc",)) c.execute("select t from issue15") - self.assertEqual(u'\xe4\xf6\xfc', c.fetchone()[0]) + self.assertEqual(u"\xe4\xf6\xfc", c.fetchone()[0]) finally: c.execute("drop table issue15") @@ -123,15 +133,21 @@ def test_issue_16(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists issue16") - c.execute("create table issue16 (name varchar(32) primary key, email varchar(32))") + c.execute( + "create table issue16 (name varchar(32) primary key, email varchar(32))" + ) try: - c.execute("insert into issue16 (name, email) values ('pete', 'floydophone')") + c.execute( + "insert into issue16 (name, email) values ('pete', 'floydophone')" + ) c.execute("select email from issue16 where name=%s", ("pete",)) self.assertEqual("floydophone", c.fetchone()[0]) finally: c.execute("drop table issue16") - @pytest.mark.skip("test_issue_17() requires a custom, legacy MySQL configuration and will not be run.") + @pytest.mark.skip( + "test_issue_17() requires a custom, legacy MySQL configuration and will not be run." + ) def test_issue_17(self): """could not connect mysql use passwod""" conn = self.connect() @@ -146,7 +162,10 @@ def test_issue_17(self): c.execute("drop table if exists issue17") c.execute("create table issue17 (x varchar(32) primary key)") c.execute("insert into issue17 (x) values ('hello, world!')") - c.execute("grant all privileges on %s.issue17 to 'issue17user'@'%%' identified by '1234'" % db) + c.execute( + "grant all privileges on %s.issue17 to 'issue17user'@'%%' identified by '1234'" + % db + ) conn.commit() conn2 = pymysql.connect(host=host, user="issue17user", passwd="1234", db=db) @@ -156,6 +175,7 @@ def test_issue_17(self): finally: c.execute("drop table issue17") + class TestNewIssues(base.PyMySQLTestCase): def test_issue_34(self): try: @@ -168,8 +188,9 @@ def test_issue_34(self): def test_issue_33(self): conn = pymysql.connect(charset="utf8", **self.databases[0]) - self.safe_create_table(conn, u'hei\xdfe', - u'create table hei\xdfe (name varchar(32))') + self.safe_create_table( + conn, u"hei\xdfe", u"create table hei\xdfe (name varchar(32))" + ) c = conn.cursor() c.execute(u"insert into hei\xdfe (name) values ('Pi\xdfata')") c.execute(u"select name from hei\xdfe") @@ -233,7 +254,7 @@ def test_issue_37(self): def test_issue_38(self): conn = self.connect() c = conn.cursor() - datum = "a" * 1024 * 1023 # reduced size for most default mysql installs + datum = "a" * 1024 * 1023 # reduced size for most default mysql installs try: with warnings.catch_warnings(): @@ -251,7 +272,7 @@ def disabled_test_issue_54(self): warnings.filterwarnings("ignore") c.execute("drop table if exists issue54") big_sql = "select * from issue54 where " - big_sql += " and ".join("%d=%d" % (i,i) for i in range(0, 100000)) + big_sql += " and ".join("%d=%d" % (i, i) for i in range(0, 100000)) try: c.execute("create table issue54 (id integer primary key)") @@ -261,6 +282,7 @@ def disabled_test_issue_54(self): finally: c.execute("drop table issue54") + class TestGitHubIssues(base.PyMySQLTestCase): def test_issue_66(self): """ 'Connection' object has no attribute 'insert_id' """ @@ -271,7 +293,9 @@ def test_issue_66(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists issue66") - c.execute("create table issue66 (id integer primary key auto_increment, x integer)") + c.execute( + "create table issue66 (id integer primary key auto_increment, x integer)" + ) c.execute("insert into issue66 (x) values (1)") c.execute("insert into issue66 (x) values (1)") self.assertEqual(2, conn.insert_id()) @@ -290,17 +314,17 @@ def test_issue_79(self): c.execute("""CREATE TABLE a (id int, value int)""") c.execute("""CREATE TABLE b (id int, value int)""") - a=(1,11) - b=(1,22) + a = (1, 11) + b = (1, 22) try: c.execute("insert into a values (%s, %s)", a) c.execute("insert into b values (%s, %s)", b) c.execute("SELECT * FROM a inner join b on a.id = b.id") r = c.fetchall()[0] - self.assertEqual(r['id'], 1) - self.assertEqual(r['value'], 11) - self.assertEqual(r['b.value'], 22) + self.assertEqual(r["id"], 1) + self.assertEqual(r["value"], 11) + self.assertEqual(r["b.value"], 22) finally: c.execute("drop table a") c.execute("drop table b") @@ -312,10 +336,12 @@ def test_issue_95(self): with warnings.catch_warnings(): warnings.filterwarnings("ignore") cur.execute("DROP PROCEDURE IF EXISTS `foo`") - cur.execute("""CREATE PROCEDURE `foo` () + cur.execute( + """CREATE PROCEDURE `foo` () BEGIN SELECT 1; - END""") + END""" + ) try: cur.execute("""CALL foo()""") cur.execute("""SELECT 1""") @@ -355,40 +381,42 @@ def test_issue_175(self): conn = self.connect() cur = conn.cursor() for length in (200, 300): - columns = ', '.join('c{0} integer'.format(i) for i in range(length)) - sql = 'create table test_field_count ({0})'.format(columns) + columns = ", ".join("c{0} integer".format(i) for i in range(length)) + sql = "create table test_field_count ({0})".format(columns) try: cur.execute(sql) - cur.execute('select * from test_field_count') + cur.execute("select * from test_field_count") assert len(cur.description) == length finally: with warnings.catch_warnings(): warnings.filterwarnings("ignore") - cur.execute('drop table if exists test_field_count') + cur.execute("drop table if exists test_field_count") def test_issue_321(self): """ Test iterable as query argument. """ conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( - conn, "issue321", - "create table issue321 (value_1 varchar(1), value_2 varchar(1))") + conn, + "issue321", + "create table issue321 (value_1 varchar(1), value_2 varchar(1))", + ) sql_insert = "insert into issue321 (value_1, value_2) values (%s, %s)" - sql_dict_insert = ("insert into issue321 (value_1, value_2) " - "values (%(value_1)s, %(value_2)s)") - sql_select = ("select * from issue321 where " - "value_1 in %s and value_2=%s") + sql_dict_insert = ( + "insert into issue321 (value_1, value_2) " + "values (%(value_1)s, %(value_2)s)" + ) + sql_select = "select * from issue321 where " "value_1 in %s and value_2=%s" data = [ - [(u"a", ), u"\u0430"], + [(u"a",), u"\u0430"], [[u"b"], u"\u0430"], - {"value_1": [[u"c"]], "value_2": u"\u0430"} + {"value_1": [[u"c"]], "value_2": u"\u0430"}, ] cur = conn.cursor() self.assertEqual(cur.execute(sql_insert, data[0]), 1) self.assertEqual(cur.execute(sql_insert, data[1]), 1) self.assertEqual(cur.execute(sql_dict_insert, data[2]), 1) - self.assertEqual( - cur.execute(sql_select, [(u"a", u"b", u"c"), u"\u0430"]), 3) + self.assertEqual(cur.execute(sql_select, [(u"a", u"b", u"c"), u"\u0430"]), 3) self.assertEqual(cur.fetchone(), (u"a", u"\u0430")) self.assertEqual(cur.fetchone(), (u"b", u"\u0430")) self.assertEqual(cur.fetchone(), (u"c", u"\u0430")) @@ -397,9 +425,11 @@ def test_issue_364(self): """ Test mixed unicode/binary arguments in executemany. """ conn = pymysql.connect(charset="utf8mb4", **self.databases[0]) self.safe_create_table( - conn, "issue364", + conn, + "issue364", "create table issue364 (value_1 binary(3), value_2 varchar(3)) " - "engine=InnoDB default charset=utf8mb4") + "engine=InnoDB default charset=utf8mb4", + ) sql = "insert into issue364 (value_1, value_2) values (_binary %s, %s)" usql = u"insert into issue364 (value_1, value_2) values (_binary %s, %s)" @@ -427,11 +457,13 @@ def test_issue_363(self): """ Test binary / geometry types. """ conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( - conn, "issue363", + conn, + "issue363", "CREATE TABLE issue363 ( " "id INTEGER PRIMARY KEY, geom LINESTRING NOT NULL /*!80003 SRID 0 */, " "SPATIAL KEY geom (geom)) " - "ENGINE=MyISAM") + "ENGINE=MyISAM", + ) cur = conn.cursor() # From MySQL 5.7, ST_GeomFromText is added and GeomFromText is deprecated. @@ -443,26 +475,32 @@ def test_issue_363(self): geom_from_text = "GeomFromText" geom_as_text = "AsText" geom_as_bin = "AsBinary" - query = ("INSERT INTO issue363 (id, geom) VALUES" - "(1998, %s('LINESTRING(1.1 1.1,2.2 2.2)'))" % geom_from_text) + query = ( + "INSERT INTO issue363 (id, geom) VALUES" + "(1998, %s('LINESTRING(1.1 1.1,2.2 2.2)'))" % geom_from_text + ) cur.execute(query) # select WKT query = "SELECT %s(geom) FROM issue363" % geom_as_text cur.execute(query) row = cur.fetchone() - self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)", )) + self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)",)) # select WKB query = "SELECT %s(geom) FROM issue363" % geom_as_bin cur.execute(query) row = cur.fetchone() - self.assertEqual(row, - (b"\x01\x02\x00\x00\x00\x02\x00\x00\x00" - b"\x9a\x99\x99\x99\x99\x99\xf1?" - b"\x9a\x99\x99\x99\x99\x99\xf1?" - b"\x9a\x99\x99\x99\x99\x99\x01@" - b"\x9a\x99\x99\x99\x99\x99\x01@", )) + self.assertEqual( + row, + ( + b"\x01\x02\x00\x00\x00\x02\x00\x00\x00" + b"\x9a\x99\x99\x99\x99\x99\xf1?" + b"\x9a\x99\x99\x99\x99\x99\xf1?" + b"\x9a\x99\x99\x99\x99\x99\x01@" + b"\x9a\x99\x99\x99\x99\x99\x01@", + ), + ) # select internal binary cur.execute("SELECT geom FROM issue363") diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index 30186e3a..bb856305 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -16,8 +16,10 @@ def test_no_file(self): self.assertRaises( OperationalError, c.execute, - ("LOAD DATA LOCAL INFILE 'no_data.txt' INTO TABLE " - "test_load_local fields terminated by ','") + ( + "LOAD DATA LOCAL INFILE 'no_data.txt' INTO TABLE " + "test_load_local fields terminated by ','" + ), ) finally: c.execute("DROP TABLE test_load_local") @@ -28,13 +30,15 @@ def test_load_file(self): conn = self.connect() c = conn.cursor() c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") - filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'data', - 'load_local_data.txt') + filename = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "data", "load_local_data.txt" + ) try: c.execute( - ("LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + - "test_load_local FIELDS TERMINATED BY ','").format(filename) + ( + "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + + "test_load_local FIELDS TERMINATED BY ','" + ).format(filename) ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -46,13 +50,15 @@ def test_unbuffered_load_file(self): conn = self.connect() c = conn.cursor(cursors.SSCursor) c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") - filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'data', - 'load_local_data.txt') + filename = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "data", "load_local_data.txt" + ) try: c.execute( - ("LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + - "test_load_local FIELDS TERMINATED BY ','").format(filename) + ( + "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + + "test_load_local FIELDS TERMINATED BY ','" + ).format(filename) ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -66,4 +72,5 @@ def test_unbuffered_load_file(self): if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index d5467b11..2679edd5 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -7,11 +7,11 @@ class TestNextset(base.PyMySQLTestCase): - def test_nextset(self): con = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) cur = con.cursor() cur.execute("SELECT 1; SELECT 2;") self.assertEqual([(1,)], list(cur)) @@ -71,14 +71,14 @@ def test_multi_cursor(self): def test_multi_statement_warnings(self): con = self.connect( init_command='SELECT "bar"; SELECT "baz"', - client_flag=CLIENT.MULTI_STATEMENTS) + client_flag=CLIENT.MULTI_STATEMENTS, + ) cursor = con.cursor() try: - cursor.execute('DROP TABLE IF EXISTS a; ' - 'DROP TABLE IF EXISTS b;') + cursor.execute("DROP TABLE IF EXISTS a; " "DROP TABLE IF EXISTS b;") except TypeError: self.fail() - #TODO: How about SSCursor and nextset? + # TODO: How about SSCursor and nextset? # It's very hard to implement correctly... diff --git a/pymysql/tests/test_optionfile.py b/pymysql/tests/test_optionfile.py index 81bd1fe4..39bd47c4 100644 --- a/pymysql/tests/test_optionfile.py +++ b/pymysql/tests/test_optionfile.py @@ -3,20 +3,19 @@ from pymysql.optionfile import Parser -__all__ = ['TestParser'] +__all__ = ["TestParser"] -_cfg_file = (r""" +_cfg_file = r""" [default] string = foo quoted = "bar" single_quoted = 'foobar' skip-slave-start -""") +""" class TestParser(TestCase): - def test_string(self): parser = Parser() parser.read_file(StringIO(_cfg_file)) diff --git a/pymysql/tests/thirdparty/__init__.py b/pymysql/tests/thirdparty/__init__.py index 7a613478..d5f05371 100644 --- a/pymysql/tests/thirdparty/__init__.py +++ b/pymysql/tests/thirdparty/__init__.py @@ -2,4 +2,5 @@ if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py index e4237c69..57c42ce7 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py @@ -4,4 +4,5 @@ if __name__ == "__main__": import unittest + unittest.main() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index e261a78e..ffead0ca 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -22,7 +22,7 @@ def setUp(self): db = self.db_module.connect(*self.connect_args, **self.connect_kwargs) self.connection = db self.cursor = db.cursor() - self.BLOBText = ''.join([chr(i) for i in range(256)] * 100); + self.BLOBText = "".join([chr(i) for i in range(256)] * 100) self.BLOBUText = "".join(chr(i) for i in range(16834)) data = bytearray(range(256)) * 16 self.BLOBBinary = self.db_module.Binary(data) @@ -32,17 +32,22 @@ def setUp(self): def tearDown(self): if self.leak_test: import gc + del self.cursor orphans = gc.collect() - self.assertFalse(orphans, "%d orphaned objects found after deleting cursor" % orphans) + self.assertFalse( + orphans, "%d orphaned objects found after deleting cursor" % orphans + ) del self.connection orphans = gc.collect() - self.assertFalse(orphans, "%d orphaned objects found after deleting connection" % orphans) + self.assertFalse( + orphans, "%d orphaned objects found after deleting connection" % orphans + ) def table_exists(self, name): try: - self.cursor.execute('select * from %s where 1=0' % name) + self.cursor.execute("select * from %s where 1=0" % name) except Exception: return False else: @@ -54,7 +59,7 @@ def quote_identifier(self, ident): def new_table_name(self): i = id(self.cursor) while True: - name = self.quote_identifier('tb%08x' % i) + name = self.quote_identifier("tb%08x" % i) if not self.table_exists(name): return name i = i + 1 @@ -68,25 +73,27 @@ def create_table(self, columndefs): into the table. """ self.table = self.new_table_name() - self.cursor.execute('CREATE TABLE %s (%s) %s' % - (self.table, - ',\n'.join(columndefs), - self.create_table_extra)) + self.cursor.execute( + "CREATE TABLE %s (%s) %s" + % (self.table, ",\n".join(columndefs), self.create_table_extra) + ) def check_data_integrity(self, columndefs, generator): # insert self.create_table(columndefs) - insert_statement = ('INSERT INTO %s VALUES (%s)' % - (self.table, - ','.join(['%s'] * len(columndefs)))) - data = [ [ generator(i,j) for j in range(len(columndefs)) ] - for i in range(self.rows) ] + insert_statement = "INSERT INTO %s VALUES (%s)" % ( + self.table, + ",".join(["%s"] * len(columndefs)), + ) + data = [ + [generator(i, j) for j in range(len(columndefs))] for i in range(self.rows) + ] if self.debug: print(data) self.cursor.executemany(insert_statement, data) self.connection.commit() # verify - self.cursor.execute('select * from %s' % self.table) + self.cursor.execute("select * from %s" % self.table) l = self.cursor.fetchall() if self.debug: print(l) @@ -94,62 +101,74 @@ def check_data_integrity(self, columndefs, generator): try: for i in range(self.rows): for j in range(len(columndefs)): - self.assertEqual(l[i][j], generator(i,j)) + self.assertEqual(l[i][j], generator(i, j)) finally: if not self.debug: - self.cursor.execute('drop table %s' % (self.table)) + self.cursor.execute("drop table %s" % (self.table)) def test_transactions(self): - columndefs = ( 'col1 INT', 'col2 VARCHAR(255)') + columndefs = ("col1 INT", "col2 VARCHAR(255)") + def generator(row, col): - if col == 0: return row - else: return ('%i' % (row%10))*255 + if col == 0: + return row + else: + return ("%i" % (row % 10)) * 255 + self.create_table(columndefs) - insert_statement = ('INSERT INTO %s VALUES (%s)' % - (self.table, - ','.join(['%s'] * len(columndefs)))) - data = [ [ generator(i,j) for j in range(len(columndefs)) ] - for i in range(self.rows) ] + insert_statement = "INSERT INTO %s VALUES (%s)" % ( + self.table, + ",".join(["%s"] * len(columndefs)), + ) + data = [ + [generator(i, j) for j in range(len(columndefs))] for i in range(self.rows) + ] self.cursor.executemany(insert_statement, data) # verify self.connection.commit() - self.cursor.execute('select * from %s' % self.table) + self.cursor.execute("select * from %s" % self.table) l = self.cursor.fetchall() self.assertEqual(len(l), self.rows) for i in range(self.rows): for j in range(len(columndefs)): - self.assertEqual(l[i][j], generator(i,j)) - delete_statement = 'delete from %s where col1=%%s' % self.table + self.assertEqual(l[i][j], generator(i, j)) + delete_statement = "delete from %s where col1=%%s" % self.table self.cursor.execute(delete_statement, (0,)) - self.cursor.execute('select col1 from %s where col1=%s' % \ - (self.table, 0)) + self.cursor.execute("select col1 from %s where col1=%s" % (self.table, 0)) l = self.cursor.fetchall() self.assertFalse(l, "DELETE didn't work") self.connection.rollback() - self.cursor.execute('select col1 from %s where col1=%s' % \ - (self.table, 0)) + self.cursor.execute("select col1 from %s where col1=%s" % (self.table, 0)) l = self.cursor.fetchall() self.assertTrue(len(l) == 1, "ROLLBACK didn't work") - self.cursor.execute('drop table %s' % (self.table)) + self.cursor.execute("drop table %s" % (self.table)) def test_truncation(self): - columndefs = ( 'col1 INT', 'col2 VARCHAR(255)') + columndefs = ("col1 INT", "col2 VARCHAR(255)") + def generator(row, col): - if col == 0: return row - else: return ('%i' % (row%10))*((255-self.rows//2)+row) + if col == 0: + return row + else: + return ("%i" % (row % 10)) * ((255 - self.rows // 2) + row) + self.create_table(columndefs) - insert_statement = ('INSERT INTO %s VALUES (%s)' % - (self.table, - ','.join(['%s'] * len(columndefs)))) + insert_statement = "INSERT INTO %s VALUES (%s)" % ( + self.table, + ",".join(["%s"] * len(columndefs)), + ) try: - self.cursor.execute(insert_statement, (0, '0'*256)) + self.cursor.execute(insert_statement, (0, "0" * 256)) except Warning: - if self.debug: print(self.cursor.messages) + if self.debug: + print(self.cursor.messages) except self.connection.DataError: pass else: - self.fail("Over-long column did not generate warnings/exception with single insert") + self.fail( + "Over-long column did not generate warnings/exception with single insert" + ) self.connection.rollback() @@ -157,132 +176,136 @@ def generator(row, col): for i in range(self.rows): data = [] for j in range(len(columndefs)): - data.append(generator(i,j)) - self.cursor.execute(insert_statement,tuple(data)) + data.append(generator(i, j)) + self.cursor.execute(insert_statement, tuple(data)) except Warning: - if self.debug: print(self.cursor.messages) + if self.debug: + print(self.cursor.messages) except self.connection.DataError: pass else: - self.fail("Over-long columns did not generate warnings/exception with execute()") + self.fail( + "Over-long columns did not generate warnings/exception with execute()" + ) self.connection.rollback() try: - data = [ [ generator(i,j) for j in range(len(columndefs)) ] - for i in range(self.rows) ] + data = [ + [generator(i, j) for j in range(len(columndefs))] + for i in range(self.rows) + ] self.cursor.executemany(insert_statement, data) except Warning: - if self.debug: print(self.cursor.messages) + if self.debug: + print(self.cursor.messages) except self.connection.DataError: pass else: - self.fail("Over-long columns did not generate warnings/exception with executemany()") + self.fail( + "Over-long columns did not generate warnings/exception with executemany()" + ) self.connection.rollback() - self.cursor.execute('drop table %s' % (self.table)) + self.cursor.execute("drop table %s" % (self.table)) def test_CHAR(self): # Character data - def generator(row,col): - return ('%i' % ((row+col) % 10)) * 255 - self.check_data_integrity( - ('col1 char(255)','col2 char(255)'), - generator) + def generator(row, col): + return ("%i" % ((row + col) % 10)) * 255 + + self.check_data_integrity(("col1 char(255)", "col2 char(255)"), generator) def test_INT(self): # Number data - def generator(row,col): - return row*row - self.check_data_integrity( - ('col1 INT',), - generator) + def generator(row, col): + return row * row + + self.check_data_integrity(("col1 INT",), generator) def test_DECIMAL(self): # DECIMAL - def generator(row,col): + def generator(row, col): from decimal import Decimal + return Decimal("%d.%02d" % (row, col)) - self.check_data_integrity( - ('col1 DECIMAL(5,2)',), - generator) + + self.check_data_integrity(("col1 DECIMAL(5,2)",), generator) def test_DATE(self): ticks = time() - def generator(row,col): - return self.db_module.DateFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 DATE',), - generator) + + def generator(row, col): + return self.db_module.DateFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 DATE",), generator) def test_TIME(self): ticks = time() - def generator(row,col): - return self.db_module.TimeFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 TIME',), - generator) + + def generator(row, col): + return self.db_module.TimeFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 TIME",), generator) def test_DATETIME(self): ticks = time() - def generator(row,col): - return self.db_module.TimestampFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 DATETIME',), - generator) + + def generator(row, col): + return self.db_module.TimestampFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 DATETIME",), generator) def test_TIMESTAMP(self): ticks = time() - def generator(row,col): - return self.db_module.TimestampFromTicks(ticks+row*86400-col*1313) - self.check_data_integrity( - ('col1 TIMESTAMP',), - generator) + + def generator(row, col): + return self.db_module.TimestampFromTicks(ticks + row * 86400 - col * 1313) + + self.check_data_integrity(("col1 TIMESTAMP",), generator) def test_fractional_TIMESTAMP(self): ticks = time() - def generator(row,col): - return self.db_module.TimestampFromTicks(ticks+row*86400-col*1313+row*0.7*col/3.0) - self.check_data_integrity( - ('col1 TIMESTAMP',), - generator) + + def generator(row, col): + return self.db_module.TimestampFromTicks( + ticks + row * 86400 - col * 1313 + row * 0.7 * col / 3.0 + ) + + self.check_data_integrity(("col1 TIMESTAMP",), generator) def test_LONG(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBUText # 'BLOB Text ' * 1024 - self.check_data_integrity( - ('col1 INT', 'col2 LONG'), - generator) + return self.BLOBUText # 'BLOB Text ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 LONG"), generator) def test_TEXT(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBUText[:5192] # 'BLOB Text ' * 1024 - self.check_data_integrity( - ('col1 INT', 'col2 TEXT'), - generator) + return self.BLOBUText[:5192] # 'BLOB Text ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 TEXT"), generator) def test_LONG_BYTE(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBBinary # 'BLOB\000Binary ' * 1024 - self.check_data_integrity( - ('col1 INT','col2 LONG BYTE'), - generator) + return self.BLOBBinary # 'BLOB\000Binary ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 LONG BYTE"), generator) def test_BLOB(self): - def generator(row,col): + def generator(row, col): if col == 0: return row else: - return self.BLOBBinary # 'BLOB\000Binary ' * 1024 - self.check_data_integrity( - ('col1 INT','col2 BLOB'), - generator) + return self.BLOBBinary # 'BLOB\000Binary ' * 1024 + + self.check_data_integrity(("col1 INT", "col2 BLOB"), generator) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 1cc202e2..6766aff3 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -1,4 +1,4 @@ -''' Python DB API 2.0 driver compliance unit test suite. +""" Python DB API 2.0 driver compliance unit test suite. This software is Public Domain and may be used without restrictions. @@ -8,11 +8,11 @@ this is turning out to be a thoroughly unwholesome unit test." -- Ian Bicking -''' +""" -__rcs_id__ = '$Id$' -__version__ = '$Revision$'[11:-2] -__author__ = 'Stuart Bishop ' +__rcs_id__ = "$Id$" +__version__ = "$Revision$"[11:-2] +__author__ = "Stuart Bishop " import time import unittest @@ -63,65 +63,66 @@ # - Fix bugs in test_setoutputsize_basic and test_setinputsizes # + class DatabaseAPI20Test(unittest.TestCase): - ''' Test a database self.driver for DB API 2.0 compatibility. - This implementation tests Gadfly, but the TestCase - is structured so that other self.drivers can subclass this - test case to ensure compiliance with the DB-API. It is - expected that this TestCase may be expanded in the future - if ambiguities or edge conditions are discovered. + """Test a database self.driver for DB API 2.0 compatibility. + This implementation tests Gadfly, but the TestCase + is structured so that other self.drivers can subclass this + test case to ensure compiliance with the DB-API. It is + expected that this TestCase may be expanded in the future + if ambiguities or edge conditions are discovered. - The 'Optional Extensions' are not yet being tested. + The 'Optional Extensions' are not yet being tested. - self.drivers should subclass this test, overriding setUp, tearDown, - self.driver, connect_args and connect_kw_args. Class specification - should be as follows: + self.drivers should subclass this test, overriding setUp, tearDown, + self.driver, connect_args and connect_kw_args. Class specification + should be as follows: - import dbapi20 - class mytest(dbapi20.DatabaseAPI20Test): - [...] + import dbapi20 + class mytest(dbapi20.DatabaseAPI20Test): + [...] - Don't 'import DatabaseAPI20Test from dbapi20', or you will - confuse the unit tester - just 'import dbapi20'. - ''' + Don't 'import DatabaseAPI20Test from dbapi20', or you will + confuse the unit tester - just 'import dbapi20'. + """ # The self.driver module. This should be the module where the 'connect' # method is to be found driver = None - connect_args = () # List of arguments to pass to connect - connect_kw_args = {} # Keyword arguments for connect - table_prefix = 'dbapi20test_' # If you need to specify a prefix for tables + connect_args = () # List of arguments to pass to connect + connect_kw_args = {} # Keyword arguments for connect + table_prefix = "dbapi20test_" # If you need to specify a prefix for tables - ddl1 = 'create table %sbooze (name varchar(20))' % table_prefix - ddl2 = 'create table %sbarflys (name varchar(20))' % table_prefix - xddl1 = 'drop table %sbooze' % table_prefix - xddl2 = 'drop table %sbarflys' % table_prefix + ddl1 = "create table %sbooze (name varchar(20))" % table_prefix + ddl2 = "create table %sbarflys (name varchar(20))" % table_prefix + xddl1 = "drop table %sbooze" % table_prefix + xddl2 = "drop table %sbarflys" % table_prefix - lowerfunc = 'lower' # Name of stored procedure to convert string->lowercase + lowerfunc = "lower" # Name of stored procedure to convert string->lowercase # Some drivers may need to override these helpers, for example adding # a 'commit' after the execute. - def executeDDL1(self,cursor): + def executeDDL1(self, cursor): cursor.execute(self.ddl1) - def executeDDL2(self,cursor): + def executeDDL2(self, cursor): cursor.execute(self.ddl2) def setUp(self): - ''' self.drivers should override this method to perform required setup - if any is necessary, such as creating the database. - ''' + """self.drivers should override this method to perform required setup + if any is necessary, such as creating the database. + """ pass def tearDown(self): - ''' self.drivers should override this method to perform required cleanup - if any is necessary, such as deleting the test database. - The default drops the tables that may be created. - ''' + """self.drivers should override this method to perform required cleanup + if any is necessary, such as deleting the test database. + The default drops the tables that may be created. + """ con = self._connect() try: cur = con.cursor() - for ddl in (self.xddl1,self.xddl2): + for ddl in (self.xddl1, self.xddl2): try: cur.execute(ddl) con.commit() @@ -134,9 +135,7 @@ def tearDown(self): def _connect(self): try: - return self.driver.connect( - *self.connect_args,**self.connect_kw_args - ) + return self.driver.connect(*self.connect_args, **self.connect_kw_args) except AttributeError: self.fail("No connect method found in self.driver module") @@ -149,7 +148,7 @@ def test_apilevel(self): # Must exist apilevel = self.driver.apilevel # Must equal 2.0 - self.assertEqual(apilevel,'2.0') + self.assertEqual(apilevel, "2.0") except AttributeError: self.fail("Driver doesn't define apilevel") @@ -158,7 +157,7 @@ def test_threadsafety(self): # Must exist threadsafety = self.driver.threadsafety # Must be a valid value - self.assertTrue(threadsafety in (0,1,2,3)) + self.assertTrue(threadsafety in (0, 1, 2, 3)) except AttributeError: self.fail("Driver doesn't define threadsafety") @@ -167,38 +166,24 @@ def test_paramstyle(self): # Must exist paramstyle = self.driver.paramstyle # Must be a valid value - self.assertTrue(paramstyle in ( - 'qmark','numeric','named','format','pyformat' - )) + self.assertTrue( + paramstyle in ("qmark", "numeric", "named", "format", "pyformat") + ) except AttributeError: self.fail("Driver doesn't define paramstyle") def test_Exceptions(self): # Make sure required exceptions exist, and are in the # defined heirarchy. - self.assertTrue(issubclass(self.driver.Warning,Exception)) - self.assertTrue(issubclass(self.driver.Error,Exception)) - self.assertTrue( - issubclass(self.driver.InterfaceError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.DatabaseError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.OperationalError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.IntegrityError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.InternalError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.ProgrammingError,self.driver.Error) - ) - self.assertTrue( - issubclass(self.driver.NotSupportedError,self.driver.Error) - ) + self.assertTrue(issubclass(self.driver.Warning, Exception)) + self.assertTrue(issubclass(self.driver.Error, Exception)) + self.assertTrue(issubclass(self.driver.InterfaceError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.DatabaseError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.OperationalError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.IntegrityError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.InternalError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.ProgrammingError, self.driver.Error)) + self.assertTrue(issubclass(self.driver.NotSupportedError, self.driver.Error)) def test_ExceptionsAsConnectionAttributes(self): # OPTIONAL EXTENSION @@ -219,7 +204,6 @@ def test_ExceptionsAsConnectionAttributes(self): self.assertTrue(con.ProgrammingError is drv.ProgrammingError) self.assertTrue(con.NotSupportedError is drv.NotSupportedError) - def test_commit(self): con = self._connect() try: @@ -232,7 +216,7 @@ def test_rollback(self): con = self._connect() # If rollback is defined, it should either work or throw # the documented exception - if hasattr(con,'rollback'): + if hasattr(con, "rollback"): try: con.rollback() except self.driver.NotSupportedError: @@ -253,14 +237,14 @@ def test_cursor_isolation(self): cur1 = con.cursor() cur2 = con.cursor() self.executeDDL1(cur1) - cur1.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) + cur1.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) cur2.execute("select name from %sbooze" % self.table_prefix) booze = cur2.fetchall() - self.assertEqual(len(booze),1) - self.assertEqual(len(booze[0]),1) - self.assertEqual(booze[0][0],'Victoria Bitter') + self.assertEqual(len(booze), 1) + self.assertEqual(len(booze[0]), 1) + self.assertEqual(booze[0][0], "Victoria Bitter") finally: con.close() @@ -269,31 +253,41 @@ def test_description(self): try: cur = con.cursor() self.executeDDL1(cur) - self.assertEqual(cur.description,None, - 'cursor.description should be none after executing a ' - 'statement that can return no rows (such as DDL)' - ) - cur.execute('select name from %sbooze' % self.table_prefix) - self.assertEqual(len(cur.description),1, - 'cursor.description describes too many columns' - ) - self.assertEqual(len(cur.description[0]),7, - 'cursor.description[x] tuples must have 7 elements' - ) - self.assertEqual(cur.description[0][0].lower(),'name', - 'cursor.description[x][0] must return column name' - ) - self.assertEqual(cur.description[0][1],self.driver.STRING, - 'cursor.description[x][1] must return column type. Got %r' - % cur.description[0][1] - ) + self.assertEqual( + cur.description, + None, + "cursor.description should be none after executing a " + "statement that can return no rows (such as DDL)", + ) + cur.execute("select name from %sbooze" % self.table_prefix) + self.assertEqual( + len(cur.description), 1, "cursor.description describes too many columns" + ) + self.assertEqual( + len(cur.description[0]), + 7, + "cursor.description[x] tuples must have 7 elements", + ) + self.assertEqual( + cur.description[0][0].lower(), + "name", + "cursor.description[x][0] must return column name", + ) + self.assertEqual( + cur.description[0][1], + self.driver.STRING, + "cursor.description[x][1] must return column type. Got %r" + % cur.description[0][1], + ) # Make sure self.description gets reset self.executeDDL2(cur) - self.assertEqual(cur.description,None, - 'cursor.description not being set to None when executing ' - 'no-result statements (eg. DDL)' - ) + self.assertEqual( + cur.description, + None, + "cursor.description not being set to None when executing " + "no-result statements (eg. DDL)", + ) finally: con.close() @@ -302,47 +296,49 @@ def test_rowcount(self): try: cur = con.cursor() self.executeDDL1(cur) - self.assertEqual(cur.rowcount,-1, - 'cursor.rowcount should be -1 after executing no-result ' - 'statements' - ) - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) - self.assertTrue(cur.rowcount in (-1,1), - 'cursor.rowcount should == number or rows inserted, or ' - 'set to -1 after executing an insert statement' - ) + self.assertEqual( + cur.rowcount, + -1, + "cursor.rowcount should be -1 after executing no-result " "statements", + ) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + self.assertTrue( + cur.rowcount in (-1, 1), + "cursor.rowcount should == number or rows inserted, or " + "set to -1 after executing an insert statement", + ) cur.execute("select name from %sbooze" % self.table_prefix) - self.assertTrue(cur.rowcount in (-1,1), - 'cursor.rowcount should == number of rows returned, or ' - 'set to -1 after executing a select statement' - ) + self.assertTrue( + cur.rowcount in (-1, 1), + "cursor.rowcount should == number of rows returned, or " + "set to -1 after executing a select statement", + ) self.executeDDL2(cur) - self.assertEqual(cur.rowcount,-1, - 'cursor.rowcount not being reset to -1 after executing ' - 'no-result statements' - ) + self.assertEqual( + cur.rowcount, + -1, + "cursor.rowcount not being reset to -1 after executing " + "no-result statements", + ) finally: con.close() - lower_func = 'lower' + lower_func = "lower" + def test_callproc(self): con = self._connect() try: cur = con.cursor() - if self.lower_func and hasattr(cur,'callproc'): - r = cur.callproc(self.lower_func,('FOO',)) - self.assertEqual(len(r),1) - self.assertEqual(r[0],'FOO') + if self.lower_func and hasattr(cur, "callproc"): + r = cur.callproc(self.lower_func, ("FOO",)) + self.assertEqual(len(r), 1) + self.assertEqual(r[0], "FOO") r = cur.fetchall() - self.assertEqual(len(r),1,'callproc produced no result set') - self.assertEqual(len(r[0]),1, - 'callproc produced invalid result set' - ) - self.assertEqual(r[0][0],'foo', - 'callproc produced invalid results' - ) + self.assertEqual(len(r), 1, "callproc produced no result set") + self.assertEqual(len(r[0]), 1, "callproc produced invalid result set") + self.assertEqual(r[0][0], "foo", "callproc produced invalid results") finally: con.close() @@ -355,14 +351,14 @@ def test_close(self): # cursor.execute should raise an Error if called after connection # closed - self.assertRaises(self.driver.Error,self.executeDDL1,cur) + self.assertRaises(self.driver.Error, self.executeDDL1, cur) # connection.commit should raise an Error if called after connection' # closed.' - self.assertRaises(self.driver.Error,con.commit) + self.assertRaises(self.driver.Error, con.commit) # connection.close should raise an Error if called more than once - self.assertRaises(self.driver.Error,con.close) + self.assertRaises(self.driver.Error, con.close) def test_execute(self): con = self._connect() @@ -372,105 +368,99 @@ def test_execute(self): finally: con.close() - def _paraminsert(self,cur): + def _paraminsert(self, cur): self.executeDDL1(cur) - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) - self.assertTrue(cur.rowcount in (-1,1)) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + self.assertTrue(cur.rowcount in (-1, 1)) - if self.driver.paramstyle == 'qmark': + if self.driver.paramstyle == "qmark": cur.execute( - 'insert into %sbooze values (?)' % self.table_prefix, - ("Cooper's",) - ) - elif self.driver.paramstyle == 'numeric': + "insert into %sbooze values (?)" % self.table_prefix, ("Cooper's",) + ) + elif self.driver.paramstyle == "numeric": cur.execute( - 'insert into %sbooze values (:1)' % self.table_prefix, - ("Cooper's",) - ) - elif self.driver.paramstyle == 'named': + "insert into %sbooze values (:1)" % self.table_prefix, ("Cooper's",) + ) + elif self.driver.paramstyle == "named": cur.execute( - 'insert into %sbooze values (:beer)' % self.table_prefix, - {'beer':"Cooper's"} - ) - elif self.driver.paramstyle == 'format': + "insert into %sbooze values (:beer)" % self.table_prefix, + {"beer": "Cooper's"}, + ) + elif self.driver.paramstyle == "format": cur.execute( - 'insert into %sbooze values (%%s)' % self.table_prefix, - ("Cooper's",) - ) - elif self.driver.paramstyle == 'pyformat': + "insert into %sbooze values (%%s)" % self.table_prefix, ("Cooper's",) + ) + elif self.driver.paramstyle == "pyformat": cur.execute( - 'insert into %sbooze values (%%(beer)s)' % self.table_prefix, - {'beer':"Cooper's"} - ) + "insert into %sbooze values (%%(beer)s)" % self.table_prefix, + {"beer": "Cooper's"}, + ) else: - self.fail('Invalid paramstyle') - self.assertTrue(cur.rowcount in (-1,1)) + self.fail("Invalid paramstyle") + self.assertTrue(cur.rowcount in (-1, 1)) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) res = cur.fetchall() - self.assertEqual(len(res),2,'cursor.fetchall returned too few rows') - beers = [res[0][0],res[1][0]] + self.assertEqual(len(res), 2, "cursor.fetchall returned too few rows") + beers = [res[0][0], res[1][0]] beers.sort() - self.assertEqual(beers[0],"Cooper's", - 'cursor.fetchall retrieved incorrect data, or data inserted ' - 'incorrectly' - ) - self.assertEqual(beers[1],"Victoria Bitter", - 'cursor.fetchall retrieved incorrect data, or data inserted ' - 'incorrectly' - ) + self.assertEqual( + beers[0], + "Cooper's", + "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + ) + self.assertEqual( + beers[1], + "Victoria Bitter", + "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + ) def test_executemany(self): con = self._connect() try: cur = con.cursor() self.executeDDL1(cur) - largs = [ ("Cooper's",) , ("Boag's",) ] - margs = [ {'beer': "Cooper's"}, {'beer': "Boag's"} ] - if self.driver.paramstyle == 'qmark': + largs = [("Cooper's",), ("Boag's",)] + margs = [{"beer": "Cooper's"}, {"beer": "Boag's"}] + if self.driver.paramstyle == "qmark": cur.executemany( - 'insert into %sbooze values (?)' % self.table_prefix, - largs - ) - elif self.driver.paramstyle == 'numeric': + "insert into %sbooze values (?)" % self.table_prefix, largs + ) + elif self.driver.paramstyle == "numeric": cur.executemany( - 'insert into %sbooze values (:1)' % self.table_prefix, - largs - ) - elif self.driver.paramstyle == 'named': + "insert into %sbooze values (:1)" % self.table_prefix, largs + ) + elif self.driver.paramstyle == "named": cur.executemany( - 'insert into %sbooze values (:beer)' % self.table_prefix, - margs - ) - elif self.driver.paramstyle == 'format': + "insert into %sbooze values (:beer)" % self.table_prefix, margs + ) + elif self.driver.paramstyle == "format": cur.executemany( - 'insert into %sbooze values (%%s)' % self.table_prefix, - largs - ) - elif self.driver.paramstyle == 'pyformat': + "insert into %sbooze values (%%s)" % self.table_prefix, largs + ) + elif self.driver.paramstyle == "pyformat": cur.executemany( - 'insert into %sbooze values (%%(beer)s)' % ( - self.table_prefix - ), - margs - ) - else: - self.fail('Unknown paramstyle') - self.assertTrue(cur.rowcount in (-1,2), - 'insert using cursor.executemany set cursor.rowcount to ' - 'incorrect value %r' % cur.rowcount + "insert into %sbooze values (%%(beer)s)" % (self.table_prefix), + margs, ) - cur.execute('select name from %sbooze' % self.table_prefix) + else: + self.fail("Unknown paramstyle") + self.assertTrue( + cur.rowcount in (-1, 2), + "insert using cursor.executemany set cursor.rowcount to " + "incorrect value %r" % cur.rowcount, + ) + cur.execute("select name from %sbooze" % self.table_prefix) res = cur.fetchall() - self.assertEqual(len(res),2, - 'cursor.fetchall retrieved incorrect number of rows' - ) - beers = [res[0][0],res[1][0]] + self.assertEqual( + len(res), 2, "cursor.fetchall retrieved incorrect number of rows" + ) + beers = [res[0][0], res[1][0]] beers.sort() - self.assertEqual(beers[0],"Boag's",'incorrect data retrieved') - self.assertEqual(beers[1],"Cooper's",'incorrect data retrieved') + self.assertEqual(beers[0], "Boag's", "incorrect data retrieved") + self.assertEqual(beers[1], "Cooper's", "incorrect data retrieved") finally: con.close() @@ -481,59 +471,62 @@ def test_fetchone(self): # cursor.fetchone should raise an Error if called before # executing a select-type query - self.assertRaises(self.driver.Error,cur.fetchone) + self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows self.executeDDL1(cur) - self.assertRaises(self.driver.Error,cur.fetchone) + self.assertRaises(self.driver.Error, cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) - self.assertEqual(cur.fetchone(),None, - 'cursor.fetchone should return None if a query retrieves ' - 'no rows' - ) - self.assertTrue(cur.rowcount in (-1,0)) + cur.execute("select name from %sbooze" % self.table_prefix) + self.assertEqual( + cur.fetchone(), + None, + "cursor.fetchone should return None if a query retrieves " "no rows", + ) + self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) - self.assertRaises(self.driver.Error,cur.fetchone) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + self.assertRaises(self.driver.Error, cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchone() - self.assertEqual(len(r),1, - 'cursor.fetchone should have retrieved a single row' - ) - self.assertEqual(r[0],'Victoria Bitter', - 'cursor.fetchone retrieved incorrect data' - ) - self.assertEqual(cur.fetchone(),None, - 'cursor.fetchone should return None if no more rows available' - ) - self.assertTrue(cur.rowcount in (-1,1)) + self.assertEqual( + len(r), 1, "cursor.fetchone should have retrieved a single row" + ) + self.assertEqual( + r[0], "Victoria Bitter", "cursor.fetchone retrieved incorrect data" + ) + self.assertEqual( + cur.fetchone(), + None, + "cursor.fetchone should return None if no more rows available", + ) + self.assertTrue(cur.rowcount in (-1, 1)) finally: con.close() samples = [ - 'Carlton Cold', - 'Carlton Draft', - 'Mountain Goat', - 'Redback', - 'Victoria Bitter', - 'XXXX' - ] + "Carlton Cold", + "Carlton Draft", + "Mountain Goat", + "Redback", + "Victoria Bitter", + "XXXX", + ] def _populate(self): - ''' Return a list of sql commands to setup the DB for the fetch - tests. - ''' + """Return a list of sql commands to setup the DB for the fetch + tests. + """ populate = [ - "insert into %sbooze values ('%s')" % (self.table_prefix,s) - for s in self.samples - ] + "insert into %sbooze values ('%s')" % (self.table_prefix, s) + for s in self.samples + ] return populate def test_fetchmany(self): @@ -542,78 +535,88 @@ def test_fetchmany(self): cur = con.cursor() # cursor.fetchmany should raise an Error if called without - #issuing a query - self.assertRaises(self.driver.Error,cur.fetchmany,4) + # issuing a query + self.assertRaises(self.driver.Error, cur.fetchmany, 4) self.executeDDL1(cur) for sql in self._populate(): cur.execute(sql) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchmany() - self.assertEqual(len(r),1, - 'cursor.fetchmany retrieved incorrect number of rows, ' - 'default of arraysize is one.' - ) - cur.arraysize=10 - r = cur.fetchmany(3) # Should get 3 rows - self.assertEqual(len(r),3, - 'cursor.fetchmany retrieved incorrect number of rows' - ) - r = cur.fetchmany(4) # Should get 2 more - self.assertEqual(len(r),2, - 'cursor.fetchmany retrieved incorrect number of rows' - ) - r = cur.fetchmany(4) # Should be an empty sequence - self.assertEqual(len(r),0, - 'cursor.fetchmany should return an empty sequence after ' - 'results are exhausted' + self.assertEqual( + len(r), + 1, + "cursor.fetchmany retrieved incorrect number of rows, " + "default of arraysize is one.", + ) + cur.arraysize = 10 + r = cur.fetchmany(3) # Should get 3 rows + self.assertEqual( + len(r), 3, "cursor.fetchmany retrieved incorrect number of rows" + ) + r = cur.fetchmany(4) # Should get 2 more + self.assertEqual( + len(r), 2, "cursor.fetchmany retrieved incorrect number of rows" + ) + r = cur.fetchmany(4) # Should be an empty sequence + self.assertEqual( + len(r), + 0, + "cursor.fetchmany should return an empty sequence after " + "results are exhausted", ) - self.assertTrue(cur.rowcount in (-1,6)) + self.assertTrue(cur.rowcount in (-1, 6)) # Same as above, using cursor.arraysize - cur.arraysize=4 - cur.execute('select name from %sbooze' % self.table_prefix) - r = cur.fetchmany() # Should get 4 rows - self.assertEqual(len(r),4, - 'cursor.arraysize not being honoured by fetchmany' - ) - r = cur.fetchmany() # Should get 2 more - self.assertEqual(len(r),2) - r = cur.fetchmany() # Should be an empty sequence - self.assertEqual(len(r),0) - self.assertTrue(cur.rowcount in (-1,6)) - - cur.arraysize=6 - cur.execute('select name from %sbooze' % self.table_prefix) - rows = cur.fetchmany() # Should get all rows - self.assertTrue(cur.rowcount in (-1,6)) - self.assertEqual(len(rows),6) - self.assertEqual(len(rows),6) + cur.arraysize = 4 + cur.execute("select name from %sbooze" % self.table_prefix) + r = cur.fetchmany() # Should get 4 rows + self.assertEqual( + len(r), 4, "cursor.arraysize not being honoured by fetchmany" + ) + r = cur.fetchmany() # Should get 2 more + self.assertEqual(len(r), 2) + r = cur.fetchmany() # Should be an empty sequence + self.assertEqual(len(r), 0) + self.assertTrue(cur.rowcount in (-1, 6)) + + cur.arraysize = 6 + cur.execute("select name from %sbooze" % self.table_prefix) + rows = cur.fetchmany() # Should get all rows + self.assertTrue(cur.rowcount in (-1, 6)) + self.assertEqual(len(rows), 6) + self.assertEqual(len(rows), 6) rows = [r[0] for r in rows] rows.sort() # Make sure we get the right data back out - for i in range(0,6): - self.assertEqual(rows[i],self.samples[i], - 'incorrect data retrieved by cursor.fetchmany' - ) - - rows = cur.fetchmany() # Should return an empty list - self.assertEqual(len(rows),0, - 'cursor.fetchmany should return an empty sequence if ' - 'called after the whole result set has been fetched' + for i in range(0, 6): + self.assertEqual( + rows[i], + self.samples[i], + "incorrect data retrieved by cursor.fetchmany", ) - self.assertTrue(cur.rowcount in (-1,6)) + + rows = cur.fetchmany() # Should return an empty list + self.assertEqual( + len(rows), + 0, + "cursor.fetchmany should return an empty sequence if " + "called after the whole result set has been fetched", + ) + self.assertTrue(cur.rowcount in (-1, 6)) self.executeDDL2(cur) - cur.execute('select name from %sbarflys' % self.table_prefix) - r = cur.fetchmany() # Should get empty sequence - self.assertEqual(len(r),0, - 'cursor.fetchmany should return an empty sequence if ' - 'query retrieved no rows' - ) - self.assertTrue(cur.rowcount in (-1,0)) + cur.execute("select name from %sbarflys" % self.table_prefix) + r = cur.fetchmany() # Should get empty sequence + self.assertEqual( + len(r), + 0, + "cursor.fetchmany should return an empty sequence if " + "query retrieved no rows", + ) + self.assertTrue(cur.rowcount in (-1, 0)) finally: con.close() @@ -633,36 +636,41 @@ def test_fetchall(self): # cursor.fetchall should raise an Error if called # after executing a a statement that cannot return rows - self.assertRaises(self.driver.Error,cur.fetchall) + self.assertRaises(self.driver.Error, cur.fetchall) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,len(self.samples))) - self.assertEqual(len(rows),len(self.samples), - 'cursor.fetchall did not retrieve all rows' - ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) + self.assertEqual( + len(rows), + len(self.samples), + "cursor.fetchall did not retrieve all rows", + ) rows = [r[0] for r in rows] rows.sort() - for i in range(0,len(self.samples)): - self.assertEqual(rows[i],self.samples[i], - 'cursor.fetchall retrieved incorrect rows' + for i in range(0, len(self.samples)): + self.assertEqual( + rows[i], self.samples[i], "cursor.fetchall retrieved incorrect rows" ) rows = cur.fetchall() self.assertEqual( - len(rows),0, - 'cursor.fetchall should return an empty list if called ' - 'after the whole result set has been fetched' - ) - self.assertTrue(cur.rowcount in (-1,len(self.samples))) + len(rows), + 0, + "cursor.fetchall should return an empty list if called " + "after the whole result set has been fetched", + ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) self.executeDDL2(cur) - cur.execute('select name from %sbarflys' % self.table_prefix) + cur.execute("select name from %sbarflys" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,0)) - self.assertEqual(len(rows),0, - 'cursor.fetchall should return an empty list if ' - 'a select query returns no rows' - ) + self.assertTrue(cur.rowcount in (-1, 0)) + self.assertEqual( + len(rows), + 0, + "cursor.fetchall should return an empty list if " + "a select query returns no rows", + ) finally: con.close() @@ -675,74 +683,74 @@ def test_mixedfetch(self): for sql in self._populate(): cur.execute(sql) - cur.execute('select name from %sbooze' % self.table_prefix) - rows1 = cur.fetchone() + cur.execute("select name from %sbooze" % self.table_prefix) + rows1 = cur.fetchone() rows23 = cur.fetchmany(2) - rows4 = cur.fetchone() + rows4 = cur.fetchone() rows56 = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,6)) - self.assertEqual(len(rows23),2, - 'fetchmany returned incorrect number of rows' - ) - self.assertEqual(len(rows56),2, - 'fetchall returned incorrect number of rows' - ) + self.assertTrue(cur.rowcount in (-1, 6)) + self.assertEqual( + len(rows23), 2, "fetchmany returned incorrect number of rows" + ) + self.assertEqual( + len(rows56), 2, "fetchall returned incorrect number of rows" + ) rows = [rows1[0]] - rows.extend([rows23[0][0],rows23[1][0]]) + rows.extend([rows23[0][0], rows23[1][0]]) rows.append(rows4[0]) - rows.extend([rows56[0][0],rows56[1][0]]) + rows.extend([rows56[0][0], rows56[1][0]]) rows.sort() - for i in range(0,len(self.samples)): - self.assertEqual(rows[i],self.samples[i], - 'incorrect data retrieved or inserted' - ) + for i in range(0, len(self.samples)): + self.assertEqual( + rows[i], self.samples[i], "incorrect data retrieved or inserted" + ) finally: con.close() - def help_nextset_setUp(self,cur): - ''' Should create a procedure called deleteme - that returns two result sets, first the - number of rows in booze then "name from booze" - ''' - raise NotImplementedError('Helper not implemented') - #sql=""" + def help_nextset_setUp(self, cur): + """Should create a procedure called deleteme + that returns two result sets, first the + number of rows in booze then "name from booze" + """ + raise NotImplementedError("Helper not implemented") + # sql=""" # create procedure deleteme as # begin # select count(*) from booze # select name from booze # end - #""" - #cur.execute(sql) + # """ + # cur.execute(sql) - def help_nextset_tearDown(self,cur): - 'If cleaning up is needed after nextSetTest' - raise NotImplementedError('Helper not implemented') - #cur.execute("drop procedure deleteme") + def help_nextset_tearDown(self, cur): + "If cleaning up is needed after nextSetTest" + raise NotImplementedError("Helper not implemented") + # cur.execute("drop procedure deleteme") def test_nextset(self): con = self._connect() try: cur = con.cursor() - if not hasattr(cur,'nextset'): + if not hasattr(cur, "nextset"): return try: self.executeDDL1(cur) - sql=self._populate() + sql = self._populate() for sql in self._populate(): cur.execute(sql) self.help_nextset_setUp(cur) - cur.callproc('deleteme') - numberofrows=cur.fetchone() - assert numberofrows[0]== len(self.samples) + cur.callproc("deleteme") + numberofrows = cur.fetchone() + assert numberofrows[0] == len(self.samples) assert cur.nextset() - names=cur.fetchall() + names = cur.fetchall() assert len(names) == len(self.samples) - s=cur.nextset() - assert s == None,'No more return sets, should return None' + s = cur.nextset() + assert s == None, "No more return sets, should return None" finally: self.help_nextset_tearDown(cur) @@ -750,16 +758,16 @@ def test_nextset(self): con.close() def test_nextset(self): - raise NotImplementedError('Drivers need to override this test') + raise NotImplementedError("Drivers need to override this test") def test_arraysize(self): # Not much here - rest of the tests for this are in test_fetchmany con = self._connect() try: cur = con.cursor() - self.assertTrue(hasattr(cur,'arraysize'), - 'cursor.arraysize must be defined' - ) + self.assertTrue( + hasattr(cur, "arraysize"), "cursor.arraysize must be defined" + ) finally: con.close() @@ -767,8 +775,8 @@ def test_setinputsizes(self): con = self._connect() try: cur = con.cursor() - cur.setinputsizes( (25,) ) - self._paraminsert(cur) # Make sure cursor still works + cur.setinputsizes((25,)) + self._paraminsert(cur) # Make sure cursor still works finally: con.close() @@ -778,74 +786,70 @@ def test_setoutputsize_basic(self): try: cur = con.cursor() cur.setoutputsize(1000) - cur.setoutputsize(2000,0) - self._paraminsert(cur) # Make sure the cursor still works + cur.setoutputsize(2000, 0) + self._paraminsert(cur) # Make sure the cursor still works finally: con.close() def test_setoutputsize(self): # Real test for setoutputsize is driver dependant - raise NotImplementedError('Driver need to override this test') + raise NotImplementedError("Driver need to override this test") def test_None(self): con = self._connect() try: cur = con.cursor() self.executeDDL1(cur) - cur.execute('insert into %sbooze values (NULL)' % self.table_prefix) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("insert into %sbooze values (NULL)" % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchall() - self.assertEqual(len(r),1) - self.assertEqual(len(r[0]),1) - self.assertEqual(r[0][0],None,'NULL value not returned as None') + self.assertEqual(len(r), 1) + self.assertEqual(len(r[0]), 1) + self.assertEqual(r[0][0], None, "NULL value not returned as None") finally: con.close() def test_Date(self): - d1 = self.driver.Date(2002,12,25) - d2 = self.driver.DateFromTicks(time.mktime((2002,12,25,0,0,0,0,0,0))) + d1 = self.driver.Date(2002, 12, 25) + d2 = self.driver.DateFromTicks(time.mktime((2002, 12, 25, 0, 0, 0, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(d1),str(d2)) def test_Time(self): - t1 = self.driver.Time(13,45,30) - t2 = self.driver.TimeFromTicks(time.mktime((2001,1,1,13,45,30,0,0,0))) + t1 = self.driver.Time(13, 45, 30) + t2 = self.driver.TimeFromTicks(time.mktime((2001, 1, 1, 13, 45, 30, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Timestamp(self): - t1 = self.driver.Timestamp(2002,12,25,13,45,30) + t1 = self.driver.Timestamp(2002, 12, 25, 13, 45, 30) t2 = self.driver.TimestampFromTicks( - time.mktime((2002,12,25,13,45,30,0,0,0)) - ) + time.mktime((2002, 12, 25, 13, 45, 30, 0, 0, 0)) + ) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Binary(self): - b = self.driver.Binary(b'Something') - b = self.driver.Binary(b'') + b = self.driver.Binary(b"Something") + b = self.driver.Binary(b"") def test_STRING(self): - self.assertTrue(hasattr(self.driver,'STRING'), - 'module.STRING must be defined' - ) + self.assertTrue(hasattr(self.driver, "STRING"), "module.STRING must be defined") def test_BINARY(self): - self.assertTrue(hasattr(self.driver,'BINARY'), - 'module.BINARY must be defined.' - ) + self.assertTrue( + hasattr(self.driver, "BINARY"), "module.BINARY must be defined." + ) def test_NUMBER(self): - self.assertTrue(hasattr(self.driver,'NUMBER'), - 'module.NUMBER must be defined.' - ) + self.assertTrue( + hasattr(self.driver, "NUMBER"), "module.NUMBER must be defined." + ) def test_DATETIME(self): - self.assertTrue(hasattr(self.driver,'DATETIME'), - 'module.DATETIME must be defined.' - ) + self.assertTrue( + hasattr(self.driver, "DATETIME"), "module.DATETIME must be defined." + ) def test_ROWID(self): - self.assertTrue(hasattr(self.driver,'ROWID'), - 'module.ROWID must be defined.' - ) + self.assertTrue(hasattr(self.driver, "ROWID"), "module.ROWID must be defined.") diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 8c1dd535..139089ab 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -4,16 +4,23 @@ from pymysql.tests import base import warnings -warnings.filterwarnings('error') +warnings.filterwarnings("error") + class test_MySQLdb(capabilities.DatabaseTest): db_module = pymysql connect_args = () connect_kwargs = base.PyMySQLTestCase.databases[0].copy() - connect_kwargs.update(dict(read_default_file='~/.my.cnf', - use_unicode=True, binary_prefix=True, - charset='utf8mb4', sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL")) + connect_kwargs.update( + dict( + read_default_file="~/.my.cnf", + use_unicode=True, + binary_prefix=True, + charset="utf8mb4", + sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL", + ) + ) leak_test = False @@ -22,64 +29,70 @@ def quote_identifier(self, ident): def test_TIME(self): from datetime import timedelta - def generator(row,col): - return timedelta(0, row*8000) - self.check_data_integrity( - ('col1 TIME',), - generator) + + def generator(row, col): + return timedelta(0, row * 8000) + + self.check_data_integrity(("col1 TIME",), generator) def test_TINYINT(self): # Number data - def generator(row,col): - v = (row*row) % 256 + def generator(row, col): + v = (row * row) % 256 if v > 127: - v = v-256 + v = v - 256 return v - self.check_data_integrity( - ('col1 TINYINT',), - generator) + + self.check_data_integrity(("col1 TINYINT",), generator) def test_stored_procedures(self): db = self.connection c = self.cursor try: - self.create_table(('pos INT', 'tree CHAR(20)')) - c.executemany("INSERT INTO %s (pos,tree) VALUES (%%s,%%s)" % self.table, - list(enumerate('ash birch cedar larch pine'.split()))) + self.create_table(("pos INT", "tree CHAR(20)")) + c.executemany( + "INSERT INTO %s (pos,tree) VALUES (%%s,%%s)" % self.table, + list(enumerate("ash birch cedar larch pine".split())), + ) db.commit() - c.execute(""" + c.execute( + """ CREATE PROCEDURE test_sp(IN t VARCHAR(255)) BEGIN SELECT pos FROM %s WHERE tree = t; END - """ % self.table) + """ + % self.table + ) db.commit() - c.callproc('test_sp', ('larch',)) + c.callproc("test_sp", ("larch",)) rows = c.fetchall() self.assertEqual(len(rows), 1) self.assertEqual(rows[0][0], 3) c.nextset() finally: c.execute("DROP PROCEDURE IF EXISTS test_sp") - c.execute('drop table %s' % (self.table)) + c.execute("drop table %s" % (self.table)) def test_small_CHAR(self): # Character data - def generator(row,col): - i = ((row+1)*(col+1)+62)%256 - if i == 62: return '' - if i == 63: return None + def generator(row, col): + i = ((row + 1) * (col + 1) + 62) % 256 + if i == 62: + return "" + if i == 63: + return None return chr(i) - self.check_data_integrity( - ('col1 char(1)','col2 char(1)'), - generator) + + self.check_data_integrity(("col1 char(1)", "col2 char(1)"), generator) def test_bug_2671682(self): from pymysql.constants import ER + try: - self.cursor.execute("describe some_non_existent_table"); + self.cursor.execute("describe some_non_existent_table") except self.connection.ProgrammingError as msg: self.assertEqual(msg.args[0], ER.NO_SUCH_TABLE) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index 2c9a0600..e882c5eb 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -9,13 +9,22 @@ class test_MySQLdb(dbapi20.DatabaseAPI20Test): driver = pymysql connect_args = () connect_kw_args = base.PyMySQLTestCase.databases[0].copy() - connect_kw_args.update(dict(read_default_file='~/.my.cnf', - charset='utf8', - sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL")) + connect_kw_args.update( + dict( + read_default_file="~/.my.cnf", + charset="utf8", + sql_mode="ANSI,STRICT_TRANS_TABLES,TRADITIONAL", + ) + ) - def test_setoutputsize(self): pass - def test_setoutputsize_basic(self): pass - def test_nextset(self): pass + def test_setoutputsize(self): + pass + + def test_setoutputsize_basic(self): + pass + + def test_nextset(self): + pass """The tests on fetchone and fetchall and rowcount bogusly test for an exception if the statement cannot return a @@ -37,36 +46,41 @@ def test_fetchall(self): # cursor.fetchall should raise an Error if called # after executing a a statement that cannot return rows -## self.assertRaises(self.driver.Error,cur.fetchall) + ## self.assertRaises(self.driver.Error,cur.fetchall) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,len(self.samples))) - self.assertEqual(len(rows),len(self.samples), - 'cursor.fetchall did not retrieve all rows' - ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) + self.assertEqual( + len(rows), + len(self.samples), + "cursor.fetchall did not retrieve all rows", + ) rows = [r[0] for r in rows] rows.sort() - for i in range(0,len(self.samples)): - self.assertEqual(rows[i],self.samples[i], - 'cursor.fetchall retrieved incorrect rows' + for i in range(0, len(self.samples)): + self.assertEqual( + rows[i], self.samples[i], "cursor.fetchall retrieved incorrect rows" ) rows = cur.fetchall() self.assertEqual( - len(rows),0, - 'cursor.fetchall should return an empty list if called ' - 'after the whole result set has been fetched' - ) - self.assertTrue(cur.rowcount in (-1,len(self.samples))) + len(rows), + 0, + "cursor.fetchall should return an empty list if called " + "after the whole result set has been fetched", + ) + self.assertTrue(cur.rowcount in (-1, len(self.samples))) self.executeDDL2(cur) - cur.execute('select name from %sbarflys' % self.table_prefix) + cur.execute("select name from %sbarflys" % self.table_prefix) rows = cur.fetchall() - self.assertTrue(cur.rowcount in (-1,0)) - self.assertEqual(len(rows),0, - 'cursor.fetchall should return an empty list if ' - 'a select query returns no rows' - ) + self.assertTrue(cur.rowcount in (-1, 0)) + self.assertEqual( + len(rows), + 0, + "cursor.fetchall should return an empty list if " + "a select query returns no rows", + ) finally: con.close() @@ -78,39 +92,40 @@ def test_fetchone(self): # cursor.fetchone should raise an Error if called before # executing a select-type query - self.assertRaises(self.driver.Error,cur.fetchone) + self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows self.executeDDL1(cur) -## self.assertRaises(self.driver.Error,cur.fetchone) + ## self.assertRaises(self.driver.Error,cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) - self.assertEqual(cur.fetchone(),None, - 'cursor.fetchone should return None if a query retrieves ' - 'no rows' - ) - self.assertTrue(cur.rowcount in (-1,0)) + cur.execute("select name from %sbooze" % self.table_prefix) + self.assertEqual( + cur.fetchone(), + None, + "cursor.fetchone should return None if a query retrieves " "no rows", + ) + self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after # executing a query that cannnot return rows - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) -## self.assertRaises(self.driver.Error,cur.fetchone) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + ## self.assertRaises(self.driver.Error,cur.fetchone) - cur.execute('select name from %sbooze' % self.table_prefix) + cur.execute("select name from %sbooze" % self.table_prefix) r = cur.fetchone() - self.assertEqual(len(r),1, - 'cursor.fetchone should have retrieved a single row' - ) - self.assertEqual(r[0],'Victoria Bitter', - 'cursor.fetchone retrieved incorrect data' - ) -## self.assertEqual(cur.fetchone(),None, -## 'cursor.fetchone should return None if no more rows available' -## ) - self.assertTrue(cur.rowcount in (-1,1)) + self.assertEqual( + len(r), 1, "cursor.fetchone should have retrieved a single row" + ) + self.assertEqual( + r[0], "Victoria Bitter", "cursor.fetchone retrieved incorrect data" + ) + ## self.assertEqual(cur.fetchone(),None, + ## 'cursor.fetchone should return None if no more rows available' + ## ) + self.assertTrue(cur.rowcount in (-1, 1)) finally: con.close() @@ -120,81 +135,86 @@ def test_rowcount(self): try: cur = con.cursor() self.executeDDL1(cur) -## self.assertEqual(cur.rowcount,-1, -## 'cursor.rowcount should be -1 after executing no-result ' -## 'statements' -## ) - cur.execute("insert into %sbooze values ('Victoria Bitter')" % ( - self.table_prefix - )) -## self.assertTrue(cur.rowcount in (-1,1), -## 'cursor.rowcount should == number or rows inserted, or ' -## 'set to -1 after executing an insert statement' -## ) + ## self.assertEqual(cur.rowcount,-1, + ## 'cursor.rowcount should be -1 after executing no-result ' + ## 'statements' + ## ) + cur.execute( + "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) + ) + ## self.assertTrue(cur.rowcount in (-1,1), + ## 'cursor.rowcount should == number or rows inserted, or ' + ## 'set to -1 after executing an insert statement' + ## ) cur.execute("select name from %sbooze" % self.table_prefix) - self.assertTrue(cur.rowcount in (-1,1), - 'cursor.rowcount should == number of rows returned, or ' - 'set to -1 after executing a select statement' - ) + self.assertTrue( + cur.rowcount in (-1, 1), + "cursor.rowcount should == number of rows returned, or " + "set to -1 after executing a select statement", + ) self.executeDDL2(cur) -## self.assertEqual(cur.rowcount,-1, -## 'cursor.rowcount not being reset to -1 after executing ' -## 'no-result statements' -## ) + ## self.assertEqual(cur.rowcount,-1, + ## 'cursor.rowcount not being reset to -1 after executing ' + ## 'no-result statements' + ## ) finally: con.close() def test_callproc(self): - pass # performed in test_MySQL_capabilities - - def help_nextset_setUp(self,cur): - ''' Should create a procedure called deleteme - that returns two result sets, first the - number of rows in booze then "name from booze" - ''' - sql=""" + pass # performed in test_MySQL_capabilities + + def help_nextset_setUp(self, cur): + """Should create a procedure called deleteme + that returns two result sets, first the + number of rows in booze then "name from booze" + """ + sql = """ create procedure deleteme() begin select count(*) from %(tp)sbooze; select name from %(tp)sbooze; end - """ % dict(tp=self.table_prefix) + """ % dict( + tp=self.table_prefix + ) cur.execute(sql) - def help_nextset_tearDown(self,cur): - 'If cleaning up is needed after nextSetTest' + def help_nextset_tearDown(self, cur): + "If cleaning up is needed after nextSetTest" cur.execute("drop procedure deleteme") def test_nextset(self): from warnings import warn + con = self._connect() try: cur = con.cursor() - if not hasattr(cur,'nextset'): + if not hasattr(cur, "nextset"): return try: self.executeDDL1(cur) - sql=self._populate() + sql = self._populate() for sql in self._populate(): cur.execute(sql) self.help_nextset_setUp(cur) - cur.callproc('deleteme') - numberofrows=cur.fetchone() - assert numberofrows[0]== len(self.samples) + cur.callproc("deleteme") + numberofrows = cur.fetchone() + assert numberofrows[0] == len(self.samples) assert cur.nextset() - names=cur.fetchall() + names = cur.fetchall() assert len(names) == len(self.samples) - s=cur.nextset() + s = cur.nextset() if s: empty = cur.fetchall() - self.assertEqual(len(empty), 0, - "non-empty result set after other result sets") - #warn("Incompatibility: MySQL returns an empty result set for the CALL itself", + self.assertEqual( + len(empty), 0, "non-empty result set after other result sets" + ) + # warn("Incompatibility: MySQL returns an empty result set for the CALL itself", # Warning) - #assert s == None,'No more return sets, should return None' + # assert s == None,'No more return sets, should return None' finally: self.help_nextset_tearDown(cur) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py index 747ea4b0..b8d4bb1e 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py @@ -2,6 +2,7 @@ import unittest import pymysql + _mysql = pymysql from pymysql.constants import FIELD_TYPE from pymysql.tests import base @@ -26,7 +27,7 @@ class CoreModule(unittest.TestCase): def test_NULL(self): """Should have a NULL constant.""" - self.assertEqual(_mysql.NULL, 'NULL') + self.assertEqual(_mysql.NULL, "NULL") def test_version(self): """Version information sanity.""" @@ -55,36 +56,45 @@ def tearDown(self): def test_thread_id(self): tid = self.conn.thread_id() - self.assertTrue(isinstance(tid, int), - "thread_id didn't return an integral value.") + self.assertTrue( + isinstance(tid, int), "thread_id didn't return an integral value." + ) - self.assertRaises(TypeError, self.conn.thread_id, ('evil',), - "thread_id shouldn't accept arguments.") + self.assertRaises( + TypeError, + self.conn.thread_id, + ("evil",), + "thread_id shouldn't accept arguments.", + ) def test_affected_rows(self): - self.assertEqual(self.conn.affected_rows(), 0, - "Should return 0 before we do anything.") - + self.assertEqual( + self.conn.affected_rows(), 0, "Should return 0 before we do anything." + ) - #def test_debug(self): - ## FIXME Only actually tests if you lack SUPER - #self.assertRaises(pymysql.OperationalError, - #self.conn.dump_debug_info) + # def test_debug(self): + ## FIXME Only actually tests if you lack SUPER + # self.assertRaises(pymysql.OperationalError, + # self.conn.dump_debug_info) def test_charset_name(self): - self.assertTrue(isinstance(self.conn.character_set_name(), str), - "Should return a string.") + self.assertTrue( + isinstance(self.conn.character_set_name(), str), "Should return a string." + ) def test_host_info(self): assert isinstance(self.conn.get_host_info(), str), "should return a string" def test_proto_info(self): - self.assertTrue(isinstance(self.conn.get_proto_info(), int), - "Should return an int.") + self.assertTrue( + isinstance(self.conn.get_proto_info(), int), "Should return an int." + ) def test_server_info(self): - self.assertTrue(isinstance(self.conn.get_server_info(), str), - "Should return an str.") + self.assertTrue( + isinstance(self.conn.get_server_info(), str), "Should return an str." + ) + if __name__ == "__main__": unittest.main() diff --git a/pymysql/util.py b/pymysql/util.py index 04683f83..1349ec7b 100644 --- a/pymysql/util.py +++ b/pymysql/util.py @@ -10,4 +10,3 @@ def byte2int(b): def int2byte(i): return struct.pack("!B", i) - diff --git a/tests/test_auth.py b/tests/test_auth.py index 61957655..e5e2a64e 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -10,7 +10,7 @@ port = 3306 ca = os.path.expanduser("~/ca.pem") -ssl = {'ca': ca, 'check_hostname': False} +ssl = {"ca": ca, "check_hostname": False} pass_sha256 = "pass_sha256_01234567890123456789" pass_caching_sha2 = "pass_caching_sha2_01234567890123456789" @@ -27,12 +27,16 @@ def test_sha256_no_passowrd_ssl(): def test_sha256_password(): - con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_sha256", password=pass_sha256, host=host, port=port, ssl=None + ) con.close() def test_sha256_password_ssl(): - con = pymysql.connect(user="user_sha256", password=pass_sha256, host=host, port=port, ssl=ssl) + con = pymysql.connect( + user="user_sha256", password=pass_sha256, host=host, port=port, ssl=ssl + ) con.close() @@ -47,20 +51,44 @@ def test_caching_sha2_no_password_ssl(): def test_caching_sha2_password(): - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=None, + ) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=None, + ) con.query("FLUSH PRIVILEGES") con.close() def test_caching_sha2_password_ssl(): - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=ssl) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=ssl, + ) con.close() # Fast path of caching sha2 - con = pymysql.connect(user="user_caching_sha2", password=pass_caching_sha2, host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=None, + ) con.query("FLUSH PRIVILEGES") con.close() diff --git a/tests/test_mariadb_auth.py b/tests/test_mariadb_auth.py index 2f336fec..b3a2719c 100644 --- a/tests/test_mariadb_auth.py +++ b/tests/test_mariadb_auth.py @@ -15,8 +15,9 @@ def test_ed25519_no_password(): def test_ed25519_password(): # nosec - con = pymysql.connect(user="user_ed25519", password="pass_ed25519", - host=host, port=port, ssl=None) + con = pymysql.connect( + user="user_ed25519", password="pass_ed25519", host=host, port=port, ssl=None + ) con.close() From 175a3e0bc826fbf0a1d3cf6f73aac46a01672bba Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:07:27 +0900 Subject: [PATCH 122/292] Remove _socketio --- pymysql/_socketio.py | 130 ------------------------------------------- 1 file changed, 130 deletions(-) delete mode 100644 pymysql/_socketio.py diff --git a/pymysql/_socketio.py b/pymysql/_socketio.py deleted file mode 100644 index 6b2d65a3..00000000 --- a/pymysql/_socketio.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -SocketIO imported from socket module in Python 3. - -Copyright (c) 2001-2013 Python Software Foundation; All Rights Reserved. -""" - -from socket import * -import io -import errno - -__all__ = ["SocketIO"] - -EINTR = errno.EINTR -_blocking_errnos = (errno.EAGAIN, errno.EWOULDBLOCK) - - -class SocketIO(io.RawIOBase): - - """Raw I/O implementation for stream sockets. - - This class supports the makefile() method on sockets. It provides - the raw I/O interface on top of a socket object. - """ - - # One might wonder why not let FileIO do the job instead. There are two - # main reasons why FileIO is not adapted: - # - it wouldn't work under Windows (where you can't used read() and - # write() on a socket handle) - # - it wouldn't work with socket timeouts (FileIO would ignore the - # timeout and consider the socket non-blocking) - - # XXX More docs - - def __init__(self, sock, mode): - if mode not in ("r", "w", "rw", "rb", "wb", "rwb"): - raise ValueError("invalid mode: %r" % mode) - io.RawIOBase.__init__(self) - self._sock = sock - if "b" not in mode: - mode += "b" - self._mode = mode - self._reading = "r" in mode - self._writing = "w" in mode - self._timeout_occurred = False - - def readinto(self, b): - """Read up to len(b) bytes into the writable buffer *b* and return - the number of bytes read. If the socket is non-blocking and no bytes - are available, None is returned. - - If *b* is non-empty, a 0 return value indicates that the connection - was shutdown at the other end. - """ - self._checkClosed() - self._checkReadable() - if self._timeout_occurred: - raise IOError("cannot read from timed out object") - while True: - try: - return self._sock.recv_into(b) - except timeout: - self._timeout_occurred = True - raise - except error as e: - n = e.args[0] - if n == EINTR: - continue - if n in _blocking_errnos: - return None - raise - - def write(self, b): - """Write the given bytes or bytearray object *b* to the socket - and return the number of bytes written. This can be less than - len(b) if not all data could be written. If the socket is - non-blocking and no bytes could be written None is returned. - """ - self._checkClosed() - self._checkWritable() - try: - return self._sock.send(b) - except error as e: - # XXX what about EINTR? - if e.args[0] in _blocking_errnos: - return None - raise - - def readable(self): - """True if the SocketIO is open for reading.""" - if self.closed: - raise ValueError("I/O operation on closed socket.") - return self._reading - - def writable(self): - """True if the SocketIO is open for writing.""" - if self.closed: - raise ValueError("I/O operation on closed socket.") - return self._writing - - def seekable(self): - """True if the SocketIO is open for seeking.""" - if self.closed: - raise ValueError("I/O operation on closed socket.") - return super().seekable() - - def fileno(self): - """Return the file descriptor of the underlying socket.""" - self._checkClosed() - return self._sock.fileno() - - @property - def name(self): - if not self.closed: - return self.fileno() - else: - return -1 - - @property - def mode(self): - return self._mode - - def close(self): - """Close the SocketIO object. This doesn't close the underlying - socket, except if all references to it have disappeared. - """ - if self.closed: - return - io.RawIOBase.close(self) - self._sock._decref_socketios() - self._sock = None From 3299afd1f1402b0df464d13333473005298ea387 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:09:58 +0900 Subject: [PATCH 123/292] Simplify --- pymysql/__init__.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 5b49262e..790cb9fc 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -25,6 +25,7 @@ from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string +from . import connections from .err import ( Warning, Error, @@ -109,20 +110,10 @@ def Binary(x): def Connect(*args, **kwargs): - """ - Connect to the database; see connections.Connection.__init__() for - more information. - """ - from .connections import Connection - - return Connection(*args, **kwargs) - + return connections.Connection(*args, **kwargs) -from . import connections as _orig_conn -if _orig_conn.Connection.__init__.__doc__ is not None: - Connect.__doc__ = _orig_conn.Connection.__init__.__doc__ -del _orig_conn +Connect.__doc__ = connections.Connection.__init__.__doc__ def get_client_info(): # for MySQLdb compatibility From 587a59670ea1e10e3cc36d73ad47484cb67ebe4f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:13:33 +0900 Subject: [PATCH 124/292] Update flake8 setting --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index db1af545..9d74b3a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,6 @@ [flake8] -ignore = E226,E301,E701 +ignore = E203,E501,W503,E722 exclude = tests,build -max-line-length = 119 [bdist_wheel] universal = 1 From 62108f59fe7d517c1586c6506a04c2963e6fe5f7 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:19:15 +0900 Subject: [PATCH 125/292] Update flake8 setting --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9d74b3a8..8efb0850 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [flake8] ignore = E203,E501,W503,E722 -exclude = tests,build +exclude = tests,build,.venv,docs [bdist_wheel] universal = 1 From 4185f7fe95ee498e61abbca9e02402318874ffb1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:20:13 +0900 Subject: [PATCH 126/292] Actions: Add lint --- .github/workflows/lint.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/lint.yaml diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..894a2d7c --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,17 @@ +name: Lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: psf/black@stable + - name: Setup flake8 annotations + uses: rbialon/flake8-annotations@v1 + - name: flake8 + run: | + pip install flake8 + flake8 pymysql From df14c55377867b7a5a159a3ed5f0280b1cf10aea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:23:22 +0900 Subject: [PATCH 127/292] black setup.py --- setup.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index e35e7b29..37dcbf95 100755 --- a/setup.py +++ b/setup.py @@ -4,38 +4,38 @@ version = "0.10.1" -with io.open('./README.rst', encoding='utf-8') as f: +with io.open("./README.rst", encoding="utf-8") as f: readme = f.read() setup( name="PyMySQL", version=version, - url='https://github.com/PyMySQL/PyMySQL/', + url="https://github.com/PyMySQL/PyMySQL/", project_urls={ "Documentation": "https://pymysql.readthedocs.io/", }, - description='Pure Python MySQL Driver', + description="Pure Python MySQL Driver", long_description=readme, - packages=find_packages(exclude=['tests*', 'pymysql.tests*']), + packages=find_packages(exclude=["tests*", "pymysql.tests*"]), extras_require={ "rsa": ["cryptography"], "ed25519": ["PyNaCl>=1.4.0"], }, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Topic :: Database', + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Database", ], keywords="MySQL", ) From 9dc65c04a0fb60054161bdd7f46fb5c3baf39949 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:37:50 +0900 Subject: [PATCH 128/292] reformat black --- .github/workflows/lint.yaml | 2 + docs/source/conf.py | 156 ++++++++++++++++++++---------------- example.py | 2 +- 3 files changed, 88 insertions(+), 72 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 894a2d7c..a1804050 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -9,6 +9,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: psf/black@stable + with: + args: ". --diff --check" - name: Setup flake8 annotations uses: rbialon/flake8-annotations@v1 - name: flake8 diff --git a/docs/source/conf.py b/docs/source/conf.py index bbadcbed..77d7073a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,55 +18,55 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath("../../")) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'PyMySQL' -copyright = u'2016, Yutaka Matsubara and GitHub contributors' +project = u"PyMySQL" +copyright = u"2016, Yutaka Matsubara and GitHub contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.7' +version = "0.7" # The full version, including alpha/beta/rc tags. -release = '0.7.2' +release = "0.7.2" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -74,154 +74,157 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = "default" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'PyMySQLdoc' +htmlhelp_basename = "PyMySQLdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'PyMySQL.tex', u'PyMySQL Documentation', - u'Yutaka Matsubara and GitHub contributors', 'manual'), + ( + "index", + "PyMySQL.tex", + u"PyMySQL Documentation", + u"Yutaka Matsubara and GitHub contributors", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -229,12 +232,17 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'pymysql', u'PyMySQL Documentation', - [u'Yutaka Matsubara and GitHub contributors'], 1) + ( + "index", + "pymysql", + u"PyMySQL Documentation", + [u"Yutaka Matsubara and GitHub contributors"], + 1, + ) ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -243,23 +251,29 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'PyMySQL', u'PyMySQL Documentation', - u'Yutaka Matsubara and GitHub contributors', 'PyMySQL', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "PyMySQL", + u"PyMySQL Documentation", + u"Yutaka Matsubara and GitHub contributors", + "PyMySQL", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {"http://docs.python.org/": None} diff --git a/example.py b/example.py index 68582138..d40e94ab 100644 --- a/example.py +++ b/example.py @@ -3,7 +3,7 @@ import pymysql -conn = pymysql.connect(host='localhost', port=3306, user='root', passwd='', db='mysql') +conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="", db="mysql") cur = conn.cursor() From e28c96eef07471f288f7308c2db73dc47f595436 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:45:41 +0900 Subject: [PATCH 129/292] Update README --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 269928b8..f8a854a6 100644 --- a/README.rst +++ b/README.rst @@ -35,13 +35,13 @@ Requirements * Python -- one of the following: - - CPython_ : 2.7 and >= 3.5 - - PyPy_ : Latest version + - CPython_ : 3.6 and newer + - PyPy_ : Latest 3.x version * MySQL Server -- one of the following: - - MySQL_ >= 5.5 - - MariaDB_ >= 5.5 + - MySQL_ >= 5.6 + - MariaDB_ >= 10.0 .. _CPython: https://www.python.org/ .. _PyPy: https://pypy.org/ @@ -77,6 +77,7 @@ Documentation is available online: https://pymysql.readthedocs.io/ For support, please refer to the `StackOverflow `_. + Example ------- From 7f44cd71f253be32d79d72dd4193f7a8a3557e8d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:51:01 +0900 Subject: [PATCH 130/292] Actions: Use cache for pip --- .github/workflows/test.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 71cc4e82..5b35716f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,9 +46,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.py }} + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-1 + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install dependency + run: | + pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls + - name: Set up MySQL run: | - sleep 10 mysql -h 127.0.0.1 -uroot -e "select version()" mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' @@ -59,7 +70,6 @@ jobs: - name: Run test run: | - pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls pytest -v --cov --cov-config .coveragerc pymysql - name: Run MySQL8 auth test From 96b7583e5cc4d476d8071893eec9a0f479e835ec Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:55:14 +0900 Subject: [PATCH 131/292] Fix circular import --- pymysql/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 790cb9fc..451012c8 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -25,7 +25,6 @@ from .constants import FIELD_TYPE from .converters import escape_dict, escape_sequence, escape_string -from . import connections from .err import ( Warning, Error, @@ -58,6 +57,8 @@ apilevel = "2.0" paramstyle = "pyformat" +from . import connections # noqa: E402 + class DBAPISet(frozenset): def __ne__(self, other): From 0e5afb12bcaee74c59dc5edb0d211e0e87a4536b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 12:59:58 +0900 Subject: [PATCH 132/292] Actions: Wait MySQL --- .github/workflows/test.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5b35716f..0253ab0c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -60,7 +60,11 @@ jobs: - name: Set up MySQL run: | - mysql -h 127.0.0.1 -uroot -e "select version()" + while : + do + sleep 1 + mysql --protocol=tcp -e 'select version()' && break + done mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' mysql -h 127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' From 8810ea977fa638c1a4db6f3a3047dbd2d8cc0b2d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:05:55 +0900 Subject: [PATCH 133/292] Actions: Run Lint only when py files are changed --- .github/workflows/lint.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a1804050..887a8f26 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,6 +1,12 @@ name: Lint -on: [push, pull_request] +on: + push: + paths: + - '**.py' + pull_request: + paths: + - '**.py' jobs: lint: From b637c37d87f66b2fbb93bc341e551fb55d9eba49 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:09:33 +0900 Subject: [PATCH 134/292] Actions: fix --- .github/workflows/test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0253ab0c..e43df4b2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -63,13 +63,13 @@ jobs: while : do sleep 1 - mysql --protocol=tcp -e 'select version()' && break + mysql -h127.0.0.1 -uroot -e 'select version()' && break done - mysql -h 127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" - mysql -h 127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' - mysql -h 127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - mysql -h 127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" - mysql -h 127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" + mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" + mysql -h127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' + mysql -h127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' + mysql -h127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" + mysql -h127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" cp .travis/docker.json pymysql/tests/databases.json - name: Run test From acce32fb2d2c6c5a438d7237d4744f13822b76c6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:16:56 +0900 Subject: [PATCH 135/292] Remove .travis.yml --- .travis.yml | 59 ----------------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index aa1f0f34..00000000 --- a/.travis.yml +++ /dev/null @@ -1,59 +0,0 @@ -# vim: sw=2 ts=2 sts=2 expandtab - -dist: bionic -language: python -cache: pip - -services: - - docker - -matrix: - include: - - env: - - DB=mariadb:10.2 - python: "3.6" - - env: - - DB=mariadb:10.3 - - TEST_MARIADB_AUTH=yes - python: "pypy3" - - env: - - DB=mariadb:10.5 - - TEST_MARIADB_AUTH=yes - python: "3.7" - - env: - - DB=mysql:5.6 - python: "3.9" - - env: - - DB=mysql:5.7 - python: "3.7" - - env: - - DB=mysql:8.0 - - TEST_AUTH=yes - python: "3.8" - -# different py version from 5.6 and 5.7 as cache seems to be based on py version -# http://dev.mysql.com/downloads/mysql/5.7.html has latest development release version -# really only need libaio1 for DB builds however libaio-dev is whitelisted for container builds and liaio1 isn't -install: - - pip install -U coveralls coverage cryptography PyNaCl pytest pytest-cov - -before_script: - - ./.travis/initializedb.sh - - python -VV - - rm -f ~/.my.cnf # set in .travis.initialize.db.sh for the above commands - we should be using database.json however - - export COVERALLS_PARALLEL=true - -script: - - pytest -v --cov --cov-config .coveragerc pymysql - - if [ "${TEST_AUTH}" = "yes" ]; - then pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - fi - - if [ "${TEST_MARIADB_AUTH}" = "yes" ]; - then pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py; - fi - - if [ ! -z "${DB}" ]; - then docker logs mysqld; - fi - -after_success: - - coveralls From 27c72285d82620d07707c38224e205b866ba9c99 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:18:32 +0900 Subject: [PATCH 136/292] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 95430ae8..fef58a82 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,35,36,37,38,py,py3} +envlist = py{36,37,38,39,py3} [testenv] commands = pytest -v pymysql/tests/ From 0b2dd7e85984d5624ba0c972463add6d5696417c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:20:40 +0900 Subject: [PATCH 137/292] Update example.py --- example.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/example.py b/example.py index d40e94ab..c12f103b 100644 --- a/example.py +++ b/example.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function - import pymysql conn = pymysql.connect(host="localhost", port=3306, user="root", passwd="", db="mysql") @@ -10,7 +8,6 @@ cur.execute("SELECT Host,User FROM user") print(cur.description) - print() for row in cur: From 58b331e2b1bb9f096e17487fc9f9a616e02b161c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:38:36 +0900 Subject: [PATCH 138/292] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..b6a7238d --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,67 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '34 7 * * 2' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 3481889b140cd621ea4b49266b4ea327b8a146cc Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:38:45 +0900 Subject: [PATCH 139/292] Cleanup (#921) * Cleanup * black --- README.rst | 8 +++----- pymysql/connections.py | 25 +++++++------------------ setup.py | 6 +----- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/README.rst b/README.rst index f8a854a6..06f3ed7b 100644 --- a/README.rst +++ b/README.rst @@ -90,7 +90,7 @@ The following examples make use of a simple table `email` varchar(255) COLLATE utf8_bin NOT NULL, `password` varchar(255) COLLATE utf8_bin NOT NULL, PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_bin AUTO_INCREMENT=1 ; @@ -103,10 +103,9 @@ The following examples make use of a simple table user='user', password='passwd', db='db', - charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) - try: + with connection: with connection.cursor() as cursor: # Create a new record sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" @@ -122,8 +121,7 @@ The following examples make use of a simple table cursor.execute(sql, ('webmaster@python.org',)) result = cursor.fetchone() print(result) - finally: - connection.close() + This example will print: diff --git a/pymysql/connections.py b/pymysql/connections.py index dc69868b..32bf509b 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -47,17 +47,6 @@ DEBUG = False -_py_version = sys.version_info[:2] - - -def _fast_surrogateescape(s): - return s.decode("ascii", "surrogateescape") - - -def _makefile(sock, mode): - return sock.makefile(mode) - - TEXT_TYPES = { FIELD_TYPE.BIT, FIELD_TYPE.BLOB, @@ -76,12 +65,12 @@ def _makefile(sock, mode): MAX_PACKET_LEN = 2 ** 24 - 1 -def pack_int24(n): +def _pack_int24(n): return struct.pack("=5.0) diff --git a/setup.py b/setup.py index 37dcbf95..08aa62f7 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -import io from setuptools import setup, find_packages version = "0.10.1" -with io.open("./README.rst", encoding="utf-8") as f: +with open("./README.rst", encoding="utf-8") as f: readme = f.read() setup( @@ -23,10 +22,7 @@ }, classifiers=[ "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From cd61e56190c3ec6ab82934d9475712cd7a170656 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 13:50:11 +0900 Subject: [PATCH 140/292] Remove old_password support (#922) --- pymysql/_auth.py | 63 ------------------------------------------------ 1 file changed, 63 deletions(-) diff --git a/pymysql/_auth.py b/pymysql/_auth.py index d16a0895..33fd9df8 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -2,7 +2,6 @@ Implements auth methods """ from .err import OperationalError -from .util import byte2int, int2byte try: @@ -16,9 +15,6 @@ from functools import partial import hashlib -import io -import struct -import warnings DEBUG = False @@ -53,65 +49,6 @@ def _my_crypt(message1, message2): return bytes(result) -# old_passwords support ported from libmysql/password.c -# https://dev.mysql.com/doc/internals/en/old-password-authentication.html - -SCRAMBLE_LENGTH_323 = 8 - - -class RandStruct_323: - def __init__(self, seed1, seed2): - self.max_value = 0x3FFFFFFF - self.seed1 = seed1 % self.max_value - self.seed2 = seed2 % self.max_value - - def my_rnd(self): - self.seed1 = (self.seed1 * 3 + self.seed2) % self.max_value - self.seed2 = (self.seed1 + self.seed2 + 33) % self.max_value - return float(self.seed1) / float(self.max_value) - - -def scramble_old_password(password, message): - """Scramble for old_password""" - warnings.warn( - "old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" - "old password support will be removed in future PyMySQL version" - ) - hash_pass = _hash_password_323(password) - hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) - hash_pass_n = struct.unpack(">LL", hash_pass) - hash_message_n = struct.unpack(">LL", hash_message) - - rand_st = RandStruct_323( - hash_pass_n[0] ^ hash_message_n[0], hash_pass_n[1] ^ hash_message_n[1] - ) - outbuf = io.BytesIO() - for _ in range(min(SCRAMBLE_LENGTH_323, len(message))): - outbuf.write(int2byte(int(rand_st.my_rnd() * 31) + 64)) - extra = int2byte(int(rand_st.my_rnd() * 31)) - out = outbuf.getvalue() - outbuf = io.BytesIO() - for c in out: - outbuf.write(int2byte(byte2int(c) ^ byte2int(extra))) - return outbuf.getvalue() - - -def _hash_password_323(password): - nr = 1345345333 - add = 7 - nr2 = 0x12345671 - - # x in py3 is numbers, p27 is chars - for c in [byte2int(x) for x in password if x not in (" ", "\t", 32, 9)]: - nr ^= (((nr & 63) + add) * c) + (nr << 8) & 0xFFFFFFFF - nr2 = (nr2 + ((nr2 << 8) ^ nr)) & 0xFFFFFFFF - add = (add + c) & 0xFFFFFFFF - - r1 = nr & ((1 << 31) - 1) # kill sign bits - r2 = nr2 & ((1 << 31) - 1) - return struct.pack(">LL", r1, r2) - - # MariaDB's client_ed25519-plugin # https://mariadb.com/kb/en/library/connection/#client_ed25519-plugin From 8d3e079aed805ba18fea61014a61b8042225ac5d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 14:15:18 +0900 Subject: [PATCH 141/292] Add LGTM badge --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 06f3ed7b..324010ef 100644 --- a/README.rst +++ b/README.rst @@ -11,6 +11,9 @@ .. image:: https://img.shields.io/badge/license-MIT-blue.svg :target: https://github.com/PyMySQL/PyMySQL/blob/master/LICENSE +.. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 + :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python + PyMySQL ======= From 744da2f5b853702c27be0ab10dad3312bed11030 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 14:21:20 +0900 Subject: [PATCH 142/292] remove util.py (#923) * remove util.py * black * fix * fix --- pymysql/connections.py | 7 +++---- pymysql/protocol.py | 9 +++------ pymysql/tests/test_basic.py | 3 +-- pymysql/tests/test_nextset.py | 1 - pymysql/util.py | 12 ------------ 5 files changed, 7 insertions(+), 25 deletions(-) delete mode 100644 pymysql/util.py diff --git a/pymysql/connections.py b/pymysql/connections.py index 32bf509b..63a8b3a9 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -25,7 +25,6 @@ EOFPacketWrapper, LoadLocalPacketWrapper, ) -from .util import byte2int, int2byte from . import err, VERSION_STRING try: @@ -76,7 +75,7 @@ def _lenenc_int(i): "Encoding %d is less than 0 - no representation in LengthEncodedInteger" % i ) elif i < 0xFB: - return int2byte(i) + return bytes([i]) elif i < (1 << 16): return b"\xfc" + struct.pack(" Date: Sun, 3 Jan 2021 14:41:50 +0900 Subject: [PATCH 143/292] Update docs (#924) --- README.rst | 2 +- docs/source/user/development.rst | 3 ++- docs/source/user/examples.rst | 7 +++---- docs/source/user/installation.rst | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 324010ef..82303d05 100644 --- a/README.rst +++ b/README.rst @@ -93,7 +93,7 @@ The following examples make use of a simple table `email` varchar(255) COLLATE utf8_bin NOT NULL, `password` varchar(255) COLLATE utf8_bin NOT NULL, PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8_bin + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1 ; diff --git a/docs/source/user/development.rst b/docs/source/user/development.rst index 39c40e1a..09907318 100644 --- a/docs/source/user/development.rst +++ b/docs/source/user/development.rst @@ -30,7 +30,8 @@ and edit the new file to match your MySQL configuration:: To run all the tests, execute the script ``runtests.py``:: - $ python runtests.py + $ pip install pytest + $ pytest -v pymysql A ``tox.ini`` file is also provided for conveniently running tests on multiple Python versions:: diff --git a/docs/source/user/examples.rst b/docs/source/user/examples.rst index 87af40c3..966d46bd 100644 --- a/docs/source/user/examples.rst +++ b/docs/source/user/examples.rst @@ -18,7 +18,7 @@ The following examples make use of a simple table `email` varchar(255) COLLATE utf8_bin NOT NULL, `password` varchar(255) COLLATE utf8_bin NOT NULL, PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1 ; @@ -34,7 +34,7 @@ The following examples make use of a simple table charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) - try: + with connection: with connection.cursor() as cursor: # Create a new record sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" @@ -50,8 +50,7 @@ The following examples make use of a simple table cursor.execute(sql, ('webmaster@python.org',)) result = cursor.fetchone() print(result) - finally: - connection.close() + This example will print: diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index d95961c6..0fea2726 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -18,13 +18,13 @@ Requirements * Python -- one of the following: - - CPython_ >= 2.7 or >= 3.5 - - Latest PyPy_ + - CPython_ >= 3.6 + - Latest PyPy_ 3 * MySQL Server -- one of the following: - - MySQL_ >= 5.5 - - MariaDB_ >= 5.5 + - MySQL_ >= 5.6 + - MariaDB_ >= 10.0 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ From 1a6b82d461037fdecf0c22476bde8b86884c8831 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 17:33:14 +0900 Subject: [PATCH 144/292] Update CHANGELOG --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d1313aa..cb6e73cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changes +## v1.0.0 + +Release date: TBD + +Backward incompatible changes: + +* Python 2.7 and 3.5 are not supported. +* old_password (used by MySQL older than 4.1) is not supported. + +Other changes: + +* Connection supports context manager API. ``__exit__`` closes the connection. (#886) +* Add MySQL Connector/Python compatible TLS options (#903) + + ## v0.10.1 Release date: 2020-09-10 From f9489ed163a4196ba9218d268901a6240fffe755 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 3 Jan 2021 17:37:37 +0900 Subject: [PATCH 145/292] Test with MariaDB 10.0 (#925) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e43df4b2..dd45bcab 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,7 +10,7 @@ jobs: strategy: matrix: include: - - db: "mariadb:10.2" + - db: "mariadb:10.0" py: "3.9" - db: "mariadb:10.3" From d9b67a397b8fa839d0ec9c812fd7c0fcffc0fd30 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 4 Jan 2021 15:06:47 +0900 Subject: [PATCH 146/292] Code cleanup (#927) * cleanup * 2to3 -f unicode * black --- pymysql/converters.py | 14 +++++++------- pymysql/protocol.py | 14 +++++++------- pymysql/tests/test_basic.py | 8 ++++---- pymysql/tests/test_connection.py | 12 ++++++------ pymysql/tests/test_converters.py | 2 +- pymysql/tests/test_issues.py | 30 +++++++++++++++--------------- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 113dd298..d910f5c5 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -64,13 +64,13 @@ def escape_float(value, mapping=None): _escape_table = [chr(x) for x in range(128)] -_escape_table[0] = u"\\0" -_escape_table[ord("\\")] = u"\\\\" -_escape_table[ord("\n")] = u"\\n" -_escape_table[ord("\r")] = u"\\r" -_escape_table[ord("\032")] = u"\\Z" -_escape_table[ord('"')] = u'\\"' -_escape_table[ord("'")] = u"\\'" +_escape_table[0] = "\\0" +_escape_table[ord("\\")] = "\\\\" +_escape_table[ord("\n")] = "\\n" +_escape_table[ord("\r")] = "\\r" +_escape_table[ord("\032")] = "\\Z" +_escape_table[ord('"')] = '\\"' +_escape_table[ord("'")] = "\\'" def escape_string(value, mapping=None): diff --git a/pymysql/protocol.py b/pymysql/protocol.py index aa5feade..559ba624 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -182,31 +182,31 @@ def read_struct(self, fmt): def is_ok_packet(self): # https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html - return self._data[0:1] == b"\0" and len(self._data) >= 7 + return self._data[0] == 0 and len(self._data) >= 7 def is_eof_packet(self): # http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet # Caution: \xFE may be LengthEncodedInteger. # If \xFE is LengthEncodedInteger header, 8bytes followed. - return self._data[0:1] == b"\xfe" and len(self._data) < 9 + return self._data[0] == 0xFE and len(self._data) < 9 def is_auth_switch_request(self): # http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest - return self._data[0:1] == b"\xfe" + return self._data[0] == 0xFE def is_extra_auth_data(self): # https://dev.mysql.com/doc/internals/en/successful-authentication.html - return self._data[0:1] == b"\x01" + return self._data[0] == 1 def is_resultset_packet(self): - field_count = ord(self._data[0:1]) + field_count = self._data[0] return 1 <= field_count <= 250 def is_load_local_packet(self): - return self._data[0:1] == b"\xfb" + return self._data[0] == 0xFB def is_error_packet(self): - return self._data[0:1] == b"\xff" + return self._data[0] == 0xFF def check_error(self): if self.is_error_packet(): diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index fc195312..c2590bf2 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -29,7 +29,7 @@ def test_datatypes(self): 123456789012, 5.7, "hello'\" world", - u"Espa\xc3\xb1ol", + "Espa\xc3\xb1ol", "binary\x00data".encode(conn.encoding), datetime.date(1988, 2, 2), datetime.datetime(2014, 5, 15, 7, 45, 57), @@ -147,9 +147,9 @@ def test_untyped(self): conn = self.connect() c = conn.cursor() c.execute("select null,''") - self.assertEqual((None, u""), c.fetchone()) + self.assertEqual((None, ""), c.fetchone()) c.execute("select '',null") - self.assertEqual((u"", None), c.fetchone()) + self.assertEqual(("", None), c.fetchone()) def test_timedelta(self): """ test timedelta conversion """ @@ -300,7 +300,7 @@ def test_json(self): ) cur = conn.cursor() - json_str = u'{"hello": "こんãĢãĄã¯"}' + json_str = '{"hello": "こんãĢãĄã¯"}' cur.execute("INSERT INTO test_json (id, `json`) values (42, %s)", (json_str,)) cur.execute("SELECT `json` from `test_json` WHERE `id`=42") res = cur.fetchone()[0] diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index abd30e0b..8303083d 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -70,17 +70,17 @@ class TestAuthentication(base.PyMySQLTestCase): del db["user"] cur.execute("SHOW PLUGINS") for r in cur: - if (r[1], r[2]) != (u"ACTIVE", u"AUTHENTICATION"): + if (r[1], r[2]) != ("ACTIVE", "AUTHENTICATION"): continue - if r[3] == u"auth_socket.so" or r[0] == u"unix_socket": + if r[3] == "auth_socket.so" or r[0] == "unix_socket": socket_plugin_name = r[0] socket_found = True - elif r[3] == u"dialog_examples.so": + elif r[3] == "dialog_examples.so": if r[0] == "two_questions": two_questions_found = True elif r[0] == "three_attempts": three_attempts_found = True - elif r[0] == u"pam": + elif r[0] == "pam": pam_found = True pam_plugin_name = r[3].split(".")[0] if pam_plugin_name == "auth_pam": @@ -92,9 +92,9 @@ class TestAuthentication(base.PyMySQLTestCase): # https://mariadb.com/kb/en/mariadb/pam-authentication-plugin/ # Names differ but functionality is close - elif r[0] == u"mysql_old_password": + elif r[0] == "mysql_old_password": mysql_old_password_found = True - elif r[0] == u"sha256_password": + elif r[0] == "sha256_password": sha256_password_found = True # else: # print("plugin: %r" % r[0]) diff --git a/pymysql/tests/test_converters.py b/pymysql/tests/test_converters.py index dc194a9e..b36ee4b3 100644 --- a/pymysql/tests/test_converters.py +++ b/pymysql/tests/test_converters.py @@ -8,7 +8,7 @@ class TestConverter(TestCase): def test_escape_string(self): - self.assertEqual(converters.escape_string(u"foo\nbar"), u"foo\\nbar") + self.assertEqual(converters.escape_string("foo\nbar"), "foo\\nbar") def test_convert_datetime(self): expected = datetime.datetime(2007, 2, 24, 23, 6, 20) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 95765e54..77d37481 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -120,9 +120,9 @@ def test_issue_15(self): c.execute("drop table if exists issue15") c.execute("create table issue15 (t varchar(32))") try: - c.execute("insert into issue15 (t) values (%s)", (u"\xe4\xf6\xfc",)) + c.execute("insert into issue15 (t) values (%s)", ("\xe4\xf6\xfc",)) c.execute("select t from issue15") - self.assertEqual(u"\xe4\xf6\xfc", c.fetchone()[0]) + self.assertEqual("\xe4\xf6\xfc", c.fetchone()[0]) finally: c.execute("drop table issue15") @@ -189,12 +189,12 @@ def test_issue_34(self): def test_issue_33(self): conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( - conn, u"hei\xdfe", u"create table hei\xdfe (name varchar(32))" + conn, "hei\xdfe", "create table hei\xdfe (name varchar(32))" ) c = conn.cursor() - c.execute(u"insert into hei\xdfe (name) values ('Pi\xdfata')") - c.execute(u"select name from hei\xdfe") - self.assertEqual(u"Pi\xdfata", c.fetchone()[0]) + c.execute("insert into hei\xdfe (name) values ('Pi\xdfata')") + c.execute("select name from hei\xdfe") + self.assertEqual("Pi\xdfata", c.fetchone()[0]) @pytest.mark.skip("This test requires manual intervention") def test_issue_35(self): @@ -408,18 +408,18 @@ def test_issue_321(self): ) sql_select = "select * from issue321 where " "value_1 in %s and value_2=%s" data = [ - [(u"a",), u"\u0430"], - [[u"b"], u"\u0430"], - {"value_1": [[u"c"]], "value_2": u"\u0430"}, + [("a",), "\u0430"], + [["b"], "\u0430"], + {"value_1": [["c"]], "value_2": "\u0430"}, ] cur = conn.cursor() self.assertEqual(cur.execute(sql_insert, data[0]), 1) self.assertEqual(cur.execute(sql_insert, data[1]), 1) self.assertEqual(cur.execute(sql_dict_insert, data[2]), 1) - self.assertEqual(cur.execute(sql_select, [(u"a", u"b", u"c"), u"\u0430"]), 3) - self.assertEqual(cur.fetchone(), (u"a", u"\u0430")) - self.assertEqual(cur.fetchone(), (u"b", u"\u0430")) - self.assertEqual(cur.fetchone(), (u"c", u"\u0430")) + self.assertEqual(cur.execute(sql_select, [("a", "b", "c"), "\u0430"]), 3) + self.assertEqual(cur.fetchone(), ("a", "\u0430")) + self.assertEqual(cur.fetchone(), ("b", "\u0430")) + self.assertEqual(cur.fetchone(), ("c", "\u0430")) def test_issue_364(self): """ Test mixed unicode/binary arguments in executemany. """ @@ -432,8 +432,8 @@ def test_issue_364(self): ) sql = "insert into issue364 (value_1, value_2) values (_binary %s, %s)" - usql = u"insert into issue364 (value_1, value_2) values (_binary %s, %s)" - values = [pymysql.Binary(b"\x00\xff\x00"), u"\xe4\xf6\xfc"] + usql = "insert into issue364 (value_1, value_2) values (_binary %s, %s)" + values = [pymysql.Binary(b"\x00\xff\x00"), "\xe4\xf6\xfc"] # test single insert and select cur = conn.cursor() From 3818ad0d4c802d1e190cd4b0bc2be746ab3fa1f0 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 4 Jan 2021 15:26:03 +0900 Subject: [PATCH 147/292] Use f-string (#928) --- pymysql/connections.py | 6 ++---- pymysql/cursors.py | 2 +- pymysql/protocol.py | 8 ++------ pymysql/tests/test_load_local.py | 10 ++-------- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 63a8b3a9..7bc87a52 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1331,7 +1331,7 @@ def _get_descriptions(self): if converter is converters.through: converter = None if DEBUG: - print("DEBUG: field={}, converter={}".format(field, converter)) + print(f"DEBUG: field={field}, converter={converter}") self.converters.append((encoding, converter)) eof_packet = self.connection._read_packet() @@ -1361,9 +1361,7 @@ def send_data(self): break conn.write_packet(chunk) except IOError: - raise err.OperationalError( - 1017, "Can't find file '{0}'".format(self.filename) - ) + raise err.OperationalError(1017, f"Can't find file '{self.filename}'") finally: # send the empty packet to signify we are done sending data conn.write_packet(b"") diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 68ac78e7..666970b9 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -242,7 +242,7 @@ def callproc(self, procname, args=()): """ conn = self._get_db() if args: - fmt = "@_{0}_%d=%s".format(procname) + fmt = f"@_{procname}_%d=%s" self._query( "SET %s" % ",".join( diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 559ba624..41c81673 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -323,9 +323,7 @@ class EOFPacketWrapper: def __init__(self, from_packet): if not from_packet.is_eof_packet(): raise ValueError( - "Cannot create '{0}' object from invalid packet type".format( - self.__class__ - ) + f"Cannot create '{self.__class__}' object from invalid packet type" ) self.packet = from_packet @@ -348,9 +346,7 @@ class LoadLocalPacketWrapper: def __init__(self, from_packet): if not from_packet.is_load_local_packet(): raise ValueError( - "Cannot create '{0}' object from invalid packet type".format( - self.__class__ - ) + f"Cannot create '{self.__class__}' object from invalid packet type" ) self.packet = from_packet diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index bb856305..b1b8128e 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -35,10 +35,7 @@ def test_load_file(self): ) try: c.execute( - ( - "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " - + "test_load_local FIELDS TERMINATED BY ','" - ).format(filename) + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -55,10 +52,7 @@ def test_unbuffered_load_file(self): ) try: c.execute( - ( - "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " - + "test_load_local FIELDS TERMINATED BY ','" - ).format(filename) + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) From 255b5dd931cbe3f9dda846ae99bed6b0c0ecf778 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 4 Jan 2021 16:18:17 +0900 Subject: [PATCH 148/292] code cleanup (#929) --- pymysql/connections.py | 15 +++------------ pymysql/tests/test_connection.py | 4 ++-- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 7bc87a52..99a9575a 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -174,7 +174,7 @@ def __init__( sql_mode=None, read_default_file=None, conv=None, - use_unicode=None, + use_unicode=True, client_flag=0, cursorclass=Cursor, init_command=None, @@ -203,9 +203,6 @@ def __init__( ssl_verify_cert=None, ssl_verify_identity=None, ): - if use_unicode is None and sys.version_info[0] > 2: - use_unicode = True - if db is not None and database is None: database = db if passwd is not None and not password: @@ -298,15 +295,9 @@ def _config(key, arg): if write_timeout is not None and write_timeout <= 0: raise ValueError("write_timeout should be > 0") self._write_timeout = write_timeout - if charset: - self.charset = charset - self.use_unicode = True - else: - self.charset = DEFAULT_CHARSET - self.use_unicode = False - if use_unicode is not None: - self.use_unicode = use_unicode + self.charset = charset or DEFAULT_CHARSET + self.use_unicode = use_unicode self.encoding = charset_by_name(self.charset).encoding diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 8303083d..d89d04e9 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -403,7 +403,7 @@ def testMySQLOldPasswordAuth(self): c = conn.cursor() # deprecated in 5.6 - if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): + if self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) else: @@ -420,7 +420,7 @@ def testMySQLOldPasswordAuth(self): secure_auth_setting = c.fetchone()[0] c.execute("set old_passwords=1") # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if sys.version_info[0:2] >= (3, 2) and self.mysql_server_is(conn, (5, 6, 0)): + if self.mysql_server_is(conn, (5, 6, 0)): with self.assertWarns(pymysql.err.Warning) as cm: c.execute("set global secure_auth=0") else: From 511b6a2af6031b234cd3cadfbdef8807eec797af Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 5 Jan 2021 14:43:04 +0900 Subject: [PATCH 149/292] Use keyword only argument (#930) * Use keyword only argument for constructor. * Remove old password test --- pymysql/__init__.py | 8 +---- pymysql/connections.py | 21 ++++++++----- pymysql/tests/test_connection.py | 52 -------------------------------- 3 files changed, 14 insertions(+), 67 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 451012c8..478fdf6a 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -110,11 +110,7 @@ def Binary(x): return bytes(x) -def Connect(*args, **kwargs): - return connections.Connection(*args, **kwargs) - - -Connect.__doc__ = connections.Connection.__init__.__doc__ +Connect = connect = Connection = connections.Connection def get_client_info(): # for MySQLdb compatibility @@ -124,8 +120,6 @@ def get_client_info(): # for MySQLdb compatibility return ".".join(map(str, version)) -connect = Connection = Connect - # we include a doctored version_info here for MySQLdb compatibility version_info = (1, 4, 0, "final", 0) diff --git a/pymysql/connections.py b/pymysql/connections.py index 99a9575a..141381fe 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -120,7 +120,7 @@ class Connection: See converters. :param use_unicode: Whether or not to default to unicode strings. - This option defaults to true for Py3k. + This option defaults to true. :param client_flag: Custom flags to send to MySQL. Find potential values in constants.CLIENT. :param cursorclass: Custom cursor class to use. :param init_command: Initial SQL statement to run when connection is established. @@ -164,12 +164,13 @@ class Connection: def __init__( self, - host=None, user=None, password="", + host=None, database=None, - port=0, + *, unix_socket=None, + port=0, charset="", sql_mode=None, read_default_file=None, @@ -179,13 +180,8 @@ def __init__( cursorclass=Cursor, init_command=None, connect_timeout=10, - ssl=None, read_default_group=None, - compress=None, - named_pipe=None, autocommit=False, - db=None, - passwd=None, local_infile=False, max_allowed_packet=16 * 1024 * 1024, defer_connect=False, @@ -196,16 +192,25 @@ def __init__( binary_prefix=False, program_name=None, server_public_key=None, + ssl=None, ssl_ca=None, ssl_cert=None, ssl_disabled=None, ssl_key=None, ssl_verify_cert=None, ssl_verify_identity=None, + compress=None, # not supported + named_pipe=None, # not supported + passwd=None, # deprecated + db=None, # deprecated ): if db is not None and database is None: + warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) database = db if passwd is not None and not password: + warnings.warn( + "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 + ) password = passwd if compress or named_pipe: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index d89d04e9..afbf014f 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -383,58 +383,6 @@ def realTestPamAuth(self): # recreate the user cur.execute(grants) - # select old_password("crummy p\tassword"); - # | old_password("crummy p\tassword") | - # | 2a01785203b08770 | - @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") - @pytest.mark.skipif( - not mysql_old_password_found, reason="no mysql_old_password plugin" - ) - def testMySQLOldPasswordAuth(self): - conn = self.connect() - if self.mysql_server_is(conn, (5, 7, 0)): - pytest.skip("Old passwords aren't supported in 5.7") - # pymysql.err.OperationalError: (1045, "Access denied for user 'old_pass_user'@'localhost' (using password: YES)") - # from login in MySQL-5.6 - if self.mysql_server_is(conn, (5, 6, 0)): - pytest.skip("Old passwords don't authenticate in 5.6") - db = self.db.copy() - db["password"] = "crummy p\tassword" - c = conn.cursor() - - # deprecated in 5.6 - if self.mysql_server_is(conn, (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) - else: - c.execute("SELECT OLD_PASSWORD('%s')" % db["password"]) - v = c.fetchone()[0] - self.assertEqual(v, "2a01785203b08770") - # only works in MariaDB and MySQL-5.6 - can't separate out by version - # if self.mysql_server_is(self.connect(), (5, 5, 0)): - # with TempUser(c, 'old_pass_user@localhost', - # self.databases[0]['db'], 'mysql_old_password', '2a01785203b08770') as u: - # cur = pymysql.connect(user='old_pass_user', **db).cursor() - # cur.execute("SELECT VERSION()") - c.execute("SELECT @@secure_auth") - secure_auth_setting = c.fetchone()[0] - c.execute("set old_passwords=1") - # pymysql.err.Warning: 'pre-4.1 password hash' is deprecated and will be removed in a future release. Please use post-4.1 password hash instead - if self.mysql_server_is(conn, (5, 6, 0)): - with self.assertWarns(pymysql.err.Warning) as cm: - c.execute("set global secure_auth=0") - else: - c.execute("set global secure_auth=0") - with TempUser( - c, - "old_pass_user@localhost", - self.databases[0]["db"], - password=db["password"], - ) as u: - cur = pymysql.connect(user="old_pass_user", **db).cursor() - cur.execute("SELECT VERSION()") - c.execute("set global secure_auth=%r" % secure_auth_setting) - @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @pytest.mark.skipif( not sha256_password_found, From f5cbb6dea0a77c5e3055a299ed9a5b458c29cb12 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 6 Jan 2021 17:16:02 +0900 Subject: [PATCH 150/292] Remvoe escape_* functions from pymysql.__all__ (#931) * .travis -> ci * remove initializedb.sh * Remvoe escape functions from __all__ * fix test * don't use deprecated keyword * fix tests * fix tests * black --- .github/workflows/test.yaml | 2 +- .travis/database.json | 4 --- .travis/docker.json | 4 --- .travis/initializedb.sh | 54 -------------------------------- ci/database.json | 4 +++ ci/docker.json | 4 +++ docs/source/user/development.rst | 4 +-- pymysql/__init__.py | 5 --- pymysql/tests/base.py | 4 +-- pymysql/tests/test_connection.py | 19 ++++++----- pymysql/tests/test_issues.py | 4 +-- 11 files changed, 26 insertions(+), 82 deletions(-) delete mode 100644 .travis/database.json delete mode 100644 .travis/docker.json delete mode 100755 .travis/initializedb.sh create mode 100644 ci/database.json create mode 100644 ci/docker.json diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dd45bcab..8f53c28d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -70,7 +70,7 @@ jobs: mysql -h127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' mysql -h127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" mysql -h127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - cp .travis/docker.json pymysql/tests/databases.json + cp ci/docker.json pymysql/tests/databases.json - name: Run test run: | diff --git a/.travis/database.json b/.travis/database.json deleted file mode 100644 index ab1f60a3..00000000 --- a/.travis/database.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - {"host": "localhost", "unix_socket": "/var/run/mysqld/mysqld.sock", "user": "root", "passwd": "", "db": "test1", "use_unicode": true, "local_infile": true}, - {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "db": "test2" } -] diff --git a/.travis/docker.json b/.travis/docker.json deleted file mode 100644 index b851fb6d..00000000 --- a/.travis/docker.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - {"host": "127.0.0.1", "port": 3306, "user": "root", "passwd": "", "db": "test1", "use_unicode": true, "local_infile": true}, - {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "db": "test2" } -] diff --git a/.travis/initializedb.sh b/.travis/initializedb.sh deleted file mode 100755 index 6991cfe6..00000000 --- a/.travis/initializedb.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -set -ex - -docker pull ${DB} -docker run -it --name=mysqld -d -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -p 3306:3306 ${DB} - -mysql() { - docker exec -i mysqld mysql "${@}" -} -while : -do - sleep 3 - mysql --protocol=tcp -e 'select version()' && break -done -docker logs mysqld - -if [ $DB == 'mysql:8.0' ]; then - WITH_PLUGIN='with mysql_native_password' - mysql -e 'SET GLOBAL local_infile=on' - docker cp mysqld:/var/lib/mysql/public_key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/ca.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" - docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - - # Test user for auth test - mysql -e ' - CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", - nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", - nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" - PASSWORD EXPIRE NEVER;' - mysql -e 'GRANT RELOAD ON *.* TO user_caching_sha2;' -elif [[ $DB == mariadb:10.* ]] && [ ${DB#mariadb:10.} -ge 3 ]; then - mysql -e ' - INSTALL SONAME "auth_ed25519"; - CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' - # we need to pass the hashed password manually until 10.4, so hide it here - mysql -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql - mysql -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql - WITH_PLUGIN='' -else - WITH_PLUGIN='' -fi - -mysql -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' -mysql -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - -mysql -u root -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" -mysql -u root -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" - -cp .travis/docker.json pymysql/tests/databases.json diff --git a/ci/database.json b/ci/database.json new file mode 100644 index 00000000..aad0bfb2 --- /dev/null +++ b/ci/database.json @@ -0,0 +1,4 @@ +[ + {"host": "localhost", "unix_socket": "/var/run/mysqld/mysqld.sock", "user": "root", "password": "", "database": "test1", "use_unicode": true, "local_infile": true}, + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" } +] diff --git a/ci/docker.json b/ci/docker.json new file mode 100644 index 00000000..34a5c7b7 --- /dev/null +++ b/ci/docker.json @@ -0,0 +1,4 @@ +[ + {"host": "127.0.0.1", "port": 3306, "user": "root", "password": "", "database": "test1", "use_unicode": true, "local_infile": true}, + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" } +] diff --git a/docs/source/user/development.rst b/docs/source/user/development.rst index 09907318..af057622 100644 --- a/docs/source/user/development.rst +++ b/docs/source/user/development.rst @@ -22,10 +22,10 @@ If you would like to run the test suite, create a database for testing like this mysql -e 'create database test_pymysql DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' mysql -e 'create database test_pymysql2 DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;' -Then, copy the file ``.travis/database.json`` to ``pymysql/tests/databases.json`` +Then, copy the file ``ci/database.json`` to ``pymysql/tests/databases.json`` and edit the new file to match your MySQL configuration:: - $ cp .travis/database.json pymysql/tests/databases.json + $ cp ci/database.json pymysql/tests/databases.json $ $EDITOR pymysql/tests/databases.json To run all the tests, execute the script ``runtests.py``:: diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 478fdf6a..6473f48d 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -24,7 +24,6 @@ import sys from .constants import FIELD_TYPE -from .converters import escape_dict, escape_sequence, escape_string from .err import ( Warning, Error, @@ -177,14 +176,10 @@ def install_as_MySQLdb(): "constants", "converters", "cursors", - "escape_dict", - "escape_sequence", - "escape_string", "get_client_info", "paramstyle", "threadsafety", "version_info", "install_as_MySQLdb", - "NULL", "__version__", ] diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index 16cd23c0..6f93a831 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -21,11 +21,11 @@ class PyMySQLTestCase(unittest.TestCase): "host": "localhost", "user": "root", "passwd": "", - "db": "test1", + "database": "test1", "use_unicode": True, "local_infile": True, }, - {"host": "localhost", "user": "root", "passwd": "", "db": "test2"}, + {"host": "localhost", "user": "root", "passwd": "", "database": "test2"}, ] def mysql_server_is(self, conn, version_tuple): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index afbf014f..be4006f6 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -142,7 +142,7 @@ def realtestSocketAuth(self): with TempUser( self.connect().cursor(), TestAuthentication.osuser + "@localhost", - self.databases[0]["db"], + self.databases[0]["database"], self.socket_plugin_name, ) as u: c = pymysql.connect(user=TestAuthentication.osuser, **self.db) @@ -216,7 +216,7 @@ def realTestDialogAuthTwoQuestions(self): with TempUser( self.connect().cursor(), "pymysql_2q@localhost", - self.databases[0]["db"], + self.databases[0]["database"], "two_questions", "notverysecret", ) as u: @@ -258,7 +258,7 @@ def realTestDialogAuthThreeAttempts(self): with TempUser( self.connect().cursor(), "pymysql_3a@localhost", - self.databases[0]["db"], + self.databases[0]["database"], "three_attempts", "stillnotverysecret", ) as u: @@ -353,7 +353,7 @@ def realTestPamAuth(self): with TempUser( cur, TestAuthentication.osuser + "@localhost", - self.databases[0]["db"], + self.databases[0]["database"], "pam", os.environ.get("PAMSERVICE"), ) as u: @@ -392,7 +392,10 @@ def testAuthSHA256(self): conn = self.connect() c = conn.cursor() with TempUser( - c, "pymysql_sha256@localhost", self.databases[0]["db"], "sha256_password" + c, + "pymysql_sha256@localhost", + self.databases[0]["database"], + "sha256_password", ) as u: if self.mysql_server_is(conn, (5, 7, 0)): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") @@ -442,8 +445,8 @@ def test_autocommit(self): def test_select_db(self): con = self.connect() - current_db = self.databases[0]["db"] - other_db = self.databases[1]["db"] + current_db = self.databases[0]["database"] + other_db = self.databases[1]["database"] cur = con.cursor() cur.execute("SELECT database()") @@ -754,7 +757,7 @@ def test_escape_fallback_encoder(self): class Custom(str): pass - mapping = {str: pymysql.escape_string} + mapping = {str: pymysql.converters.escape_string} self.assertEqual(con.escape(Custom("foobar"), mapping), "'foobar'") def test_escape_no_default(self): diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 77d37481..b4ced4b0 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -66,7 +66,7 @@ def test_issue_6(self): """ exception: TypeError: ord() expected a character, but string of length 0 found """ # ToDo: this test requires access to db 'mysql'. kwargs = self.databases[0].copy() - kwargs["db"] = "mysql" + kwargs["database"] = "mysql" conn = pymysql.connect(**kwargs) c = conn.cursor() c.execute("select * from user") @@ -152,7 +152,7 @@ def test_issue_17(self): """could not connect mysql use passwod""" conn = self.connect() host = self.databases[0]["host"] - db = self.databases[0]["db"] + db = self.databases[0]["database"] c = conn.cursor() # grant access to a table to a user with a password From e24da41280af04e48423d00454fdd17343b63841 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 6 Jan 2021 21:58:04 +0900 Subject: [PATCH 151/292] Use `database` in examples. (#933) --- README.rst | 2 +- docs/source/user/examples.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 82303d05..46e60ff9 100644 --- a/README.rst +++ b/README.rst @@ -105,7 +105,7 @@ The following examples make use of a simple table connection = pymysql.connect(host='localhost', user='user', password='passwd', - db='db', + database='db', cursorclass=pymysql.cursors.DictCursor) with connection: diff --git a/docs/source/user/examples.rst b/docs/source/user/examples.rst index 966d46bd..e9e02410 100644 --- a/docs/source/user/examples.rst +++ b/docs/source/user/examples.rst @@ -30,7 +30,7 @@ The following examples make use of a simple table connection = pymysql.connect(host='localhost', user='user', password='passwd', - db='db', + database='db', charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) From 66e29fb789dd6a3c3c677c476ee9dc745efd2d04 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:25:20 +0900 Subject: [PATCH 152/292] Update CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6e73cb..ccf1805e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,18 @@ Release date: TBD Backward incompatible changes: * Python 2.7 and 3.5 are not supported. -* old_password (used by MySQL older than 4.1) is not supported. +* ``connect()`` uses keyword-only arguments. User must use keyword argument. +* ``connect()`` kwargs ``db`` and ``passwd`` are now deprecated; Use ``database`` and ``password`` instead. +* old_password authentication method (used by MySQL older than 4.1) is not supported. +* MySQL 5.5 and MariaDB 5.5 are not officially supported, although it may still works. +* Removed ``escape_dict``, ``escape_sequence``, and ``escape_string`` from ``pymysql`` + module. They are still in ``pymysql.converters``. Other changes: * Connection supports context manager API. ``__exit__`` closes the connection. (#886) * Add MySQL Connector/Python compatible TLS options (#903) +* Major code cleanup; PyMySQL now uses black and flake8. ## v0.10.1 From 6e5d5bd94af056c66a1ed05de754a83f8628faea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:28:35 +0900 Subject: [PATCH 153/292] v1.0.0 --- CHANGELOG.md | 2 +- pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf1805e..001b2631 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v1.0.0 -Release date: TBD +Release date: 2021-01-07 Backward incompatible changes: diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 6473f48d..45581468 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (0, 10, 1, None) +VERSION = (1, 0, 0, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 08aa62f7..6e1f732c 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from setuptools import setup, find_packages -version = "0.10.1" +version = "1.0.0" with open("./README.rst", encoding="utf-8") as f: readme = f.read() From f65351b1bd6c02eab07f20cbedada6ebfbf6d56d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:53:34 +0900 Subject: [PATCH 154/292] Do not create universal wheel --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8efb0850..b40802e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,9 +2,6 @@ ignore = E203,E501,W503,E722 exclude = tests,build,.venv,docs -[bdist_wheel] -universal = 1 - [metadata] license = "MIT" license_files = LICENSE From 5a02e5780f615ac7793373d63c407b979c33cd1c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 09:59:12 +0900 Subject: [PATCH 155/292] remove badges --- README.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.rst b/README.rst index 46e60ff9..279181f1 100644 --- a/README.rst +++ b/README.rst @@ -2,15 +2,9 @@ :target: https://pymysql.readthedocs.io/ :alt: Documentation Status -.. image:: https://badge.fury.io/py/PyMySQL.svg - :target: https://badge.fury.io/py/PyMySQL - .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master -.. image:: https://img.shields.io/badge/license-MIT-blue.svg - :target: https://github.com/PyMySQL/PyMySQL/blob/master/LICENSE - .. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python From 5d1e27de3f35a936f7baf63036098d44f4a41a58 Mon Sep 17 00:00:00 2001 From: Nicusor Picatureanu <33037485+Nicusor97@users.noreply.github.com> Date: Thu, 7 Jan 2021 10:06:32 +0200 Subject: [PATCH 156/292] Set python_requires='>=3.6' (#936) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6e1f732c..0224339e 100755 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ description="Pure Python MySQL Driver", long_description=readme, packages=find_packages(exclude=["tests*", "pymysql.tests*"]), + python_requires=">=3.6", extras_require={ "rsa": ["cryptography"], "ed25519": ["PyNaCl>=1.4.0"], From 7c4700bd66b36e6e50e7f8c7df57635f0dafb006 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 7 Jan 2021 17:55:31 +0900 Subject: [PATCH 157/292] Remove tox --- docs/source/user/development.rst | 5 ----- tox.ini | 9 --------- 2 files changed, 14 deletions(-) delete mode 100644 tox.ini diff --git a/docs/source/user/development.rst b/docs/source/user/development.rst index af057622..1f8a2637 100644 --- a/docs/source/user/development.rst +++ b/docs/source/user/development.rst @@ -32,8 +32,3 @@ To run all the tests, execute the script ``runtests.py``:: $ pip install pytest $ pytest -v pymysql - -A ``tox.ini`` file is also provided for conveniently running tests on multiple -Python versions:: - - $ tox diff --git a/tox.ini b/tox.ini deleted file mode 100644 index fef58a82..00000000 --- a/tox.ini +++ /dev/null @@ -1,9 +0,0 @@ -[tox] -envlist = py{36,37,38,39,py3} - -[testenv] -commands = pytest -v pymysql/tests/ -deps = coverage pytest -passenv = USER - PASSWORD - PAMSERVICE From 0acaa7f4fa4e2a9a30c835fc1be0b74eec3aaf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 8 Jan 2021 02:08:27 +0100 Subject: [PATCH 158/292] Use built-in unittest.mock (#938) Use built-in Python 3 unittest.mock instead of relying on mock package that is only necessary for ancient versions of Python. --- .github/workflows/test.yaml | 2 +- pymysql/tests/test_connection.py | 5 +++-- requirements-dev.txt | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f53c28d..09846c94 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -56,7 +56,7 @@ jobs: - name: Install dependency run: | - pip install -U cryptography PyNaCl pytest pytest-cov mock coveralls + pip install -U cryptography PyNaCl pytest pytest-cov coveralls - name: Set up MySQL run: | diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index be4006f6..75db73cd 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,9 +1,10 @@ import datetime import ssl import sys -import time -import mock import pytest +import time +from unittest import mock + import pymysql from pymysql.tests import base from pymysql.constants import CLIENT diff --git a/requirements-dev.txt b/requirements-dev.txt index 69d3f68a..d65512fb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ cryptography PyNaCl>=1.4.0 pytest -mock From 2d36a195060b46e12f16d8b776468bab53ea6919 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 8 Jan 2021 10:38:14 +0900 Subject: [PATCH 159/292] Remove warning for db and passwd. (#940) * update doc * Remove warning. --- docs/source/user/examples.rst | 2 +- pymysql/connections.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/source/user/examples.rst b/docs/source/user/examples.rst index e9e02410..3946db9b 100644 --- a/docs/source/user/examples.rst +++ b/docs/source/user/examples.rst @@ -56,4 +56,4 @@ This example will print: .. code:: python - {'password': 'very-secret', 'id': 1} + {'id': 1, 'password': 'very-secret'} diff --git a/pymysql/connections.py b/pymysql/connections.py index 141381fe..cb203589 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -135,8 +135,6 @@ class Connection: :param ssl_verify_cert: Set to true to check the validity of server certificates :param ssl_verify_identity: Set to true to check the server's identity :param read_default_group: Group to read from in the configuration file. - :param compress: Not supported - :param named_pipe: Not supported :param autocommit: Autocommit mode. None means use server default. (default: False) :param local_infile: Boolean to enable the use of LOAD DATA LOCAL command. (default: False) :param max_allowed_packet: Max size of packet sent to server in bytes. (default: 16MB) @@ -149,9 +147,11 @@ class Connection: an argument. For the dialog plugin, a prompt(echo, prompt) method can be used (if no authenticate method) for returning a string from the user. (experimental) :param server_public_key: SHA256 authentication plugin public key value. (default: None) - :param db: Alias for database. (for compatibility to MySQLdb) - :param passwd: Alias for password. (for compatibility to MySQLdb) :param binary_prefix: Add _binary prefix on bytes and bytearray. (default: False) + :param compress: Not supported + :param named_pipe: Not supported + :param db: **DEPRECATED** Alias for database. + :param passwd: **DEPRECATED** Alias for password. See `Connection `_ in the specification. @@ -205,12 +205,16 @@ def __init__( db=None, # deprecated ): if db is not None and database is None: - warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) + # We will raise warining in 2022 or later. + # See https://github.com/PyMySQL/PyMySQL/issues/939 + # warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) database = db if passwd is not None and not password: - warnings.warn( - "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 - ) + # We will raise warining in 2022 or later. + # See https://github.com/PyMySQL/PyMySQL/issues/939 + # warnings.warn( + # "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 + # ) password = passwd if compress or named_pipe: From 5c6f8bcb741c32719a07e8c95eb8050cb9249511 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 8 Jan 2021 11:47:02 +0900 Subject: [PATCH 160/292] v1.0.1 --- CHANGELOG.md | 9 +++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 001b2631..beb4b2f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changes +## v1.0.1 + +Release date: 2021-01-08 + +* Stop emitting DeprecationWarning for use of ``db`` and ``passwd``. + Note that they are still deprecated. (#939) +* Add ``python_requires=">=3.6"`` to setup.py. (#936) + + ## v1.0.0 Release date: 2021-01-07 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 45581468..ee59924a 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 0, None) +VERSION = (1, 0, 1, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index 0224339e..f9962c75 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from setuptools import setup, find_packages -version = "1.0.0" +version = "1.0.1" with open("./README.rst", encoding="utf-8") as f: readme = f.read() From abe83c262ea647a09e0f13587fa91d6a14a71598 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 8 Jan 2021 23:00:40 +0900 Subject: [PATCH 161/292] Make 4 more arguments to keyword-only. (#941) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index cb203589..92b7a77e 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -164,11 +164,11 @@ class Connection: def __init__( self, - user=None, + *, + user=None, # The first four arguments is based on DB-API 2.0 recommendation. password="", host=None, database=None, - *, unix_socket=None, port=0, charset="", From b12efdb6c1baa55e58a4384271e33a7351d554d5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 9 Jan 2021 20:32:51 +0900 Subject: [PATCH 162/292] v1.0.2 --- CHANGELOG.md | 8 ++++++++ pymysql/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beb4b2f9..9885af52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changes +## v1.0.2 + +Release date: 2021-01-09 + +* Fix `user`, `password`, `host`, `database` are still positional arguments. + All arguments of `connect()` are now keyword-only. (#941) + + ## v1.0.1 Release date: 2021-01-08 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index ee59924a..5fe2aec5 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 1, None) +VERSION = (1, 0, 2, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/setup.py b/setup.py index f9962c75..1510a0cf 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from setuptools import setup, find_packages -version = "1.0.1" +version = "1.0.2" with open("./README.rst", encoding="utf-8") as f: readme = f.read() From 1fd5292f33868f9f9c8b90e1e53f82dd4aa992b4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 18 Jan 2021 17:08:55 +0900 Subject: [PATCH 163/292] Update README.rst --- README.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.rst b/README.rst index 279181f1..f514d901 100644 --- a/README.rst +++ b/README.rst @@ -17,13 +17,6 @@ PyMySQL This package contains a pure-Python MySQL client library, based on `PEP 249`_. -Most public APIs are compatible with mysqlclient and MySQLdb. - -NOTE: PyMySQL doesn't support low level APIs `_mysql` provides like `data_seek`, -`store_result`, and `use_result`. You should use high level APIs defined in `PEP 249`_. -But some APIs like `autocommit` and `ping` are supported because `PEP 249`_ doesn't cover -their usecase. - .. _`PEP 249`: https://www.python.org/dev/peps/pep-0249/ From 96d738a051673deff4d6b85d0d263c404e37e181 Mon Sep 17 00:00:00 2001 From: Rajat Jain Date: Tue, 19 Jan 2021 17:21:28 +0530 Subject: [PATCH 164/292] Remove Cursor._last_executed (#948) Fixes: #947. --- pymysql/cursors.py | 2 -- pymysql/tests/test_basic.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 666970b9..727a28e0 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -305,7 +305,6 @@ def scroll(self, value, mode="relative"): def _query(self, q): conn = self._get_db() - self._last_executed = q self._clear_result() conn.query(q) self._do_get_result() @@ -410,7 +409,6 @@ def close(self): def _query(self, q): conn = self._get_db() - self._last_executed = q self._clear_result() conn.query(q, unbuffered=True) self._do_get_result() diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index c2590bf2..678ea923 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -353,7 +353,7 @@ def test_bulk_insert(self): data, ) self.assertEqual( - cursor._last_executed, + cursor._executed, bytearray( b"insert into bulkinsert (id, name, age, height) values " b"(0,'bob',21,123),(1,'jim',56,45),(2,'fred',100,180)" @@ -377,7 +377,7 @@ def test_bulk_insert_multiline_statement(self): data, ) self.assertEqual( - cursor._last_executed.strip(), + cursor._executed.strip(), bytearray( b"""insert into bulkinsert (id, name, @@ -422,7 +422,7 @@ def test_issue_288(self): data, ) self.assertEqual( - cursor._last_executed.strip(), + cursor._executed.strip(), bytearray( b"""insert into bulkinsert (id, name, From 381e6aba21687cba18ca002db062f2fab3a04a9b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 19 Jan 2021 22:12:11 +0900 Subject: [PATCH 165/292] Actions: Fix 422 error on Coveralls (#949) * Actions: Update coveralls flag name * fix 422 error See https://github.com/TheKevJames/coveralls-python/issues/252 --- .github/workflows/test.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 09846c94..26b3f9c9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,21 +106,23 @@ jobs: pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py - name: Report coverage - run: coveralls + run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_FLAG_NAME: ${{ matrix.test-name }} + COVERALLS_FLAG_NAME: ${{ matrix.py }}-${{ matrix.db }} COVERALLS_PARALLEL: true coveralls: name: Finish coveralls runs-on: ubuntu-20.04 needs: test - container: python:3-slim steps: + - uses: actions/setup-python@v2 + with: + python-version: 3.9 - name: Finished run: | - pip3 install --upgrade coveralls - coveralls --finish + pip install --upgrade coveralls + coveralls --finish --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 565dc36985a0d2c38a5a85cb4aa5b53e5c086f7c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 19 Jan 2021 22:25:28 +0900 Subject: [PATCH 166/292] Actions: Use cache in finish (#950) --- .github/workflows/test.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 26b3f9c9..158188cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -120,6 +120,14 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.9 + + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: finish-pip-1 + restore-keys: | + finish-pip- + - name: Finished run: | pip install --upgrade coveralls From 5a11bab69075a5b9120877aa70f5b86f930809c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scheibe?= Date: Sun, 24 Jan 2021 04:00:02 +0100 Subject: [PATCH 167/292] Fix docstring for converter functions (#952) Co-authored-by: Rene Scheibe --- pymysql/converters.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index d910f5c5..200cae5f 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -155,16 +155,16 @@ def _convert_second_fraction(s): def convert_datetime(obj): """Returns a DATETIME or TIMESTAMP column value as a datetime object: - >>> datetime_or_None('2007-02-25 23:06:20') + >>> convert_datetime('2007-02-25 23:06:20') datetime.datetime(2007, 2, 25, 23, 6, 20) - >>> datetime_or_None('2007-02-25T23:06:20') + >>> convert_datetime('2007-02-25T23:06:20') datetime.datetime(2007, 2, 25, 23, 6, 20) Illegal values are returned as None: - >>> datetime_or_None('2007-02-31T23:06:20') is None + >>> convert_datetime('2007-02-31T23:06:20') is None True - >>> datetime_or_None('0000-00-00 00:00:00') is None + >>> convert_datetime('0000-00-00 00:00:00') is None True """ @@ -189,14 +189,14 @@ def convert_datetime(obj): def convert_timedelta(obj): """Returns a TIME column as a timedelta object: - >>> timedelta_or_None('25:06:17') + >>> convert_timedelta('25:06:17') datetime.timedelta(1, 3977) - >>> timedelta_or_None('-25:06:17') + >>> convert_timedelta('-25:06:17') datetime.timedelta(-2, 83177) Illegal values are returned as None: - >>> timedelta_or_None('random crap') is None + >>> convert_timedelta('random crap') is None True Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but @@ -236,14 +236,14 @@ def convert_timedelta(obj): def convert_time(obj): """Returns a TIME column as a time object: - >>> time_or_None('15:06:17') + >>> convert_time('15:06:17') datetime.time(15, 6, 17) Illegal values are returned as None: - >>> time_or_None('-25:06:17') is None + >>> convert_time('-25:06:17') is None True - >>> time_or_None('random crap') is None + >>> convert_time('random crap') is None True Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but @@ -279,14 +279,14 @@ def convert_time(obj): def convert_date(obj): """Returns a DATE column as a date object: - >>> date_or_None('2007-02-26') + >>> convert_date('2007-02-26') datetime.date(2007, 2, 26) Illegal values are returned as None: - >>> date_or_None('2007-02-31') is None + >>> convert_date('2007-02-31') is None True - >>> date_or_None('0000-00-00') is None + >>> convert_date('0000-00-00') is None True """ From 6ccbecc1a0dfd04065b081950d2d35b1dac0aaa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scheibe?= Date: Tue, 2 Feb 2021 07:23:09 +0100 Subject: [PATCH 168/292] Improve docstrings (#954) - dot at the end of descriptions - 3rd instead of 2nd person - more type information - minor rephrasing Co-authored-by: Rene Scheibe --- pymysql/connections.py | 46 +++++++++++++++++----------------- pymysql/cursors.py | 56 +++++++++++++++++++++++++++--------------- 2 files changed, 59 insertions(+), 43 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 92b7a77e..b525014c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -99,18 +99,18 @@ class Connection: Establish a connection to the MySQL database. Accepts several arguments: - :param host: Host where the database server is located - :param user: Username to log in as + :param host: Host where the database server is located. + :param user: Username to log in as. :param password: Password to use. :param database: Database to use, None to not use a particular one. :param port: MySQL port to use, default is usually OK. (default: 3306) :param bind_address: When the client has multiple network interfaces, specify the interface from which to connect to the host. Argument can be a hostname or an IP address. - :param unix_socket: Optionally, you can use a unix socket rather than TCP/IP. + :param unix_socket: Use a unix socket rather than TCP/IP. :param read_timeout: The timeout for reading from the connection in seconds (default: None - no timeout) :param write_timeout: The timeout for writing to the connection in seconds (default: None - no timeout) - :param charset: Charset you want to use. + :param charset: Charset to use. :param sql_mode: Default SQL_MODE to use. :param read_default_file: Specifies my.cnf file to read these parameters from under the [client] section. @@ -124,16 +124,15 @@ class Connection: :param client_flag: Custom flags to send to MySQL. Find potential values in constants.CLIENT. :param cursorclass: Custom cursor class to use. :param init_command: Initial SQL statement to run when connection is established. - :param connect_timeout: Timeout before throwing an exception when connecting. + :param connect_timeout: The timeout for connecting to the database in seconds. (default: 10, min: 1, max: 31536000) - :param ssl: - A dict of arguments similar to mysql_ssl_set()'s parameters. - :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate - :param ssl_cert: Path to the file that contains a PEM-formatted client certificate - :param ssl_disabled: A boolean value that disables usage of TLS - :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate - :param ssl_verify_cert: Set to true to check the validity of server certificates - :param ssl_verify_identity: Set to true to check the server's identity + :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. + :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate. + :param ssl_cert: Path to the file that contains a PEM-formatted client certificate. + :param ssl_disabled: A boolean value that disables usage of TLS. + :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate. + :param ssl_verify_cert: Set to true to check the server certificate's validity. + :param ssl_verify_identity: Set to true to check the server's identity. :param read_default_group: Group to read from in the configuration file. :param autocommit: Autocommit mode. None means use server default. (default: False) :param local_infile: Boolean to enable the use of LOAD DATA LOCAL command. (default: False) @@ -148,8 +147,8 @@ class Connection: (if no authenticate method) for returning a string from the user. (experimental) :param server_public_key: SHA256 authentication plugin public key value. (default: None) :param binary_prefix: Add _binary prefix on bytes and bytearray. (default: False) - :param compress: Not supported - :param named_pipe: Not supported + :param compress: Not supported. + :param named_pipe: Not supported. :param db: **DEPRECATED** Alias for database. :param passwd: **DEPRECATED** Alias for password. @@ -415,11 +414,11 @@ def close(self): @property def open(self): - """Return True if the connection is open""" + """Return True if the connection is open.""" return self._sock is not None def _force_close(self): - """Close connection without QUIT message""" + """Close connection without QUIT message.""" if self._sock: try: self._sock.close() @@ -448,7 +447,7 @@ def _read_ok_packet(self): return ok def _send_autocommit_mode(self): - """Set whether or not to commit after every execute()""" + """Set whether or not to commit after every execute().""" self._execute_command( COMMAND.COM_QUERY, "SET AUTOCOMMIT = %s" % self.escape(self.autocommit_mode) ) @@ -496,7 +495,7 @@ def select_db(self, db): self._read_ok_packet() def escape(self, obj, mapping=None): - """Escape whatever value you pass to it. + """Escape whatever value is passed. Non-standard, for internal use; do not use this in your applications. """ @@ -510,7 +509,7 @@ def escape(self, obj, mapping=None): return converters.escape_item(obj, self.charset, mapping=mapping) def literal(self, obj): - """Alias for escape() + """Alias for escape(). Non-standard, for internal use; do not use this in your applications. """ @@ -530,9 +529,8 @@ def cursor(self, cursor=None): """ Create a new cursor to execute queries with. - :param cursor: The type of cursor to create; one of :py:class:`Cursor`, - :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. - None means use Cursor. + :param cursor: The type of cursor to create. None means use Cursor. + :type cursor: :py:class:`Cursor`, :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. """ if cursor: return cursor(self) @@ -565,6 +563,8 @@ def ping(self, reconnect=True): Check if the server is alive. :param reconnect: If the connection is closed, reconnect. + :type reconnect: boolean + :raise Error: If the connection is closed and reconnect=False. """ if self._sock is None: diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 727a28e0..2b5ccca9 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -15,7 +15,7 @@ class Cursor: """ - This is the object you use to interact with the database. + This is the object used to interact with the database. Do not create an instance of a Cursor yourself. Call connections.Connection.cursor(). @@ -79,7 +79,7 @@ def setoutputsizes(self, *args): """Does nothing, required by DB API.""" def _nextset(self, unbuffered=False): - """Get the next query set""" + """Get the next query set.""" conn = self._get_db() current_result = self._result if current_result is None or current_result is not conn._result: @@ -114,9 +114,18 @@ def _escape_args(self, args, conn): def mogrify(self, query, args=None): """ - Returns the exact string that is sent to the database by calling the + Returns the exact string that would be sent to the database by calling the execute() method. + :param query: Query to mogrify. + :type query: str + + :param args: Parameters used with query. (optional) + :type args: tuple, list or dict + + :return: The query with argument binding applied. + :rtype: str + This method follows the extension to the DB API 2.0 followed by Psycopg. """ conn = self._get_db() @@ -127,14 +136,15 @@ def mogrify(self, query, args=None): return query def execute(self, query, args=None): - """Execute a query + """Execute a query. - :param str query: Query to execute. + :param query: Query to execute. + :type query: str - :param args: parameters used with query. (optional) + :param args: Parameters used with query. (optional) :type args: tuple, list or dict - :return: Number of affected rows + :return: Number of affected rows. :rtype: int If args is a list or tuple, %s can be used as a placeholder in the query. @@ -150,12 +160,16 @@ def execute(self, query, args=None): return result def executemany(self, query, args): - # type: (str, list) -> int - """Run several data against one query + """Run several data against one query. + + :param query: Query to execute. + :type query: str + + :param args: Sequence of sequences or mappings. It is used as parameter. + :type args: tuple or list - :param query: query to execute on server - :param args: Sequence of sequences or mappings. It is used as parameter. :return: Number of rows affected, if any. + :rtype: int or None This method improves performance on multiple-row INSERT and REPLACE. Otherwise it is equivalent to looping over args with @@ -213,11 +227,13 @@ def _do_execute_many( return rows def callproc(self, procname, args=()): - """Execute stored procedure procname with args + """Execute stored procedure procname with args. - procname -- string, name of procedure to execute on server + :param procname: Name of procedure to execute on server. + :type procname: str - args -- Sequence of parameters to use with procedure + :param args: Sequence of parameters to use with procedure. + :type args: tuple or list Returns the original args. @@ -260,7 +276,7 @@ def callproc(self, procname, args=()): return args def fetchone(self): - """Fetch the next row""" + """Fetch the next row.""" self._check_executed() if self._rows is None or self.rownumber >= len(self._rows): return None @@ -269,7 +285,7 @@ def fetchone(self): return result def fetchmany(self, size=None): - """Fetch several rows""" + """Fetch several rows.""" self._check_executed() if self._rows is None: return () @@ -279,7 +295,7 @@ def fetchmany(self, size=None): return result def fetchall(self): - """Fetch all the rows""" + """Fetch all the rows.""" self._check_executed() if self._rows is None: return () @@ -418,11 +434,11 @@ def nextset(self): return self._nextset(unbuffered=True) def read_next(self): - """Read next row""" + """Read next row.""" return self._conv_row(self._result._read_rowdata_packet_unbuffered()) def fetchone(self): - """Fetch next row""" + """Fetch next row.""" self._check_executed() row = self.read_next() if row is None: @@ -450,7 +466,7 @@ def __iter__(self): return self.fetchall_unbuffered() def fetchmany(self, size=None): - """Fetch many""" + """Fetch many.""" self._check_executed() if size is None: size = self.arraysize From fb10477caf21122a89d7f216a0670d49dd2aa5d2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 27 Jun 2021 10:55:03 +0900 Subject: [PATCH 169/292] black --- pymysql/tests/test_basic.py | 16 ++++++++-------- pymysql/tests/test_connection.py | 18 +++++++++--------- pymysql/tests/test_issues.py | 32 ++++++++++++++++---------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 678ea923..a0dea9c8 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -14,7 +14,7 @@ class TestConversion(base.PyMySQLTestCase): def test_datatypes(self): - """ test every data type """ + """test every data type""" conn = self.connect() c = conn.cursor() c.execute( @@ -80,7 +80,7 @@ def test_datatypes(self): c.execute("drop table test_datatypes") def test_dict(self): - """ test dict escaping """ + """test dict escaping""" conn = self.connect() c = conn.cursor() c.execute("create table test_dict (a integer, b integer, c integer)") @@ -143,7 +143,7 @@ def test_blob(self): self.assertEqual(data, c.fetchone()[0]) def test_untyped(self): - """ test conversion of null, empty string """ + """test conversion of null, empty string""" conn = self.connect() c = conn.cursor() c.execute("select null,''") @@ -152,7 +152,7 @@ def test_untyped(self): self.assertEqual(("", None), c.fetchone()) def test_timedelta(self): - """ test timedelta conversion """ + """test timedelta conversion""" conn = self.connect() c = conn.cursor() c.execute( @@ -172,7 +172,7 @@ def test_timedelta(self): ) def test_datetime_microseconds(self): - """ test datetime conversion w microseconds""" + """test datetime conversion w microseconds""" conn = self.connect() if not self.mysql_server_is(conn, (5, 6, 4)): @@ -243,7 +243,7 @@ class TestCursor(base.PyMySQLTestCase): # self.assertEqual(r, c.description) def test_fetch_no_result(self): - """ test a fetchone() with no rows """ + """test a fetchone() with no rows""" conn = self.connect() c = conn.cursor() c.execute("create table test_nr (b varchar(32))") @@ -255,7 +255,7 @@ def test_fetch_no_result(self): c.execute("drop table test_nr") def test_aggregates(self): - """ test aggregate functions """ + """test aggregate functions""" conn = self.connect() c = conn.cursor() try: @@ -269,7 +269,7 @@ def test_aggregates(self): c.execute("drop table test_aggregates") def test_single_tuple(self): - """ test a single tuple """ + """test a single tuple""" conn = self.connect() c = conn.cursor() self.safe_create_table( diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 75db73cd..a469be5a 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -226,7 +226,7 @@ def realTestDialogAuthTwoQuestions(self): pymysql.connect( user="pymysql_2q", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @@ -266,12 +266,12 @@ def realTestDialogAuthThreeAttempts(self): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.DialogHandler}, - **self.db + **self.db, ) with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( @@ -282,27 +282,27 @@ def realTestDialogAuthThreeAttempts(self): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.DefectiveHandler}, - **self.db + **self.db, ) with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"notdialogplugin": TestAuthentication.Dialog}, - **self.db + **self.db, ) TestAuthentication.Dialog.m = {b"Password, please:": b"I do not know"} with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) TestAuthentication.Dialog.m = {b"Password, please:": None} with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, - **self.db + **self.db, ) @pytest.mark.skipif(not socket_auth, reason="connection to unix_socket required") @@ -367,7 +367,7 @@ def realTestPamAuth(self): auth_plugin_map={ b"mysql_cleartext_password": TestAuthentication.DefectiveHandler }, - **self.db + **self.db, ) except pymysql.OperationalError as e: self.assertEqual(1045, e.args[0]) @@ -378,7 +378,7 @@ def realTestPamAuth(self): auth_plugin_map={ b"mysql_cleartext_password": TestAuthentication.DefectiveHandler }, - **self.db + **self.db, ) if grants: # recreate the user diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index b4ced4b0..76d4b133 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -14,7 +14,7 @@ class TestOldIssues(base.PyMySQLTestCase): def test_issue_3(self): - """ undefined methods datetime_or_None, date_or_None """ + """undefined methods datetime_or_None, date_or_None""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -42,7 +42,7 @@ def test_issue_3(self): c.execute("drop table issue3") def test_issue_4(self): - """ can't retrieve TIMESTAMP fields """ + """can't retrieve TIMESTAMP fields""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -57,13 +57,13 @@ def test_issue_4(self): c.execute("drop table issue4") def test_issue_5(self): - """ query on information_schema.tables fails """ + """query on information_schema.tables fails""" con = self.connect() cur = con.cursor() cur.execute("select * from information_schema.tables") def test_issue_6(self): - """ exception: TypeError: ord() expected a character, but string of length 0 found """ + """exception: TypeError: ord() expected a character, but string of length 0 found""" # ToDo: this test requires access to db 'mysql'. kwargs = self.databases[0].copy() kwargs["database"] = "mysql" @@ -73,7 +73,7 @@ def test_issue_6(self): conn.close() def test_issue_8(self): - """ Primary Key and Index error when selecting data """ + """Primary Key and Index error when selecting data""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -93,7 +93,7 @@ def test_issue_8(self): c.execute("drop table test") def test_issue_13(self): - """ can't handle large result fields """ + """can't handle large result fields""" conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): @@ -112,7 +112,7 @@ def test_issue_13(self): cur.execute("drop table issue13") def test_issue_15(self): - """ query should be expanded before perform character encoding """ + """query should be expanded before perform character encoding""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -127,7 +127,7 @@ def test_issue_15(self): c.execute("drop table issue15") def test_issue_16(self): - """ Patch for string and tuple escaping """ + """Patch for string and tuple escaping""" conn = self.connect() c = conn.cursor() with warnings.catch_warnings(): @@ -285,7 +285,7 @@ def disabled_test_issue_54(self): class TestGitHubIssues(base.PyMySQLTestCase): def test_issue_66(self): - """ 'Connection' object has no attribute 'insert_id' """ + """'Connection' object has no attribute 'insert_id'""" conn = self.connect() c = conn.cursor() self.assertEqual(0, conn.insert_id()) @@ -303,7 +303,7 @@ def test_issue_66(self): c.execute("drop table issue66") def test_issue_79(self): - """ Duplicate field overwrites the previous one in the result of DictCursor """ + """Duplicate field overwrites the previous one in the result of DictCursor""" conn = self.connect() c = conn.cursor(pymysql.cursors.DictCursor) @@ -330,7 +330,7 @@ def test_issue_79(self): c.execute("drop table b") def test_issue_95(self): - """ Leftover trailing OK packet for "CALL my_sp" queries """ + """Leftover trailing OK packet for "CALL my_sp" queries""" conn = self.connect() cur = conn.cursor() with warnings.catch_warnings(): @@ -352,7 +352,7 @@ def test_issue_95(self): cur.execute("DROP PROCEDURE IF EXISTS `foo`") def test_issue_114(self): - """ autocommit is not set after reconnecting with ping() """ + """autocommit is not set after reconnecting with ping()""" conn = pymysql.connect(charset="utf8", **self.databases[0]) conn.autocommit(False) c = conn.cursor() @@ -377,7 +377,7 @@ def test_issue_114(self): conn.close() def test_issue_175(self): - """ The number of fields returned by server is read in wrong way """ + """The number of fields returned by server is read in wrong way""" conn = self.connect() cur = conn.cursor() for length in (200, 300): @@ -393,7 +393,7 @@ def test_issue_175(self): cur.execute("drop table if exists test_field_count") def test_issue_321(self): - """ Test iterable as query argument. """ + """Test iterable as query argument.""" conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( conn, @@ -422,7 +422,7 @@ def test_issue_321(self): self.assertEqual(cur.fetchone(), ("c", "\u0430")) def test_issue_364(self): - """ Test mixed unicode/binary arguments in executemany. """ + """Test mixed unicode/binary arguments in executemany.""" conn = pymysql.connect(charset="utf8mb4", **self.databases[0]) self.safe_create_table( conn, @@ -454,7 +454,7 @@ def test_issue_364(self): cur.executemany(usql, args=(values, values, values)) def test_issue_363(self): - """ Test binary / geometry types. """ + """Test binary / geometry types.""" conn = pymysql.connect(charset="utf8", **self.databases[0]) self.safe_create_table( conn, From 46d17402afaa07369b954eee026f68c5b96207ba Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 30 Jul 2021 12:44:50 +0900 Subject: [PATCH 170/292] Use dessant/lock-threads. --- .github/workflows/lock.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/lock.yml diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 00000000..1b25b4c7 --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,16 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '0 0 * * *' + +permissions: + issues: write + pull-requests: write + +jobs: + action: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v2 + From d0cd254bb4886d04b74f868b4e63f2c595bebe2b Mon Sep 17 00:00:00 2001 From: Valentin Nechayev Date: Tue, 3 Aug 2021 08:57:21 +0300 Subject: [PATCH 171/292] Fix generating authentication response with long strings (#988) Connection attributes shall be encoded using lenenc-str approach for a separate string and the whole section. --- pymysql/connections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index b525014c..00605dd9 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -898,10 +898,10 @@ def _request_authentication(self): connect_attrs = b"" for k, v in self._connect_attrs.items(): k = k.encode("utf-8") - connect_attrs += struct.pack("B", len(k)) + k + connect_attrs += _lenenc_int(len(k)) + k v = v.encode("utf-8") - connect_attrs += struct.pack("B", len(v)) + v - data += struct.pack("B", len(connect_attrs)) + connect_attrs + connect_attrs += _lenenc_int(len(v)) + v + data += _lenenc_int(len(connect_attrs)) + connect_attrs self.write_packet(data) auth_packet = self._read_packet() From f0091e09889a3db2400f821bee6a411fa1822a44 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Aug 2021 15:06:22 +0900 Subject: [PATCH 172/292] Fix doctest in pymysql.converters (#994) Fixes #993 --- pymysql/converters.py | 44 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 200cae5f..da63ceb7 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -160,13 +160,12 @@ def convert_datetime(obj): >>> convert_datetime('2007-02-25T23:06:20') datetime.datetime(2007, 2, 25, 23, 6, 20) - Illegal values are returned as None: - - >>> convert_datetime('2007-02-31T23:06:20') is None - True - >>> convert_datetime('0000-00-00 00:00:00') is None - True + Illegal values are returned as str: + >>> convert_datetime('2007-02-31T23:06:20') + '2007-02-31T23:06:20' + >>> convert_datetime('0000-00-00 00:00:00') + '0000-00-00 00:00:00' """ if isinstance(obj, (bytes, bytearray)): obj = obj.decode("ascii") @@ -190,14 +189,14 @@ def convert_timedelta(obj): """Returns a TIME column as a timedelta object: >>> convert_timedelta('25:06:17') - datetime.timedelta(1, 3977) + datetime.timedelta(days=1, seconds=3977) >>> convert_timedelta('-25:06:17') - datetime.timedelta(-2, 83177) + datetime.timedelta(days=-2, seconds=82423) - Illegal values are returned as None: + Illegal values are returned as string: - >>> convert_timedelta('random crap') is None - True + >>> convert_timedelta('random crap') + 'random crap' Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but can accept values as (+|-)DD HH:MM:SS. The latter format will not @@ -239,12 +238,12 @@ def convert_time(obj): >>> convert_time('15:06:17') datetime.time(15, 6, 17) - Illegal values are returned as None: + Illegal values are returned as str: - >>> convert_time('-25:06:17') is None - True - >>> convert_time('random crap') is None - True + >>> convert_time('-25:06:17') + '-25:06:17' + >>> convert_time('random crap') + 'random crap' Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but can accept values as (+|-)DD HH:MM:SS. The latter format will not @@ -282,13 +281,12 @@ def convert_date(obj): >>> convert_date('2007-02-26') datetime.date(2007, 2, 26) - Illegal values are returned as None: - - >>> convert_date('2007-02-31') is None - True - >>> convert_date('0000-00-00') is None - True + Illegal values are returned as str: + >>> convert_date('2007-02-31') + '2007-02-31' + >>> convert_date('0000-00-00') + '0000-00-00' """ if isinstance(obj, (bytes, bytearray)): obj = obj.decode("ascii") @@ -362,3 +360,5 @@ def through(x): conversions = encoders.copy() conversions.update(decoders) Thing2Literal = escape_str + +# Run doctests with `pytest --doctest-modules pymysql/converters.py` From eba874bd771901b54440b40265b26b0597ea6146 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 4 Aug 2021 13:08:47 +0900 Subject: [PATCH 173/292] Actions: Run test with Python 3.10 (#996) --- .github/workflows/test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 158188cd..6f6f97a5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -31,6 +31,9 @@ jobs: py: "3.9" mysql_auth: true + - db: "mysql:8.0" + py: "3.10-dev" + services: mysql: image: "${{ matrix.db }}" From 33d165dc3087d298ed0e2d7c4e306ccfdab1ec2c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 28 Aug 2021 12:28:44 +0900 Subject: [PATCH 174/292] Fix calling undefined function (#1003) Fixes #981. --- pymysql/connections.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 00605dd9..32b37bbf 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -920,10 +920,7 @@ def _request_authentication(self): ): auth_packet = self._process_auth(plugin_name, auth_packet) else: - # send legacy handshake - data = _auth.scramble_old_password(self.password, self.salt) + b"\0" - self.write_packet(data) - auth_packet = self._read_packet() + raise err.OperationalError("received unknown auth swich request") elif auth_packet.is_extra_auth_data(): if DEBUG: print("received extra data") From 78f0cf99e5d5351df0821442e4dc35c49a6390c6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 28 Aug 2021 13:19:08 +0900 Subject: [PATCH 175/292] Stop showing handler name when hander is not set. (#1004) Fixes #987. --- pymysql/connections.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 32b37bbf..199558ec 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -998,8 +998,7 @@ def _process_auth(self, plugin_name, auth_packet): else: raise err.OperationalError( 2059, - "Authentication plugin '%s' (%r) not configured" - % (plugin_name, handler), + "Authentication plugin '%s' not configured" % (plugin_name,), ) pkt = self._read_packet() pkt.check_error() From f24cb9aa7295921bcd8f34f752c8a05b981d3125 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 2 Oct 2021 17:23:14 +1000 Subject: [PATCH 176/292] tests: container docker-entrypoint-initdb.d for ease of testing (#1009) This allows easier local testing in a container image. mysql (mysql in ubuntu) --comments is needed to push mariab comments to the server side for processing. --- .github/workflows/test.yaml | 30 +++------------------ ci/docker-entrypoint-initdb.d/README | 12 +++++++++ ci/docker-entrypoint-initdb.d/init.sql | 7 +++++ ci/docker-entrypoint-initdb.d/mariadb.sql | 2 ++ ci/docker-entrypoint-initdb.d/mysql.sql | 8 ++++++ pymysql/tests/test_connection.py | 32 +++++++++++++++++++++++ tests/test_mariadb_auth.py | 24 ----------------- 7 files changed, 65 insertions(+), 50 deletions(-) create mode 100644 ci/docker-entrypoint-initdb.d/README create mode 100644 ci/docker-entrypoint-initdb.d/init.sql create mode 100644 ci/docker-entrypoint-initdb.d/mariadb.sql create mode 100644 ci/docker-entrypoint-initdb.d/mysql.sql delete mode 100644 tests/test_mariadb_auth.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6f6f97a5..1269ad05 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,16 +10,14 @@ jobs: strategy: matrix: include: - - db: "mariadb:10.0" + - db: "mariadb:10.2" py: "3.9" - db: "mariadb:10.3" py: "3.8" - mariadb_auth: true - db: "mariadb:10.5" py: "3.7" - mariadb_auth: true - db: "mysql:5.6" py: "3.6" @@ -69,10 +67,9 @@ jobs: mysql -h127.0.0.1 -uroot -e 'select version()' && break done mysql -h127.0.0.1 -uroot -e "SET GLOBAL local_infile=on" - mysql -h127.0.0.1 -uroot -e 'create database test1 DEFAULT CHARACTER SET utf8mb4' - mysql -h127.0.0.1 -uroot -e 'create database test2 DEFAULT CHARACTER SET utf8mb4' - mysql -h127.0.0.1 -uroot -e "create user test2 identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2;" - mysql -h127.0.0.1 -uroot -e "create user test2@localhost identified ${WITH_PLUGIN} by 'some password'; grant all on test2.* to test2@localhost;" + mysql -h127.0.0.1 -uroot --comments < ci/docker-entrypoint-initdb.d/init.sql + mysql -h127.0.0.1 -uroot --comments < ci/docker-entrypoint-initdb.d/mysql.sql + mysql -h127.0.0.1 -uroot --comments < ci/docker-entrypoint-initdb.d/mariadb.sql cp ci/docker.json pymysql/tests/databases.json - name: Run test @@ -87,27 +84,8 @@ jobs: docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - mysql -uroot -h127.0.0.1 -e ' - CREATE USER - user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", - nopass_sha256 IDENTIFIED WITH "sha256_password", - user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", - nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" - PASSWORD EXPIRE NEVER; - GRANT RELOAD ON *.* TO user_caching_sha2;' pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - - name: Run MariaDB auth test - if: ${{ matrix.mariadb_auth }} - run: | - mysql -uroot -h127.0.0.1 -e ' - INSTALL SONAME "auth_ed25519"; - CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so";' - # we need to pass the hashed password manually until 10.4, so hide it here - mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER nopass_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"\"),'\";');" | mysql -uroot -h127.0.0.1 - mysql -uroot -h127.0.0.1 -sNe "SELECT CONCAT('CREATE USER user_ed25519 IDENTIFIED VIA ed25519 USING \"',ed25519_password(\"pass_ed25519\"),'\";');" | mysql -uroot -h127.0.0.1 - pytest -v --cov --cov-config .coveragerc tests/test_mariadb_auth.py - - name: Report coverage run: coveralls --service=github env: diff --git a/ci/docker-entrypoint-initdb.d/README b/ci/docker-entrypoint-initdb.d/README new file mode 100644 index 00000000..6a54b93d --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/README @@ -0,0 +1,12 @@ +To test with a MariaDB or MySQL container image: + +docker run -d -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=1 \ + --name=mysqld -v ./ci/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d:z \ + mysql:8.0.26 --local-infile=1 + +cp ci/docker.json pymysql/tests/databases.json + +pytest + + +Note: Some authentication tests that don't match the image version will fail. diff --git a/ci/docker-entrypoint-initdb.d/init.sql b/ci/docker-entrypoint-initdb.d/init.sql new file mode 100644 index 00000000..b741d41c --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/init.sql @@ -0,0 +1,7 @@ +create database test1 DEFAULT CHARACTER SET utf8mb4; +create database test2 DEFAULT CHARACTER SET utf8mb4; +create user test2 identified by 'some password'; +grant all on test2.* to test2; +create user test2@localhost identified by 'some password'; +grant all on test2.* to test2@localhost; + diff --git a/ci/docker-entrypoint-initdb.d/mariadb.sql b/ci/docker-entrypoint-initdb.d/mariadb.sql new file mode 100644 index 00000000..912d365a --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/mariadb.sql @@ -0,0 +1,2 @@ +/*M!100122 INSTALL SONAME "auth_ed25519" */; +/*M!100122 CREATE FUNCTION ed25519_password RETURNS STRING SONAME "auth_ed25519.so" */; diff --git a/ci/docker-entrypoint-initdb.d/mysql.sql b/ci/docker-entrypoint-initdb.d/mysql.sql new file mode 100644 index 00000000..a4ba0927 --- /dev/null +++ b/ci/docker-entrypoint-initdb.d/mysql.sql @@ -0,0 +1,8 @@ +/*!80001 CREATE USER + user_sha256 IDENTIFIED WITH "sha256_password" BY "pass_sha256_01234567890123456789", + nopass_sha256 IDENTIFIED WITH "sha256_password", + user_caching_sha2 IDENTIFIED WITH "caching_sha2_password" BY "pass_caching_sha2_01234567890123456789", + nopass_caching_sha2 IDENTIFIED WITH "caching_sha2_password" + PASSWORD EXPIRE NEVER */; + +/*!80001 GRANT RELOAD ON *.* TO user_caching_sha2 */; diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index a469be5a..e95b75d6 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -53,6 +53,7 @@ class TestAuthentication(base.PyMySQLTestCase): pam_found = False mysql_old_password_found = False sha256_password_found = False + ed25519_found = False import os @@ -97,6 +98,8 @@ class TestAuthentication(base.PyMySQLTestCase): mysql_old_password_found = True elif r[0] == "sha256_password": sha256_password_found = True + elif r[0] == "ed25519": + ed25519_found = True # else: # print("plugin: %r" % r[0]) @@ -412,6 +415,35 @@ def testAuthSHA256(self): with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user="pymysql_sha256", **db) + @pytest.mark.skipif(not ed25519_found, reason="no ed25519 authention plugin") + def testAuthEd25519(self): + db = self.db.copy() + del db["password"] + conn = self.connect() + c = conn.cursor() + c.execute("select ed25519_password(''), ed25519_password('ed25519_password')") + for r in c: + empty_pass = r[0].decode("ascii") + non_empty_pass = r[1].decode("ascii") + + with TempUser( + c, + "pymysql_ed25519", + self.databases[0]["database"], + "ed25519", + empty_pass, + ) as u: + pymysql.connect(user="pymysql_ed25519", password="", **db) + + with TempUser( + c, + "pymysql_ed25519", + self.databases[0]["database"], + "ed25519", + non_empty_pass, + ) as u: + pymysql.connect(user="pymysql_ed25519", password="ed25519_password", **db) + class TestConnection(base.PyMySQLTestCase): def test_utf8mb4(self): diff --git a/tests/test_mariadb_auth.py b/tests/test_mariadb_auth.py deleted file mode 100644 index b3a2719c..00000000 --- a/tests/test_mariadb_auth.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Test for auth methods supported by MariaDB 10.3+""" - -import pymysql - -# pymysql.connections.DEBUG = True -# pymysql._auth.DEBUG = True - -host = "127.0.0.1" -port = 3306 - - -def test_ed25519_no_password(): - con = pymysql.connect(user="nopass_ed25519", host=host, port=port, ssl=None) - con.close() - - -def test_ed25519_password(): # nosec - con = pymysql.connect( - user="user_ed25519", password="pass_ed25519", host=host, port=port, ssl=None - ) - con.close() - - -# default mariadb docker images aren't configured with SSL From 534f4a6f53097384842b55ac7466a8033c0d1375 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Mon, 31 Jan 2022 05:32:17 +0100 Subject: [PATCH 177/292] fix typo in comment (#1024) --- pymysql/connections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 199558ec..bfe8b10a 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -204,12 +204,12 @@ def __init__( db=None, # deprecated ): if db is not None and database is None: - # We will raise warining in 2022 or later. + # We will raise warning in 2022 or later. # See https://github.com/PyMySQL/PyMySQL/issues/939 # warnings.warn("'db' is deprecated, use 'database'", DeprecationWarning, 3) database = db if passwd is not None and not password: - # We will raise warining in 2022 or later. + # We will raise warning in 2022 or later. # See https://github.com/PyMySQL/PyMySQL/issues/939 # warnings.warn( # "'passwd' is deprecated, use 'password'", DeprecationWarning, 3 From 72f70c9ff81103b4a2e0b8531663a80d44595c2d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 31 Jan 2022 13:50:32 +0900 Subject: [PATCH 178/292] Update black version (#1026) --- docs/source/conf.py | 16 ++++++++-------- pymysql/connections.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 77d7073a..a57a03c4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -46,8 +46,8 @@ master_doc = "index" # General information about the project. -project = u"PyMySQL" -copyright = u"2016, Yutaka Matsubara and GitHub contributors" +project = "PyMySQL" +copyright = "2016, Yutaka Matsubara and GitHub contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -200,8 +200,8 @@ ( "index", "PyMySQL.tex", - u"PyMySQL Documentation", - u"Yutaka Matsubara and GitHub contributors", + "PyMySQL Documentation", + "Yutaka Matsubara and GitHub contributors", "manual", ), ] @@ -235,8 +235,8 @@ ( "index", "pymysql", - u"PyMySQL Documentation", - [u"Yutaka Matsubara and GitHub contributors"], + "PyMySQL Documentation", + ["Yutaka Matsubara and GitHub contributors"], 1, ) ] @@ -254,8 +254,8 @@ ( "index", "PyMySQL", - u"PyMySQL Documentation", - u"Yutaka Matsubara and GitHub contributors", + "PyMySQL Documentation", + "Yutaka Matsubara and GitHub contributors", "PyMySQL", "One line description of project.", "Miscellaneous", diff --git a/pymysql/connections.py b/pymysql/connections.py index bfe8b10a..2edeb508 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -61,7 +61,7 @@ DEFAULT_CHARSET = "utf8mb4" -MAX_PACKET_LEN = 2 ** 24 - 1 +MAX_PACKET_LEN = 2**24 - 1 def _pack_int24(n): From afbef5ea0d1bc4c5c2d5d15c5ce519ecdfd29a1d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 31 Jan 2022 14:35:31 +0900 Subject: [PATCH 179/292] Actions: Use actions/setup-python cache (#1027) --- .github/workflows/test.yaml | 25 +++++++++---------------- requirements-dev.txt | 2 ++ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1269ad05..2a9ff0a6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,17 +47,12 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.py }} - - - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-1 - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' + cache-dependency-path: 'requirements-dev.txt' - name: Install dependency run: | - pip install -U cryptography PyNaCl pytest pytest-cov coveralls + pip install -U -r requirements-dev.txt - name: Set up MySQL run: | @@ -98,16 +93,14 @@ jobs: runs-on: ubuntu-20.04 needs: test steps: - - uses: actions/setup-python@v2 - with: - python-version: 3.9 + - name: requirements. + run: | + echo coveralls > requirements.txt - - uses: actions/cache@v2 + - uses: actions/setup-python@v2 with: - path: ~/.cache/pip - key: finish-pip-1 - restore-keys: | - finish-pip- + python-version: '3.9' + cache: 'pip' - name: Finished run: | diff --git a/requirements-dev.txt b/requirements-dev.txt index d65512fb..13d7f7fb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,5 @@ cryptography PyNaCl>=1.4.0 pytest +pytest-cov +coveralls From 2beebd92b8ad3fb59a93714c799450dbfebe3922 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 1 Feb 2022 01:04:50 +0100 Subject: [PATCH 180/292] update pymysql.constants.CR (#1029) values from https://github.com/mysql/mysql-server/blob/mysql-8.0.28/include/errmsg.h --- pymysql/constants/CR.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pymysql/constants/CR.py b/pymysql/constants/CR.py index 25579a7c..deae977e 100644 --- a/pymysql/constants/CR.py +++ b/pymysql/constants/CR.py @@ -65,4 +65,15 @@ CR_AUTH_PLUGIN_CANNOT_LOAD = 2059 CR_DUPLICATE_CONNECTION_ATTR = 2060 CR_AUTH_PLUGIN_ERR = 2061 -CR_ERROR_LAST = 2061 +CR_INSECURE_API_ERR = 2062 +CR_FILE_NAME_TOO_LONG = 2063 +CR_SSL_FIPS_MODE_ERR = 2064 +CR_DEPRECATED_COMPRESSION_NOT_SUPPORTED = 2065 +CR_COMPRESSION_WRONGLY_CONFIGURED = 2066 +CR_KERBEROS_USER_NOT_FOUND = 2067 +CR_LOAD_DATA_LOCAL_INFILE_REJECTED = 2068 +CR_LOAD_DATA_LOCAL_INFILE_REALPATH_FAIL = 2069 +CR_DNS_SRV_LOOKUP_FAILED = 2070 +CR_MANDATORY_TRACKER_NOT_FOUND = 2071 +CR_INVALID_FACTOR_NO = 2072 +CR_ERROR_LAST = 2072 From 3fb9dd9b1f88334bb8014969a7b7f7027632dcca Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 1 Feb 2022 04:57:02 +0100 Subject: [PATCH 181/292] Use constants (#1028) --- pymysql/connections.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 2edeb508..04e3c53f 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -13,7 +13,7 @@ from . import _auth from .charset import charset_by_name, charset_by_id -from .constants import CLIENT, COMMAND, CR, FIELD_TYPE, SERVER_STATUS +from .constants import CLIENT, COMMAND, CR, ER, FIELD_TYPE, SERVER_STATUS from . import converters from .cursors import Cursor from .optionfile import Parser @@ -441,7 +441,10 @@ def get_autocommit(self): def _read_ok_packet(self): pkt = self._read_packet() if not pkt.is_ok_packet(): - raise err.OperationalError(2014, "Command Out of Sync") + raise err.OperationalError( + CR.CR_COMMANDS_OUT_OF_SYNC, + "Command Out of Sync", + ) ok = OKPacketWrapper(pkt) self.server_status = ok.server_status return ok @@ -654,7 +657,8 @@ def connect(self, sock=None): if isinstance(e, (OSError, IOError, socket.error)): exc = err.OperationalError( - 2003, "Can't connect to MySQL server on %r (%s)" % (self.host, e) + CR.CR_CONN_HOST_ERROR, + "Can't connect to MySQL server on %r (%s)" % (self.host, e), ) # Keep original exception and traceback to investigate error. exc.original_exception = e @@ -945,7 +949,7 @@ def _process_auth(self, plugin_name, auth_packet): except AttributeError: if plugin_name != b"dialog": raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s'" " not loaded: - %r missing authenticate method" % (plugin_name, type(handler)), @@ -983,21 +987,21 @@ def _process_auth(self, plugin_name, auth_packet): self.write_packet(resp + b"\0") except AttributeError: raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s'" " not loaded: - %r missing prompt method" % (plugin_name, handler), ) except TypeError: raise err.OperationalError( - 2061, + CR.CR_AUTH_PLUGIN_ERR, "Authentication plugin '%s'" " %r didn't respond with string. Returned '%r' to prompt %r" % (plugin_name, handler, resp, prompt), ) else: raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s' not configured" % (plugin_name,), ) pkt = self._read_packet() @@ -1007,7 +1011,8 @@ def _process_auth(self, plugin_name, auth_packet): return pkt else: raise err.OperationalError( - 2059, "Authentication plugin '%s' not configured" % plugin_name + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, + "Authentication plugin '%s' not configured" % plugin_name, ) self.write_packet(data) @@ -1024,7 +1029,7 @@ def _get_auth_plugin_handler(self, plugin_name): handler = plugin_class(self) except TypeError: raise err.OperationalError( - 2059, + CR.CR_AUTH_PLUGIN_CANNOT_LOAD, "Authentication plugin '%s'" " not loaded: - %r cannot be constructed with connection object" % (plugin_name, plugin_class), @@ -1211,7 +1216,10 @@ def _read_load_local_packet(self, first_packet): if ( not ok_packet.is_ok_packet() ): # pragma: no cover - upstream induced protocol error - raise err.OperationalError(2014, "Commands Out of Sync") + raise err.OperationalError( + CR.CR_COMMANDS_OUT_OF_SYNC, + "Commands Out of Sync", + ) self._read_ok_packet(ok_packet) def _check_packet_is_eof(self, packet): @@ -1357,7 +1365,10 @@ def send_data(self): break conn.write_packet(chunk) except IOError: - raise err.OperationalError(1017, f"Can't find file '{self.filename}'") + raise err.OperationalError( + ER.FILE_NOT_FOUND, + f"Can't find file '{self.filename}'", + ) finally: # send the empty packet to signify we are done sending data conn.write_packet(b"") From cebba92d338d89ac46381f3e1ca637416a77c0e2 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 6 Feb 2022 08:50:49 +0100 Subject: [PATCH 182/292] Improve GitHub workflow (#1031) - concurrency cancels builds in progress e.g. on pull requests - matrix jobs no longer fail fast, allowing to see failure reasons for all matrix jobs - coveralls no longer runs on forks, this would fail anyways --- .github/workflows/test.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2a9ff0a6..d9b9e2af 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,10 +4,15 @@ on: push: pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: include: - db: "mariadb:10.2" @@ -82,6 +87,7 @@ jobs: pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - name: Report coverage + if: github.repository == 'PyMySQL/PyMySQL' run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -89,6 +95,7 @@ jobs: COVERALLS_PARALLEL: true coveralls: + if: github.repository == 'PyMySQL/PyMySQL' name: Finish coveralls runs-on: ubuntu-20.04 needs: test From 062384c26d10556529af91d0f0946e302b727d18 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 6 Feb 2022 08:52:15 +0100 Subject: [PATCH 183/292] Drop support of EOL Python and DB versions (#1030) - Python now requires 3.7+, reflected in python_requires - MySQL now requires 5.7+ in tests - MariaDB unchanged in tests, only dropped support in documentation - Added Python 3.11 to test matrix - Added MariaDB 10.7 to test matrix - DB version checks have been removed from various tests where no longer needed this also results in running a few tests on MariaDB which were previously only running on MySQL. --- .github/workflows/test.yaml | 8 ++++---- CHANGELOG.md | 9 +++++++++ README.rst | 6 +++--- docs/source/user/installation.rst | 6 +++--- pymysql/tests/base.py | 5 +++++ pymysql/tests/test_basic.py | 6 +++--- pymysql/tests/test_connection.py | 10 +--------- pymysql/tests/test_issues.py | 15 +++------------ setup.py | 5 +++-- 9 files changed, 34 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d9b9e2af..0d2e9998 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -24,18 +24,18 @@ jobs: - db: "mariadb:10.5" py: "3.7" - - db: "mysql:5.6" - py: "3.6" + - db: "mariadb:10.7" + py: "3.11-dev" - db: "mysql:5.7" - py: "pypy-3.6" + py: "pypy-3.8" - db: "mysql:8.0" py: "3.9" mysql_auth: true - db: "mysql:8.0" - py: "3.10-dev" + py: "3.10" services: mysql: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9885af52..abf38b3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changes +## v1.0.3 + +Release date: TBD + +* Dropped support of end of life MySQL version 5.6 +* Dropped support of end of life MariaDB versions below 10.2 +* Dropped support of end of life Python version 3.6 + + ## v1.0.2 Release date: 2021-01-09 diff --git a/README.rst b/README.rst index f514d901..e7c9419e 100644 --- a/README.rst +++ b/README.rst @@ -25,13 +25,13 @@ Requirements * Python -- one of the following: - - CPython_ : 3.6 and newer + - CPython_ : 3.7 and newer - PyPy_ : Latest 3.x version * MySQL Server -- one of the following: - - MySQL_ >= 5.6 - - MariaDB_ >= 10.0 + - MySQL_ >= 5.7 + - MariaDB_ >= 10.2 .. _CPython: https://www.python.org/ .. _PyPy: https://pypy.org/ diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index 0fea2726..c66aae3d 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -18,13 +18,13 @@ Requirements * Python -- one of the following: - - CPython_ >= 3.6 + - CPython_ >= 3.7 - Latest PyPy_ 3 * MySQL Server -- one of the following: - - MySQL_ >= 5.6 - - MariaDB_ >= 10.0 + - MySQL_ >= 5.7 + - MariaDB_ >= 10.2 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index 6f93a831..a87307a5 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -32,6 +32,11 @@ def mysql_server_is(self, conn, version_tuple): """Return True if the given connection is on the version given or greater. + This only checks the server version string provided when the + connection is established, therefore any check for a version tuple + greater than (5, 5, 5) will always fail on MariaDB, as it always + starts with 5.5.5, e.g. 5.5.5-10.7.1-MariaDB-1:10.7.1+maria~focal. + e.g.:: if self.mysql_server_is(conn, (5, 6, 4)): diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index a0dea9c8..d37d1976 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -175,8 +175,6 @@ def test_datetime_microseconds(self): """test datetime conversion w microseconds""" conn = self.connect() - if not self.mysql_server_is(conn, (5, 6, 4)): - pytest.skip("target backend does not support microseconds") c = conn.cursor() dt = datetime.datetime(2013, 11, 12, 9, 9, 9, 123450) c.execute("create table test_datetime (id int, ts datetime(6))") @@ -285,8 +283,10 @@ def test_json(self): args = self.databases[0].copy() args["charset"] = "utf8mb4" conn = pymysql.connect(**args) + # MariaDB only has limited JSON support, stores data as longtext + # https://mariadb.com/kb/en/json-data-type/ if not self.mysql_server_is(conn, (5, 7, 0)): - pytest.skip("JSON type is not supported on MySQL <= 5.6") + pytest.skip("JSON type is only supported on MySQL >= 5.7") self.safe_create_table( conn, diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index e95b75d6..23a2aa04 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -105,8 +105,6 @@ class TestAuthentication(base.PyMySQLTestCase): def test_plugin(self): conn = self.connect() - if not self.mysql_server_is(conn, (5, 5, 0)): - pytest.skip("MySQL-5.5 required for plugins") cur = conn.cursor() cur.execute( "select plugin from mysql.user where concat(user, '@', host)=current_user()" @@ -401,13 +399,7 @@ def testAuthSHA256(self): self.databases[0]["database"], "sha256_password", ) as u: - if self.mysql_server_is(conn, (5, 7, 0)): - c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") - else: - c.execute("SET old_passwords = 2") - c.execute( - "SET PASSWORD FOR 'pymysql_sha256'@'localhost' = PASSWORD('Sh@256Pa33')" - ) + c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") c.execute("FLUSH PRIVILEGES") db = self.db.copy() db["password"] = "Sh@256Pa33" diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 76d4b133..3ea2c2c4 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -466,29 +466,20 @@ def test_issue_363(self): ) cur = conn.cursor() - # From MySQL 5.7, ST_GeomFromText is added and GeomFromText is deprecated. - if self.mysql_server_is(conn, (5, 7, 0)): - geom_from_text = "ST_GeomFromText" - geom_as_text = "ST_AsText" - geom_as_bin = "ST_AsBinary" - else: - geom_from_text = "GeomFromText" - geom_as_text = "AsText" - geom_as_bin = "AsBinary" query = ( "INSERT INTO issue363 (id, geom) VALUES" - "(1998, %s('LINESTRING(1.1 1.1,2.2 2.2)'))" % geom_from_text + "(1998, ST_GeomFromText('LINESTRING(1.1 1.1,2.2 2.2)'))" ) cur.execute(query) # select WKT - query = "SELECT %s(geom) FROM issue363" % geom_as_text + query = "SELECT ST_AsText(geom) FROM issue363" cur.execute(query) row = cur.fetchone() self.assertEqual(row, ("LINESTRING(1.1 1.1,2.2 2.2)",)) # select WKB - query = "SELECT %s(geom) FROM issue363" % geom_as_bin + query = "SELECT ST_AsBinary(geom) FROM issue363" cur.execute(query) row = cur.fetchone() self.assertEqual( diff --git a/setup.py b/setup.py index 1510a0cf..7cdc692f 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ description="Pure Python MySQL Driver", long_description=readme, packages=find_packages(exclude=["tests*", "pymysql.tests*"]), - python_requires=">=3.6", + python_requires=">=3.7", extras_require={ "rsa": ["cryptography"], "ed25519": ["PyNaCl>=1.4.0"], @@ -24,10 +24,11 @@ classifiers=[ "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From ee88d0f0e6499ad3054edbf057e08abfe25993c4 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Sun, 6 Feb 2022 08:53:30 +0100 Subject: [PATCH 184/292] Fix coveralls branch in README.rst (#1034) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e7c9419e..f1384c92 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ :target: https://pymysql.readthedocs.io/ :alt: Documentation Status -.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=master&service=github - :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=master +.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github + :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=main .. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python From eb108a61669f8883426d35f153dc48c6348d4b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A4=80=EA=B7=9C?= Date: Tue, 22 Mar 2022 14:54:05 +0900 Subject: [PATCH 185/292] Fix minor typo in error message (#1038) --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 04e3c53f..9de40dea 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -924,7 +924,7 @@ def _request_authentication(self): ): auth_packet = self._process_auth(plugin_name, auth_packet) else: - raise err.OperationalError("received unknown auth swich request") + raise err.OperationalError("received unknown auth switch request") elif auth_packet.is_extra_auth_data(): if DEBUG: print("received extra data") From b9e07c5bb56806a167003ced8d3c5e704657e503 Mon Sep 17 00:00:00 2001 From: Daniel Golding Date: Sat, 16 Apr 2022 07:23:52 +0200 Subject: [PATCH 186/292] Document that the ssl connection parameter can be an SSLContext (#1045) --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 9de40dea..94ea545f 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -126,7 +126,7 @@ class Connection: :param init_command: Initial SQL statement to run when connection is established. :param connect_timeout: The timeout for connecting to the database in seconds. (default: 10, min: 1, max: 31536000) - :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. + :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters or an ssl.SSLContext. :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate. :param ssl_cert: Path to the file that contains a PEM-formatted client certificate. :param ssl_disabled: A boolean value that disables usage of TLS. From 72ee1f3804082442fcbc5c0b1a054ed5c284cd7d Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 14 Jun 2022 06:40:21 +0200 Subject: [PATCH 187/292] Update mariadb tests to 10.8, remove end of life mariadb 10.2 (#1049) --- .github/workflows/test.yaml | 6 +++--- CHANGELOG.md | 2 +- README.rst | 2 +- docs/source/user/installation.rst | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0d2e9998..e07a4c9b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,9 +15,6 @@ jobs: fail-fast: false matrix: include: - - db: "mariadb:10.2" - py: "3.9" - - db: "mariadb:10.3" py: "3.8" @@ -27,6 +24,9 @@ jobs: - db: "mariadb:10.7" py: "3.11-dev" + - db: "mariadb:10.8" + py: "3.9" + - db: "mysql:5.7" py: "pypy-3.8" diff --git a/CHANGELOG.md b/CHANGELOG.md index abf38b3f..5a429244 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Release date: TBD * Dropped support of end of life MySQL version 5.6 -* Dropped support of end of life MariaDB versions below 10.2 +* Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 diff --git a/README.rst b/README.rst index f1384c92..318e9460 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Requirements * MySQL Server -- one of the following: - MySQL_ >= 5.7 - - MariaDB_ >= 10.2 + - MariaDB_ >= 10.3 .. _CPython: https://www.python.org/ .. _PyPy: https://pypy.org/ diff --git a/docs/source/user/installation.rst b/docs/source/user/installation.rst index c66aae3d..9313f14d 100644 --- a/docs/source/user/installation.rst +++ b/docs/source/user/installation.rst @@ -24,7 +24,7 @@ Requirements * MySQL Server -- one of the following: - MySQL_ >= 5.7 - - MariaDB_ >= 10.2 + - MariaDB_ >= 10.3 .. _CPython: http://www.python.org/ .. _PyPy: http://pypy.org/ From 0ab388939ae96fa32acc59ebcc2e7b1a2a4da8c1 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 14 Jul 2022 07:57:13 +0200 Subject: [PATCH 188/292] Fix CodeQL target branch (#1054) master branch was renamed to main some time ago, leading to this action no longer working properly, at least for PRs --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b6a7238d..94165437 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,10 +13,10 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [ main ] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ main ] schedule: - cron: '34 7 * * 2' From 7f47ac0184294b15a3b53cdcbe96b9895d0c6f4c Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Thu, 14 Jul 2022 07:57:25 +0200 Subject: [PATCH 189/292] Update CodeQL GitHub action to v2 (#1055) v1 has been deprecated: https://github.blog/changelog/2022-04-27-code-scanning-deprecation-of-codeql-action-v1/ --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 94165437..d559b1cd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -50,7 +50,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -64,4 +64,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From d1748350b9b6b4efdcead428fad2fbcdb7cfddd0 Mon Sep 17 00:00:00 2001 From: WangDi Date: Fri, 22 Jul 2022 13:12:12 +0800 Subject: [PATCH 190/292] tests: remove duplicate test (#1057) --- pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index e882c5eb..9ac190f2 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -23,9 +23,6 @@ def test_setoutputsize(self): def test_setoutputsize_basic(self): pass - def test_nextset(self): - pass - """The tests on fetchone and fetchall and rowcount bogusly test for an exception if the statement cannot return a result set. MySQL always returns a result set; it's just that From dd47caae95011e79b9e2ee12549d23f05a7f839d Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Wed, 24 Aug 2022 04:50:30 +0200 Subject: [PATCH 191/292] Remove deprecated socket.error from Connection.connect exception handler (#1062) Since python 3.3, `socket.error` is a deprecated alias for OSError, which is already included. --- pymysql/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 94ea545f..3265d32e 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -655,7 +655,7 @@ def connect(self, sock=None): except: # noqa pass - if isinstance(e, (OSError, IOError, socket.error)): + if isinstance(e, (OSError, IOError)): exc = err.OperationalError( CR.CR_CONN_HOST_ERROR, "Can't connect to MySQL server on %r (%s)" % (self.host, e), From e77b21898ab46887067df981eaa19809533ec4bf Mon Sep 17 00:00:00 2001 From: Chuck Cadman <51368516+cdcadman@users.noreply.github.com> Date: Mon, 19 Sep 2022 00:06:49 -0700 Subject: [PATCH 192/292] Raise ProgrammingError on -inf in addition to inf (#1067) Co-authored-by: Chuck Cadman --- pymysql/converters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index da63ceb7..2acc3e58 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -56,7 +56,7 @@ def escape_int(value, mapping=None): def escape_float(value, mapping=None): s = repr(value) - if s in ("inf", "nan"): + if s in ("inf", "-inf", "nan"): raise ProgrammingError("%s can not be used with MySQL" % s) if "e" not in s: s += "e0" From 3dc1abbdaf7af99357c834c58f0e27f871ebe885 Mon Sep 17 00:00:00 2001 From: SergeantMenacingGarlic <87030047+SergeantMenacingGarlic@users.noreply.github.com> Date: Tue, 11 Oct 2022 03:06:18 -0400 Subject: [PATCH 193/292] Add unix socket test (#1061) --- .github/workflows/test.yaml | 9 +++++++++ ci/docker.json | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e07a4c9b..5a8f6dab 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,9 +45,18 @@ jobs: env: MYSQL_ALLOW_EMPTY_PASSWORD: yes options: "--name=mysqld" + volumes: + - /run/mysqld:/run/mysqld steps: - uses: actions/checkout@v2 + + - name: Workaround MySQL container permissions + if: startsWith(matrix.db, 'mysql') + run: | + sudo chown 999:999 /run/mysqld + /usr/bin/docker ps --all --filter status=exited --no-trunc --format "{{.ID}}" | xargs -r /usr/bin/docker start + - name: Set up Python ${{ matrix.py }} uses: actions/setup-python@v2 with: diff --git a/ci/docker.json b/ci/docker.json index 34a5c7b7..63d19a68 100644 --- a/ci/docker.json +++ b/ci/docker.json @@ -1,4 +1,5 @@ [ {"host": "127.0.0.1", "port": 3306, "user": "root", "password": "", "database": "test1", "use_unicode": true, "local_infile": true}, - {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" } + {"host": "127.0.0.1", "port": 3306, "user": "test2", "password": "some password", "database": "test2" }, + {"host": "localhost", "port": 3306, "user": "test2", "password": "some password", "database": "test2", "unix_socket": "/run/mysqld/mysqld.sock"} ] From 90317924e8f4ae5af871d4ef32cfadf963a795f4 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Fri, 11 Nov 2022 03:27:42 +0100 Subject: [PATCH 194/292] Use Python 3.11 release instead of -dev in tests (#1076) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5a8f6dab..39afc579 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,7 +22,7 @@ jobs: py: "3.7" - db: "mariadb:10.7" - py: "3.11-dev" + py: "3.11" - db: "mariadb:10.8" py: "3.9" From ed56379dcc165f8810c8678c56bff7bb544a710f Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Fri, 11 Nov 2022 13:28:06 +1100 Subject: [PATCH 195/292] docs: Fix a few typos (#1053) --- pymysql/tests/test_connection.py | 2 +- pymysql/tests/test_issues.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 23a2aa04..94a8dea0 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -492,7 +492,7 @@ def test_connection_gone_away(self): time.sleep(2) with self.assertRaises(pymysql.OperationalError) as cm: cur.execute("SELECT 1+1") - # error occures while reading, not writing because of socket buffer. + # error occurs while reading, not writing because of socket buffer. # self.assertEqual(cm.exception.args[0], 2006) self.assertIn(cm.exception.args[0], (2006, 2013)) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 3ea2c2c4..733d56a1 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -149,7 +149,7 @@ def test_issue_16(self): "test_issue_17() requires a custom, legacy MySQL configuration and will not be run." ) def test_issue_17(self): - """could not connect mysql use passwod""" + """could not connect mysql use password""" conn = self.connect() host = self.databases[0]["host"] db = self.databases[0]["database"] From e3a1beba22234f419d68c6947d7a1a0bf5d2eae4 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 9 Jan 2023 09:36:10 +0100 Subject: [PATCH 196/292] flake8: Use max_line_length instead of ignoring E501 (#1081) --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b40802e4..e487e5e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,7 @@ [flake8] -ignore = E203,E501,W503,E722 exclude = tests,build,.venv,docs +ignore = E203,W503,E722 +max_line_length=129 [metadata] license = "MIT" From e91d097029f90055237741b5e56f81933ec1c981 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 9 Jan 2023 13:10:32 +0100 Subject: [PATCH 197/292] Fix typos discovered by codespell (#1082) --- CHANGELOG.md | 2 +- pymysql/_auth.py | 2 +- pymysql/tests/test_DictCursor.py | 2 +- pymysql/tests/test_basic.py | 2 +- pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py | 12 ++++++------ .../thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a429244..87c3f9e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -204,7 +204,7 @@ Release date: 2016-08-30 Release date: 2016-07-29 * Fix SELECT JSON type cause UnicodeError -* Avoid float convertion while parsing microseconds +* Avoid float conversion while parsing microseconds * Warning has number * SSCursor supports warnings diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 33fd9df8..f6c9eb96 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -241,7 +241,7 @@ def caching_sha2_password_auth(conn, pkt): return pkt if n != 4: - raise OperationalError("caching sha2: Unknwon result for fast auth: %s" % n) + raise OperationalError("caching sha2: Unknown result for fast auth: %s" % n) if DEBUG: print("caching sha2: Trying full auth...") diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index 581a0c4a..bbc87d03 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -17,7 +17,7 @@ def setUp(self): self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) - # create a table ane some data to query + # create a table and some data to query with warnings.catch_warnings(): warnings.filterwarnings("ignore") c.execute("drop table if exists dictcursor") diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index d37d1976..bc88e5a5 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -320,7 +320,7 @@ def setUp(self): self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) - # create a table ane some data to query + # create a table and some data to query self.safe_create_table( conn, "bulkinsert", diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 6766aff3..30620ce4 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -51,9 +51,9 @@ # - Now a subclass of TestCase, to avoid requiring the driver stub # to use multiple inheritance # - Reversed the polarity of buggy test in test_description -# - Test exception heirarchy correctly +# - Test exception hierarchy correctly # - self.populate is now self._populate(), so if a driver stub -# overrides self.ddl1 this change propogates +# overrides self.ddl1 this change propagates # - VARCHAR columns now have a width, which will hopefully make the # DDL even more portible (this will be reversed if it causes more problems) # - cursor.rowcount being checked after various execute and fetchXXX methods @@ -174,7 +174,7 @@ def test_paramstyle(self): def test_Exceptions(self): # Make sure required exceptions exist, and are in the - # defined heirarchy. + # defined hierarchy. self.assertTrue(issubclass(self.driver.Warning, Exception)) self.assertTrue(issubclass(self.driver.Error, Exception)) self.assertTrue(issubclass(self.driver.InterfaceError, self.driver.Error)) @@ -474,7 +474,7 @@ def test_fetchone(self): self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows self.executeDDL1(cur) self.assertRaises(self.driver.Error, cur.fetchone) @@ -487,7 +487,7 @@ def test_fetchone(self): self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows cur.execute( "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) ) @@ -792,7 +792,7 @@ def test_setoutputsize_basic(self): con.close() def test_setoutputsize(self): - # Real test for setoutputsize is driver dependant + # Real test for setoutputsize is driver dependent raise NotImplementedError("Driver need to override this test") def test_None(self): diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index 9ac190f2..bc1e1b2e 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -92,7 +92,7 @@ def test_fetchone(self): self.assertRaises(self.driver.Error, cur.fetchone) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows self.executeDDL1(cur) ## self.assertRaises(self.driver.Error,cur.fetchone) @@ -105,7 +105,7 @@ def test_fetchone(self): self.assertTrue(cur.rowcount in (-1, 0)) # cursor.fetchone should raise an Error if called after - # executing a query that cannnot return rows + # executing a query that cannot return rows cur.execute( "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) ) From 15c2e4c88bfffacce3cc7eaa5a89fdf25c58edea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Jan 2023 10:10:37 +0900 Subject: [PATCH 198/292] Action: Update to dessant/lock-threads@v4 --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 1b25b4c7..7806b7db 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -12,5 +12,5 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v2 + - uses: dessant/lock-threads@v4 From 67af9a55b4f6fa9fe7d0cc13877b4f6016db3680 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 19 Jan 2023 13:27:07 +0900 Subject: [PATCH 199/292] Action: Run 'Lock Threads' weekly. --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 7806b7db..c8f2ca24 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -2,7 +2,7 @@ name: 'Lock Threads' on: schedule: - - cron: '0 0 * * *' + - cron: '9 30 * * 1' permissions: issues: write From d734f15bd8ed20a7442c6bac59d3894181cc326e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 14:35:02 +0900 Subject: [PATCH 200/292] Action: Add doctest (#1086) --- .github/workflows/test.yaml | 1 + pymysql/tests/test_basic.py | 1 - pymysql/tests/test_connection.py | 1 - pymysql/tests/thirdparty/test_MySQLdb/capabilities.py | 1 - .../tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py | 1 - 5 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 39afc579..aee9e1bc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -84,6 +84,7 @@ jobs: - name: Run test run: | pytest -v --cov --cov-config .coveragerc pymysql + pytest -v --cov --cov-config .coveragerc --doctest-modules pymysql/converters.py - name: Run MySQL8 auth test if: ${{ matrix.mysql_auth }} diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index bc88e5a5..8af07da0 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -312,7 +312,6 @@ def test_json(self): class TestBulkInserts(base.PyMySQLTestCase): - cursor_type = pymysql.cursors.DictCursor def setUp(self): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 94a8dea0..d6fb5e52 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -45,7 +45,6 @@ def __exit__(self, exc_type, exc_value, traceback): class TestAuthentication(base.PyMySQLTestCase): - socket_auth = False socket_found = False two_questions_found = False diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index ffead0ca..0276a558 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -10,7 +10,6 @@ class DatabaseTest(unittest.TestCase): - db_module = None connect_args = () connect_kwargs = dict(use_unicode=True, charset="utf8mb4", binary_prefix=True) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 139089ab..11bfdbe2 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -8,7 +8,6 @@ class test_MySQLdb(capabilities.DatabaseTest): - db_module = pymysql connect_args = () connect_kwargs = base.PyMySQLTestCase.databases[0].copy() From 958a195d20551821db34b0c6b2d79739bc5543cf Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 15:58:08 +0900 Subject: [PATCH 201/292] Action: Fix lock --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index c8f2ca24..5dde1354 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -2,7 +2,7 @@ name: 'Lock Threads' on: schedule: - - cron: '9 30 * * 1' + - cron: '30 9 * * 1' permissions: issues: write From 6270177c19fcb29e9d48c5178f91601a0e1a1fb1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 16:58:15 +0900 Subject: [PATCH 202/292] README: Remove LGTM label --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 318e9460..592b295a 100644 --- a/README.rst +++ b/README.rst @@ -5,9 +5,6 @@ .. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=main -.. image:: https://img.shields.io/lgtm/grade/python/g/PyMySQL/PyMySQL.svg?logo=lgtm&logoWidth=18 - :target: https://lgtm.com/projects/g/PyMySQL/PyMySQL/context:python - PyMySQL ======= From 592c4d2cf29702d36ad56469d74de4510fb5a376 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Feb 2023 17:01:16 +0900 Subject: [PATCH 203/292] Action: Fix test coverage --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index aee9e1bc..2b334503 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -84,7 +84,7 @@ jobs: - name: Run test run: | pytest -v --cov --cov-config .coveragerc pymysql - pytest -v --cov --cov-config .coveragerc --doctest-modules pymysql/converters.py + pytest -v --cov-append --cov-config .coveragerc --doctest-modules pymysql/converters.py - name: Run MySQL8 auth test if: ${{ matrix.mysql_auth }} From ded5f5a2d20f6eb033ade4096e88e291e432740b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 6 Feb 2023 20:39:57 +0900 Subject: [PATCH 204/292] Use pyproject.toml (#1087) --- .flake8 | 4 ++++ pyproject.toml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 14 -------------- setup.py | 39 --------------------------------------- 4 files changed, 53 insertions(+), 53 deletions(-) create mode 100644 .flake8 create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100755 setup.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..3f1c38a3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +exclude = tests,build,.venv,docs +ignore = E203,W503,E722 +max_line_length=129 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..3793a8c1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[project] +name = "PyMySQL" +version = "1.0.2" +description = "Pure Python MySQL Driver" +authors = [ + {name = "Inada Naoki", email = "songofacandy@gmail.com"}, + {name = "Yutaka Matsubara", email = "yutaka.matsubara@gmail.com"} +] +dependencies = [] + +requires-python = ">=3.7" +readme = "README.rst" +license = {text = "MIT License"} +keywords = ["MySQL"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Database", +] + +[project.optional-dependencies] +"rsa" = [ + "cryptography" +] +"ed25519" = [ + "PyNaCl>=1.4.0" +] + +[project.urls] +"Project" = "https://github.com/PyMySQL/PyMySQL" +"Documentation" = "https://pymysql.readthedocs.io/" + +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +namespaces = false +include = ["pymysql"] +exclude = ["tests*", "pymysql.tests*"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e487e5e7..00000000 --- a/setup.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[flake8] -exclude = tests,build,.venv,docs -ignore = E203,W503,E722 -max_line_length=129 - -[metadata] -license = "MIT" -license_files = LICENSE - -author=yutaka.matsubara -author_email=yutaka.matsubara@gmail.com - -maintainer=Inada Naoki -maintainer_email=songofacandy@gmail.com diff --git a/setup.py b/setup.py deleted file mode 100755 index 7cdc692f..00000000 --- a/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup, find_packages - -version = "1.0.2" - -with open("./README.rst", encoding="utf-8") as f: - readme = f.read() - -setup( - name="PyMySQL", - version=version, - url="https://github.com/PyMySQL/PyMySQL/", - project_urls={ - "Documentation": "https://pymysql.readthedocs.io/", - }, - description="Pure Python MySQL Driver", - long_description=readme, - packages=find_packages(exclude=["tests*", "pymysql.tests*"]), - python_requires=">=3.7", - extras_require={ - "rsa": ["cryptography"], - "ed25519": ["PyNaCl>=1.4.0"], - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Topic :: Database", - ], - keywords="MySQL", -) From 5fa787694107c5a5dd7742852a0f830dc7bcf560 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 6 Feb 2023 12:40:18 +0100 Subject: [PATCH 205/292] Upgrade GitHub Actions (#1080) --- .github/workflows/lint.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 887a8f26..a3131ce2 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,10 +10,12 @@ on: jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.x - uses: psf/black@stable with: args: ". --diff --check" From b1399c95bcde8ef73cbc3a6d4e8bf767094bbd9e Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 7 Feb 2023 00:55:20 +0100 Subject: [PATCH 206/292] Upgrade more GitHub Actions (#1088) Followup to #1080 --- .github/workflows/test.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2b334503..993347f6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,7 +10,7 @@ concurrency: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -49,7 +49,7 @@ jobs: - /run/mysqld:/run/mysqld steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Workaround MySQL container permissions if: startsWith(matrix.db, 'mysql') @@ -58,7 +58,7 @@ jobs: /usr/bin/docker ps --all --filter status=exited --no-trunc --format "{{.ID}}" | xargs -r /usr/bin/docker start - name: Set up Python ${{ matrix.py }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} cache: 'pip' @@ -66,7 +66,7 @@ jobs: - name: Install dependency run: | - pip install -U -r requirements-dev.txt + pip install --upgrade -r requirements-dev.txt - name: Set up MySQL run: | @@ -107,16 +107,16 @@ jobs: coveralls: if: github.repository == 'PyMySQL/PyMySQL' name: Finish coveralls - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: test steps: - name: requirements. run: | echo coveralls > requirements.txt - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.x' cache: 'pip' - name: Finished From d894ab5c045fd4bc86edbe8321454b86410e12c4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 22 Mar 2023 19:54:05 +0900 Subject: [PATCH 207/292] Convert README to Markdown (#1093) --- README.md | 105 +++++++++++++++++++++++++++++++++++++ README.rst | 138 ------------------------------------------------- pyproject.toml | 2 +- 3 files changed, 106 insertions(+), 139 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 00000000..dec84080 --- /dev/null +++ b/README.md @@ -0,0 +1,105 @@ +[![Documentation Status](https://readthedocs.org/projects/pymysql/badge/?version=latest)](https://pymysql.readthedocs.io/) +[![image](https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github)](https://coveralls.io/github/PyMySQL/PyMySQL?branch=main) + +# PyMySQL + +This package contains a pure-Python MySQL client library, based on [PEP +249](https://www.python.org/dev/peps/pep-0249/). + +## Requirements + +- Python -- one of the following: + - [CPython](https://www.python.org/) : 3.7 and newer + - [PyPy](https://pypy.org/) : Latest 3.x version +- MySQL Server -- one of the following: + - [MySQL](https://www.mysql.com/) \>= 5.7 + - [MariaDB](https://mariadb.org/) \>= 10.3 + +## Installation + +Package is uploaded on [PyPI](https://pypi.org/project/PyMySQL). + +You can install it with pip: + + $ python3 -m pip install PyMySQL + +To use "sha256_password" or "caching_sha2_password" for authenticate, +you need to install additional dependency: + + $ python3 -m pip install PyMySQL[rsa] + +To use MariaDB's "ed25519" authentication method, you need to install +additional dependency: + + $ python3 -m pip install PyMySQL[ed25519] + +## Documentation + +Documentation is available online: + +For support, please refer to the +[StackOverflow](https://stackoverflow.com/questions/tagged/pymysql). + +## Example + +The following examples make use of a simple table + +``` sql +CREATE TABLE `users` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `email` varchar(255) COLLATE utf8_bin NOT NULL, + `password` varchar(255) COLLATE utf8_bin NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +AUTO_INCREMENT=1 ; +``` + +``` python +import pymysql.cursors + +# Connect to the database +connection = pymysql.connect(host='localhost', + user='user', + password='passwd', + database='db', + cursorclass=pymysql.cursors.DictCursor) + +with connection: + with connection.cursor() as cursor: + # Create a new record + sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" + cursor.execute(sql, ('webmaster@python.org', 'very-secret')) + + # connection is not autocommit by default. So you must commit to save + # your changes. + connection.commit() + + with connection.cursor() as cursor: + # Read a single record + sql = "SELECT `id`, `password` FROM `users` WHERE `email`=%s" + cursor.execute(sql, ('webmaster@python.org',)) + result = cursor.fetchone() + print(result) +``` + +This example will print: + +``` python +{'password': 'very-secret', 'id': 1} +``` + +## Resources + +- DB-API 2.0: +- MySQL Reference Manuals: +- MySQL client/server protocol: + +- "Connector" channel in MySQL Community Slack: + +- PyMySQL mailing list: + + +## License + +PyMySQL is released under the MIT License. See LICENSE for more +information. diff --git a/README.rst b/README.rst deleted file mode 100644 index 592b295a..00000000 --- a/README.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. image:: https://readthedocs.org/projects/pymysql/badge/?version=latest - :target: https://pymysql.readthedocs.io/ - :alt: Documentation Status - -.. image:: https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github - :target: https://coveralls.io/github/PyMySQL/PyMySQL?branch=main - - -PyMySQL -======= - -.. contents:: Table of Contents - :local: - -This package contains a pure-Python MySQL client library, based on `PEP 249`_. - -.. _`PEP 249`: https://www.python.org/dev/peps/pep-0249/ - - -Requirements -------------- - -* Python -- one of the following: - - - CPython_ : 3.7 and newer - - PyPy_ : Latest 3.x version - -* MySQL Server -- one of the following: - - - MySQL_ >= 5.7 - - MariaDB_ >= 10.3 - -.. _CPython: https://www.python.org/ -.. _PyPy: https://pypy.org/ -.. _MySQL: https://www.mysql.com/ -.. _MariaDB: https://mariadb.org/ - - -Installation ------------- - -Package is uploaded on `PyPI `_. - -You can install it with pip:: - - $ python3 -m pip install PyMySQL - -To use "sha256_password" or "caching_sha2_password" for authenticate, -you need to install additional dependency:: - - $ python3 -m pip install PyMySQL[rsa] - -To use MariaDB's "ed25519" authentication method, you need to install -additional dependency:: - - $ python3 -m pip install PyMySQL[ed25519] - - -Documentation -------------- - -Documentation is available online: https://pymysql.readthedocs.io/ - -For support, please refer to the `StackOverflow -`_. - - -Example -------- - -The following examples make use of a simple table - -.. code:: sql - - CREATE TABLE `users` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `email` varchar(255) COLLATE utf8_bin NOT NULL, - `password` varchar(255) COLLATE utf8_bin NOT NULL, - PRIMARY KEY (`id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin - AUTO_INCREMENT=1 ; - - -.. code:: python - - import pymysql.cursors - - # Connect to the database - connection = pymysql.connect(host='localhost', - user='user', - password='passwd', - database='db', - cursorclass=pymysql.cursors.DictCursor) - - with connection: - with connection.cursor() as cursor: - # Create a new record - sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)" - cursor.execute(sql, ('webmaster@python.org', 'very-secret')) - - # connection is not autocommit by default. So you must commit to save - # your changes. - connection.commit() - - with connection.cursor() as cursor: - # Read a single record - sql = "SELECT `id`, `password` FROM `users` WHERE `email`=%s" - cursor.execute(sql, ('webmaster@python.org',)) - result = cursor.fetchone() - print(result) - - -This example will print: - -.. code:: python - - {'password': 'very-secret', 'id': 1} - - -Resources ---------- - -* DB-API 2.0: https://www.python.org/dev/peps/pep-0249/ - -* MySQL Reference Manuals: https://dev.mysql.com/doc/ - -* MySQL client/server protocol: - https://dev.mysql.com/doc/internals/en/client-server-protocol.html - -* "Connector" channel in MySQL Community Slack: - https://lefred.be/mysql-community-on-slack/ - -* PyMySQL mailing list: https://groups.google.com/forum/#!forum/pymysql-users - -License -------- - -PyMySQL is released under the MIT License. See LICENSE for more information. diff --git a/pyproject.toml b/pyproject.toml index 3793a8c1..a0a36105 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors = [ dependencies = [] requires-python = ">=3.7" -readme = "README.rst" +readme = "README.md" license = {text = "MIT License"} keywords = ["MySQL"] classifiers = [ From adff5ee6bf62be0d1bbc7eb8cb49e310d258ad51 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 23 Mar 2023 18:11:35 +0900 Subject: [PATCH 208/292] Update MANIFEST.in --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index e9e1eebc..e2e577a9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst LICENSE CHANGELOG.md +include README.md LICENSE CHANGELOG.md From d0c2871192b9a53733f32158dade3ea2e1847eab Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 24 Mar 2023 01:41:54 +0900 Subject: [PATCH 209/292] Release v1.0.3rc1 (#1094) --- pymysql/__init__.py | 2 +- pyproject.toml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 5fe2aec5..291d5c6a 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 2, None) +VERSION = (1, 0, 3, "rc1") if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/pyproject.toml b/pyproject.toml index a0a36105..dbb82c8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,5 @@ [project] name = "PyMySQL" -version = "1.0.2" description = "Pure Python MySQL Driver" authors = [ {name = "Inada Naoki", email = "songofacandy@gmail.com"}, @@ -26,6 +25,7 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Topic :: Database", ] +dynamic = ["version"] [project.optional-dependencies] "rsa" = [ @@ -47,3 +47,6 @@ build-backend = "setuptools.build_meta" namespaces = false include = ["pymysql"] exclude = ["tests*", "pymysql.tests*"] + +[tool.setuptools.dynamic] +version = {attr = "pymysql.VERSION"} From 35bf026a7fda258277548ab93195972aeb867322 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Mar 2023 13:59:34 +0900 Subject: [PATCH 210/292] Fix setuptools didn't include pymysql.constants (#1096) Fix #1095 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index dbb82c8d..0f043181 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] namespaces = false -include = ["pymysql"] +include = ["pymysql*"] exclude = ["tests*", "pymysql.tests*"] [tool.setuptools.dynamic] From 7b0e0eab5fe0293a24adcdbdf479043eef939793 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 28 Mar 2023 12:34:54 +0900 Subject: [PATCH 211/292] v1.0.3 (#1097) --- pymysql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 291d5c6a..4b6cc2a9 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,7 +47,7 @@ ) -VERSION = (1, 0, 3, "rc1") +VERSION = (1, 0, 3, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: From 930b25034f1a3b6e3a202e072675f163770b25cb Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 28 Mar 2023 12:53:08 +0900 Subject: [PATCH 212/292] Fix VERSION for dynamic version (#1098) --- pymysql/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 4b6cc2a9..c0039c3f 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,11 +47,11 @@ ) -VERSION = (1, 0, 3, None) -if VERSION[3] is not None: +VERSION = (1, 0, 3) +if len(VERSION) > 3: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: - VERSION_STRING = "%d.%d.%d" % VERSION[:3] + VERSION_STRING = "%d.%d.%d" % VERSION threadsafety = 1 apilevel = "2.0" paramstyle = "pyformat" @@ -113,10 +113,7 @@ def Binary(x): def get_client_info(): # for MySQLdb compatibility - version = VERSION - if VERSION[3] is None: - version = VERSION[:3] - return ".".join(map(str, version)) + return VERSION_STRING # we include a doctored version_info here for MySQLdb compatibility From 57e2e93276c7b48e6ec5b99c1712e48661d92183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 28 Mar 2023 16:48:57 +0200 Subject: [PATCH 213/292] Remove redundant wheel dep from pyproject.toml (#1099) Remove the redundant `wheel` dependency, as it is added by the backend automatically. Listing it explicitly in the documentation was a historical mistake and has been fixed since, see: https://github.com/pypa/setuptools/commit/f7d30a9529378cf69054b5176249e5457aaf640a --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0f043181..a67031b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dynamic = ["version"] "Documentation" = "https://pymysql.readthedocs.io/" [build-system] -requires = ["setuptools>=61", "wheel"] +requires = ["setuptools>=61"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] From 885841f3fee416c222a75d83a81f74d3dcd71b51 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 31 Mar 2023 23:42:11 +0900 Subject: [PATCH 214/292] Add security policy --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..da9c516d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). +Tidelift will coordinate the fix and disclosure. From 72e7c580515588f0646c3322c3dba63dbcc90810 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 1 May 2023 19:22:22 +0900 Subject: [PATCH 215/292] Run lock-threads only on PyMySQL/PyMySQL --- .github/workflows/lock.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 5dde1354..780dd92d 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -9,7 +9,8 @@ permissions: pull-requests: write jobs: - action: + lock-threads: + if: github.repository == 'PyMySQL/PyMySQL' runs-on: ubuntu-latest steps: - uses: dessant/lock-threads@v4 From 101f6e970cb2df47f1363bca590aab88a809804c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 22 May 2023 10:44:35 +0000 Subject: [PATCH 216/292] Update FUNDING.yml --- .github/FUNDING.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 89fc5cf8..253a13ac 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,10 +1,10 @@ # These are supported funding model platforms -github: [methane] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: ["methane"] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +tidelift: "pypi/PyMySQL" # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username From a5e837f9de3b13abcef3500a1dc35fdbfa2f5784 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 23 May 2023 19:28:20 +0900 Subject: [PATCH 217/292] ci: Fix black options (#1109) --- .github/workflows/lint.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index a3131ce2..9d9eafb0 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,7 +18,8 @@ jobs: python-version: 3.x - uses: psf/black@stable with: - args: ". --diff --check" + options: "--check --verbose" + src: "." - name: Setup flake8 annotations uses: rbialon/flake8-annotations@v1 - name: flake8 From 1448310e1400a87267f2707eadceab00af4dedad Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 23 May 2023 19:28:34 +0900 Subject: [PATCH 218/292] Remove unused function (#1108) --- pymysql/cursors.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 2b5ccca9..b36f473c 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -95,13 +95,6 @@ def _nextset(self, unbuffered=False): def nextset(self): return self._nextset(False) - def _ensure_bytes(self, x, encoding=None): - if isinstance(x, str): - x = x.encode(encoding) - elif isinstance(x, (tuple, list)): - x = type(x)(self._ensure_bytes(v, encoding=encoding) for v in x) - return x - def _escape_args(self, args, conn): if isinstance(args, (tuple, list)): return tuple(conn.literal(arg) for arg in args) From 01ddf9d1b26d78d5d03e483d076544a5a50d7c47 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 23 May 2023 14:01:02 +0200 Subject: [PATCH 219/292] Expose `Cursor.warning_count` (#1056) In #774 automatic warnings were removed. This provides a way to check for existence of warnings without having to perform an additional query over the network. Co-authored-by: Inada Naoki --- CHANGELOG.md | 7 +++++++ pymysql/cursors.py | 5 +++++ pymysql/tests/test_SSCursor.py | 33 ++++++++++++++++++++++++++++++-- pymysql/tests/test_cursor.py | 20 +++++++++++++++++-- pymysql/tests/test_load_local.py | 32 +++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87c3f9e8..76fdb6a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changes +## v1.1.0 + +Release date: TBD + +* Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) + ## v1.0.3 Release date: TBD @@ -7,6 +13,7 @@ Release date: TBD * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 +* Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) ## v1.0.2 diff --git a/pymysql/cursors.py b/pymysql/cursors.py index b36f473c..e57fba76 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -32,6 +32,7 @@ class Cursor: def __init__(self, connection): self.connection = connection + self.warning_count = 0 self.description = None self.rownumber = 0 self.rowcount = -1 @@ -324,6 +325,7 @@ def _clear_result(self): self._result = None self.rowcount = 0 + self.warning_count = 0 self.description = None self.lastrowid = None self._rows = None @@ -334,6 +336,7 @@ def _do_get_result(self): self._result = result = conn._result self.rowcount = result.affected_rows + self.warning_count = result.warning_count self.description = result.description self.lastrowid = result.insert_id self._rows = result.rows @@ -435,6 +438,7 @@ def fetchone(self): self._check_executed() row = self.read_next() if row is None: + self.warning_count = self._result.warning_count return None self.rownumber += 1 return row @@ -468,6 +472,7 @@ def fetchmany(self, size=None): for i in range(size): row = self.read_next() if row is None: + self.warning_count = self._result.warning_count break rows.append(row) self.rownumber += 1 diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index a68a7769..d19d3e5d 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -3,13 +3,13 @@ try: from pymysql.tests import base import pymysql.cursors - from pymysql.constants import CLIENT + from pymysql.constants import CLIENT, ER except Exception: # For local testing from top-level directory, without installing sys.path.append("../pymysql") from pymysql.tests import base import pymysql.cursors - from pymysql.constants import CLIENT + from pymysql.constants import CLIENT, ER class TestSSCursor(base.PyMySQLTestCase): @@ -122,6 +122,35 @@ def test_SSCursor(self): cursor.execute("DROP TABLE IF EXISTS tz_data") cursor.close() + def test_warnings(self): + con = self.connect() + cur = con.cursor(pymysql.cursors.SSCursor) + cur.execute("DROP TABLE IF EXISTS `no_exists_table`") + self.assertEqual(cur.warning_count, 1) + + cur.execute("SHOW WARNINGS") + w = cur.fetchone() + self.assertEqual(w[1], ER.BAD_TABLE_ERROR) + self.assertIn( + "no_exists_table", + w[2], + ) + + # ensure unbuffered result is finished + self.assertIsNone(cur.fetchone()) + + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + self.assertIsNone(cur.fetchone()) + + self.assertEqual(cur.warning_count, 0) + + cur.execute("SELECT CAST('abc' AS SIGNED)") + # this ensures fully retrieving the unbuffered result + rows = cur.fetchmany(2) + self.assertEqual(len(rows), 1) + self.assertEqual(cur.warning_count, 1) + __all__ = ["TestSSCursor"] diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 783caf88..63ecce02 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -1,5 +1,4 @@ -import warnings - +from pymysql.constants import ER from pymysql.tests import base import pymysql.cursors @@ -129,3 +128,20 @@ def test_executemany(self): ) finally: cursor.execute("DROP TABLE IF EXISTS percent_test") + + def test_warnings(self): + con = self.connect() + cur = con.cursor() + cur.execute("DROP TABLE IF EXISTS `no_exists_table`") + self.assertEqual(cur.warning_count, 1) + + cur.execute("SHOW WARNINGS") + w = cur.fetchone() + self.assertEqual(w[1], ER.BAD_TABLE_ERROR) + self.assertIn( + "no_exists_table", + w[2], + ) + + cur.execute("SELECT 1") + self.assertEqual(cur.warning_count, 0) diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index b1b8128e..194c5be9 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -1,4 +1,5 @@ from pymysql import cursors, OperationalError, Warning +from pymysql.constants import ER from pymysql.tests import base import os @@ -63,6 +64,37 @@ def test_unbuffered_load_file(self): c = conn.cursor() c.execute("DROP TABLE test_load_local") + def test_load_warnings(self): + """Test load local infile produces the appropriate warnings""" + conn = self.connect() + c = conn.cursor() + c.execute("CREATE TABLE test_load_local (a INTEGER, b INTEGER)") + filename = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "data", + "load_local_warn_data.txt", + ) + try: + c.execute( + ( + "LOAD DATA LOCAL INFILE '{0}' INTO TABLE " + + "test_load_local FIELDS TERMINATED BY ','" + ).format(filename) + ) + self.assertEqual(1, c.warning_count) + + c.execute("SHOW WARNINGS") + w = c.fetchone() + + self.assertEqual(ER.TRUNCATED_WRONG_VALUE_FOR_FIELD, w[1]) + self.assertIn( + "incorrect integer value", + w[2].lower(), + ) + finally: + c.execute("DROP TABLE test_load_local") + c.close() + if __name__ == "__main__": import unittest From ea79b3216e948ca1095bc7802e798bc3eb9dd599 Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 23 May 2023 14:18:40 +0200 Subject: [PATCH 220/292] Add constants and tests related to query timeouts (#1033) --- pymysql/constants/ER.py | 3 + pymysql/tests/base.py | 8 +++ pymysql/tests/test_SSCursor.py | 101 +++++++++++++++++++++++++++++---- pymysql/tests/test_cursor.py | 67 ++++++++++++++++++++++ 4 files changed, 168 insertions(+), 11 deletions(-) diff --git a/pymysql/constants/ER.py b/pymysql/constants/ER.py index ddcc4e90..98729d12 100644 --- a/pymysql/constants/ER.py +++ b/pymysql/constants/ER.py @@ -470,5 +470,8 @@ WRONG_STRING_LENGTH = 1468 ERROR_LAST = 1468 +# MariaDB only +STATEMENT_TIMEOUT = 1969 +QUERY_TIMEOUT = 3024 # https://github.com/PyMySQL/PyMySQL/issues/607 CONSTRAINT_FAILED = 4025 diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index a87307a5..ff33bc4e 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -49,6 +49,14 @@ def mysql_server_is(self, conn, version_tuple): ) return server_version_tuple >= version_tuple + def get_mysql_vendor(self, conn): + server_version = conn.get_server_info() + + if "MariaDB" in server_version: + return "mariadb" + + return "mysql" + _connections = None @property diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index d19d3e5d..9cb5bafe 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -1,15 +1,8 @@ -import sys +import pytest -try: - from pymysql.tests import base - import pymysql.cursors - from pymysql.constants import CLIENT, ER -except Exception: - # For local testing from top-level directory, without installing - sys.path.append("../pymysql") - from pymysql.tests import base - import pymysql.cursors - from pymysql.constants import CLIENT, ER +from pymysql.tests import base +import pymysql.cursors +from pymysql.constants import CLIENT, ER class TestSSCursor(base.PyMySQLTestCase): @@ -122,6 +115,92 @@ def test_SSCursor(self): cursor.execute("DROP TABLE IF EXISTS tz_data") cursor.close() + def test_execution_time_limit(self): + # this method is similarly implemented in test_cursor + + conn = self.connect() + + # table creation and filling is SSCursor only as it's not provided by self.setUp() + self.safe_create_table( + conn, + "test", + "create table test (data varchar(10))", + ) + with conn.cursor() as cur: + cur.execute( + "insert into test (data) values " + "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" + ) + conn.commit() + + db_type = self.get_mysql_vendor(conn) + + with conn.cursor(pymysql.cursors.SSCursor) as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + + cur.execute(sql) + # unlike Cursor, SSCursor returns a list of tuples here + self.assertEqual( + cur.fetchall(), + [ + ("row1", 0), + ("row2", 0), + ("row3", 0), + ("row4", 0), + ("row5", 0), + ], + ) + + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + cur.execute(sql) + self.assertEqual(cur.fetchone(), ("row1", 0)) + + # this discards the previous unfinished query and raises an + # incomplete unbuffered query warning + with pytest.warns(UserWarning): + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + + # SSCursor will not read the EOF packet until we try to read + # another row. Skipping this will raise an incomplete unbuffered + # query warning in the next cur.execute(). + self.assertEqual(cur.fetchone(), None) + + if db_type == "mysql": + sql = "SELECT /*+ MAX_EXECUTION_TIME(1) */ data, sleep(1) FROM test" + else: + sql = "SET STATEMENT max_statement_time=0.001 FOR SELECT data, sleep(1) FROM test" + with pytest.raises(pymysql.err.OperationalError) as cm: + # in an unbuffered cursor the OperationalError may not show up + # until fetching the entire result + cur.execute(sql) + cur.fetchall() + + if db_type == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + self.assertEqual(cm.value.args[0], ER.QUERY_TIMEOUT) + else: + self.assertEqual(cm.value.args[0], ER.STATEMENT_TIMEOUT) + + # connection should still be fine at this point + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + def test_warnings(self): con = self.connect() cur = con.cursor(pymysql.cursors.SSCursor) diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 63ecce02..66d968df 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -2,6 +2,8 @@ from pymysql.tests import base import pymysql.cursors +import pytest + class CursorTest(base.PyMySQLTestCase): def setUp(self): @@ -18,6 +20,7 @@ def setUp(self): "insert into test (data) values " "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" ) + conn.commit() cursor.close() self.test_connection = pymysql.connect(**self.databases[0]) self.addCleanup(self.test_connection.close) @@ -129,6 +132,70 @@ def test_executemany(self): finally: cursor.execute("DROP TABLE IF EXISTS percent_test") + def test_execution_time_limit(self): + # this method is similarly implemented in test_SScursor + + conn = self.test_connection + db_type = self.get_mysql_vendor(conn) + + with conn.cursor(pymysql.cursors.Cursor) as cur: + # MySQL MAX_EXECUTION_TIME takes ms + # MariaDB max_statement_time takes seconds as int/float, introduced in 10.1 + + # this will sleep 0.01 seconds per row + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + + cur.execute(sql) + # unlike SSCursor, Cursor returns a tuple of tuples here + self.assertEqual( + cur.fetchall(), + ( + ("row1", 0), + ("row2", 0), + ("row3", 0), + ("row4", 0), + ("row5", 0), + ), + ) + + if db_type == "mysql": + sql = ( + "SELECT /*+ MAX_EXECUTION_TIME(2000) */ data, sleep(0.01) FROM test" + ) + else: + sql = "SET STATEMENT max_statement_time=2 FOR SELECT data, sleep(0.01) FROM test" + cur.execute(sql) + self.assertEqual(cur.fetchone(), ("row1", 0)) + + # this discards the previous unfinished query + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + + if db_type == "mysql": + sql = "SELECT /*+ MAX_EXECUTION_TIME(1) */ data, sleep(1) FROM test" + else: + sql = "SET STATEMENT max_statement_time=0.001 FOR SELECT data, sleep(1) FROM test" + with pytest.raises(pymysql.err.OperationalError) as cm: + # in a buffered cursor this should reliably raise an + # OperationalError + cur.execute(sql) + + if db_type == "mysql": + # this constant was only introduced in MySQL 5.7, not sure + # what was returned before, may have been ER_QUERY_INTERRUPTED + self.assertEqual(cm.value.args[0], ER.QUERY_TIMEOUT) + else: + self.assertEqual(cm.value.args[0], ER.STATEMENT_TIMEOUT) + + # connection should still be fine at this point + cur.execute("SELECT 1") + self.assertEqual(cur.fetchone(), (1,)) + def test_warnings(self): con = self.connect() cur = con.cursor() From 2ee4f706d34412a6d39417b92360bfa13ddc4e14 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 23 May 2023 21:46:00 +0900 Subject: [PATCH 221/292] Fix wrong merge --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76fdb6a7..ce74e84b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,6 @@ Release date: TBD * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 -* Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) ## v1.0.2 From 3cd76d7256416e3aa9575b3b9823c9491f92369c Mon Sep 17 00:00:00 2001 From: Richard Schwab Date: Tue, 23 May 2023 14:47:38 +0200 Subject: [PATCH 222/292] Fix SSCursor raising query timeout error on wrong query on MySQL DB (#1035) Fixes https://github.com/PyMySQL/PyMySQL/issues/1032#issuecomment-1030764742 --- CHANGELOG.md | 2 ++ pymysql/connections.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce74e84b..6dc75225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,10 @@ Release date: TBD +* Fixed SSCursor raising OperationalError for query timeouts on wrong statement (#1032) * Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) + ## v1.0.3 Release date: TBD diff --git a/pymysql/connections.py b/pymysql/connections.py index 3265d32e..f82b1951 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1262,7 +1262,20 @@ def _finish_unbuffered_query(self): # in fact, no way to stop MySQL from sending all the data after # executing a query, so we just spin, and wait for an EOF packet. while self.unbuffered_active: - packet = self.connection._read_packet() + try: + packet = self.connection._read_packet() + except err.OperationalError as e: + if e.args[0] in ( + ER.QUERY_TIMEOUT, + ER.STATEMENT_TIMEOUT, + ): + # if the query timed out we can simply ignore this error + self.unbuffered_active = False + self.connection = None + return + + raise + if self._check_packet_is_eof(packet): self.unbuffered_active = False self.connection = None # release reference to kill cyclic reference. From a6f53dbffa5ee6986b0c48c32e43bd071a04217d Mon Sep 17 00:00:00 2001 From: Gonzalo Sanchez Date: Tue, 23 May 2023 12:43:50 -0300 Subject: [PATCH 223/292] Make Cursor an iterator (#995) Fix #992 Co-authored-by: Gonzalo Sanchez Co-authored-by: Inada Naoki --- pymysql/cursors.py | 11 +++++++---- pymysql/tests/test_cursor.py | 8 ++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index e57fba76..d8a93c78 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -342,7 +342,13 @@ def _do_get_result(self): self._rows = result.rows def __iter__(self): - return iter(self.fetchone, None) + return self + + def __next__(self): + row = self.fetchone() + if row is None: + raise StopIteration + return row Warning = err.Warning Error = err.Error @@ -459,9 +465,6 @@ def fetchall_unbuffered(self): """ return iter(self.fetchone, None) - def __iter__(self): - return self.fetchall_unbuffered() - def fetchmany(self, size=None): """Fetch many.""" self._check_executed() diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 66d968df..16d297f6 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -25,6 +25,14 @@ def setUp(self): self.test_connection = pymysql.connect(**self.databases[0]) self.addCleanup(self.test_connection.close) + def test_cursor_is_iterator(self): + """Test that the cursor is an iterator""" + conn = self.test_connection + cursor = conn.cursor() + cursor.execute("select * from test") + self.assertEqual(cursor.__iter__(), cursor) + self.assertEqual(cursor.__next__(), ("row1",)) + def test_cleanup_rows_unbuffered(self): conn = self.test_connection cursor = conn.cursor(pymysql.cursors.SSCursor) From 4072c7fff9871f6eb811b9b4442bbb5411b6d01b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 01:17:31 +0900 Subject: [PATCH 224/292] ci: Update CodeQL workflow (#1110) --- .github/workflows/codeql-analysis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d559b1cd..a4c434c5 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,21 +27,16 @@ jobs: strategy: fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: - languages: ${{ matrix.language }} + languages: "python" # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. From 2fe0b1293d1a24140f6d35f5ff37d7b5a46a28e1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 14:17:19 +0900 Subject: [PATCH 225/292] Use Ruff instead of flake8 (#1112) --- .flake8 | 4 - .github/workflows/lint.yaml | 13 +--- pymysql/_auth.py | 3 +- pymysql/connections.py | 15 ++-- pymysql/converters.py | 5 +- pymysql/tests/__init__.py | 19 ----- pymysql/tests/base.py | 1 - pymysql/tests/test_basic.py | 28 +++++-- pymysql/tests/test_connection.py | 75 +++++++------------ pymysql/tests/test_cursor.py | 3 +- pymysql/tests/test_issues.py | 2 - pymysql/tests/test_load_local.py | 8 +- .../tests/thirdparty/test_MySQLdb/__init__.py | 2 - .../thirdparty/test_MySQLdb/capabilities.py | 1 - .../tests/thirdparty/test_MySQLdb/dbapi20.py | 20 +++-- .../test_MySQLdb/test_MySQLdb_capabilities.py | 1 - .../test_MySQLdb/test_MySQLdb_dbapi20.py | 4 - .../test_MySQLdb/test_MySQLdb_nonstandard.py | 1 - pyproject.toml | 6 ++ 19 files changed, 89 insertions(+), 122 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 3f1c38a3..00000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -exclude = tests,build,.venv,docs -ignore = E203,W503,E722 -max_line_length=129 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 9d9eafb0..77edb0c3 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -2,6 +2,7 @@ name: Lint on: push: + branches: ["main"] paths: - '**.py' pull_request: @@ -13,16 +14,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.x + - uses: psf/black@stable with: options: "--check --verbose" src: "." - - name: Setup flake8 annotations - uses: rbialon/flake8-annotations@v1 - - name: flake8 - run: | - pip install flake8 - flake8 pymysql + + - uses: chartboost/ruff-action@v1 diff --git a/pymysql/_auth.py b/pymysql/_auth.py index f6c9eb96..99987b77 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -141,7 +141,8 @@ def sha2_rsa_encrypt(password, salt, public_key): """ if not _have_cryptography: raise RuntimeError( - "'cryptography' package is required for sha256_password or caching_sha2_password auth methods" + "'cryptography' package is required for sha256_password or" + + " caching_sha2_password auth methods" ) message = _xor_password(password + b"\0", salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) diff --git a/pymysql/connections.py b/pymysql/connections.py index f82b1951..7bbc089f 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -108,8 +108,10 @@ class Connection: the interface from which to connect to the host. Argument can be a hostname or an IP address. :param unix_socket: Use a unix socket rather than TCP/IP. - :param read_timeout: The timeout for reading from the connection in seconds (default: None - no timeout) - :param write_timeout: The timeout for writing to the connection in seconds (default: None - no timeout) + :param read_timeout: The timeout for reading from the connection in seconds. + (default: None - no timeout) + :param write_timeout: The timeout for writing to the connection in seconds. + (default: None - no timeout) :param charset: Charset to use. :param sql_mode: Default SQL_MODE to use. :param read_default_file: @@ -130,7 +132,8 @@ class Connection: :param ssl_ca: Path to the file that contains a PEM-formatted CA certificate. :param ssl_cert: Path to the file that contains a PEM-formatted client certificate. :param ssl_disabled: A boolean value that disables usage of TLS. - :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate. + :param ssl_key: Path to the file that contains a PEM-formatted private key for + the client certificate. :param ssl_verify_cert: Set to true to check the server certificate's validity. :param ssl_verify_identity: Set to true to check the server's identity. :param read_default_group: Group to read from in the configuration file. @@ -533,7 +536,8 @@ def cursor(self, cursor=None): Create a new cursor to execute queries with. :param cursor: The type of cursor to create. None means use Cursor. - :type cursor: :py:class:`Cursor`, :py:class:`SSCursor`, :py:class:`DictCursor`, or :py:class:`SSDictCursor`. + :type cursor: :py:class:`Cursor`, :py:class:`SSCursor`, :py:class:`DictCursor`, + or :py:class:`SSDictCursor`. """ if cursor: return cursor(self) @@ -1228,7 +1232,8 @@ def _check_packet_is_eof(self, packet): # TODO: Support CLIENT.DEPRECATE_EOF # 1) Add DEPRECATE_EOF to CAPABILITIES # 2) Mask CAPABILITIES with server_capabilities - # 3) if server_capabilities & CLIENT.DEPRECATE_EOF: use OKPacketWrapper instead of EOFPacketWrapper + # 3) if server_capabilities & CLIENT.DEPRECATE_EOF: + # use OKPacketWrapper instead of EOFPacketWrapper wp = EOFPacketWrapper(packet) self.warning_count = wp.warning_count self.has_next = wp.has_next diff --git a/pymysql/converters.py b/pymysql/converters.py index 2acc3e58..1adac752 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -120,7 +120,10 @@ def escape_time(obj, mapping=None): def escape_datetime(obj, mapping=None): if obj.microsecond: - fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" + fmt = ( + "'{0.year:04}-{0.month:02}-{0.day:02}" + + " {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'" + ) else: fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}'" return fmt.format(obj) diff --git a/pymysql/tests/__init__.py b/pymysql/tests/__init__.py index fe3b1d0f..e69de29b 100644 --- a/pymysql/tests/__init__.py +++ b/pymysql/tests/__init__.py @@ -1,19 +0,0 @@ -# Sorted by alphabetical order -from pymysql.tests.test_DictCursor import * -from pymysql.tests.test_SSCursor import * -from pymysql.tests.test_basic import * -from pymysql.tests.test_connection import * -from pymysql.tests.test_converters import * -from pymysql.tests.test_cursor import * -from pymysql.tests.test_err import * -from pymysql.tests.test_issues import * -from pymysql.tests.test_load_local import * -from pymysql.tests.test_nextset import * -from pymysql.tests.test_optionfile import * - -from pymysql.tests.thirdparty import * - -if __name__ == "__main__": - import unittest - - unittest.main() diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index ff33bc4e..b5094563 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -1,4 +1,3 @@ -import gc import json import os import re diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index 8af07da0..ecf043f6 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -6,7 +6,6 @@ import pymysql.cursors from pymysql.tests import base -from pymysql.err import ProgrammingError __all__ = ["TestConversion", "TestCursor", "TestBulkInserts"] @@ -18,7 +17,22 @@ def test_datatypes(self): conn = self.connect() c = conn.cursor() c.execute( - "create table test_datatypes (b bit, i int, l bigint, f real, s varchar(32), u varchar(32), bb blob, d date, dt datetime, ts timestamp, td time, t time, st datetime)" + """ +create table test_datatypes ( + b bit, + i int, + l bigint, + f real, + s varchar(32), + u varchar(32), + bb blob, + d date, + dt datetime, + ts timestamp, + td time, + t time, + st datetime) +""" ) try: # insert values @@ -38,7 +52,8 @@ def test_datatypes(self): time.localtime(), ) c.execute( - "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values" + " (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", v, ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") @@ -54,7 +69,8 @@ def test_datatypes(self): # check nulls c.execute( - "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + "insert into test_datatypes (b,i,l,f,s,u,bb,d,dt,td,t,st)" + " values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", [None] * 12, ) c.execute("select b,i,l,f,s,u,bb,d,dt,td,t,st from test_datatypes") @@ -156,7 +172,8 @@ def test_timedelta(self): conn = self.connect() c = conn.cursor() c.execute( - "select time('12:30'), time('23:12:59'), time('23:12:59.05100'), time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')" + "select time('12:30'), time('23:12:59'), time('23:12:59.05100')," + + " time('-12:30'), time('-23:12:59'), time('-23:12:59.05100'), time('-00:30')" ) self.assertEqual( ( @@ -317,7 +334,6 @@ class TestBulkInserts(base.PyMySQLTestCase): def setUp(self): super(TestBulkInserts, self).setUp() self.conn = conn = self.connect() - c = conn.cursor(self.cursor_type) # create a table and some data to query self.safe_create_table( diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index d6fb5e52..bbaf3dec 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -1,6 +1,5 @@ import datetime import ssl -import sys import pytest import time from unittest import mock @@ -145,8 +144,8 @@ def realtestSocketAuth(self): TestAuthentication.osuser + "@localhost", self.databases[0]["database"], self.socket_plugin_name, - ) as u: - c = pymysql.connect(user=TestAuthentication.osuser, **self.db) + ): + pymysql.connect(user=TestAuthentication.osuser, **self.db) class Dialog: fail = False @@ -168,7 +167,7 @@ def __init__(self, con): def authenticate(self, pkt): while True: flag = pkt.read_uint8() - echo = (flag & 0x06) == 0x02 + # echo = (flag & 0x06) == 0x02 last = (flag & 0x01) == 0x01 prompt = pkt.read_all() @@ -220,7 +219,7 @@ def realTestDialogAuthTwoQuestions(self): self.databases[0]["database"], "two_questions", "notverysecret", - ) as u: + ): with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user="pymysql_2q", **self.db) pymysql.connect( @@ -262,7 +261,7 @@ def realTestDialogAuthThreeAttempts(self): self.databases[0]["database"], "three_attempts", "stillnotverysecret", - ) as u: + ): pymysql.connect( user="pymysql_3a", auth_plugin_map={b"dialog": TestAuthentication.Dialog}, @@ -357,9 +356,9 @@ def realTestPamAuth(self): self.databases[0]["database"], "pam", os.environ.get("PAMSERVICE"), - ) as u: + ): try: - c = pymysql.connect(user=TestAuthentication.osuser, **db) + pymysql.connect(user=TestAuthentication.osuser, **db) db["password"] = "very bad guess at password" with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( @@ -371,7 +370,8 @@ def realTestPamAuth(self): ) except pymysql.OperationalError as e: self.assertEqual(1045, e.args[0]) - # we had 'bad guess at password' work with pam. Well at least we get a permission denied here + # we had 'bad guess at password' work with pam. Well at least we get + # a permission denied here with self.assertRaises(pymysql.err.OperationalError): pymysql.connect( user=TestAuthentication.osuser, @@ -397,12 +397,13 @@ def testAuthSHA256(self): "pymysql_sha256@localhost", self.databases[0]["database"], "sha256_password", - ) as u: + ): c.execute("SET PASSWORD FOR 'pymysql_sha256'@'localhost' ='Sh@256Pa33'") c.execute("FLUSH PRIVILEGES") db = self.db.copy() db["password"] = "Sh@256Pa33" - # Although SHA256 is supported, need the configuration of public key of the mysql server. Currently will get error by this test. + # Although SHA256 is supported, need the configuration of public key of + # the mysql server. Currently will get error by this test. with self.assertRaises(pymysql.err.OperationalError): pymysql.connect(user="pymysql_sha256", **db) @@ -423,7 +424,7 @@ def testAuthEd25519(self): self.databases[0]["database"], "ed25519", empty_pass, - ) as u: + ): pymysql.connect(user="pymysql_ed25519", password="", **db) with TempUser( @@ -432,7 +433,7 @@ def testAuthEd25519(self): self.databases[0]["database"], "ed25519", non_empty_pass, - ) as u: + ): pymysql.connect(user="pymysql_ed25519", password="ed25519_password", **db) @@ -441,7 +442,7 @@ def test_utf8mb4(self): """This test requires MySQL >= 5.5""" arg = self.databases[0].copy() arg["charset"] = "utf8mb4" - conn = pymysql.connect(**arg) + pymysql.connect(**arg) def test_largedata(self): """Large query and response (>=16MB)""" @@ -544,9 +545,7 @@ def test_defer_connect(self): def test_ssl_connect(self): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -565,9 +564,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_called_with("cipher") dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -585,9 +582,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -601,9 +596,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -620,9 +613,7 @@ def test_ssl_connect(self): for ssl_verify_cert in (True, "1", "yes", "true"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -641,9 +632,7 @@ def test_ssl_connect(self): for ssl_verify_cert in (None, False, "0", "no", "false"): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -663,9 +652,7 @@ def test_ssl_connect(self): for ssl_ca in ("ca", None): for ssl_verify_cert in ("foo", "bar", ""): dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -686,9 +673,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -705,9 +690,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -722,9 +705,7 @@ def test_ssl_connect(self): assert not create_default_context.called dummy_ssl_context = mock.Mock(options=0) - with mock.patch( - "pymysql.connections.Connection.connect" - ) as connect, mock.patch( + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -762,21 +743,18 @@ def test_escape_string(self): def test_escape_builtin_encoders(self): con = self.connect() - cur = con.cursor() val = datetime.datetime(2012, 3, 4, 5, 6) self.assertEqual(con.escape(val, con.encoders), "'2012-03-04 05:06:00'") def test_escape_custom_object(self): con = self.connect() - cur = con.cursor() mapping = {Foo: escape_foo} self.assertEqual(con.escape(Foo(), mapping), "bar") def test_escape_fallback_encoder(self): con = self.connect() - cur = con.cursor() class Custom(str): pass @@ -786,13 +764,11 @@ class Custom(str): def test_escape_no_default(self): con = self.connect() - cur = con.cursor() self.assertRaises(TypeError, con.escape, 42, {}) def test_escape_dict_value(self): con = self.connect() - cur = con.cursor() mapping = con.encoders.copy() mapping[Foo] = escape_foo @@ -800,7 +776,6 @@ def test_escape_dict_value(self): def test_escape_list_item(self): con = self.connect() - cur = con.cursor() mapping = con.encoders.copy() mapping[Foo] = escape_foo diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 16d297f6..6666ab88 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -105,7 +105,8 @@ def test_executemany(self): ) assert m is not None - # cursor._executed must bee "insert into test (data) values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)" + # cursor._executed must bee "insert into test (data) + # values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)" # list args data = range(10) cursor.executemany("insert into test (data) values (%s)", data) diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 733d56a1..7f361c94 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -1,12 +1,10 @@ import datetime import time import warnings -import sys import pytest import pymysql -from pymysql import cursors from pymysql.tests import base __all__ = ["TestOldIssues", "TestNewIssues", "TestGitHubIssues"] diff --git a/pymysql/tests/test_load_local.py b/pymysql/tests/test_load_local.py index 194c5be9..50922142 100644 --- a/pymysql/tests/test_load_local.py +++ b/pymysql/tests/test_load_local.py @@ -1,4 +1,4 @@ -from pymysql import cursors, OperationalError, Warning +from pymysql import cursors, OperationalError from pymysql.constants import ER from pymysql.tests import base @@ -36,7 +36,8 @@ def test_load_file(self): ) try: c.execute( - f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local" + + " FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) @@ -53,7 +54,8 @@ def test_unbuffered_load_file(self): ) try: c.execute( - f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local FIELDS TERMINATED BY ','" + f"LOAD DATA LOCAL INFILE '{filename}' INTO TABLE test_load_local" + + " FIELDS TERMINATED BY ','" ) c.execute("SELECT COUNT(*) FROM test_load_local") self.assertEqual(22749, c.fetchone()[0]) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py index 57c42ce7..501bfd2d 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/__init__.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/__init__.py @@ -1,6 +1,4 @@ -from .test_MySQLdb_capabilities import test_MySQLdb as test_capabilities from .test_MySQLdb_nonstandard import * -from .test_MySQLdb_dbapi20 import test_MySQLdb as test_dbapi2 if __name__ == "__main__": import unittest diff --git a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py index 0276a558..bb47cc5f 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/capabilities.py @@ -4,7 +4,6 @@ Adapted from a script by M-A Lemburg. """ -import sys from time import time import unittest diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 30620ce4..83851295 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -225,7 +225,7 @@ def test_rollback(self): def test_cursor(self): con = self._connect() try: - cur = con.cursor() + con.cursor() finally: con.close() @@ -810,28 +810,26 @@ def test_None(self): con.close() def test_Date(self): - d1 = self.driver.Date(2002, 12, 25) - d2 = self.driver.DateFromTicks(time.mktime((2002, 12, 25, 0, 0, 0, 0, 0, 0))) + self.driver.Date(2002, 12, 25) + self.driver.DateFromTicks(time.mktime((2002, 12, 25, 0, 0, 0, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(d1),str(d2)) def test_Time(self): - t1 = self.driver.Time(13, 45, 30) - t2 = self.driver.TimeFromTicks(time.mktime((2001, 1, 1, 13, 45, 30, 0, 0, 0))) + self.driver.Time(13, 45, 30) + self.driver.TimeFromTicks(time.mktime((2001, 1, 1, 13, 45, 30, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Timestamp(self): - t1 = self.driver.Timestamp(2002, 12, 25, 13, 45, 30) - t2 = self.driver.TimestampFromTicks( - time.mktime((2002, 12, 25, 13, 45, 30, 0, 0, 0)) - ) + self.driver.Timestamp(2002, 12, 25, 13, 45, 30) + self.driver.TimestampFromTicks(time.mktime((2002, 12, 25, 13, 45, 30, 0, 0, 0))) # Can we assume this? API doesn't specify, but it seems implied # self.assertEqual(str(t1),str(t2)) def test_Binary(self): - b = self.driver.Binary(b"Something") - b = self.driver.Binary(b"") + self.driver.Binary(b"Something") + self.driver.Binary(b"") def test_STRING(self): self.assertTrue(hasattr(self.driver, "STRING"), "module.STRING must be defined") diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py index 11bfdbe2..6a2894a5 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py @@ -1,5 +1,4 @@ from . import capabilities -import unittest import pymysql from pymysql.tests import base import warnings diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index bc1e1b2e..c68289fe 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -2,8 +2,6 @@ import pymysql from pymysql.tests import base -import unittest - class test_MySQLdb(dbapi20.DatabaseAPI20Test): driver = pymysql @@ -181,8 +179,6 @@ def help_nextset_tearDown(self, cur): cur.execute("drop procedure deleteme") def test_nextset(self): - from warnings import warn - con = self._connect() try: cur = con.cursor() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py index b8d4bb1e..1545fbb5 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py @@ -1,4 +1,3 @@ -import sys import unittest import pymysql diff --git a/pyproject.toml b/pyproject.toml index a67031b3..48fe3660 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,9 @@ exclude = ["tests*", "pymysql.tests*"] [tool.setuptools.dynamic] version = {attr = "pymysql.VERSION"} + +[tool.ruff] +line-length = 99 +exclude = [ + "pymysql/tests/thirdparty", +] From d02e090e7a4766584750720d058bcc8e46eec48f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 14:50:22 +0900 Subject: [PATCH 226/292] Use Codecov instead of coveralls. (#1113) --- .github/workflows/test.yaml | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 993347f6..bea7747c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,6 +44,7 @@ jobs: - 3306:3306 env: MYSQL_ALLOW_EMPTY_PASSWORD: yes + MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: yes options: "--name=mysqld" volumes: - /run/mysqld:/run/mysqld @@ -96,32 +97,6 @@ jobs: docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" pytest -v --cov --cov-config .coveragerc tests/test_auth.py; - - name: Report coverage + - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - run: coveralls --service=github - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_FLAG_NAME: ${{ matrix.py }}-${{ matrix.db }} - COVERALLS_PARALLEL: true - - coveralls: - if: github.repository == 'PyMySQL/PyMySQL' - name: Finish coveralls - runs-on: ubuntu-latest - needs: test - steps: - - name: requirements. - run: | - echo coveralls > requirements.txt - - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - cache: 'pip' - - - name: Finished - run: | - pip install --upgrade coveralls - coveralls --finish --service=github - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: codecov/codecov-action@v3 From f5c0ac217b08e8a59f382bd252491de9f73d6f6a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 14:52:36 +0900 Subject: [PATCH 227/292] Update README codecov badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dec84080..6e6a6bf2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Documentation Status](https://readthedocs.org/projects/pymysql/badge/?version=latest)](https://pymysql.readthedocs.io/) -[![image](https://coveralls.io/repos/PyMySQL/PyMySQL/badge.svg?branch=main&service=github)](https://coveralls.io/github/PyMySQL/PyMySQL?branch=main) +[![codecov](https://codecov.io/gh/PyMySQL/PyMySQL/branch/main/graph/badge.svg?token=ppEuaNXBW4)](https://codecov.io/gh/PyMySQL/PyMySQL) # PyMySQL From b39a43ade46eaacb081615a82bdc14ef62974ccf Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 16:53:04 +0900 Subject: [PATCH 228/292] ci: Fix MySQL 8 build overwrite previous coverage --- .github/workflows/test.yaml | 2 +- pyproject.toml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bea7747c..c3275cca 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -95,7 +95,7 @@ jobs: docker cp mysqld:/var/lib/mysql/server-cert.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-key.pem "${HOME}" docker cp mysqld:/var/lib/mysql/client-cert.pem "${HOME}" - pytest -v --cov --cov-config .coveragerc tests/test_auth.py; + pytest -v --cov-append --cov-config .coveragerc tests/test_auth.py; - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' diff --git a/pyproject.toml b/pyproject.toml index 48fe3660..18714779 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,3 +56,8 @@ line-length = 99 exclude = [ "pymysql/tests/thirdparty", ] + +[tool.pdm.dev-dependencies] +dev = [ + "pytest-cov>=4.0.0", +] From 92287000831deed476e6d4a8341c6210f984bda5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 24 May 2023 22:24:24 +0900 Subject: [PATCH 229/292] optionfile: Replace `_` with `-` (#1114) Fix #1020 --- pymysql/optionfile.py | 3 +++ pymysql/tests/test_optionfile.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pymysql/optionfile.py b/pymysql/optionfile.py index 432621b7..c36f1625 100644 --- a/pymysql/optionfile.py +++ b/pymysql/optionfile.py @@ -13,6 +13,9 @@ def __remove_quotes(self, value): return value[1:-1] return value + def optionxform(self, key): + return key.lower().replace("_", "-") + def get(self, section, option): value = configparser.RawConfigParser.get(self, section, option) return self.__remove_quotes(value) diff --git a/pymysql/tests/test_optionfile.py b/pymysql/tests/test_optionfile.py index 39bd47c4..d13553dd 100644 --- a/pymysql/tests/test_optionfile.py +++ b/pymysql/tests/test_optionfile.py @@ -21,4 +21,4 @@ def test_string(self): parser.read_file(StringIO(_cfg_file)) self.assertEqual(parser.get("default", "string"), "foo") self.assertEqual(parser.get("default", "quoted"), "bar") - self.assertEqual(parser.get("default", "single_quoted"), "foobar") + self.assertEqual(parser.get("default", "single-quoted"), "foobar") From bfbc6a53db56d37993837ea59146995e7410b41b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 00:08:34 +0900 Subject: [PATCH 230/292] Cursor.fetchall() always return list. (#1115) Cursor.fetchmany() returns empty tuple when exhausted all rows. It is for Django compatibility. Fix #1042. --- pymysql/cursors.py | 8 +++++++- pymysql/tests/test_nextset.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index d8a93c78..e098e7de 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -282,6 +282,8 @@ def fetchmany(self, size=None): """Fetch several rows.""" self._check_executed() if self._rows is None: + # Django expects () for EOF. + # https://github.com/django/django/blob/0c1518ee429b01c145cf5b34eab01b0b92f8c246/django/db/backends/mysql/features.py#L8 return () end = self.rownumber + (size or self.arraysize) result = self._rows[self.rownumber : end] @@ -292,7 +294,7 @@ def fetchall(self): """Fetch all the rows.""" self._check_executed() if self._rows is None: - return () + return [] if self.rownumber: result = self._rows[self.rownumber :] else: @@ -479,6 +481,10 @@ def fetchmany(self, size=None): break rows.append(row) self.rownumber += 1 + if not rows: + # Django expects () for EOF. + # https://github.com/django/django/blob/0c1518ee429b01c145cf5b34eab01b0b92f8c246/django/db/backends/mysql/features.py#L8 + return () return rows def scroll(self, value, mode="relative"): diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index 28972325..4b6b2a77 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -38,7 +38,7 @@ def test_nextset_error(self): self.assertEqual([(i,)], list(cur)) with self.assertRaises(pymysql.ProgrammingError): cur.nextset() - self.assertEqual((), cur.fetchall()) + self.assertEqual([], cur.fetchall()) def test_ok_and_next(self): cur = self.connect(client_flag=CLIENT.MULTI_STATEMENTS).cursor() From bd3bd014999475242b5963b1af7990beaa6af6b5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 00:33:50 +0900 Subject: [PATCH 231/292] Fix LOAD DATA LOCAL INFILE write EOF packet on closed connection. (#1116) Fix #989 --- .gitignore | 1 + pymysql/connections.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 98f4d45c..09a5654f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ /pymysql/tests/databases.json __pycache__ Pipfile.lock +pdm.lock diff --git a/pymysql/connections.py b/pymysql/connections.py index 7bbc089f..ef3342aa 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -1370,7 +1370,7 @@ def send_data(self): """Send data packets from the local file to the server""" if not self.connection._sock: raise err.InterfaceError(0, "") - conn = self.connection + conn: Connection = self.connection try: with open(self.filename, "rb") as open_file: @@ -1388,5 +1388,6 @@ def send_data(self): f"Can't find file '{self.filename}'", ) finally: - # send the empty packet to signify we are done sending data - conn.write_packet(b"") + if not conn._closed: + # send the empty packet to signify we are done sending data + conn.write_packet(b"") From 9a694a16a3a98ebf53cd14a1361db6c9faadba8f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 00:52:43 +0900 Subject: [PATCH 232/292] Deprecate Cursor.Error access (#1117) Fix #1111. --- pymysql/cursors.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/pymysql/cursors.py b/pymysql/cursors.py index e098e7de..84564a08 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -1,4 +1,5 @@ import re +import warnings from . import err @@ -352,16 +353,29 @@ def __next__(self): raise StopIteration return row - Warning = err.Warning - Error = err.Error - InterfaceError = err.InterfaceError - DatabaseError = err.DatabaseError - DataError = err.DataError - OperationalError = err.OperationalError - IntegrityError = err.IntegrityError - InternalError = err.InternalError - ProgrammingError = err.ProgrammingError - NotSupportedError = err.NotSupportedError + def __getattr__(self, name): + # DB-API 2.0 optional extension says these errors can be accessed + # via Connection object. But MySQLdb had defined them on Cursor object. + if name in ( + "Warning", + "Error", + "InterfaceError", + "DatabaseError", + "DataError", + "OperationalError", + "IntegrityError", + "InternalError", + "ProgrammingError", + "NotSupportedError", + ): + # Deprecated since v1.1 + warnings.warn( + "PyMySQL errors hould be accessed from `pymysql` package", + DeprecationWarning, + stacklevel=2, + ) + return getattr(err, name) + raise AttributeError(name) class DictCursorMixin: From 103004d6ed59d8eef95fe069e8ca4f60d4965be3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 01:02:27 +0900 Subject: [PATCH 233/292] Run pyupgrade (#1118) --- pymysql/charset.py | 2 +- pymysql/connections.py | 20 +++++++++++--------- pymysql/cursors.py | 4 ++-- pymysql/protocol.py | 4 ++-- pymysql/tests/base.py | 4 ++-- pymysql/tests/test_DictCursor.py | 4 ++-- pymysql/tests/test_basic.py | 2 +- pymysql/tests/test_connection.py | 4 ++-- pymysql/tests/test_cursor.py | 2 +- pymysql/tests/test_issues.py | 4 ++-- 10 files changed, 26 insertions(+), 24 deletions(-) diff --git a/pymysql/charset.py b/pymysql/charset.py index ac87c53d..cdc02164 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -7,7 +7,7 @@ def __init__(self, id, name, collation, is_default): self.is_default = is_default == "Yes" def __repr__(self): - return "Charset(id=%s, name=%r, collation=%r)" % ( + return "Charset(id={}, name={!r}, collation={!r})".format( self.id, self.name, self.collation, diff --git a/pymysql/connections.py b/pymysql/connections.py index ef3342aa..d161e789 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -528,7 +528,9 @@ def escape_string(self, s): def _quote_bytes(self, s): if self.server_status & SERVER_STATUS.SERVER_STATUS_NO_BACKSLASH_ESCAPES: - return "'%s'" % (s.replace(b"'", b"''").decode("ascii", "surrogateescape"),) + return "'{}'".format( + s.replace(b"'", b"''").decode("ascii", "surrogateescape") + ) return converters.escape_bytes(s) def cursor(self, cursor=None): @@ -621,7 +623,7 @@ def connect(self, sock=None): (self.host, self.port), self.connect_timeout, **kwargs ) break - except (OSError, IOError) as e: + except OSError as e: if e.errno == errno.EINTR: continue raise @@ -662,7 +664,7 @@ def connect(self, sock=None): if isinstance(e, (OSError, IOError)): exc = err.OperationalError( CR.CR_CONN_HOST_ERROR, - "Can't connect to MySQL server on %r (%s)" % (self.host, e), + f"Can't connect to MySQL server on {self.host!r} ({e})", ) # Keep original exception and traceback to investigate error. exc.original_exception = e @@ -739,13 +741,13 @@ def _read_bytes(self, num_bytes): try: data = self._rfile.read(num_bytes) break - except (IOError, OSError) as e: + except OSError as e: if e.errno == errno.EINTR: continue self._force_close() raise err.OperationalError( CR.CR_SERVER_LOST, - "Lost connection to MySQL server during query (%s)" % (e,), + f"Lost connection to MySQL server during query ({e})", ) except BaseException: # Don't convert unknown exception to MySQLError. @@ -762,10 +764,10 @@ def _write_bytes(self, data): self._sock.settimeout(self._write_timeout) try: self._sock.sendall(data) - except IOError as e: + except OSError as e: self._force_close() raise err.OperationalError( - CR.CR_SERVER_GONE_ERROR, "MySQL server has gone away (%r)" % (e,) + CR.CR_SERVER_GONE_ERROR, f"MySQL server has gone away ({e!r})" ) def _read_query_result(self, unbuffered=False): @@ -1006,7 +1008,7 @@ def _process_auth(self, plugin_name, auth_packet): else: raise err.OperationalError( CR.CR_AUTH_PLUGIN_CANNOT_LOAD, - "Authentication plugin '%s' not configured" % (plugin_name,), + f"Authentication plugin '{plugin_name}' not configured", ) pkt = self._read_packet() pkt.check_error() @@ -1382,7 +1384,7 @@ def send_data(self): if not chunk: break conn.write_packet(chunk) - except IOError: + except OSError: raise err.OperationalError( ER.FILE_NOT_FOUND, f"Can't find file '{self.filename}'", diff --git a/pymysql/cursors.py b/pymysql/cursors.py index 84564a08..8be05ca2 100644 --- a/pymysql/cursors.py +++ b/pymysql/cursors.py @@ -262,7 +262,7 @@ def callproc(self, procname, args=()): ) self.nextset() - q = "CALL %s(%s)" % ( + q = "CALL {}({})".format( procname, ",".join(["@_%s_%d" % (procname, i) for i in range(len(args))]), ) @@ -383,7 +383,7 @@ class DictCursorMixin: dict_type = dict def _do_get_result(self): - super(DictCursorMixin, self)._do_get_result() + super()._do_get_result() fields = [] if self.description: for f in self._result.fields: diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 41c81673..2db92d39 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -35,7 +35,7 @@ def printable(data): dump_data = [data[i : i + 16] for i in range(0, min(len(data), 256), 16)] for d in dump_data: print( - " ".join("{:02X}".format(x) for x in d) + " ".join(f"{x:02X}" for x in d) + " " * (16 - len(d)) + " " * 2 + "".join(printable(x) for x in d) @@ -275,7 +275,7 @@ def get_column_length(self): return self.length def __str__(self): - return "%s %r.%r.%r, type=%s, flags=%x" % ( + return "{} {!r}.{!r}.{!r}, type={}, flags={:x}".format( self.__class__, self.db, self.table_name, diff --git a/pymysql/tests/base.py b/pymysql/tests/base.py index b5094563..6dfa9590 100644 --- a/pymysql/tests/base.py +++ b/pymysql/tests/base.py @@ -98,7 +98,7 @@ def safe_create_table(self, connection, tablename, ddl, cleanup=True): with warnings.catch_warnings(): warnings.simplefilter("ignore") - cursor.execute("drop table if exists `%s`" % (tablename,)) + cursor.execute(f"drop table if exists `{tablename}`") cursor.execute(ddl) cursor.close() if cleanup: @@ -108,5 +108,5 @@ def drop_table(self, connection, tablename): cursor = connection.cursor() with warnings.catch_warnings(): warnings.simplefilter("ignore") - cursor.execute("drop table if exists `%s`" % (tablename,)) + cursor.execute(f"drop table if exists `{tablename}`") cursor.close() diff --git a/pymysql/tests/test_DictCursor.py b/pymysql/tests/test_DictCursor.py index bbc87d03..4e545792 100644 --- a/pymysql/tests/test_DictCursor.py +++ b/pymysql/tests/test_DictCursor.py @@ -13,7 +13,7 @@ class TestDictCursor(base.PyMySQLTestCase): cursor_type = pymysql.cursors.DictCursor def setUp(self): - super(TestDictCursor, self).setUp() + super().setUp() self.conn = conn = self.connect() c = conn.cursor(self.cursor_type) @@ -36,7 +36,7 @@ def setUp(self): def tearDown(self): c = self.conn.cursor() c.execute("drop table dictcursor") - super(TestDictCursor, self).tearDown() + super().tearDown() def _ensure_cursor_expired(self, cursor): pass diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index ecf043f6..e77605fd 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -332,7 +332,7 @@ class TestBulkInserts(base.PyMySQLTestCase): cursor_type = pymysql.cursors.DictCursor def setUp(self): - super(TestBulkInserts, self).setUp() + super().setUp() self.conn = conn = self.connect() # create a table and some data to query diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index bbaf3dec..869ff0f8 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -28,7 +28,7 @@ def __init__(self, c, user, db, auth=None, authdata=None, password=None): # already exists - TODO need to check the same plugin applies self._created = False try: - c.execute("GRANT SELECT ON %s.* TO %s" % (db, user)) + c.execute(f"GRANT SELECT ON {db}.* TO {user}") self._grant = True except pymysql.err.InternalError: self._grant = False @@ -38,7 +38,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): if self._grant: - self._c.execute("REVOKE SELECT ON %s.* FROM %s" % (self._db, self._user)) + self._c.execute(f"REVOKE SELECT ON {self._db}.* FROM {self._user}") if self._created: self._c.execute("DROP USER %s" % self._user) diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index 6666ab88..b292c206 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -7,7 +7,7 @@ class CursorTest(base.PyMySQLTestCase): def setUp(self): - super(CursorTest, self).setUp() + super().setUp() conn = self.connect() self.safe_create_table( diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 7f361c94..3564d3a6 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -379,8 +379,8 @@ def test_issue_175(self): conn = self.connect() cur = conn.cursor() for length in (200, 300): - columns = ", ".join("c{0} integer".format(i) for i in range(length)) - sql = "create table test_field_count ({0})".format(columns) + columns = ", ".join(f"c{i} integer" for i in range(length)) + sql = f"create table test_field_count ({columns})" try: cur.execute(sql) cur.execute("select * from test_field_count") From 69290924144f961167c257ae33959c46e298efd2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 02:01:00 +0900 Subject: [PATCH 234/292] Add `collation` option and `set_character_set()` to Connection (#1119) Send `SET NAMES` on every new connection to ensure charset/collation are correctly configured. Fix #1092 --- pymysql/connections.py | 43 +++++++++++++++++++++++++++++--- pymysql/tests/test_connection.py | 14 +++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index d161e789..f4782939 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -112,7 +112,8 @@ class Connection: (default: None - no timeout) :param write_timeout: The timeout for writing to the connection in seconds. (default: None - no timeout) - :param charset: Charset to use. + :param str charset: Charset to use. + :param str collation: Collation name to use. :param sql_mode: Default SQL_MODE to use. :param read_default_file: Specifies my.cnf file to read these parameters from under the [client] section. @@ -174,6 +175,7 @@ def __init__( unix_socket=None, port=0, charset="", + collation=None, sql_mode=None, read_default_file=None, conv=None, @@ -308,6 +310,7 @@ def _config(key, arg): self._write_timeout = write_timeout self.charset = charset or DEFAULT_CHARSET + self.collation = collation self.use_unicode = use_unicode self.encoding = charset_by_name(self.charset).encoding @@ -593,13 +596,32 @@ def ping(self, reconnect=True): raise def set_charset(self, charset): + """Deprecated. Use set_character_set() instead.""" + # This function has been implemented in old PyMySQL. + # But this name is different from MySQLdb. + # So we keep this function for compatibility and add + # new set_character_set() function. + self.set_character_set(charset) + + def set_character_set(self, charset, collation=None): + """ + Set charaset (and collation) + + Send "SET NAMES charset [COLLATE collation]" query. + Update Connection.encoding based on charset. + """ # Make sure charset is supported. encoding = charset_by_name(charset).encoding - self._execute_command(COMMAND.COM_QUERY, "SET NAMES %s" % self.escape(charset)) + if collation: + query = f"SET NAMES {charset} COLLATE {collation}" + else: + query = f"SET NAMES {charset}" + self._execute_command(COMMAND.COM_QUERY, query) self._read_packet() self.charset = charset self.encoding = encoding + self.collation = collation def connect(self, sock=None): self._closed = False @@ -641,15 +663,30 @@ def connect(self, sock=None): self._get_server_information() self._request_authentication() + # Send "SET NAMES" query on init for: + # - Ensure charaset (and collation) is set to the server. + # - collation_id in handshake packet may be ignored. + # - If collation is not specified, we don't know what is server's + # default collation for the charset. For example, default collation + # of utf8mb4 is: + # - MySQL 5.7, MariaDB 10.x: utf8mb4_general_ci + # - MySQL 8.0: utf8mb4_0900_ai_ci + # + # Reference: + # - https://github.com/PyMySQL/PyMySQL/issues/1092 + # - https://github.com/wagtail/wagtail/issues/9477 + # - https://zenn.dev/methane/articles/2023-mysql-collation (Japanese) + self.set_character_set(self.charset, self.collation) + if self.sql_mode is not None: c = self.cursor() c.execute("SET sql_mode=%s", (self.sql_mode,)) + c.close() if self.init_command is not None: c = self.cursor() c.execute(self.init_command) c.close() - self.commit() if self.autocommit_mode is not None: self.autocommit(self.autocommit_mode) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 869ff0f8..0803efc9 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -444,6 +444,20 @@ def test_utf8mb4(self): arg["charset"] = "utf8mb4" pymysql.connect(**arg) + def test_set_character_set(self): + con = self.connect() + cur = con.cursor() + + con.set_character_set("latin1") + cur.execute("SELECT @@character_set_connection") + self.assertEqual(cur.fetchone(), ("latin1",)) + self.assertEqual(con.encoding, "cp1252") + + con.set_character_set("utf8mb4", "utf8mb4_general_ci") + cur.execute("SELECT @@character_set_connection, @@collation_connection") + self.assertEqual(cur.fetchone(), ("utf8mb4", "utf8mb4_general_ci")) + self.assertEqual(con.encoding, "utf8") + def test_largedata(self): """Large query and response (>=16MB)""" cur = self.connect().cursor() From fee5df0397ae99af8def8225b450e25002b8cb13 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 14:55:46 +0900 Subject: [PATCH 235/292] CI: Run Django test (#1121) There are some known difference between them so we can not pass the test for now. Fix #1100 --- .github/workflows/django.yaml | 66 +++++++++++++++++++++++++++++++++++ ci/test_mysql.py | 47 +++++++++++++++++++++++++ pymysql/__init__.py | 51 +++++++++++++-------------- pymysql/connections.py | 4 +-- 4 files changed, 140 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/django.yaml create mode 100644 ci/test_mysql.py diff --git a/.github/workflows/django.yaml b/.github/workflows/django.yaml new file mode 100644 index 00000000..da664f85 --- /dev/null +++ b/.github/workflows/django.yaml @@ -0,0 +1,66 @@ +name: Django test + +on: + push: + # branches: ["main"] + # pull_request: + +jobs: + django-test: + name: "Run Django LTS test suite" + runs-on: ubuntu-latest + # There are some known difference between MySQLdb and PyMySQL. + continue-on-error: true + env: + PIP_NO_PYTHON_VERSION_WARNING: 1 + PIP_DISABLE_PIP_VERSION_CHECK: 1 + # DJANGO_VERSION: "3.2.19" + strategy: + fail-fast: false + matrix: + include: + # Django 3.2.9+ supports Python 3.10 + # https://docs.djangoproject.com/ja/3.2/releases/3.2/ + - django: "3.2.19" + python: "3.10" + + - django: "4.2.1" + python: "3.11" + + steps: + - name: Start MySQL + run: | + sudo systemctl start mysql.service + mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -uroot -proot mysql + mysql -uroot -proot -e "set global innodb_flush_log_at_trx_commit=0;" + mysql -uroot -proot -e "CREATE USER 'scott'@'%' IDENTIFIED BY 'tiger'; GRANT ALL ON *.* TO scott;" + mysql -uroot -proot -e "CREATE DATABASE django_default; CREATE DATABASE django_other;" + + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install mysqlclient + run: | + #pip install mysqlclient # Use stable version + pip install .[rsa] + + - name: Setup Django + run: | + sudo apt-get install libmemcached-dev + wget https://github.com/django/django/archive/${{ matrix.django }}.tar.gz + tar xf ${{ matrix.django }}.tar.gz + cp ci/test_mysql.py django-${{ matrix.django }}/tests/ + cd django-${{ matrix.django }} + pip install . -r tests/requirements/py3.txt + + - name: Run Django test + run: | + cd django-${{ matrix.django }}/tests/ + # test_runner does not using our test_mysql.py + # We can't run whole django test suite for now. + # Run olly backends test + DJANGO_SETTINGS_MODULE=test_mysql python runtests.py backends diff --git a/ci/test_mysql.py b/ci/test_mysql.py new file mode 100644 index 00000000..b97978a2 --- /dev/null +++ b/ci/test_mysql.py @@ -0,0 +1,47 @@ +# This is an example test settings file for use with the Django test suite. +# +# The 'sqlite3' backend requires only the ENGINE setting (an in- +# memory database will be used). All other backends will require a +# NAME and potentially authentication information. See the +# following section in the docs for more information: +# +# https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/ +# +# The different databases that Django supports behave differently in certain +# situations, so it is recommended to run the test suite against as many +# database backends as possible. You may want to create a separate settings +# file for each of the backends you test against. + +import pymysql + +pymysql.install_as_MySQLdb() + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": "django_default", + "HOST": "127.0.0.1", + "USER": "scott", + "PASSWORD": "tiger", + "TEST": {"CHARSET": "utf8mb3", "COLLATION": "utf8mb3_general_ci"}, + }, + "other": { + "ENGINE": "django.db.backends.mysql", + "NAME": "django_other", + "HOST": "127.0.0.1", + "USER": "scott", + "PASSWORD": "tiger", + "TEST": {"CHARSET": "utf8mb3", "COLLATION": "utf8mb3_general_ci"}, + }, +} + +SECRET_KEY = "django_tests_secret_key" + +# Use a fast hasher to speed up tests. +PASSWORD_HASHERS = [ + "django.contrib.auth.hashers.MD5PasswordHasher", +] + +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" + +USE_TZ = False diff --git a/pymysql/__init__.py b/pymysql/__init__.py index c0039c3f..ab43c1a9 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -46,12 +46,30 @@ TimestampFromTicks, ) +# PyMySQL version. +# Used by setuptools. +VERSION = (1, 1, 0, "dev", 1) + +### for mysqlclient compatibility +### Django checks mysqlclient version. +version_info = (1, 4, 3, "final", 0) +__version__ = "1.4.3" + + +def get_client_info(): # for MySQLdb compatibility + return __version__ + + +def install_as_MySQLdb(): + """ + After this function is called, any application that imports MySQLdb + will unwittingly actually use pymysql. + """ + sys.modules["MySQLdb"] = sys.modules["pymysql"] + + +# end of mysqlclient compatibility code -VERSION = (1, 0, 3) -if len(VERSION) > 3: - VERSION_STRING = "%d.%d.%d_%s" % VERSION -else: - VERSION_STRING = "%d.%d.%d" % VERSION threadsafety = 1 apilevel = "2.0" paramstyle = "pyformat" @@ -109,31 +127,12 @@ def Binary(x): return bytes(x) -Connect = connect = Connection = connections.Connection - - -def get_client_info(): # for MySQLdb compatibility - return VERSION_STRING - - -# we include a doctored version_info here for MySQLdb compatibility -version_info = (1, 4, 0, "final", 0) - -NULL = "NULL" - -__version__ = get_client_info() - - def thread_safe(): return True # match MySQLdb.thread_safe() -def install_as_MySQLdb(): - """ - After this function is called, any application that imports MySQLdb or - _mysql will unwittingly actually use pymysql. - """ - sys.modules["MySQLdb"] = sys.modules["_mysql"] = sys.modules["pymysql"] +Connect = connect = Connection = connections.Connection +NULL = "NULL" __all__ = [ diff --git a/pymysql/connections.py b/pymysql/connections.py index f4782939..6edac04c 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -25,7 +25,7 @@ EOFPacketWrapper, LoadLocalPacketWrapper, ) -from . import err, VERSION_STRING +from . import err, __version__ try: import ssl @@ -346,7 +346,7 @@ def _config(key, arg): self._connect_attrs = { "_client_name": "pymysql", "_pid": str(os.getpid()), - "_client_version": VERSION_STRING, + "_client_version": __version__, } if program_name: From a5849526821c2d085b94e25ef0b2499ae04dad84 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 15:02:25 +0900 Subject: [PATCH 236/292] Update CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc75225..0e94843c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,11 +10,17 @@ Release date: TBD ## v1.0.3 -Release date: TBD +Release date: 2023-03-28 * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 +* Removed _last_executed because of duplication with _executed by @rajat315315 in https://github.com/PyMySQL/PyMySQL/pull/948 +* Fix generating authentication response with long strings by @netch80 in https://github.com/PyMySQL/PyMySQL/pull/988 +* update pymysql.constants.CR by @Nothing4You in https://github.com/PyMySQL/PyMySQL/pull/1029 +* Document that the ssl connection parameter can be an SSLContext by @cakemanny in https://github.com/PyMySQL/PyMySQL/pull/1045 +* Raise ProgrammingError on -np.inf in addition to np.inf by @cdcadman in https://github.com/PyMySQL/PyMySQL/pull/1067 +* Use Python 3.11 release instead of -dev in tests by @Nothing4You in https://github.com/PyMySQL/PyMySQL/pull/1076 ## v1.0.2 From 2596bbb5b796aae5bb0759b403d6d28cc22b720c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 25 May 2023 15:26:39 +0900 Subject: [PATCH 237/292] Release v1.1.0rc1 (#1122) --- CHANGELOG.md | 8 ++++++++ pymysql/__init__.py | 5 +++-- pymysql/connections.py | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e94843c..dc5ff161 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ Release date: TBD * Fixed SSCursor raising OperationalError for query timeouts on wrong statement (#1032) * Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) +* Make Cursor iterator (#995) +* Support '_' in key name in my.cnf (#1114) +* `Cursor.fetchall()` returns empty list instead of tuple (#1115). Note that `Cursor.fetchmany()` still return empty tuple after reading all rows for compatibility with Django. +* Deprecate Error classes in Cursor class (#1117) +* Add `Connection.set_character_set(charset, collation=None)` (#1119) +* Deprecate `Connection.set_charset(charset)` (#1119) +* New connection always send "SET NAMES charset [COLLATE collation]" query. (#1119) + Since collation table is vary on MySQL server versions, collation in handshake is fragile. ## v1.0.3 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index ab43c1a9..b9971ff0 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -47,8 +47,9 @@ ) # PyMySQL version. -# Used by setuptools. -VERSION = (1, 1, 0, "dev", 1) +# Used by setuptools and connection_attrs +VERSION = (1, 1, 0, "rc", 1) +VERSION_STRING = "1.1.0rc1" ### for mysqlclient compatibility ### Django checks mysqlclient version. diff --git a/pymysql/connections.py b/pymysql/connections.py index 6edac04c..843bea5e 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -25,7 +25,7 @@ EOFPacketWrapper, LoadLocalPacketWrapper, ) -from . import err, __version__ +from . import err, VERSION_STRING try: import ssl @@ -345,8 +345,8 @@ def _config(key, arg): self._connect_attrs = { "_client_name": "pymysql", + "_client_version": VERSION_STRING, "_pid": str(os.getpid()), - "_client_version": __version__, } if program_name: From 2df6c068b7a0dd733e72a068b3aca3e8738177ad Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 25 May 2023 17:36:52 +1000 Subject: [PATCH 238/292] Bump mariadb version (#1123) In README and GH actions. --- .github/workflows/test.yaml | 6 +++--- README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c3275cca..6b1e0f32 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,16 +15,16 @@ jobs: fail-fast: false matrix: include: - - db: "mariadb:10.3" + - db: "mariadb:10.4" py: "3.8" - db: "mariadb:10.5" py: "3.7" - - db: "mariadb:10.7" + - db: "mariadb:10.6" py: "3.11" - - db: "mariadb:10.8" + - db: "mariadb:lts" py: "3.9" - db: "mysql:5.7" diff --git a/README.md b/README.md index 6e6a6bf2..32f5df2f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ This package contains a pure-Python MySQL client library, based on [PEP - [PyPy](https://pypy.org/) : Latest 3.x version - MySQL Server -- one of the following: - [MySQL](https://www.mysql.com/) \>= 5.7 - - [MariaDB](https://mariadb.org/) \>= 10.3 + - [MariaDB](https://mariadb.org/) \>= 10.4 ## Installation From f4c348fdcf4ac21a92be58b6f94e9d7a13826a38 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Jun 2023 13:55:38 +0900 Subject: [PATCH 239/292] Configure Renovate (#1124) --- renovate.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..39a2b6e9 --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base" + ] +} From c3a12f683345a97a8cc8516cf2123a5836c38f7d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 15 Jun 2023 13:20:26 +0900 Subject: [PATCH 240/292] Make charset="utf8" use utf8mb4. (#1127) Use charset="utf8mb3" to use utf8mb3 instead. Fix #1126 --- pymysql/charset.py | 319 +++++++++++++++++----------------- pymysql/tests/test_charset.py | 25 +++ 2 files changed, 188 insertions(+), 156 deletions(-) create mode 100644 pymysql/tests/test_charset.py diff --git a/pymysql/charset.py b/pymysql/charset.py index cdc02164..b1c1ca8b 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -1,16 +1,16 @@ +# Internal use only. Do not use directly. + MBLENGTH = {8: 1, 33: 3, 88: 2, 91: 2} class Charset: - def __init__(self, id, name, collation, is_default): + def __init__(self, id, name, collation, is_default=False): self.id, self.name, self.collation = id, name, collation - self.is_default = is_default == "Yes" + self.is_default = is_default def __repr__(self): - return "Charset(id={}, name={!r}, collation={!r})".format( - self.id, - self.name, - self.collation, + return ( + f"Charset(id={self.id}, name={self.name!r}, collation={self.collation!r})" ) @property @@ -45,165 +45,172 @@ def by_id(self, id): return self._by_id[id] def by_name(self, name): + if name == "utf8": + name = "utf8mb4" return self._by_name.get(name.lower()) _charsets = Charsets() +charset_by_name = _charsets.by_name +charset_by_id = _charsets.by_id + """ +TODO: update this script. + Generated with: mysql -N -s -e "select id, character_set_name, collation_name, is_default from information_schema.collations order by id;" | python -c "import sys for l in sys.stdin.readlines(): - id, name, collation, is_default = l.split(chr(9)) - print '_charsets.add(Charset(%s, \'%s\', \'%s\', \'%s\'))' \ - % (id, name, collation, is_default.strip()) -" - + id, name, collation, is_default = l.split(chr(9)) + if is_default.strip() == "Yes": + print('_charsets.add(Charset(%s, \'%s\', \'%s\', True))' \ + % (id, name, collation)) + else: + print('_charsets.add(Charset(%s, \'%s\', \'%s\'))' \ + % (id, name, collation, bool(is_default.strip())) """ -_charsets.add(Charset(1, "big5", "big5_chinese_ci", "Yes")) -_charsets.add(Charset(2, "latin2", "latin2_czech_cs", "")) -_charsets.add(Charset(3, "dec8", "dec8_swedish_ci", "Yes")) -_charsets.add(Charset(4, "cp850", "cp850_general_ci", "Yes")) -_charsets.add(Charset(5, "latin1", "latin1_german1_ci", "")) -_charsets.add(Charset(6, "hp8", "hp8_english_ci", "Yes")) -_charsets.add(Charset(7, "koi8r", "koi8r_general_ci", "Yes")) -_charsets.add(Charset(8, "latin1", "latin1_swedish_ci", "Yes")) -_charsets.add(Charset(9, "latin2", "latin2_general_ci", "Yes")) -_charsets.add(Charset(10, "swe7", "swe7_swedish_ci", "Yes")) -_charsets.add(Charset(11, "ascii", "ascii_general_ci", "Yes")) -_charsets.add(Charset(12, "ujis", "ujis_japanese_ci", "Yes")) -_charsets.add(Charset(13, "sjis", "sjis_japanese_ci", "Yes")) -_charsets.add(Charset(14, "cp1251", "cp1251_bulgarian_ci", "")) -_charsets.add(Charset(15, "latin1", "latin1_danish_ci", "")) -_charsets.add(Charset(16, "hebrew", "hebrew_general_ci", "Yes")) -_charsets.add(Charset(18, "tis620", "tis620_thai_ci", "Yes")) -_charsets.add(Charset(19, "euckr", "euckr_korean_ci", "Yes")) -_charsets.add(Charset(20, "latin7", "latin7_estonian_cs", "")) -_charsets.add(Charset(21, "latin2", "latin2_hungarian_ci", "")) -_charsets.add(Charset(22, "koi8u", "koi8u_general_ci", "Yes")) -_charsets.add(Charset(23, "cp1251", "cp1251_ukrainian_ci", "")) -_charsets.add(Charset(24, "gb2312", "gb2312_chinese_ci", "Yes")) -_charsets.add(Charset(25, "greek", "greek_general_ci", "Yes")) -_charsets.add(Charset(26, "cp1250", "cp1250_general_ci", "Yes")) -_charsets.add(Charset(27, "latin2", "latin2_croatian_ci", "")) -_charsets.add(Charset(28, "gbk", "gbk_chinese_ci", "Yes")) -_charsets.add(Charset(29, "cp1257", "cp1257_lithuanian_ci", "")) -_charsets.add(Charset(30, "latin5", "latin5_turkish_ci", "Yes")) -_charsets.add(Charset(31, "latin1", "latin1_german2_ci", "")) -_charsets.add(Charset(32, "armscii8", "armscii8_general_ci", "Yes")) -_charsets.add(Charset(33, "utf8", "utf8_general_ci", "Yes")) -_charsets.add(Charset(34, "cp1250", "cp1250_czech_cs", "")) -_charsets.add(Charset(36, "cp866", "cp866_general_ci", "Yes")) -_charsets.add(Charset(37, "keybcs2", "keybcs2_general_ci", "Yes")) -_charsets.add(Charset(38, "macce", "macce_general_ci", "Yes")) -_charsets.add(Charset(39, "macroman", "macroman_general_ci", "Yes")) -_charsets.add(Charset(40, "cp852", "cp852_general_ci", "Yes")) -_charsets.add(Charset(41, "latin7", "latin7_general_ci", "Yes")) -_charsets.add(Charset(42, "latin7", "latin7_general_cs", "")) -_charsets.add(Charset(43, "macce", "macce_bin", "")) -_charsets.add(Charset(44, "cp1250", "cp1250_croatian_ci", "")) -_charsets.add(Charset(45, "utf8mb4", "utf8mb4_general_ci", "Yes")) -_charsets.add(Charset(46, "utf8mb4", "utf8mb4_bin", "")) -_charsets.add(Charset(47, "latin1", "latin1_bin", "")) -_charsets.add(Charset(48, "latin1", "latin1_general_ci", "")) -_charsets.add(Charset(49, "latin1", "latin1_general_cs", "")) -_charsets.add(Charset(50, "cp1251", "cp1251_bin", "")) -_charsets.add(Charset(51, "cp1251", "cp1251_general_ci", "Yes")) -_charsets.add(Charset(52, "cp1251", "cp1251_general_cs", "")) -_charsets.add(Charset(53, "macroman", "macroman_bin", "")) -_charsets.add(Charset(57, "cp1256", "cp1256_general_ci", "Yes")) -_charsets.add(Charset(58, "cp1257", "cp1257_bin", "")) -_charsets.add(Charset(59, "cp1257", "cp1257_general_ci", "Yes")) -_charsets.add(Charset(63, "binary", "binary", "Yes")) -_charsets.add(Charset(64, "armscii8", "armscii8_bin", "")) -_charsets.add(Charset(65, "ascii", "ascii_bin", "")) -_charsets.add(Charset(66, "cp1250", "cp1250_bin", "")) -_charsets.add(Charset(67, "cp1256", "cp1256_bin", "")) -_charsets.add(Charset(68, "cp866", "cp866_bin", "")) -_charsets.add(Charset(69, "dec8", "dec8_bin", "")) -_charsets.add(Charset(70, "greek", "greek_bin", "")) -_charsets.add(Charset(71, "hebrew", "hebrew_bin", "")) -_charsets.add(Charset(72, "hp8", "hp8_bin", "")) -_charsets.add(Charset(73, "keybcs2", "keybcs2_bin", "")) -_charsets.add(Charset(74, "koi8r", "koi8r_bin", "")) -_charsets.add(Charset(75, "koi8u", "koi8u_bin", "")) -_charsets.add(Charset(76, "utf8", "utf8_tolower_ci", "")) -_charsets.add(Charset(77, "latin2", "latin2_bin", "")) -_charsets.add(Charset(78, "latin5", "latin5_bin", "")) -_charsets.add(Charset(79, "latin7", "latin7_bin", "")) -_charsets.add(Charset(80, "cp850", "cp850_bin", "")) -_charsets.add(Charset(81, "cp852", "cp852_bin", "")) -_charsets.add(Charset(82, "swe7", "swe7_bin", "")) -_charsets.add(Charset(83, "utf8", "utf8_bin", "")) -_charsets.add(Charset(84, "big5", "big5_bin", "")) -_charsets.add(Charset(85, "euckr", "euckr_bin", "")) -_charsets.add(Charset(86, "gb2312", "gb2312_bin", "")) -_charsets.add(Charset(87, "gbk", "gbk_bin", "")) -_charsets.add(Charset(88, "sjis", "sjis_bin", "")) -_charsets.add(Charset(89, "tis620", "tis620_bin", "")) -_charsets.add(Charset(91, "ujis", "ujis_bin", "")) -_charsets.add(Charset(92, "geostd8", "geostd8_general_ci", "Yes")) -_charsets.add(Charset(93, "geostd8", "geostd8_bin", "")) -_charsets.add(Charset(94, "latin1", "latin1_spanish_ci", "")) -_charsets.add(Charset(95, "cp932", "cp932_japanese_ci", "Yes")) -_charsets.add(Charset(96, "cp932", "cp932_bin", "")) -_charsets.add(Charset(97, "eucjpms", "eucjpms_japanese_ci", "Yes")) -_charsets.add(Charset(98, "eucjpms", "eucjpms_bin", "")) -_charsets.add(Charset(99, "cp1250", "cp1250_polish_ci", "")) -_charsets.add(Charset(192, "utf8", "utf8_unicode_ci", "")) -_charsets.add(Charset(193, "utf8", "utf8_icelandic_ci", "")) -_charsets.add(Charset(194, "utf8", "utf8_latvian_ci", "")) -_charsets.add(Charset(195, "utf8", "utf8_romanian_ci", "")) -_charsets.add(Charset(196, "utf8", "utf8_slovenian_ci", "")) -_charsets.add(Charset(197, "utf8", "utf8_polish_ci", "")) -_charsets.add(Charset(198, "utf8", "utf8_estonian_ci", "")) -_charsets.add(Charset(199, "utf8", "utf8_spanish_ci", "")) -_charsets.add(Charset(200, "utf8", "utf8_swedish_ci", "")) -_charsets.add(Charset(201, "utf8", "utf8_turkish_ci", "")) -_charsets.add(Charset(202, "utf8", "utf8_czech_ci", "")) -_charsets.add(Charset(203, "utf8", "utf8_danish_ci", "")) -_charsets.add(Charset(204, "utf8", "utf8_lithuanian_ci", "")) -_charsets.add(Charset(205, "utf8", "utf8_slovak_ci", "")) -_charsets.add(Charset(206, "utf8", "utf8_spanish2_ci", "")) -_charsets.add(Charset(207, "utf8", "utf8_roman_ci", "")) -_charsets.add(Charset(208, "utf8", "utf8_persian_ci", "")) -_charsets.add(Charset(209, "utf8", "utf8_esperanto_ci", "")) -_charsets.add(Charset(210, "utf8", "utf8_hungarian_ci", "")) -_charsets.add(Charset(211, "utf8", "utf8_sinhala_ci", "")) -_charsets.add(Charset(212, "utf8", "utf8_german2_ci", "")) -_charsets.add(Charset(213, "utf8", "utf8_croatian_ci", "")) -_charsets.add(Charset(214, "utf8", "utf8_unicode_520_ci", "")) -_charsets.add(Charset(215, "utf8", "utf8_vietnamese_ci", "")) -_charsets.add(Charset(223, "utf8", "utf8_general_mysql500_ci", "")) -_charsets.add(Charset(224, "utf8mb4", "utf8mb4_unicode_ci", "")) -_charsets.add(Charset(225, "utf8mb4", "utf8mb4_icelandic_ci", "")) -_charsets.add(Charset(226, "utf8mb4", "utf8mb4_latvian_ci", "")) -_charsets.add(Charset(227, "utf8mb4", "utf8mb4_romanian_ci", "")) -_charsets.add(Charset(228, "utf8mb4", "utf8mb4_slovenian_ci", "")) -_charsets.add(Charset(229, "utf8mb4", "utf8mb4_polish_ci", "")) -_charsets.add(Charset(230, "utf8mb4", "utf8mb4_estonian_ci", "")) -_charsets.add(Charset(231, "utf8mb4", "utf8mb4_spanish_ci", "")) -_charsets.add(Charset(232, "utf8mb4", "utf8mb4_swedish_ci", "")) -_charsets.add(Charset(233, "utf8mb4", "utf8mb4_turkish_ci", "")) -_charsets.add(Charset(234, "utf8mb4", "utf8mb4_czech_ci", "")) -_charsets.add(Charset(235, "utf8mb4", "utf8mb4_danish_ci", "")) -_charsets.add(Charset(236, "utf8mb4", "utf8mb4_lithuanian_ci", "")) -_charsets.add(Charset(237, "utf8mb4", "utf8mb4_slovak_ci", "")) -_charsets.add(Charset(238, "utf8mb4", "utf8mb4_spanish2_ci", "")) -_charsets.add(Charset(239, "utf8mb4", "utf8mb4_roman_ci", "")) -_charsets.add(Charset(240, "utf8mb4", "utf8mb4_persian_ci", "")) -_charsets.add(Charset(241, "utf8mb4", "utf8mb4_esperanto_ci", "")) -_charsets.add(Charset(242, "utf8mb4", "utf8mb4_hungarian_ci", "")) -_charsets.add(Charset(243, "utf8mb4", "utf8mb4_sinhala_ci", "")) -_charsets.add(Charset(244, "utf8mb4", "utf8mb4_german2_ci", "")) -_charsets.add(Charset(245, "utf8mb4", "utf8mb4_croatian_ci", "")) -_charsets.add(Charset(246, "utf8mb4", "utf8mb4_unicode_520_ci", "")) -_charsets.add(Charset(247, "utf8mb4", "utf8mb4_vietnamese_ci", "")) -_charsets.add(Charset(248, "gb18030", "gb18030_chinese_ci", "Yes")) -_charsets.add(Charset(249, "gb18030", "gb18030_bin", "")) -_charsets.add(Charset(250, "gb18030", "gb18030_unicode_520_ci", "")) -_charsets.add(Charset(255, "utf8mb4", "utf8mb4_0900_ai_ci", "")) -charset_by_name = _charsets.by_name -charset_by_id = _charsets.by_id +_charsets.add(Charset(1, "big5", "big5_chinese_ci", True)) +_charsets.add(Charset(2, "latin2", "latin2_czech_cs")) +_charsets.add(Charset(3, "dec8", "dec8_swedish_ci", True)) +_charsets.add(Charset(4, "cp850", "cp850_general_ci", True)) +_charsets.add(Charset(5, "latin1", "latin1_german1_ci")) +_charsets.add(Charset(6, "hp8", "hp8_english_ci", True)) +_charsets.add(Charset(7, "koi8r", "koi8r_general_ci", True)) +_charsets.add(Charset(8, "latin1", "latin1_swedish_ci", True)) +_charsets.add(Charset(9, "latin2", "latin2_general_ci", True)) +_charsets.add(Charset(10, "swe7", "swe7_swedish_ci", True)) +_charsets.add(Charset(11, "ascii", "ascii_general_ci", True)) +_charsets.add(Charset(12, "ujis", "ujis_japanese_ci", True)) +_charsets.add(Charset(13, "sjis", "sjis_japanese_ci", True)) +_charsets.add(Charset(14, "cp1251", "cp1251_bulgarian_ci")) +_charsets.add(Charset(15, "latin1", "latin1_danish_ci")) +_charsets.add(Charset(16, "hebrew", "hebrew_general_ci", True)) +_charsets.add(Charset(18, "tis620", "tis620_thai_ci", True)) +_charsets.add(Charset(19, "euckr", "euckr_korean_ci", True)) +_charsets.add(Charset(20, "latin7", "latin7_estonian_cs")) +_charsets.add(Charset(21, "latin2", "latin2_hungarian_ci")) +_charsets.add(Charset(22, "koi8u", "koi8u_general_ci", True)) +_charsets.add(Charset(23, "cp1251", "cp1251_ukrainian_ci")) +_charsets.add(Charset(24, "gb2312", "gb2312_chinese_ci", True)) +_charsets.add(Charset(25, "greek", "greek_general_ci", True)) +_charsets.add(Charset(26, "cp1250", "cp1250_general_ci", True)) +_charsets.add(Charset(27, "latin2", "latin2_croatian_ci")) +_charsets.add(Charset(28, "gbk", "gbk_chinese_ci", True)) +_charsets.add(Charset(29, "cp1257", "cp1257_lithuanian_ci")) +_charsets.add(Charset(30, "latin5", "latin5_turkish_ci", True)) +_charsets.add(Charset(31, "latin1", "latin1_german2_ci")) +_charsets.add(Charset(32, "armscii8", "armscii8_general_ci", True)) +_charsets.add(Charset(33, "utf8mb3", "utf8mb3_general_ci", True)) +_charsets.add(Charset(34, "cp1250", "cp1250_czech_cs")) +_charsets.add(Charset(36, "cp866", "cp866_general_ci", True)) +_charsets.add(Charset(37, "keybcs2", "keybcs2_general_ci", True)) +_charsets.add(Charset(38, "macce", "macce_general_ci", True)) +_charsets.add(Charset(39, "macroman", "macroman_general_ci", True)) +_charsets.add(Charset(40, "cp852", "cp852_general_ci", True)) +_charsets.add(Charset(41, "latin7", "latin7_general_ci", True)) +_charsets.add(Charset(42, "latin7", "latin7_general_cs")) +_charsets.add(Charset(43, "macce", "macce_bin")) +_charsets.add(Charset(44, "cp1250", "cp1250_croatian_ci")) +_charsets.add(Charset(45, "utf8mb4", "utf8mb4_general_ci", True)) +_charsets.add(Charset(46, "utf8mb4", "utf8mb4_bin")) +_charsets.add(Charset(47, "latin1", "latin1_bin")) +_charsets.add(Charset(48, "latin1", "latin1_general_ci")) +_charsets.add(Charset(49, "latin1", "latin1_general_cs")) +_charsets.add(Charset(50, "cp1251", "cp1251_bin")) +_charsets.add(Charset(51, "cp1251", "cp1251_general_ci", True)) +_charsets.add(Charset(52, "cp1251", "cp1251_general_cs")) +_charsets.add(Charset(53, "macroman", "macroman_bin")) +_charsets.add(Charset(57, "cp1256", "cp1256_general_ci", True)) +_charsets.add(Charset(58, "cp1257", "cp1257_bin")) +_charsets.add(Charset(59, "cp1257", "cp1257_general_ci", True)) +_charsets.add(Charset(63, "binary", "binary", True)) +_charsets.add(Charset(64, "armscii8", "armscii8_bin")) +_charsets.add(Charset(65, "ascii", "ascii_bin")) +_charsets.add(Charset(66, "cp1250", "cp1250_bin")) +_charsets.add(Charset(67, "cp1256", "cp1256_bin")) +_charsets.add(Charset(68, "cp866", "cp866_bin")) +_charsets.add(Charset(69, "dec8", "dec8_bin")) +_charsets.add(Charset(70, "greek", "greek_bin")) +_charsets.add(Charset(71, "hebrew", "hebrew_bin")) +_charsets.add(Charset(72, "hp8", "hp8_bin")) +_charsets.add(Charset(73, "keybcs2", "keybcs2_bin")) +_charsets.add(Charset(74, "koi8r", "koi8r_bin")) +_charsets.add(Charset(75, "koi8u", "koi8u_bin")) +_charsets.add(Charset(76, "utf8mb3", "utf8mb3_tolower_ci")) +_charsets.add(Charset(77, "latin2", "latin2_bin")) +_charsets.add(Charset(78, "latin5", "latin5_bin")) +_charsets.add(Charset(79, "latin7", "latin7_bin")) +_charsets.add(Charset(80, "cp850", "cp850_bin")) +_charsets.add(Charset(81, "cp852", "cp852_bin")) +_charsets.add(Charset(82, "swe7", "swe7_bin")) +_charsets.add(Charset(83, "utf8mb3", "utf8mb3_bin")) +_charsets.add(Charset(84, "big5", "big5_bin")) +_charsets.add(Charset(85, "euckr", "euckr_bin")) +_charsets.add(Charset(86, "gb2312", "gb2312_bin")) +_charsets.add(Charset(87, "gbk", "gbk_bin")) +_charsets.add(Charset(88, "sjis", "sjis_bin")) +_charsets.add(Charset(89, "tis620", "tis620_bin")) +_charsets.add(Charset(91, "ujis", "ujis_bin")) +_charsets.add(Charset(92, "geostd8", "geostd8_general_ci", True)) +_charsets.add(Charset(93, "geostd8", "geostd8_bin")) +_charsets.add(Charset(94, "latin1", "latin1_spanish_ci")) +_charsets.add(Charset(95, "cp932", "cp932_japanese_ci", True)) +_charsets.add(Charset(96, "cp932", "cp932_bin")) +_charsets.add(Charset(97, "eucjpms", "eucjpms_japanese_ci", True)) +_charsets.add(Charset(98, "eucjpms", "eucjpms_bin")) +_charsets.add(Charset(99, "cp1250", "cp1250_polish_ci")) +_charsets.add(Charset(192, "utf8mb3", "utf8mb3_unicode_ci")) +_charsets.add(Charset(193, "utf8mb3", "utf8mb3_icelandic_ci")) +_charsets.add(Charset(194, "utf8mb3", "utf8mb3_latvian_ci")) +_charsets.add(Charset(195, "utf8mb3", "utf8mb3_romanian_ci")) +_charsets.add(Charset(196, "utf8mb3", "utf8mb3_slovenian_ci")) +_charsets.add(Charset(197, "utf8mb3", "utf8mb3_polish_ci")) +_charsets.add(Charset(198, "utf8mb3", "utf8mb3_estonian_ci")) +_charsets.add(Charset(199, "utf8mb3", "utf8mb3_spanish_ci")) +_charsets.add(Charset(200, "utf8mb3", "utf8mb3_swedish_ci")) +_charsets.add(Charset(201, "utf8mb3", "utf8mb3_turkish_ci")) +_charsets.add(Charset(202, "utf8mb3", "utf8mb3_czech_ci")) +_charsets.add(Charset(203, "utf8mb3", "utf8mb3_danish_ci")) +_charsets.add(Charset(204, "utf8mb3", "utf8mb3_lithuanian_ci")) +_charsets.add(Charset(205, "utf8mb3", "utf8mb3_slovak_ci")) +_charsets.add(Charset(206, "utf8mb3", "utf8mb3_spanish2_ci")) +_charsets.add(Charset(207, "utf8mb3", "utf8mb3_roman_ci")) +_charsets.add(Charset(208, "utf8mb3", "utf8mb3_persian_ci")) +_charsets.add(Charset(209, "utf8mb3", "utf8mb3_esperanto_ci")) +_charsets.add(Charset(210, "utf8mb3", "utf8mb3_hungarian_ci")) +_charsets.add(Charset(211, "utf8mb3", "utf8mb3_sinhala_ci")) +_charsets.add(Charset(212, "utf8mb3", "utf8mb3_german2_ci")) +_charsets.add(Charset(213, "utf8mb3", "utf8mb3_croatian_ci")) +_charsets.add(Charset(214, "utf8mb3", "utf8mb3_unicode_520_ci")) +_charsets.add(Charset(215, "utf8mb3", "utf8mb3_vietnamese_ci")) +_charsets.add(Charset(223, "utf8mb3", "utf8mb3_general_mysql500_ci")) +_charsets.add(Charset(224, "utf8mb4", "utf8mb4_unicode_ci")) +_charsets.add(Charset(225, "utf8mb4", "utf8mb4_icelandic_ci")) +_charsets.add(Charset(226, "utf8mb4", "utf8mb4_latvian_ci")) +_charsets.add(Charset(227, "utf8mb4", "utf8mb4_romanian_ci")) +_charsets.add(Charset(228, "utf8mb4", "utf8mb4_slovenian_ci")) +_charsets.add(Charset(229, "utf8mb4", "utf8mb4_polish_ci")) +_charsets.add(Charset(230, "utf8mb4", "utf8mb4_estonian_ci")) +_charsets.add(Charset(231, "utf8mb4", "utf8mb4_spanish_ci")) +_charsets.add(Charset(232, "utf8mb4", "utf8mb4_swedish_ci")) +_charsets.add(Charset(233, "utf8mb4", "utf8mb4_turkish_ci")) +_charsets.add(Charset(234, "utf8mb4", "utf8mb4_czech_ci")) +_charsets.add(Charset(235, "utf8mb4", "utf8mb4_danish_ci")) +_charsets.add(Charset(236, "utf8mb4", "utf8mb4_lithuanian_ci")) +_charsets.add(Charset(237, "utf8mb4", "utf8mb4_slovak_ci")) +_charsets.add(Charset(238, "utf8mb4", "utf8mb4_spanish2_ci")) +_charsets.add(Charset(239, "utf8mb4", "utf8mb4_roman_ci")) +_charsets.add(Charset(240, "utf8mb4", "utf8mb4_persian_ci")) +_charsets.add(Charset(241, "utf8mb4", "utf8mb4_esperanto_ci")) +_charsets.add(Charset(242, "utf8mb4", "utf8mb4_hungarian_ci")) +_charsets.add(Charset(243, "utf8mb4", "utf8mb4_sinhala_ci")) +_charsets.add(Charset(244, "utf8mb4", "utf8mb4_german2_ci")) +_charsets.add(Charset(245, "utf8mb4", "utf8mb4_croatian_ci")) +_charsets.add(Charset(246, "utf8mb4", "utf8mb4_unicode_520_ci")) +_charsets.add(Charset(247, "utf8mb4", "utf8mb4_vietnamese_ci")) +_charsets.add(Charset(248, "gb18030", "gb18030_chinese_ci", True)) +_charsets.add(Charset(249, "gb18030", "gb18030_bin")) +_charsets.add(Charset(250, "gb18030", "gb18030_unicode_520_ci")) +_charsets.add(Charset(255, "utf8mb4", "utf8mb4_0900_ai_ci")) diff --git a/pymysql/tests/test_charset.py b/pymysql/tests/test_charset.py new file mode 100644 index 00000000..94e6e155 --- /dev/null +++ b/pymysql/tests/test_charset.py @@ -0,0 +1,25 @@ +import pymysql.charset + + +def test_utf8(): + utf8mb3 = pymysql.charset.charset_by_name("utf8mb3") + assert utf8mb3.name == "utf8mb3" + assert utf8mb3.collation == "utf8mb3_general_ci" + assert ( + repr(utf8mb3) + == "Charset(id=33, name='utf8mb3', collation='utf8mb3_general_ci')" + ) + + # MySQL 8.0 changed the default collation for utf8mb4. + # But we use old default for compatibility. + utf8mb4 = pymysql.charset.charset_by_name("utf8mb4") + assert utf8mb4.name == "utf8mb4" + assert utf8mb4.collation == "utf8mb4_general_ci" + assert ( + repr(utf8mb4) + == "Charset(id=45, name='utf8mb4', collation='utf8mb4_general_ci')" + ) + + # utf8 is alias of utf8mb4 since MySQL 8.0, and PyMySQL v1.1. + utf8 = pymysql.charset.charset_by_name("utf8") + assert utf8 == utf8mb4 From fed7e8069bf09d3b4e819dc8c59d6b7096e4183f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 15 Jun 2023 13:31:21 +0900 Subject: [PATCH 241/292] Add codecov.yml (#1128) --- codecov.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..919adf20 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,7 @@ +# https://docs.codecov.com/docs/common-recipe-list +coverage: + status: + project: + default: + target: auto + threshold: 3% From f3f3477682a3bbc80eb0240034abcd288d7dda63 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 15 Jun 2023 16:58:17 +0900 Subject: [PATCH 242/292] Release v1.1.0rc2 (#1129) --- CHANGELOG.md | 1 + pymysql/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5ff161..ea1d732a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Release date: TBD * Deprecate `Connection.set_charset(charset)` (#1119) * New connection always send "SET NAMES charset [COLLATE collation]" query. (#1119) Since collation table is vary on MySQL server versions, collation in handshake is fragile. +* Support `charset="utf8mb3"` option (#1127) ## v1.0.3 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index b9971ff0..68d7043b 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -48,8 +48,8 @@ # PyMySQL version. # Used by setuptools and connection_attrs -VERSION = (1, 1, 0, "rc", 1) -VERSION_STRING = "1.1.0rc1" +VERSION = (1, 1, 0, "rc", 2) +VERSION_STRING = "1.1.0rc2" ### for mysqlclient compatibility ### Django checks mysqlclient version. From 0803b539d4e370001fc93942643ab6843d3eb331 Mon Sep 17 00:00:00 2001 From: "codesee-maps[bot]" <86324825+codesee-maps[bot]@users.noreply.github.com> Date: Wed, 21 Jun 2023 18:22:40 +0000 Subject: [PATCH 243/292] Install the CodeSee workflow. Learn more at https://docs.codesee.io --- .github/workflows/codesee-arch-diagram.yml | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/codesee-arch-diagram.yml diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml new file mode 100644 index 00000000..806d41d1 --- /dev/null +++ b/.github/workflows/codesee-arch-diagram.yml @@ -0,0 +1,23 @@ +# This workflow was added by CodeSee. Learn more at https://codesee.io/ +# This is v2.0 of this workflow file +on: + push: + branches: + - main + pull_request_target: + types: [opened, synchronize, reopened] + +name: CodeSee + +permissions: read-all + +jobs: + codesee: + runs-on: ubuntu-latest + continue-on-error: true + name: Analyze the repo with CodeSee + steps: + - uses: Codesee-io/codesee-action@v2 + with: + codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} + codesee-url: https://app.codesee.io From fe856a55963eac53d5fd714d7de06328cab90293 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 26 Jun 2023 14:31:53 +0900 Subject: [PATCH 244/292] Release v1.1.0 (#1130) --- CHANGELOG.md | 13 ++++++++++--- pymysql/__init__.py | 8 ++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1d732a..c6283670 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,15 @@ # Changes +## Backward incompatible changes planned in the future. + +* Error classes in Cursor class will be removed after 2024-06 +* `Connection.set_charset(charset)` will be removed after 2024-06 +* `db` and `passwd` will emit DeprecationWarning in v1.2. See #933. + + ## v1.1.0 -Release date: TBD +Release date: 2023-06-26 * Fixed SSCursor raising OperationalError for query timeouts on wrong statement (#1032) * Exposed `Cursor.warning_count` to check for warnings without additional query (#1056) @@ -10,7 +17,7 @@ Release date: TBD * Support '_' in key name in my.cnf (#1114) * `Cursor.fetchall()` returns empty list instead of tuple (#1115). Note that `Cursor.fetchmany()` still return empty tuple after reading all rows for compatibility with Django. * Deprecate Error classes in Cursor class (#1117) -* Add `Connection.set_character_set(charset, collation=None)` (#1119) +* Add `Connection.set_character_set(charset, collation=None)`. This method is compatible with mysqlclient. (#1119) * Deprecate `Connection.set_charset(charset)` (#1119) * New connection always send "SET NAMES charset [COLLATE collation]" query. (#1119) Since collation table is vary on MySQL server versions, collation in handshake is fragile. @@ -24,7 +31,7 @@ Release date: 2023-03-28 * Dropped support of end of life MySQL version 5.6 * Dropped support of end of life MariaDB versions below 10.3 * Dropped support of end of life Python version 3.6 -* Removed _last_executed because of duplication with _executed by @rajat315315 in https://github.com/PyMySQL/PyMySQL/pull/948 +* Removed `_last_executed` because of duplication with `_executed` by @rajat315315 in https://github.com/PyMySQL/PyMySQL/pull/948 * Fix generating authentication response with long strings by @netch80 in https://github.com/PyMySQL/PyMySQL/pull/988 * update pymysql.constants.CR by @Nothing4You in https://github.com/PyMySQL/PyMySQL/pull/1029 * Document that the ssl connection parameter can be an SSLContext by @cakemanny in https://github.com/PyMySQL/PyMySQL/pull/1045 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 68d7043b..53625d37 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -48,13 +48,13 @@ # PyMySQL version. # Used by setuptools and connection_attrs -VERSION = (1, 1, 0, "rc", 2) -VERSION_STRING = "1.1.0rc2" +VERSION = (1, 1, 0, "final", 1) +VERSION_STRING = "1.1.0" ### for mysqlclient compatibility ### Django checks mysqlclient version. -version_info = (1, 4, 3, "final", 0) -__version__ = "1.4.3" +version_info = (1, 4, 6, "final", 1) +__version__ = "1.4.6" def get_client_info(): # for MySQLdb compatibility From dbf1ff52a695278cd80e179641f67bb6e2a83326 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 26 Jun 2023 14:33:12 +0900 Subject: [PATCH 245/292] Fix dynamic version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 18714779..15df9f3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ include = ["pymysql*"] exclude = ["tests*", "pymysql.tests*"] [tool.setuptools.dynamic] -version = {attr = "pymysql.VERSION"} +version = {attr = "pymysql.VERSION_STRING"} [tool.ruff] line-length = 99 From 6b10225c94087d47782049aafc8e12efa512337b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 1 Jul 2023 01:29:58 +0900 Subject: [PATCH 246/292] Disable renovate dashboard --- renovate.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index 39a2b6e9..09e16da6 100644 --- a/renovate.json +++ b/renovate.json @@ -2,5 +2,6 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:base" - ] + ], + "dependencyDashboard": false } From 8157da51e844f619eb693c5f5dd2758dca1d1c98 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 4 Sep 2023 02:04:26 -0600 Subject: [PATCH 247/292] Add support for Python 3.12 (#1134) --- .github/workflows/test.yaml | 7 +++++++ pyproject.toml | 1 + 2 files changed, 8 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6b1e0f32..1153b9e4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,6 +8,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: test: runs-on: ubuntu-latest @@ -24,6 +27,9 @@ jobs: - db: "mariadb:10.6" py: "3.11" + - db: "mariadb:10.6" + py: "3.12" + - db: "mariadb:lts" py: "3.9" @@ -62,6 +68,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} + allow-prereleases: true cache: 'pip' cache-dependency-path: 'requirements-dev.txt' diff --git a/pyproject.toml b/pyproject.toml index 15df9f3c..8e75058c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 9e956ad5212f533a0541ee4f5e9e676d8a11b6d6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:59:37 +0900 Subject: [PATCH 248/292] update actions/checkout action to v4 (#1136) --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/django.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a4c434c5..13519f18 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/django.yaml b/.github/workflows/django.yaml index da664f85..395c64fd 100644 --- a/.github/workflows/django.yaml +++ b/.github/workflows/django.yaml @@ -36,7 +36,7 @@ jobs: mysql -uroot -proot -e "CREATE USER 'scott'@'%' IDENTIFIED BY 'tiger'; GRANT ALL ON *.* TO scott;" mysql -uroot -proot -e "CREATE DATABASE django_default; CREATE DATABASE django_other;" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 77edb0c3..c0c013b0 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -13,7 +13,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1153b9e4..dcd1abea 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -56,7 +56,7 @@ jobs: - /run/mysqld:/run/mysqld steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Workaround MySQL container permissions if: startsWith(matrix.db, 'mysql') From c1d8063759a4a3968b0f7907e098554d9a8ad552 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 15 Sep 2023 06:55:02 +0900 Subject: [PATCH 249/292] Update codecov/codecov-action action to v4 (#1137) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dcd1abea..b28b63bd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,4 +106,4 @@ jobs: - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 From 39bf50e057332d27b8ccf27f11d04467fa1e3904 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 14 Nov 2023 18:34:43 +0900 Subject: [PATCH 250/292] ci: use codecov@v3 (#1142) v4 is still beta. --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b28b63bd..dcd1abea 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,4 +106,4 @@ jobs: - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v3 From 5820fa09844276477b3f6299341f9dc05d415526 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 14 Nov 2023 18:35:17 +0900 Subject: [PATCH 251/292] update CHANGELOG Add future changes. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6283670..f371ef32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * Error classes in Cursor class will be removed after 2024-06 * `Connection.set_charset(charset)` will be removed after 2024-06 * `db` and `passwd` will emit DeprecationWarning in v1.2. See #933. - +* `Connection.ping(reconnect)` change the default to not reconnect. ## v1.1.0 From 8b514a4bc103852c8031fd4e0e634ae3d2c10c22 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 15 Nov 2023 02:44:13 +0900 Subject: [PATCH 252/292] ci: update dessant/lock-threads action to v5 (#1141) --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 780dd92d..21449e3b 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -13,5 +13,5 @@ jobs: if: github.repository == 'PyMySQL/PyMySQL' runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v4 + - uses: dessant/lock-threads@v5 From 523f0949f33f481e4d41c920c2e1314faeae28ab Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 13:51:06 +0900 Subject: [PATCH 253/292] update document settings --- .readthedocs.yaml | 22 ++++ docs/Makefile | 24 ----- docs/make.bat | 242 ------------------------------------------ docs/source/conf.py | 3 +- docs/source/index.rst | 4 +- 5 files changed, 26 insertions(+), 269 deletions(-) create mode 100644 .readthedocs.yaml delete mode 100644 docs/make.bat diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..0ff55962 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt diff --git a/docs/Makefile b/docs/Makefile index d3725552..c1240d2b 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -74,30 +74,6 @@ json: @echo @echo "Build finished; now you can process the JSON files." -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyMySQL.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyMySQL.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PyMySQL" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyMySQL" - @echo "# devhelp" - epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index dcd4287c..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,242 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source -set I18NSPHINXOPTS=%SPHINXOPTS% source -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PyMySQL.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PyMySQL.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/docs/source/conf.py b/docs/source/conf.py index a57a03c4..d346fbda 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -101,7 +101,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "default" +# html_theme = "default" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/source/index.rst b/docs/source/index.rst index 97633f1a..e64b6423 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,5 @@ -Welcome to PyMySQL's documentation! -=================================== +PyMySQL documentation +===================== .. toctree:: :maxdepth: 2 From f62893a2c284468091efc95e5b744abcf274dc34 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 13:57:25 +0900 Subject: [PATCH 254/292] update document settings --- docs/source/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index d346fbda..410e9c74 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,7 +30,7 @@ # ones. extensions = [ "sphinx.ext.autodoc", - "sphinx.ext.intersphinx", + "sphinx_rtd_theme", ] # Add any paths that contain templates here, relative to this directory. @@ -47,7 +47,7 @@ # General information about the project. project = "PyMySQL" -copyright = "2016, Yutaka Matsubara and GitHub contributors" +copyright = "2023, Inada Naoki and GitHub contributors" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -104,6 +104,7 @@ # html_theme = "default" html_theme = "sphinx_rtd_theme" + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. From eb0b6e3429fcd1971be56cae32ffe5780c1c9cb6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 13:57:25 +0900 Subject: [PATCH 255/292] update document settings --- docs/source/conf.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 410e9c74..1eafbda8 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,7 +30,6 @@ # ones. extensions = [ "sphinx.ext.autodoc", - "sphinx_rtd_theme", ] # Add any paths that contain templates here, relative to this directory. @@ -101,9 +100,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# html_theme = "default" -html_theme = "sphinx_rtd_theme" - + html_theme = "default" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 7f96f9335181c5ae4992a097540348a2ae174cc6 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:12:24 +0900 Subject: [PATCH 256/292] fix conf.py --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1eafbda8..a8bee6c6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -100,7 +100,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. - html_theme = "default" +html_theme = "default" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 910e5fc1e2bec0e80f75ac5c2d955686e3a5242c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:28:25 +0900 Subject: [PATCH 257/292] update document settings --- .readthedocs.yaml | 15 +++++---------- docs/requirements.txt | 2 ++ 2 files changed, 7 insertions(+), 10 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0ff55962..59fdb65d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,22 +1,17 @@ # .readthedocs.yaml # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required version: 2 -# Set the version of Python and other tools you might need build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.12" + +python: + install: + - requirements: docs/requirements.txt # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/source/conf.py - -# We recommend specifying your dependencies to enable reproducible builds: -# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..8d45d0b6 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx~=7.2 +sphinx-rtd-theme~=1.3.0 From 0001c409524e4738a3e686c7faf65421281fbf4f Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:34:27 +0900 Subject: [PATCH 258/292] doc: use rtd theme (#1143) --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a8bee6c6..78dc55ca 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -100,7 +100,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "default" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 1ed7cffc0335442235ac103ed458ae38f95b33b1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 14:39:24 +0900 Subject: [PATCH 259/292] fix ruff error --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8e75058c..b9a3ef54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ line-length = 99 exclude = [ "pymysql/tests/thirdparty", ] +ignore = ["E721"] [tool.pdm.dev-dependencies] dev = [ From 84d3f93701341ba34c352663d3a5fc22af2f3d32 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Nov 2023 17:24:51 +0900 Subject: [PATCH 260/292] use Ruff as formatter (#1144) --- .github/workflows/lint.yaml | 14 ++++++++------ pymysql/protocol.py | 4 ++-- pymysql/tests/test_SSCursor.py | 5 +---- pymysql/tests/test_basic.py | 6 +++--- pymysql/tests/test_cursor.py | 3 +-- pymysql/tests/test_issues.py | 5 ++--- pymysql/tests/test_nextset.py | 2 +- pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py | 8 ++++---- .../test_MySQLdb/test_MySQLdb_dbapi20.py | 2 +- pyproject.toml | 1 - 10 files changed, 23 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c0c013b0..269211c2 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -13,11 +13,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: checkout + uses: actions/checkout@v4 - - uses: psf/black@stable - with: - options: "--check --verbose" - src: "." + - name: lint + uses: chartboost/ruff-action@v1 - - uses: chartboost/ruff-action@v1 + - name: check format + uses: chartboost/ruff-action@v1 + with: + args: "format --diff" diff --git a/pymysql/protocol.py b/pymysql/protocol.py index 2db92d39..340d9cf2 100644 --- a/pymysql/protocol.py +++ b/pymysql/protocol.py @@ -89,8 +89,8 @@ def advance(self, length): new_position = self._position + length if new_position < 0 or new_position > len(self._data): raise Exception( - "Invalid advance amount (%s) for cursor. " - "Position=%s" % (length, new_position) + "Invalid advance amount (%s) for cursor. Position=%s" + % (length, new_position) ) self._position = new_position diff --git a/pymysql/tests/test_SSCursor.py b/pymysql/tests/test_SSCursor.py index 9cb5bafe..d5e6e2bc 100644 --- a/pymysql/tests/test_SSCursor.py +++ b/pymysql/tests/test_SSCursor.py @@ -27,10 +27,7 @@ def test_SSCursor(self): # Create table cursor.execute( - "CREATE TABLE tz_data (" - "region VARCHAR(64)," - "zone VARCHAR(64)," - "name VARCHAR(64))" + "CREATE TABLE tz_data (region VARCHAR(64), zone VARCHAR(64), name VARCHAR(64))" ) conn.begin() diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index e77605fd..c60b0cca 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -364,7 +364,7 @@ def test_bulk_insert(self): data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] cursor.executemany( - "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + "insert into bulkinsert (id, name, age, height) values (%s,%s,%s,%s)", data, ) self.assertEqual( @@ -414,14 +414,14 @@ def test_bulk_insert_single_record(self): cursor = conn.cursor() data = [(0, "bob", 21, 123)] cursor.executemany( - "insert into bulkinsert (id, name, age, height) " "values (%s,%s,%s,%s)", + "insert into bulkinsert (id, name, age, height) values (%s,%s,%s,%s)", data, ) cursor.execute("commit") self._verify_records(data) def test_issue_288(self): - """executemany should work with "insert ... on update" """ + """executemany should work with "insert ... on update""" conn = self.connect() cursor = conn.cursor() data = [(0, "bob", 21, 123), (1, "jim", 56, 45), (2, "fred", 100, 180)] diff --git a/pymysql/tests/test_cursor.py b/pymysql/tests/test_cursor.py index b292c206..2e267fb6 100644 --- a/pymysql/tests/test_cursor.py +++ b/pymysql/tests/test_cursor.py @@ -17,8 +17,7 @@ def setUp(self): ) cursor = conn.cursor() cursor.execute( - "insert into test (data) values " - "('row1'), ('row2'), ('row3'), ('row4'), ('row5')" + "insert into test (data) values ('row1'), ('row2'), ('row3'), ('row4'), ('row5')" ) conn.commit() cursor.close() diff --git a/pymysql/tests/test_issues.py b/pymysql/tests/test_issues.py index 3564d3a6..f1fe8dd4 100644 --- a/pymysql/tests/test_issues.py +++ b/pymysql/tests/test_issues.py @@ -401,10 +401,9 @@ def test_issue_321(self): sql_insert = "insert into issue321 (value_1, value_2) values (%s, %s)" sql_dict_insert = ( - "insert into issue321 (value_1, value_2) " - "values (%(value_1)s, %(value_2)s)" + "insert into issue321 (value_1, value_2) values (%(value_1)s, %(value_2)s)" ) - sql_select = "select * from issue321 where " "value_1 in %s and value_2=%s" + sql_select = "select * from issue321 where value_1 in %s and value_2=%s" data = [ [("a",), "\u0430"], [["b"], "\u0430"], diff --git a/pymysql/tests/test_nextset.py b/pymysql/tests/test_nextset.py index 4b6b2a77..a10f8d5b 100644 --- a/pymysql/tests/test_nextset.py +++ b/pymysql/tests/test_nextset.py @@ -75,7 +75,7 @@ def test_multi_statement_warnings(self): cursor = con.cursor() try: - cursor.execute("DROP TABLE IF EXISTS a; " "DROP TABLE IF EXISTS b;") + cursor.execute("DROP TABLE IF EXISTS a; DROP TABLE IF EXISTS b;") except TypeError: self.fail() diff --git a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py index 83851295..fff14b86 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/dbapi20.py @@ -299,7 +299,7 @@ def test_rowcount(self): self.assertEqual( cur.rowcount, -1, - "cursor.rowcount should be -1 after executing no-result " "statements", + "cursor.rowcount should be -1 after executing no-result statements", ) cur.execute( "insert into %sbooze values ('Victoria Bitter')" % (self.table_prefix) @@ -409,12 +409,12 @@ def _paraminsert(self, cur): self.assertEqual( beers[0], "Cooper's", - "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + "cursor.fetchall retrieved incorrect data, or data inserted incorrectly", ) self.assertEqual( beers[1], "Victoria Bitter", - "cursor.fetchall retrieved incorrect data, or data inserted " "incorrectly", + "cursor.fetchall retrieved incorrect data, or data inserted incorrectly", ) def test_executemany(self): @@ -482,7 +482,7 @@ def test_fetchone(self): self.assertEqual( cur.fetchone(), None, - "cursor.fetchone should return None if a query retrieves " "no rows", + "cursor.fetchone should return None if a query retrieves no rows", ) self.assertTrue(cur.rowcount in (-1, 0)) diff --git a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py index c68289fe..5c34d40d 100644 --- a/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +++ b/pymysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py @@ -98,7 +98,7 @@ def test_fetchone(self): self.assertEqual( cur.fetchone(), None, - "cursor.fetchone should return None if a query retrieves " "no rows", + "cursor.fetchone should return None if a query retrieves no rows", ) self.assertTrue(cur.rowcount in (-1, 0)) diff --git a/pyproject.toml b/pyproject.toml index b9a3ef54..1c10b4b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ exclude = ["tests*", "pymysql.tests*"] version = {attr = "pymysql.VERSION_STRING"} [tool.ruff] -line-length = 99 exclude = [ "pymysql/tests/thirdparty", ] From d206182822d61650eed8ccd167171ea6131113d1 Mon Sep 17 00:00:00 2001 From: Sergei Vaskov Date: Thu, 16 Nov 2023 12:39:02 +0200 Subject: [PATCH 261/292] Add ssl_key_password param (#1145) Add support for SSL private key password in Connection class to handle encrypted keys. Co-authored-by: Sergei Vaskov --- pymysql/connections.py | 10 +++- pymysql/tests/test_connection.py | 81 +++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 843bea5e..7e12e169 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -135,6 +135,7 @@ class Connection: :param ssl_disabled: A boolean value that disables usage of TLS. :param ssl_key: Path to the file that contains a PEM-formatted private key for the client certificate. + :param ssl_key_password: The password for the client certificate private key. :param ssl_verify_cert: Set to true to check the server certificate's validity. :param ssl_verify_identity: Set to true to check the server's identity. :param read_default_group: Group to read from in the configuration file. @@ -201,6 +202,7 @@ def __init__( ssl_cert=None, ssl_disabled=None, ssl_key=None, + ssl_key_password=None, ssl_verify_cert=None, ssl_verify_identity=None, compress=None, # not supported @@ -262,7 +264,7 @@ def _config(key, arg): if not ssl: ssl = {} if isinstance(ssl, dict): - for key in ["ca", "capath", "cert", "key", "cipher"]: + for key in ["ca", "capath", "cert", "key", "password", "cipher"]: value = _config("ssl-" + key, ssl.get(key)) if value: ssl[key] = value @@ -281,6 +283,8 @@ def _config(key, arg): ssl["cert"] = ssl_cert if ssl_key is not None: ssl["key"] = ssl_key + if ssl_key_password is not None: + ssl["password"] = ssl_key_password if ssl: if not SSL_ENABLED: raise NotImplementedError("ssl module not found") @@ -389,7 +393,9 @@ def _create_ssl_ctx(self, sslp): else: ctx.verify_mode = ssl.CERT_NONE if hasnoca else ssl.CERT_REQUIRED if "cert" in sslp: - ctx.load_cert_chain(sslp["cert"], keyfile=sslp.get("key")) + ctx.load_cert_chain( + sslp["cert"], keyfile=sslp.get("key"), password=sslp.get("password") + ) if "cipher" in sslp: ctx.set_ciphers(sslp["cipher"]) ctx.options |= ssl.OP_NO_SSLv2 diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 0803efc9..ccfc4a32 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -574,7 +574,11 @@ def test_ssl_connect(self): assert create_default_context.called assert dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) dummy_ssl_context.set_ciphers.assert_called_with("cipher") dummy_ssl_context = mock.Mock(options=0) @@ -592,7 +596,34 @@ def test_ssl_connect(self): assert create_default_context.called assert dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: + pymysql.connect( + ssl={ + "ca": "ca", + "cert": "cert", + "key": "key", + "password": "password", + }, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password="password", + ) dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) @@ -622,7 +653,11 @@ def test_ssl_connect(self): assert create_default_context.called assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) dummy_ssl_context.set_ciphers.assert_not_called for ssl_verify_cert in (True, "1", "yes", "true"): @@ -640,7 +675,9 @@ def test_ssl_connect(self): assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_REQUIRED dummy_ssl_context.load_cert_chain.assert_called_with( - "cert", keyfile="key" + "cert", + keyfile="key", + password=None, ) dummy_ssl_context.set_ciphers.assert_not_called @@ -659,7 +696,9 @@ def test_ssl_connect(self): assert not dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE dummy_ssl_context.load_cert_chain.assert_called_with( - "cert", keyfile="key" + "cert", + keyfile="key", + password=None, ) dummy_ssl_context.set_ciphers.assert_not_called @@ -682,7 +721,9 @@ def test_ssl_connect(self): ssl.CERT_REQUIRED if ssl_ca is not None else ssl.CERT_NONE ), (ssl_ca, ssl_verify_cert) dummy_ssl_context.load_cert_chain.assert_called_with( - "cert", keyfile="key" + "cert", + keyfile="key", + password=None, ) dummy_ssl_context.set_ciphers.assert_not_called @@ -700,7 +741,33 @@ def test_ssl_connect(self): assert create_default_context.called assert dummy_ssl_context.check_hostname assert dummy_ssl_context.verify_mode == ssl.CERT_NONE - dummy_ssl_context.load_cert_chain.assert_called_with("cert", keyfile="key") + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password=None, + ) + dummy_ssl_context.set_ciphers.assert_not_called + + dummy_ssl_context = mock.Mock(options=0) + with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + "pymysql.connections.ssl.create_default_context", + new=mock.Mock(return_value=dummy_ssl_context), + ) as create_default_context: + pymysql.connect( + ssl_ca="ca", + ssl_cert="cert", + ssl_key="key", + ssl_key_password="password", + ssl_verify_identity=True, + ) + assert create_default_context.called + assert dummy_ssl_context.check_hostname + assert dummy_ssl_context.verify_mode == ssl.CERT_NONE + dummy_ssl_context.load_cert_chain.assert_called_with( + "cert", + keyfile="key", + password="password", + ) dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0) From f476773eca5480c75e6d418abd7e73ae6c51ac22 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 13:31:03 +0900 Subject: [PATCH 262/292] chore(deps): update dependency sphinx-rtd-theme to v2 (#1147) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 8d45d0b6..01406623 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ sphinx~=7.2 -sphinx-rtd-theme~=1.3.0 +sphinx-rtd-theme~=2.0.0 From f13f054abcc18b39855a760a84be0a517f0da658 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 22:34:08 +0900 Subject: [PATCH 263/292] chore(deps): update actions/setup-python action to v5 (#1152) --- .github/workflows/django.yaml | 2 +- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/django.yaml b/.github/workflows/django.yaml index 395c64fd..5c460954 100644 --- a/.github/workflows/django.yaml +++ b/.github/workflows/django.yaml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dcd1abea..bfe8fff1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -65,7 +65,7 @@ jobs: /usr/bin/docker ps --all --filter status=exited --no-trunc --format "{{.ID}}" | xargs -r /usr/bin/docker start - name: Set up Python ${{ matrix.py }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} allow-prereleases: true From 1e28be81c24dde66f8acbf4c5e24f60d6b5e72e7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:04:49 +0900 Subject: [PATCH 264/292] chore(deps): update github/codeql-action action to v3 (#1154) --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 13519f18..df49979e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: "python" # If you wish to specify custom queries, you can do so here or in a config file. @@ -45,7 +45,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -59,4 +59,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 1f0b7856de4008e7e4c1e8c1b215d5d4dfaecd1a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 12:55:12 +0900 Subject: [PATCH 265/292] chore(deps): update codecov/codecov-action action to v4 (#1158) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bfe8fff1..6d59d8c4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,4 +106,4 @@ jobs: - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 From 9694747ae619e88b792a8e0b4c08036572452584 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Feb 2024 15:42:24 +0900 Subject: [PATCH 266/292] pyupgrade --- docs/source/conf.py | 2 -- pymysql/connections.py | 23 +++++++++-------------- pymysql/protocol.py | 6 ++---- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 78dc55ca..158d0d12 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # PyMySQL documentation build configuration file, created by # sphinx-quickstart on Tue May 17 12:01:11 2016. # diff --git a/pymysql/connections.py b/pymysql/connections.py index 7e12e169..dc121e1b 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -84,8 +84,7 @@ def _lenenc_int(i): return b"\xfe" + struct.pack(" len(self._data): raise Exception( - "Invalid advance amount (%s) for cursor. Position=%s" - % (length, new_position) + f"Invalid advance amount ({length}) for cursor. Position={new_position}" ) self._position = new_position From bbd049f40db9c696574ce6f31669880042c56d79 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Feb 2024 17:16:41 +0900 Subject: [PATCH 267/292] Support error packet without sqlstate (#1160) Fix #1156 --- pymysql/connections.py | 2 -- pymysql/err.py | 9 ++++++++- pymysql/tests/test_err.py | 22 ++++++++++++---------- pyproject.toml | 2 ++ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index dc121e1b..3a04ddd6 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -765,8 +765,6 @@ def _read_packet(self, packet_type=MysqlPacket): dump_packet(recv_data) buff += recv_data # https://dev.mysql.com/doc/internals/en/sending-more-than-16mbyte.html - if bytes_to_read == 0xFFFFFF: - continue if bytes_to_read < MAX_PACKET_LEN: break diff --git a/pymysql/err.py b/pymysql/err.py index 3da5b166..dac65d3b 100644 --- a/pymysql/err.py +++ b/pymysql/err.py @@ -136,7 +136,14 @@ def _map_error(exc, *errors): def raise_mysql_exception(data): errno = struct.unpack(" Date: Tue, 26 Mar 2024 18:02:41 +1100 Subject: [PATCH 268/292] test json - mariadb without JSON type (#1165) MariaDB-11.0.1 removed the 5.5.5 version hack (MDEV-28910). MariaDB still doesn't support JSON as a type. Use get_mysql_vendor() == mysql for the final part of test_json. --- pymysql/tests/test_basic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pymysql/tests/test_basic.py b/pymysql/tests/test_basic.py index c60b0cca..0fe13b59 100644 --- a/pymysql/tests/test_basic.py +++ b/pymysql/tests/test_basic.py @@ -323,9 +323,10 @@ def test_json(self): res = cur.fetchone()[0] self.assertEqual(json.loads(res), json.loads(json_str)) - cur.execute("SELECT CAST(%s AS JSON) AS x", (json_str,)) - res = cur.fetchone()[0] - self.assertEqual(json.loads(res), json.loads(json_str)) + if self.get_mysql_vendor(conn) == "mysql": + cur.execute("SELECT CAST(%s AS JSON) AS x", (json_str,)) + res = cur.fetchone()[0] + self.assertEqual(json.loads(res), json.loads(json_str)) class TestBulkInserts(base.PyMySQLTestCase): From 69f6c7439bee14784e0ea70ae107af6446cc0c67 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 4 May 2024 15:55:22 +0900 Subject: [PATCH 269/292] ruff format --- pymysql/__init__.py | 1 + pymysql/_auth.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 53625d37..37395551 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -21,6 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + import sys from .constants import FIELD_TYPE diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 99987b77..8ce744fb 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -1,6 +1,7 @@ """ Implements auth methods """ + from .err import OperationalError From 7f032a699d55340f05101deb4d7d4f63db4adc11 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 20 May 2024 13:25:18 +0900 Subject: [PATCH 270/292] remove coveralls from requirements --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 13d7f7fb..140d3706 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,3 @@ cryptography PyNaCl>=1.4.0 pytest pytest-cov -coveralls From 521e40050cb386a499f68f483fefd144c493053c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 18 May 2024 11:33:30 +0900 Subject: [PATCH 271/292] forbid dict parameter --- pymysql/converters.py | 6 +----- pymysql/tests/test_connection.py | 7 +++++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 1adac752..dbf97ca7 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -27,11 +27,7 @@ def escape_item(val, charset, mapping=None): def escape_dict(val, charset, mapping=None): - n = {} - for k, v in val.items(): - quoted = escape_item(v, charset, mapping) - n[k] = quoted - return n + raise TypeError("dict can not be used as parameter") def escape_sequence(val, charset, mapping=None): diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index ccfc4a32..dcf3394c 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -848,12 +848,15 @@ def test_escape_no_default(self): self.assertRaises(TypeError, con.escape, 42, {}) - def test_escape_dict_value(self): + def test_escape_dict_raise_typeerror(self): + """con.escape(dict) should raise TypeError""" con = self.connect() mapping = con.encoders.copy() mapping[Foo] = escape_foo - self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) + #self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) + with self.assertRaises(TypeError): + con.escape({"foo": Foo()}) def test_escape_list_item(self): con = self.connect() From 2cab9ecc641e962565c6254a5091f90c47f59b35 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 21 May 2024 20:01:22 +0900 Subject: [PATCH 272/292] v1.1.1 --- CHANGELOG.md | 15 +++++++++++++++ pymysql/__init__.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f371ef32..825dc47c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ * `db` and `passwd` will emit DeprecationWarning in v1.2. See #933. * `Connection.ping(reconnect)` change the default to not reconnect. +## v1.1.1 + +Release date: 2024-05-21 + +> [!WARNING] +> This release fixes a vulnerability (CVE-2024-36039). +> All users are recommended to update to this version. +> +> If you can not update soon, check the input value from +> untrusted source has an expected type. Only dict input +> from untrusted source can be an attack vector. + +* Prohibit dict parameter for `Cursor.execute()`. It didn't produce valid SQL + and might cause SQL injection. (CVE-2024-36039) + ## v1.1.0 Release date: 2023-06-26 diff --git a/pymysql/__init__.py b/pymysql/__init__.py index 37395551..bbf9023e 100644 --- a/pymysql/__init__.py +++ b/pymysql/__init__.py @@ -49,8 +49,8 @@ # PyMySQL version. # Used by setuptools and connection_attrs -VERSION = (1, 1, 0, "final", 1) -VERSION_STRING = "1.1.0" +VERSION = (1, 1, 1, "final", 1) +VERSION_STRING = "1.1.1" ### for mysqlclient compatibility ### Django checks mysqlclient version. From a6ae2c71966fb65b071c9066e21d8c806df42f15 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 21 May 2024 20:04:54 +0900 Subject: [PATCH 273/292] fix format --- pymysql/tests/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index dcf3394c..d8e69b32 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -854,7 +854,7 @@ def test_escape_dict_raise_typeerror(self): mapping = con.encoders.copy() mapping[Foo] = escape_foo - #self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) + # self.assertEqual(con.escape({"foo": Foo()}, mapping), {"foo": "bar"}) with self.assertRaises(TypeError): con.escape({"foo": Foo()}) From 53b35f7fbe6d0e4cfc22996ab9f5523a4829b11c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 21 May 2024 20:09:01 +0900 Subject: [PATCH 274/292] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 825dc47c..a633f6c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Release date: 2024-05-21 * Prohibit dict parameter for `Cursor.execute()`. It didn't produce valid SQL and might cause SQL injection. (CVE-2024-36039) +* Added ssl_key_password param. #1145 ## v1.1.0 From 95635f587ba9076e71a223b113efb08ac34a361d Mon Sep 17 00:00:00 2001 From: Mirko Palancaji <41736842+palm002@users.noreply.github.com> Date: Wed, 19 Jun 2024 22:57:53 +1000 Subject: [PATCH 275/292] Prevent UnboundLocalError during unbuffered query (#1174) Addresses the issue of `UnboundLocalError` which occurs when `MySQLResult` class fails to initialize due to a `SystemExit` exception by initialising the `MySQLResult` object before `try/except` block. Co-authored-by: Inada Naoki --- pymysql/connections.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 3a04ddd6..f12731e1 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -812,16 +812,10 @@ def _write_bytes(self, data): def _read_query_result(self, unbuffered=False): self._result = None + result = MySQLResult(self) if unbuffered: - try: - result = MySQLResult(self) - result.init_unbuffered_query() - except: - result.unbuffered_active = False - result.connection = None - raise + result.init_unbuffered_query() else: - result = MySQLResult(self) result.read() self._result = result if result.server_status is not None: @@ -1212,17 +1206,16 @@ def init_unbuffered_query(self): :raise OperationalError: If the connection to the MySQL server is lost. :raise InternalError: """ - self.unbuffered_active = True first_packet = self.connection._read_packet() if first_packet.is_ok_packet(): - self._read_ok_packet(first_packet) - self.unbuffered_active = False self.connection = None + self._read_ok_packet(first_packet) elif first_packet.is_load_local_packet(): - self._read_load_local_packet(first_packet) - self.unbuffered_active = False - self.connection = None + try: + self._read_load_local_packet(first_packet) + finally: + self.connection = None else: self.field_count = first_packet.read_length_encoded_integer() self._get_descriptions() @@ -1231,6 +1224,7 @@ def init_unbuffered_query(self): # value of a 64bit unsigned integer. Since we're emulating MySQLdb, # we set it to this instead of None, which would be preferred. self.affected_rows = 18446744073709551615 + self.unbuffered_active = True def _read_ok_packet(self, first_packet): ok_packet = OKPacketWrapper(first_packet) From d93cde99055092b9c802a5038cf31bf98b2b87aa Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Wed, 4 Sep 2024 06:40:43 +0200 Subject: [PATCH 276/292] remove mention of runtests.py (#1182) --- docs/source/user/development.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/user/development.rst b/docs/source/user/development.rst index 1f8a2637..2d80a624 100644 --- a/docs/source/user/development.rst +++ b/docs/source/user/development.rst @@ -28,7 +28,7 @@ and edit the new file to match your MySQL configuration:: $ cp ci/database.json pymysql/tests/databases.json $ $EDITOR pymysql/tests/databases.json -To run all the tests, execute the script ``runtests.py``:: +To run all the tests, you can use pytest:: - $ pip install pytest + $ pip install -r requirements-dev.txt $ pytest -v pymysql From 9204b641f3ecff73704e10549f615d8762358652 Mon Sep 17 00:00:00 2001 From: CF Bolz-Tereick Date: Thu, 5 Sep 2024 07:20:39 +0200 Subject: [PATCH 277/292] close `connection._rfile` in `Connection._force_close` (#1184) fix #1183. --- pymysql/connections.py | 10 ++++----- pymysql/tests/test_connection.py | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index f12731e1..5f60377e 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -161,6 +161,7 @@ class Connection: """ _sock = None + _rfile = None _auth_plugin_name = "" _closed = False _secure = False @@ -430,6 +431,8 @@ def open(self): def _force_close(self): """Close connection without QUIT message.""" + if self._rfile: + self._rfile.close() if self._sock: try: self._sock.close() @@ -696,12 +699,7 @@ def connect(self, sock=None): if self.autocommit_mode is not None: self.autocommit(self.autocommit_mode) except BaseException as e: - self._rfile = None - if sock is not None: - try: - sock.close() - except: # noqa - pass + self._force_close() if isinstance(e, (OSError, IOError)): exc = err.OperationalError( diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index d8e69b32..61dba600 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -883,3 +883,40 @@ def test_commit_during_multi_result(self): con.commit() cur.execute("SELECT 3") self.assertEqual(cur.fetchone()[0], 3) + + def test_force_close_closes_socketio(self): + con = self.connect() + sock = con._sock + fileno = sock.fileno() + rfile = con._rfile + + con._force_close() + assert rfile.closed + assert sock._closed + assert sock.fileno() != fileno # should be set to -1 + + def test_socket_closed_on_exception_in_connect(self): + con = self.connect(defer_connect=True) + sock = None + rfile = None + fileno = -1 + + def _request_authentication(): + nonlocal sock, rfile, fileno + sock = con._sock + assert sock is not None + fileno = sock.fileno() + rfile = con._rfile + assert rfile is not None + raise TypeError + + con._request_authentication = _request_authentication + + with pytest.raises(TypeError): + con.connect() + assert not con.open + assert con._rfile is None + assert con._sock is None + assert rfile.closed + assert sock._closed + assert sock.fileno() != fileno # should be set to -1 From ec27bade879ad05fda214188d035c1fe3f255a35 Mon Sep 17 00:00:00 2001 From: Ujjwal Kumar Singh <95489300+theneuralcraftsman@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:21:03 +0530 Subject: [PATCH 278/292] Added MariaDB in readme description (#1186) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32f5df2f..a91c6008 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # PyMySQL -This package contains a pure-Python MySQL client library, based on [PEP +This package contains a pure-Python MySQL and MariaDB client library, based on [PEP 249](https://www.python.org/dev/peps/pep-0249/). ## Requirements From 54e68807dd1a3f67b855c1e8c4c6ce0526d2bff1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:23:07 +0900 Subject: [PATCH 279/292] chore(deps): update dependency sphinx-rtd-theme to v3 (#1189) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 01406623..d2f5c5a5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ sphinx~=7.2 -sphinx-rtd-theme~=2.0.0 +sphinx-rtd-theme~=3.0.0 From dabf0982b498112db8883dcf71a4f68c9d2d9fad Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:30:20 +0900 Subject: [PATCH 280/292] chore(deps): update dependency sphinx to v8 (#1179) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index d2f5c5a5..48319f03 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx~=7.2 +sphinx~=8.0 sphinx-rtd-theme~=3.0.0 From a1ac8239c8bf79e7f1a17347b10d6e184221f9c1 Mon Sep 17 00:00:00 2001 From: Cycloctane Date: Wed, 6 Nov 2024 11:46:44 +0800 Subject: [PATCH 281/292] Add support for Python 3.13 (#1190) - fixes #1188 - Add python 3.13 to test matrix and pyproject.toml --- .github/workflows/test.yaml | 3 +++ pymysql/connections.py | 6 ++++-- pyproject.toml | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6d59d8c4..d3693fdd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -30,6 +30,9 @@ jobs: - db: "mariadb:10.6" py: "3.12" + - db: "mariadb:10.6" + py: "3.13" + - db: "mariadb:lts" py: "3.9" diff --git a/pymysql/connections.py b/pymysql/connections.py index 5f60377e..fe4d0c45 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -40,8 +40,10 @@ DEFAULT_USER = getpass.getuser() del getpass -except (ImportError, KeyError): - # KeyError occurs when there's no entry in OS database for a current user. +except (ImportError, KeyError, OSError): + # When there's no entry in OS database for a current user: + # KeyError is raised in Python 3.12 and below. + # OSError is raised in Python 3.13+ DEFAULT_USER = None DEBUG = False diff --git a/pyproject.toml b/pyproject.toml index 8cd9ddb4..ee103916 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 8876b98b683912b46ddafa1ac2fcea9911e2c8c4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Dec 2024 18:37:23 +0900 Subject: [PATCH 282/292] ci: remove lock-threads --- .github/workflows/lock.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .github/workflows/lock.yml diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml deleted file mode 100644 index 21449e3b..00000000 --- a/.github/workflows/lock.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: 'Lock Threads' - -on: - schedule: - - cron: '30 9 * * 1' - -permissions: - issues: write - pull-requests: write - -jobs: - lock-threads: - if: github.repository == 'PyMySQL/PyMySQL' - runs-on: ubuntu-latest - steps: - - uses: dessant/lock-threads@v5 - From 7dead51f8605f315e7931bae58ea8b2126b945ba Mon Sep 17 00:00:00 2001 From: Eugene Kennedy Date: Sun, 12 Jan 2025 03:17:12 -0500 Subject: [PATCH 283/292] Resolve UTF8 charset case-insensitively (#1195) --- pymysql/charset.py | 3 ++- pymysql/tests/test_charset.py | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pymysql/charset.py b/pymysql/charset.py index b1c1ca8b..ec8e14e2 100644 --- a/pymysql/charset.py +++ b/pymysql/charset.py @@ -45,9 +45,10 @@ def by_id(self, id): return self._by_id[id] def by_name(self, name): + name = name.lower() if name == "utf8": name = "utf8mb4" - return self._by_name.get(name.lower()) + return self._by_name.get(name) _charsets = Charsets() diff --git a/pymysql/tests/test_charset.py b/pymysql/tests/test_charset.py index 94e6e155..85a310e4 100644 --- a/pymysql/tests/test_charset.py +++ b/pymysql/tests/test_charset.py @@ -21,5 +21,23 @@ def test_utf8(): ) # utf8 is alias of utf8mb4 since MySQL 8.0, and PyMySQL v1.1. - utf8 = pymysql.charset.charset_by_name("utf8") - assert utf8 == utf8mb4 + lowercase_utf8 = pymysql.charset.charset_by_name("utf8") + assert lowercase_utf8 == utf8mb4 + + # Regardless of case, UTF8 (which is special cased) should resolve to the same thing + uppercase_utf8 = pymysql.charset.charset_by_name("UTF8") + mixedcase_utf8 = pymysql.charset.charset_by_name("UtF8") + assert uppercase_utf8 == lowercase_utf8 + assert mixedcase_utf8 == lowercase_utf8 + +def test_case_sensitivity(): + lowercase_latin1 = pymysql.charset.charset_by_name("latin1") + assert lowercase_latin1 is not None + + # lowercase and uppercase should resolve to the same charset + uppercase_latin1 = pymysql.charset.charset_by_name("LATIN1") + assert uppercase_latin1 == lowercase_latin1 + + # lowercase and mixed case should resolve to the same charset + mixedcase_latin1 = pymysql.charset.charset_by_name("LaTiN1") + assert mixedcase_latin1 == lowercase_latin1 From 046d36c83a272b322b41146a326af4606df9f0d4 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 14 Jan 2025 16:51:25 +0900 Subject: [PATCH 284/292] update ci versions (#1196) --- .github/workflows/test.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d3693fdd..b67c2ea9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,14 +22,11 @@ jobs: py: "3.8" - db: "mariadb:10.5" - py: "3.7" + py: "3.8" - db: "mariadb:10.6" py: "3.11" - - db: "mariadb:10.6" - py: "3.12" - - db: "mariadb:10.6" py: "3.13" @@ -37,14 +34,15 @@ jobs: py: "3.9" - db: "mysql:5.7" - py: "pypy-3.8" + py: "pypy-3.10" - db: "mysql:8.0" - py: "3.9" + py: "3.8" mysql_auth: true - db: "mysql:8.0" py: "3.10" + mysql_auth: true services: mysql: From 0d4609c22b55ad7827ab7186cbbc44068f0a0ed2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 14 Jan 2025 19:05:42 +0900 Subject: [PATCH 285/292] use KILL instead of COM_KILL for MySQL 8.4 support (#1197) --- .github/workflows/test.yaml | 16 ++++++++-------- pymysql/connections.py | 6 +++--- pymysql/tests/test_charset.py | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b67c2ea9..a8e10af0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -19,29 +19,29 @@ jobs: matrix: include: - db: "mariadb:10.4" - py: "3.8" + py: "3.13" - db: "mariadb:10.5" - py: "3.8" + py: "3.11" - db: "mariadb:10.6" - py: "3.11" + py: "3.10" - db: "mariadb:10.6" - py: "3.13" + py: "3.9" - db: "mariadb:lts" - py: "3.9" + py: "3.8" - db: "mysql:5.7" py: "pypy-3.10" - db: "mysql:8.0" - py: "3.8" + py: "3.13" mysql_auth: true - - db: "mysql:8.0" - py: "3.10" + - db: "mysql:8.4" + py: "3.8" mysql_auth: true services: diff --git a/pymysql/connections.py b/pymysql/connections.py index fe4d0c45..91825f75 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -576,9 +576,9 @@ def affected_rows(self): return self._affected_rows def kill(self, thread_id): - arg = struct.pack(" Date: Tue, 14 Jan 2025 19:36:35 +0900 Subject: [PATCH 286/292] disable VERIFY_X509_STRICT for Python 3.13 support (#1198) --- pymysql/connections.py | 6 ++++++ pymysql/tests/test_connection.py | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/pymysql/connections.py b/pymysql/connections.py index 91825f75..2ddcb3f7 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -377,6 +377,12 @@ def _create_ssl_ctx(self, sslp): capath = sslp.get("capath") hasnoca = ca is None and capath is None ctx = ssl.create_default_context(cafile=ca, capath=capath) + + # Python 3.13 enables VERIFY_X509_STRICT by default. + # But self signed certificates that are generated by MySQL automatically + # doesn't pass the verification. + ctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + ctx.check_hostname = not hasnoca and sslp.get("check_hostname", True) verify_mode_value = sslp.get("verify_mode") if verify_mode_value is None: diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 61dba600..03e35f86 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -558,7 +558,7 @@ def test_defer_connect(self): sock.close() def test_ssl_connect(self): - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -581,7 +581,7 @@ def test_ssl_connect(self): ) dummy_ssl_context.set_ciphers.assert_called_with("cipher") - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -603,7 +603,7 @@ def test_ssl_connect(self): ) dummy_ssl_context.set_ciphers.assert_not_called - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -626,7 +626,7 @@ def test_ssl_connect(self): ) dummy_ssl_context.set_ciphers.assert_not_called - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -640,7 +640,7 @@ def test_ssl_connect(self): dummy_ssl_context.load_cert_chain.assert_not_called dummy_ssl_context.set_ciphers.assert_not_called - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -661,7 +661,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called for ssl_verify_cert in (True, "1", "yes", "true"): - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -682,7 +682,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called for ssl_verify_cert in (None, False, "0", "no", "false"): - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -704,7 +704,7 @@ def test_ssl_connect(self): for ssl_ca in ("ca", None): for ssl_verify_cert in ("foo", "bar", ""): - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -727,7 +727,7 @@ def test_ssl_connect(self): ) dummy_ssl_context.set_ciphers.assert_not_called - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -748,7 +748,7 @@ def test_ssl_connect(self): ) dummy_ssl_context.set_ciphers.assert_not_called - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -770,7 +770,7 @@ def test_ssl_connect(self): ) dummy_ssl_context.set_ciphers.assert_not_called - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), @@ -785,7 +785,7 @@ def test_ssl_connect(self): ) assert not create_default_context.called - dummy_ssl_context = mock.Mock(options=0) + dummy_ssl_context = mock.Mock(options=0, verify_flags=0) with mock.patch("pymysql.connections.Connection.connect"), mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), From 1920de3d8eca0565979e6c32dc2fdfd29c3d8db4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 19:40:04 +0900 Subject: [PATCH 287/292] chore(deps): update codecov/codecov-action action to v5 (#1191) --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a8e10af0..6abc96b7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -107,4 +107,4 @@ jobs: - name: Upload coverage reports to Codecov if: github.repository == 'PyMySQL/PyMySQL' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 From 66ad1eaa47cfde19dfe01900ceb5f6ea51483e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sil=C3=A9n?= Date: Tue, 14 Jan 2025 12:44:46 +0200 Subject: [PATCH 288/292] add MariaDB to README.md (#1181) --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a91c6008..95e4520a 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ # PyMySQL -This package contains a pure-Python MySQL and MariaDB client library, based on [PEP -249](https://www.python.org/dev/peps/pep-0249/). +This package contains a pure-Python MySQL and MariaDB client library, based on +[PEP 249](https://www.python.org/dev/peps/pep-0249/). ## Requirements @@ -92,6 +92,7 @@ This example will print: - DB-API 2.0: - MySQL Reference Manuals: +- Getting Help With MariaDB - MySQL client/server protocol: - "Connector" channel in MySQL Community Slack: From 5f6533f883535b76c2d3a776c4746027027106f8 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Jan 2025 10:45:47 +0900 Subject: [PATCH 289/292] refactor: use defer_connect instead of mock (#1199) --- pymysql/tests/test_connection.py | 36 +++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/pymysql/tests/test_connection.py b/pymysql/tests/test_connection.py index 03e35f86..1a16c982 100644 --- a/pymysql/tests/test_connection.py +++ b/pymysql/tests/test_connection.py @@ -559,7 +559,7 @@ def test_defer_connect(self): def test_ssl_connect(self): dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -570,6 +570,7 @@ def test_ssl_connect(self): "key": "key", "cipher": "cipher", }, + defer_connect=True, ) assert create_default_context.called assert dummy_ssl_context.check_hostname @@ -582,7 +583,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_called_with("cipher") dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -592,6 +593,7 @@ def test_ssl_connect(self): "cert": "cert", "key": "key", }, + defer_connect=True, ) assert create_default_context.called assert dummy_ssl_context.check_hostname @@ -604,7 +606,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -615,6 +617,7 @@ def test_ssl_connect(self): "key": "key", "password": "password", }, + defer_connect=True, ) assert create_default_context.called assert dummy_ssl_context.check_hostname @@ -627,12 +630,13 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: pymysql.connect( ssl_ca="ca", + defer_connect=True, ) assert create_default_context.called assert not dummy_ssl_context.check_hostname @@ -641,7 +645,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -649,6 +653,7 @@ def test_ssl_connect(self): ssl_ca="ca", ssl_cert="cert", ssl_key="key", + defer_connect=True, ) assert create_default_context.called assert not dummy_ssl_context.check_hostname @@ -662,7 +667,7 @@ def test_ssl_connect(self): for ssl_verify_cert in (True, "1", "yes", "true"): dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -670,6 +675,7 @@ def test_ssl_connect(self): ssl_cert="cert", ssl_key="key", ssl_verify_cert=ssl_verify_cert, + defer_connect=True, ) assert create_default_context.called assert not dummy_ssl_context.check_hostname @@ -683,7 +689,7 @@ def test_ssl_connect(self): for ssl_verify_cert in (None, False, "0", "no", "false"): dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -691,6 +697,7 @@ def test_ssl_connect(self): ssl_cert="cert", ssl_key="key", ssl_verify_cert=ssl_verify_cert, + defer_connect=True, ) assert create_default_context.called assert not dummy_ssl_context.check_hostname @@ -705,7 +712,7 @@ def test_ssl_connect(self): for ssl_ca in ("ca", None): for ssl_verify_cert in ("foo", "bar", ""): dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -714,6 +721,7 @@ def test_ssl_connect(self): ssl_cert="cert", ssl_key="key", ssl_verify_cert=ssl_verify_cert, + defer_connect=True, ) assert create_default_context.called assert not dummy_ssl_context.check_hostname @@ -728,7 +736,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -737,6 +745,7 @@ def test_ssl_connect(self): ssl_cert="cert", ssl_key="key", ssl_verify_identity=True, + defer_connect=True, ) assert create_default_context.called assert dummy_ssl_context.check_hostname @@ -749,7 +758,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -759,6 +768,7 @@ def test_ssl_connect(self): ssl_key="key", ssl_key_password="password", ssl_verify_identity=True, + defer_connect=True, ) assert create_default_context.called assert dummy_ssl_context.check_hostname @@ -771,7 +781,7 @@ def test_ssl_connect(self): dummy_ssl_context.set_ciphers.assert_not_called dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -782,11 +792,12 @@ def test_ssl_connect(self): "cert": "cert", "key": "key", }, + defer_connect=True, ) assert not create_default_context.called dummy_ssl_context = mock.Mock(options=0, verify_flags=0) - with mock.patch("pymysql.connections.Connection.connect"), mock.patch( + with mock.patch( "pymysql.connections.ssl.create_default_context", new=mock.Mock(return_value=dummy_ssl_context), ) as create_default_context: @@ -795,6 +806,7 @@ def test_ssl_connect(self): ssl_ca="ca", ssl_cert="cert", ssl_key="key", + defer_connect=True, ) assert not create_default_context.called From e88b729f8f1ddcf0851e0153188b016d0e9ec00c Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 15 Jan 2025 11:43:46 +0900 Subject: [PATCH 290/292] remove codeql and codesee actions --- .github/workflows/codeql-analysis.yml | 62 ---------------------- .github/workflows/codesee-arch-diagram.yml | 23 -------- 2 files changed, 85 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/codesee-arch-diagram.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index df49979e..00000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,62 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - schedule: - - cron: '34 7 * * 2' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: "python" - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/codesee-arch-diagram.yml b/.github/workflows/codesee-arch-diagram.yml deleted file mode 100644 index 806d41d1..00000000 --- a/.github/workflows/codesee-arch-diagram.yml +++ /dev/null @@ -1,23 +0,0 @@ -# This workflow was added by CodeSee. Learn more at https://codesee.io/ -# This is v2.0 of this workflow file -on: - push: - branches: - - main - pull_request_target: - types: [opened, synchronize, reopened] - -name: CodeSee - -permissions: read-all - -jobs: - codesee: - runs-on: ubuntu-latest - continue-on-error: true - name: Analyze the repo with CodeSee - steps: - - uses: Codesee-io/codesee-action@v2 - with: - codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} - codesee-url: https://app.codesee.io From 53efd1ec7f0e854abc62eb875b944f56bca29dd2 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 29 Jan 2025 16:57:30 +0900 Subject: [PATCH 291/292] ci: use astral-sh/ruff-action (#1201) --- .github/workflows/lint.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 269211c2..07ea6603 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -13,13 +13,12 @@ jobs: lint: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: lint - uses: chartboost/ruff-action@v1 + - uses: astral-sh/ruff-action@v3 + + - name: format + run: ruff format --diff - - name: check format - uses: chartboost/ruff-action@v1 - with: - args: "format --diff" + - name: lint + run: ruff check --diff From 01af30fea0880c3b72e6c7b3b05d66a8c28ced7a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 29 Jan 2025 18:06:45 +0900 Subject: [PATCH 292/292] fix auth_switch_request handling (#1200) --- .coveragerc | 1 + pymysql/_auth.py | 8 ++++++-- pymysql/connections.py | 4 ++++ tests/test_auth.py | 28 +++++++++++++++++++++++++++- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index a9ec9942..efa9a2ff 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,7 @@ branch = True source = pymysql + tests omit = pymysql/tests/* pymysql/tests/thirdparty/test_MySQLdb/* diff --git a/pymysql/_auth.py b/pymysql/_auth.py index 8ce744fb..4790449b 100644 --- a/pymysql/_auth.py +++ b/pymysql/_auth.py @@ -166,6 +166,8 @@ def sha256_password_auth(conn, pkt): if pkt.is_auth_switch_request(): conn.salt = pkt.read_all() + if conn.salt.endswith(b"\0"): + conn.salt = conn.salt[:-1] if not conn.server_public_key and conn.password: # Request server public key if DEBUG: @@ -215,9 +217,11 @@ def caching_sha2_password_auth(conn, pkt): if pkt.is_auth_switch_request(): # Try from fast auth - if DEBUG: - print("caching sha2: Trying fast path") conn.salt = pkt.read_all() + if conn.salt.endswith(b"\0"): # str.removesuffix is available in 3.9 + conn.salt = conn.salt[:-1] + if DEBUG: + print(f"caching sha2: Trying fast path. salt={conn.salt.hex()!r}") scrambled = scramble_caching_sha2(conn.password, conn.salt) pkt = _roundtrip(conn, scrambled) # else: fast auth is tried in initial handshake diff --git a/pymysql/connections.py b/pymysql/connections.py index 2ddcb3f7..99fcfcd0 100644 --- a/pymysql/connections.py +++ b/pymysql/connections.py @@ -47,6 +47,7 @@ DEFAULT_USER = None DEBUG = False +_DEFAULT_AUTH_PLUGIN = None # if this is not None, use it instead of server's default. TEXT_TYPES = { FIELD_TYPE.BIT, @@ -1158,6 +1159,9 @@ def _get_server_information(self): else: self._auth_plugin_name = data[i:server_end].decode("utf-8") + if _DEFAULT_AUTH_PLUGIN is not None: # for tests + self._auth_plugin_name = _DEFAULT_AUTH_PLUGIN + def get_server_info(self): return self.server_version diff --git a/tests/test_auth.py b/tests/test_auth.py index e5e2a64e..d7a0e82f 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -71,6 +71,19 @@ def test_caching_sha2_password(): con.query("FLUSH PRIVILEGES") con.close() + # Fast path after auth_switch_request + pymysql.connections._DEFAULT_AUTH_PLUGIN = "mysql_native_password" + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=ssl, + ) + con.query("FLUSH PRIVILEGES") + con.close() + pymysql.connections._DEFAULT_AUTH_PLUGIN = None + def test_caching_sha2_password_ssl(): con = pymysql.connect( @@ -88,7 +101,20 @@ def test_caching_sha2_password_ssl(): password=pass_caching_sha2, host=host, port=port, - ssl=None, + ssl=ssl, + ) + con.query("FLUSH PRIVILEGES") + con.close() + + # Fast path after auth_switch_request + pymysql.connections._DEFAULT_AUTH_PLUGIN = "mysql_native_password" + con = pymysql.connect( + user="user_caching_sha2", + password=pass_caching_sha2, + host=host, + port=port, + ssl=ssl, ) con.query("FLUSH PRIVILEGES") con.close() + pymysql.connections._DEFAULT_AUTH_PLUGIN = None