From 1fa7e13db03ce0e655611c87b1d65abb6fdace50 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 6 Aug 2021 20:46:14 +0200 Subject: [PATCH 1/9] Test that str params with embedded NULL chars are passed to SQLite functions --- Lib/sqlite3/test/userfunctions.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 9681dbdde2b092..f8a724e455cf36 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -351,6 +351,13 @@ def test_empty_blob(self): cur = self.con.execute("select isblob(x'')") self.assertTrue(cur.fetchone()[0]) + def test_set_and_get(self): + self.con.create_function("returnit", 1, lambda x: x) + for val in 1, 2.5, "text", "1\x002", "", b"data", None: + with self.subTest(val=val): + cur = self.con.execute("select returnit(?)", (val,)) + self.assertEqual(val, cur.fetchone()[0]) + # Regarding deterministic functions: # # Between 3.8.3 and 3.15.0, deterministic functions were only used to From aebc9decb84e9bdf5d36a9998f762bb8d760b985 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 6 Aug 2021 21:39:25 +0200 Subject: [PATCH 2/9] Refactor most set param tests --- Lib/sqlite3/test/userfunctions.py | 76 +++++++------------------------ 1 file changed, 17 insertions(+), 59 deletions(-) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index f8a724e455cf36..086b64e3ea33c8 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -78,18 +78,8 @@ def func_memoryerror(): def func_overflowerror(): raise OverflowError -def func_isstring(v): - return type(v) is str -def func_isint(v): - return type(v) is int -def func_isfloat(v): - return type(v) is float -def func_isnone(v): - return type(v) is type(None) def func_isblob(v): return isinstance(v, (bytes, memoryview)) -def func_islonglong(v): - return isinstance(v, int) and v >= 1<<31 def func(*args): return len(args) @@ -198,12 +188,7 @@ def setUp(self): self.con.create_function("memoryerror", 0, func_memoryerror) self.con.create_function("overflowerror", 0, func_overflowerror) - self.con.create_function("isstring", 1, func_isstring) - self.con.create_function("isint", 1, func_isint) - self.con.create_function("isfloat", 1, func_isfloat) - self.con.create_function("isnone", 1, func_isnone) self.con.create_function("isblob", 1, func_isblob) - self.con.create_function("islonglong", 1, func_islonglong) self.con.create_function("spam", -1, func) self.con.execute("create table test(t text)") @@ -303,44 +288,6 @@ def test_func_overflow_error(self): cur.execute("select overflowerror()") cur.fetchone() - def test_param_string(self): - cur = self.con.cursor() - for text in ["foo", str()]: - with self.subTest(text=text): - cur.execute("select isstring(?)", (text,)) - val = cur.fetchone()[0] - self.assertEqual(val, 1) - - def test_param_int(self): - cur = self.con.cursor() - cur.execute("select isint(?)", (42,)) - val = cur.fetchone()[0] - self.assertEqual(val, 1) - - def test_param_float(self): - cur = self.con.cursor() - cur.execute("select isfloat(?)", (3.14,)) - val = cur.fetchone()[0] - self.assertEqual(val, 1) - - def test_param_none(self): - cur = self.con.cursor() - cur.execute("select isnone(?)", (None,)) - val = cur.fetchone()[0] - self.assertEqual(val, 1) - - def test_param_blob(self): - cur = self.con.cursor() - cur.execute("select isblob(?)", (memoryview(b"blob"),)) - val = cur.fetchone()[0] - self.assertEqual(val, 1) - - def test_param_long_long(self): - cur = self.con.cursor() - cur.execute("select islonglong(?)", (1<<42,)) - val = cur.fetchone()[0] - self.assertEqual(val, 1) - def test_any_arguments(self): cur = self.con.cursor() cur.execute("select spam(?, ?)", (1, 2)) @@ -351,12 +298,23 @@ def test_empty_blob(self): cur = self.con.execute("select isblob(x'')") self.assertTrue(cur.fetchone()[0]) - def test_set_and_get(self): - self.con.create_function("returnit", 1, lambda x: x) - for val in 1, 2.5, "text", "1\x002", "", b"data", None: - with self.subTest(val=val): - cur = self.con.execute("select returnit(?)", (val,)) - self.assertEqual(val, cur.fetchone()[0]) + def test_func_params(self): + self.con.create_function("boomerang", 1, lambda x: x) + dataset = ( + (42, int), + (1<<42, int), # long long + (3.14, float), + ("text", str), + ("1\x002", str), + (b"blob", bytes), + (None, type(None)), + ) + for val, tp in dataset: + with self.subTest(val=val, tp=tp): + cur = self.con.execute("select boomerang(?)", (val,)) + retval = cur.fetchone()[0] + self.assertEqual(type(retval), tp) + self.assertEqual(retval, val) # Regarding deterministic functions: # From 13f24137639baef3c02a377987133606ba27b029 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 7 Aug 2021 19:46:22 +0200 Subject: [PATCH 3/9] expand test suite and check for PyFloat_AsDouble errors --- Lib/sqlite3/test/userfunctions.py | 34 ++++++++++++++++++++++--------- Modules/_sqlite/connection.c | 6 +++++- Modules/_sqlite/statement.c | 11 ++++++++-- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 086b64e3ea33c8..42b41de64053b8 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -78,12 +78,6 @@ def func_memoryerror(): def func_overflowerror(): raise OverflowError -def func_isblob(v): - return isinstance(v, (bytes, memoryview)) - -def func(*args): - return len(args) - class AggrNoStep: def __init__(self): pass @@ -188,8 +182,10 @@ def setUp(self): self.con.create_function("memoryerror", 0, func_memoryerror) self.con.create_function("overflowerror", 0, func_overflowerror) - self.con.create_function("isblob", 1, func_isblob) - self.con.create_function("spam", -1, func) + self.con.create_function("isblob", 1, + lambda x: isinstance(x, (bytes, memoryview))) + self.con.create_function("spam", -1, lambda *x: len(x)) + self.con.create_function("boomerang", 1, lambda x: x) self.con.execute("create table test(t text)") def tearDown(self): @@ -298,15 +294,33 @@ def test_empty_blob(self): cur = self.con.execute("select isblob(x'')") self.assertTrue(cur.fetchone()[0]) + def test_nan_float(self): + cur = self.con.execute("select boomerang(?)", (float('nan'),)) + # SQLite has no concept of nan; it is converted to NULL + self.assertIsNone(cur.fetchone()[0]) + + def test_too_large_int(self): + err = "Python int too large to convert to SQLite INTEGER" + self.assertRaisesRegex(OverflowError, err, self.con.execute, + "select boomerang(?)", (1 << 65,)) + + def test_non_contiguous_blob(self): + err = "could not convert BLOB to buffer" + self.assertRaisesRegex(ValueError, err, self.con.execute, + "select boomerang(?)", + (memoryview(b"blob")[::2],)) + def test_func_params(self): - self.con.create_function("boomerang", 1, lambda x: x) dataset = ( (42, int), - (1<<42, int), # long long + (-1, int), + (1234567890123456789, int), (3.14, float), + (float('inf'), float), ("text", str), ("1\x002", str), (b"blob", bytes), + (bytearray(range(2)), bytes), (None, type(None)), ) for val, tp in dataset: diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 0dab3e85160e82..d2e1d69de4ce52 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -517,7 +517,11 @@ _pysqlite_set_result(sqlite3_context* context, PyObject* py_val) return -1; sqlite3_result_int64(context, value); } else if (PyFloat_Check(py_val)) { - sqlite3_result_double(context, PyFloat_AsDouble(py_val)); + double value = PyFloat_AsDouble(py_val); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + sqlite3_result_double(context, value); } else if (PyUnicode_Check(py_val)) { Py_ssize_t sz; const char *str = PyUnicode_AsUTF8AndSize(py_val, &sz); diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 983df2d50c975d..9d7cd9e43e2e69 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -170,9 +170,16 @@ int pysqlite_statement_bind_parameter(pysqlite_Statement* self, int pos, PyObjec rc = sqlite3_bind_int64(self->st, pos, value); break; } - case TYPE_FLOAT: - rc = sqlite3_bind_double(self->st, pos, PyFloat_AsDouble(parameter)); + case TYPE_FLOAT: { + double value = PyFloat_AsDouble(parameter); + if (value == -1 && PyErr_Occurred()) { + rc = -1; + } + else { + rc = sqlite3_bind_double(self->st, pos, value); + } break; + } case TYPE_UNICODE: string = PyUnicode_AsUTF8AndSize(parameter, &buflen); if (string == NULL) From 3ed9af66ae74f0085b95e50874574f9225c14e71 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 8 Aug 2021 22:18:15 +0200 Subject: [PATCH 4/9] Address review: add more tests --- Lib/sqlite3/test/userfunctions.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 42b41de64053b8..4a86ea88b65a81 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -178,12 +178,14 @@ def setUp(self): self.con.create_function("returnnull", 0, func_returnnull) self.con.create_function("returnblob", 0, func_returnblob) self.con.create_function("returnlonglong", 0, func_returnlonglong) + self.con.create_function("returnnan", 0, lambda: float("nan")) + self.con.create_function("returntoolargeint", 0, lambda: 1 << 65) self.con.create_function("raiseexception", 0, func_raiseexception) self.con.create_function("memoryerror", 0, func_memoryerror) self.con.create_function("overflowerror", 0, func_overflowerror) - self.con.create_function("isblob", 1, - lambda x: isinstance(x, (bytes, memoryview))) + self.con.create_function("isblob", 1, lambda x: isinstance(x, bytes)) + self.con.create_function("isnone", 1, lambda x: x is None) self.con.create_function("spam", -1, lambda *x: len(x)) self.con.create_function("boomerang", 1, lambda x: x) self.con.execute("create table test(t text)") @@ -262,6 +264,16 @@ def test_func_return_long_long(self): val = cur.fetchone()[0] self.assertEqual(val, 1<<31) + def test_func_return_nan(self): + cur = self.con.cursor() + cur.execute("select returnnan()") + self.assertIsNone(cur.fetchone()[0]) + + def test_func_return_too_large_int(self): + cur = self.con.cursor() + self.assertRaisesRegex(sqlite.DataError, "string or blob too big", + self.con.execute, "select returntoolargeint()") + @with_tracebacks(['func_raiseexception', '5/0', 'ZeroDivisionError']) def test_func_exception(self): cur = self.con.cursor() @@ -295,14 +307,14 @@ def test_empty_blob(self): self.assertTrue(cur.fetchone()[0]) def test_nan_float(self): - cur = self.con.execute("select boomerang(?)", (float('nan'),)) + cur = self.con.execute("select isnone(?)", (float("nan"),)) # SQLite has no concept of nan; it is converted to NULL - self.assertIsNone(cur.fetchone()[0]) + self.assertTrue(cur.fetchone()[0]) def test_too_large_int(self): err = "Python int too large to convert to SQLite INTEGER" self.assertRaisesRegex(OverflowError, err, self.con.execute, - "select boomerang(?)", (1 << 65,)) + "select spam(?)", (1 << 65,)) def test_non_contiguous_blob(self): err = "could not convert BLOB to buffer" @@ -321,6 +333,7 @@ def test_func_params(self): ("1\x002", str), (b"blob", bytes), (bytearray(range(2)), bytes), + (memoryview(b"blob"), bytes), (None, type(None)), ) for val, tp in dataset: From 05c2f807097c98f2e901e140d9bbc40bbb5c59f9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 12 Aug 2021 13:44:30 +0200 Subject: [PATCH 5/9] Test surrogates and non-ascii chars --- Lib/sqlite3/test/userfunctions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index dd6adbae74cfb6..3b63cd05ee52a6 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -326,11 +326,15 @@ def test_too_large_int(self): "select spam(?)", (1 << 65,)) def test_non_contiguous_blob(self): - err = "could not convert BLOB to buffer" - self.assertRaisesRegex(ValueError, err, self.con.execute, - "select boomerang(?)", + self.assertRaisesRegex(ValueError, "could not convert BLOB to buffer", + self.con.execute, "select boomerang(?)", (memoryview(b"blob")[::2],)) + def test_param_surrogates(self): + self.assertRaisesRegex(UnicodeEncodeError, "surrogates not allowed", + self.con.execute, "select spam(?)", + ("\ud803\ude6d",)) + def test_func_params(self): dataset = ( (42, int), @@ -340,6 +344,7 @@ def test_func_params(self): (float('inf'), float), ("text", str), ("1\x002", str), + ("\u02e2q\u02e1\u2071\u1d57\u1d49", str), (b"blob", bytes), (bytearray(range(2)), bytes), (memoryview(b"blob"), bytes), From 21022c2519daffeeaefa589e9107697e39f7cdef Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 12 Aug 2021 14:04:15 +0200 Subject: [PATCH 6/9] Test 63-bit int with non-zero low bits --- Lib/sqlite3/test/userfunctions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 3b63cd05ee52a6..f1dfabb1163267 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -340,6 +340,7 @@ def test_func_params(self): (42, int), (-1, int), (1234567890123456789, int), + (4611686018427387905, int), # 63-bit int with non-zero low bits (3.14, float), (float('inf'), float), ("text", str), From 0a8ba4262162d1804087731372ca4fb097733d24 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 13 Aug 2021 09:45:58 +0200 Subject: [PATCH 7/9] import test.support.gc_collect iso. import gc --- Lib/sqlite3/test/userfunctions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index f1dfabb1163267..6897f76c9a2af1 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -23,14 +23,13 @@ import contextlib import functools -import gc import io import sys import unittest import unittest.mock import sqlite3 as sqlite -from test.support import bigmemtest +from test.support import bigmemtest, gc_collect def with_tracebacks(strings, traceback=True): @@ -413,7 +412,7 @@ def md5sum(t): y.append(y) del x,y - gc.collect() + gc_collect() def test_func_return_too_large_int(self): cur = self.con.cursor() From 02725cf970d6eb51b91bf8bf96aaa4114a91c486 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Sep 2021 10:15:51 +0200 Subject: [PATCH 8/9] Make test_func_params more specific --- Lib/sqlite3/test/test_userfunctions.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Lib/sqlite3/test/test_userfunctions.py b/Lib/sqlite3/test/test_userfunctions.py index b0ea043660b866..4a0eb8047331fb 100644 --- a/Lib/sqlite3/test/test_userfunctions.py +++ b/Lib/sqlite3/test/test_userfunctions.py @@ -335,7 +335,12 @@ def test_param_surrogates(self): ("\ud803\ude6d",)) def test_func_params(self): - dataset = ( + results = [] + def append_result(arg): + results.append((arg, type(arg))) + self.con.create_function("test_params", 1, append_result) + + dataset = [ (42, int), (-1, int), (1234567890123456789, int), @@ -349,13 +354,11 @@ def test_func_params(self): (bytearray(range(2)), bytes), (memoryview(b"blob"), bytes), (None, type(None)), - ) - for val, tp in dataset: - with self.subTest(val=val, tp=tp): - cur = self.con.execute("select boomerang(?)", (val,)) - retval = cur.fetchone()[0] - self.assertEqual(type(retval), tp) - self.assertEqual(retval, val) + ] + for val, _ in dataset: + cur = self.con.execute("select test_params(?)", (val,)) + cur.fetchone() + self.assertEqual(dataset, results) # Regarding deterministic functions: # From 5be2810e18b3c697f452094ba3d611ca08d3217d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 15 Sep 2021 10:18:29 +0200 Subject: [PATCH 9/9] Remove unneeded test UDF --- Lib/sqlite3/test/test_userfunctions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/sqlite3/test/test_userfunctions.py b/Lib/sqlite3/test/test_userfunctions.py index 4a0eb8047331fb..ae0e964d58e3bb 100644 --- a/Lib/sqlite3/test/test_userfunctions.py +++ b/Lib/sqlite3/test/test_userfunctions.py @@ -195,7 +195,6 @@ def setUp(self): self.con.create_function("isblob", 1, lambda x: isinstance(x, bytes)) self.con.create_function("isnone", 1, lambda x: x is None) self.con.create_function("spam", -1, lambda *x: len(x)) - self.con.create_function("boomerang", 1, lambda x: x) self.con.execute("create table test(t text)") def tearDown(self): @@ -326,7 +325,7 @@ def test_too_large_int(self): def test_non_contiguous_blob(self): self.assertRaisesRegex(ValueError, "could not convert BLOB to buffer", - self.con.execute, "select boomerang(?)", + self.con.execute, "select spam(?)", (memoryview(b"blob")[::2],)) def test_param_surrogates(self):