From 4fa0e91bd77290614e721718691bb76ac2dc7f0e Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 24 Feb 2017 12:23:19 +0200 Subject: [PATCH 01/77] support BLOB incremental I/O in sqlite module --- Doc/includes/sqlite3/blob.py | 13 ++ Doc/includes/sqlite3/blob_with.py | 12 ++ Doc/library/sqlite3.rst | 67 ++++++ Lib/sqlite3/test/dbapi.py | 279 +++++++++++++++++++++++- Modules/_sqlite/blob.c | 342 ++++++++++++++++++++++++++++++ Modules/_sqlite/blob.h | 25 +++ Modules/_sqlite/connection.c | 93 +++++++- Modules/_sqlite/connection.h | 3 +- Modules/_sqlite/module.c | 4 +- setup.py | 3 +- 10 files changed, 830 insertions(+), 11 deletions(-) create mode 100644 Doc/includes/sqlite3/blob.py create mode 100644 Doc/includes/sqlite3/blob_with.py create mode 100644 Modules/_sqlite/blob.c create mode 100644 Modules/_sqlite/blob.h diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py new file mode 100644 index 00000000000000..b8d2c78d1a365d --- /dev/null +++ b/Doc/includes/sqlite3/blob.py @@ -0,0 +1,13 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +blob = con.open_blob("test", "blob_col", 1, 1) +blob.write(b"a" * 5) +blob.write(b"b" * 5) +blob.seek(0) +print(blob.read()) # will print b"aaaaabbbbb" +blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py new file mode 100644 index 00000000000000..624b680591901a --- /dev/null +++ b/Doc/includes/sqlite3/blob_with.py @@ -0,0 +1,12 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +with con.open_blob("test", "blob_col", 1, 1) as blob: + blob.write(b"a" * 5) + blob.write(b"b" * 5) + blob.seek(0) + print(blob.read()) # will print b"aaaaabbbbb" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ccb82278bdaa13..ceb92b3fe90fab 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,6 +301,16 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. + .. method:: open_blob(table, column, row, readonly=False, dbname="main") + + On success a :class:`Blob` handle to the + :abbr:`BLOB (Binary Large OBject)` located in row *row*, + column *column*, table *table* in database *dbname* will be returned. + When *readonly* is :const:`True` the BLOB is opened with read + permissions. Otherwise the BLOB has read and write permissions. + + .. versionadded:: 3.7 + .. method:: commit() This method commits the current transaction. If you don't call this method, @@ -853,6 +863,63 @@ Exceptions transactions turned off. It is a subclass of :exc:`DatabaseError`. +.. _sqlite3-blob-objects: + +Blob Objects +------------ + +.. versionadded:: 3.7 + +.. class:: Blob + + A :class:`Blob` instance can read and write the data in the + :abbr:`BLOB (Binary Large OBject)`. + + .. method:: Blob.close() + + Close the BLOB now (rather than whenever __del__ is called). + + The BLOB will be unusable from this point forward; an + :class:`~sqlite3.Error` (or subclass) exception will be + raised if any operation is attempted with the BLOB. + + .. method:: Blob.__len__() + + Return the BLOB size. + + .. method:: Blob.read([size]) + + Read *size* bytes of data from the BLOB at the current offset position. + If the end of the BLOB is reached we will return the data up to end of + file. When *size* is not specified or negative we will read up to end + of BLOB. + + .. method:: Blob.write(data) + + Write *data* to the BLOB at the current offset. This function cannot + changed BLOB length. If data write will result in writing to more + then BLOB current size an error will be raised. + + .. method:: Blob.tell() + + Return the current offset of the BLOB. + + .. method:: Blob.seek(offset, [whence]) + + Set the BLOB offset. The *whence* argument is optional and defaults to + :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values + are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and + :data:`os.SEEK_END` or 2 (seek relative to the BLOB’s end). + + :class:`Blob` example: + + .. literalinclude:: ../includes/sqlite3/blob.py + + A :class:`Blob` can also be used with :term:`context manager`: + + .. literalinclude:: ../includes/sqlite3/blob_with.py + + .. _sqlite3-types: SQLite and Python types diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index be11337154bdd2..85db83ee51f246 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -511,6 +511,167 @@ def CheckLastRowIDInsertOR(self): self.assertEqual(results, expected) +class BlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.blob_data = b"a" * 100 + self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data, )) + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.second_data = b"b" * 100 + + def tearDown(self): + self.blob.close() + self.cx.close() + + def CheckLength(self): + self.assertEqual(len(self.blob), 100) + + def CheckTell(self): + self.assertEqual(self.blob.tell(), 0) + + def CheckSeekFromBlobStart(self): + self.blob.seek(10) + self.assertEqual(self.blob.tell(), 10) + self.blob.seek(10, 0) + self.assertEqual(self.blob.tell(), 10) + + def CheckSeekFromCurrentPosition(self): + self.blob.seek(10, 1) + self.blob.seek(10, 1) + self.assertEqual(self.blob.tell(), 20) + + def CheckSeekFromBlobEnd(self): + self.blob.seek(-10, 2) + self.assertEqual(self.blob.tell(), 90) + + def CheckBlobSeekOverBlobSize(self): + try: + self.blob.seek(1000) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobSeekUnderBlobSize(self): + try: + self.blob.seek(-10) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobRead(self): + self.assertEqual(self.blob.read(), self.blob_data) + + def CheckBlobReadSize(self): + self.assertEqual(len(self.blob.read(10)), 10) + + def CheckBlobReadAdvanceOffset(self): + self.blob.read(10) + self.assertEqual(self.blob.tell(), 10) + + def CheckBlobReadStartAtOffset(self): + self.blob.seek(10) + self.blob.write(self.second_data[:10]) + self.blob.seek(10) + self.assertEqual(self.blob.read(10), self.second_data[:10]) + + def CheckBlobWrite(self): + self.blob.write(self.second_data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], self.second_data) + + def CheckBlobWriteAtOffset(self): + self.blob.seek(50) + self.blob.write(self.second_data[:50]) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], + self.blob_data[:50] + self.second_data[:50]) + + def CheckBlobWriteAdvanceOffset(self): + self.blob.write(self.second_data[:50]) + self.assertEqual(self.blob.tell(), 50) + + def CheckBlobWriteMoreThenBlobSize(self): + try: + self.blob.write(b"a" * 1000) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobReadAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.read() + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteWhenReadOnly(self): + read_only_blob = \ + self.cx.open_blob("test", "blob_col", 1, readonly=True) + try: + read_only_blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + read_only_blob.close() + + def CheckBlobOpenWithBadDb(self): + try: + self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadTable(self): + try: + self.cx.open_blob("notexisintg", "blob_col", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadColumn(self): + try: + self.cx.open_blob("test", "notexisting", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadRow(self): + try: + self.cx.open_blob("test", "blob_col", 2) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + +@unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -768,6 +929,20 @@ def CheckClosedCurExecute(self): with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4") + def CheckClosedBlobRead(self): + con = sqlite.connect(":memory:") + con.execute("create table test(id integer primary key, blob_col blob)") + con.execute("insert into test(blob_col) values (zeroblob(100))") + blob = con.open_blob("test", "blob_col", 1) + con.close() + try: + blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except: + self.fail("Should have raised a ProgrammingError") + def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") con.close() @@ -921,6 +1096,99 @@ def CheckOnConflictReplace(self): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) + +class ClosedBlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckClosedRead(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedWrite(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.write(b"aaaaaaaaa") + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedSeek(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.seek(10) + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedTell(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.tell() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedClose(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + +class BlobContextManagerTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckContextExecute(self): + data = b"a" * 100 + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.write(data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) + + def CheckContextCloseBlob(self): + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.seek(10) + try: + blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def suite(): module_suite = unittest.makeSuite(ModuleTests, "Check") connection_suite = unittest.makeSuite(ConnectionTests, "Check") @@ -931,11 +1199,12 @@ def suite(): closed_con_suite = unittest.makeSuite(ClosedConTests, "Check") closed_cur_suite = unittest.makeSuite(ClosedCurTests, "Check") on_conflict_suite = unittest.makeSuite(SqliteOnConflictTests, "Check") - return unittest.TestSuite(( - module_suite, connection_suite, cursor_suite, thread_suite, - constructor_suite, ext_suite, closed_con_suite, closed_cur_suite, - on_conflict_suite, - )) + blob_suite = unittest.makeSuite(BlobTests, "Check") + closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") + blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") + return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite)) def test(): runner = unittest.TextTestRunner() diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c new file mode 100644 index 00000000000000..11da2b0ad5d691 --- /dev/null +++ b/Modules/_sqlite/blob.c @@ -0,0 +1,342 @@ +#include "blob.h" +#include "util.h" + + +int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, + sqlite3_blob *blob) +{ + Py_INCREF(connection); + self->connection = connection; + self->offset = 0; + self->blob = blob; + self->in_weakreflist = NULL; + + if (!pysqlite_check_thread(self->connection)) { + return -1; + } + return 0; +} + +static void remove_blob_from_connection_blob_list(pysqlite_Blob *self) +{ + Py_ssize_t i; + PyObject *item; + + for (i = 0; i < PyList_GET_SIZE(self->connection->blobs); i++) { + item = PyList_GET_ITEM(self->connection->blobs, i); + if (PyWeakref_GetObject(item) == (PyObject *)self) { + PyList_SetSlice(self->connection->blobs, i, i+1, NULL); + break; + } + } +} + +static void _close_blob_inner(pysqlite_Blob* self) +{ + sqlite3_blob *blob; + + /* close the blob */ + blob = self->blob; + self->blob = NULL; + if (blob) { + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + } + + /* remove from connection weaklist */ + remove_blob_from_connection_blob_list(self); + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } +} + +static void pysqlite_blob_dealloc(pysqlite_Blob* self) +{ + _close_blob_inner(self); + Py_XDECREF(self->connection); + Py_TYPE(self)->tp_free((PyObject*)self); +} + + +/* + * Checks if a blob object is usable (i. e. not closed). + * + * 0 => error; 1 => ok + */ +int pysqlite_check_blob(pysqlite_Blob *blob) +{ + + if (!blob->blob) { + PyErr_SetString(pysqlite_ProgrammingError, + "Cannot operate on a closed blob."); + return 0; + } else if (!pysqlite_check_connection(blob->connection) || + !pysqlite_check_thread(blob->connection)) { + return 0; + } else { + return 1; + } +} + + +PyObject* pysqlite_blob_close(pysqlite_Blob *self) +{ + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + _close_blob_inner(self); + Py_RETURN_NONE; +}; + + +static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) +{ + int blob_length; + if (!pysqlite_check_blob(self)) { + return -1; + } + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + return blob_length; +}; + + +PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) +{ + int read_length = -1; + int blob_length = 0; + PyObject *buffer; + char *raw_buffer; + int rc; + + if (!PyArg_ParseTuple(args, "|i", &read_length)) { + return NULL; + } + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + + /* TODO: make this multithreaded and safe! */ + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (read_length < 0) { + /* same as file read. */ + read_length = blob_length; + } + + /* making sure we don't read more then blob size */ + if (read_length > blob_length - self->offset) { + read_length = blob_length - self->offset; + } + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + + /* update offset. */ + self->offset += read_length; + + return buffer; +}; + + +PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) +{ + Py_buffer data_buffer; + int rc; + + if (PyObject_GetBuffer(data, &data_buffer, PyBUF_SIMPLE) < 0) { + return NULL; + } + + if (data_buffer.len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "data longer than INT_MAX bytes"); + PyBuffer_Release(&data_buffer); + return NULL; + } + + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } + + /* TODO: throw better error on data bigger then blob. */ + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buffer.buf, + data_buffer.len, self->offset); + Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyBuffer_Release(&data_buffer); + return NULL; + } + + self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); + Py_RETURN_NONE; +} + + +PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) +{ + int blob_length, offset, from_what = 0; + + if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { + return NULL; + } + + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + switch (from_what) { + case 0: // relative to blob begin + break; + case 1: // relative to current position + if (offset > INT_MAX - self->offset) { + goto overflow; + } + offset = self->offset + offset; + break; + case 2: // relative to blob end + if (offset > INT_MAX - blob_length) { + goto overflow; + } + offset = blob_length + offset; + break; + default: + return PyErr_Format(PyExc_ValueError, + "from_what should be 0, 1 or 2"); + } + + if (offset < 0 || offset > blob_length) { + return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + } + + self->offset = offset; + Py_RETURN_NONE; + +overflow: + return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); +} + + +PyObject* pysqlite_blob_tell(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + return PyLong_FromLong(self->offset); +} + + +PyObject* pysqlite_blob_enter(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_INCREF(self); + return (PyObject *)self; +} + + +PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) +{ + PyObject *res; + if (!pysqlite_check_blob(self)) { + return NULL; + } + + res = pysqlite_blob_close(self); + if (!res) { + return NULL; + } + Py_XDECREF(res); + + Py_RETURN_FALSE; +} + + +static PyMethodDef blob_methods[] = { + {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, + PyDoc_STR("read data from blob")}, + {"write", (PyCFunction)pysqlite_blob_write, METH_O, + PyDoc_STR("write data to blob")}, + {"close", (PyCFunction)pysqlite_blob_close, METH_NOARGS, + PyDoc_STR("close blob")}, + {"seek", (PyCFunction)pysqlite_blob_seek, METH_VARARGS, + PyDoc_STR("change blob current offset")}, + {"tell", (PyCFunction)pysqlite_blob_tell, METH_NOARGS, + PyDoc_STR("return blob current offset")}, + {"__enter__", (PyCFunction)pysqlite_blob_enter, METH_NOARGS, + PyDoc_STR("blob context manager enter")}, + {"__exit__", (PyCFunction)pysqlite_blob_exit, METH_VARARGS, + PyDoc_STR("blob context manager exit")}, + {NULL, NULL} +}; + +static PySequenceMethods blob_sequence_methods = { + (lenfunc)pysqlite_blob_length, /* sq_length */ +}; + + +PyTypeObject pysqlite_BlobType = { + PyVarObject_HEAD_INIT(NULL, 0) + MODULE_NAME ".Blob", + .tp_basicsize = sizeof(pysqlite_Blob), + .tp_dealloc = (destructor)pysqlite_blob_dealloc, + .tp_as_sequence = &blob_sequence_methods, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), + .tp_methods = blob_methods, +}; + +extern int pysqlite_blob_setup_types(void) +{ + pysqlite_BlobType.tp_new = PyType_GenericNew; + return PyType_Ready(&pysqlite_BlobType); +} diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h new file mode 100644 index 00000000000000..204918fd4f01c6 --- /dev/null +++ b/Modules/_sqlite/blob.h @@ -0,0 +1,25 @@ +#ifndef PYSQLITE_BLOB_H +#define PYSQLITE_BLOB_H +#include "Python.h" +#include "sqlite3.h" +#include "connection.h" + +typedef struct +{ + PyObject_HEAD + pysqlite_Connection* connection; + sqlite3_blob *blob; + int offset; + + PyObject* in_weakreflist; /* List of weak references */ +} pysqlite_Blob; + +extern PyTypeObject pysqlite_BlobType; + +int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, + sqlite3_blob *blob); +PyObject* pysqlite_blob_close(pysqlite_Blob *self); + +int pysqlite_blob_setup_types(void); + +#endif diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 958be7d869794a..b99fcdb24fc25f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -27,6 +27,7 @@ #include "connection.h" #include "statement.h" #include "cursor.h" +#include "blob.h" #include "prepare_protocol.h" #include "util.h" @@ -105,7 +106,8 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_CLEAR(self->statement_cache); Py_CLEAR(self->statements); Py_CLEAR(self->cursors); - + Py_CLEAR(self->blobs); + Py_INCREF(Py_None); Py_XSETREF(self->row_factory, Py_None); @@ -159,10 +161,11 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject self->created_statements = 0; self->created_cursors = 0; - /* Create lists of weak references to statements/cursors */ + /* Create lists of weak references to statements/cursors/blobs */ self->statements = PyList_New(0); self->cursors = PyList_New(0); - if (!self->statements || !self->cursors) { + self->blobs = PyList_New(0); + if (!self->statements || !self->cursors || !self->blobs) { return -1; } @@ -258,6 +261,8 @@ void pysqlite_connection_dealloc(pysqlite_Connection* self) Py_XDECREF(self->collations); Py_XDECREF(self->statements); Py_XDECREF(self->cursors); + Py_XDECREF(self->blobs); + Py_TYPE(self)->tp_free((PyObject*)self); } @@ -327,6 +332,84 @@ PyObject* pysqlite_connection_cursor(pysqlite_Connection* self, PyObject* args, return cursor; } +PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, + PyObject *kwargs) +{ + static char *kwlist[] = {"table", "column", "row", "readonly", + "dbname", NULL, NULL}; + int rc; + const char *dbname = "main", *table, *column; + long long row; + int readonly = 0; + sqlite3_blob *blob; + pysqlite_Blob *pyblob = NULL; + PyObject *weakref; + + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + &table, &column, &row, &readonly, + &dbname)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_open(self->db, dbname, table, column, row, + !readonly, &blob); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + _pysqlite_seterror(self->db, NULL); + return NULL; + } + + pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (!pyblob) { + goto error; + } + + rc = pysqlite_blob_init(pyblob, self, blob); + if (rc) { + Py_CLEAR(pyblob); + goto error; + } + + // Add our blob to connection blobs list + weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + if (!weakref) { + Py_CLEAR(pyblob); + goto error; + } + if (PyList_Append(self->blobs, weakref) != 0) { + Py_CLEAR(weakref); + Py_CLEAR(pyblob); + goto error; + } + Py_DECREF(weakref); + + return (PyObject*)pyblob; + +error: + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + return NULL; +} + +static void pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + int i; + PyObject *weakref; + PyObject *blob; + + for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + weakref = PyList_GET_ITEM(self->blobs, i); + blob = PyWeakref_GetObject(weakref); + if (blob != Py_None) { + pysqlite_blob_close((pysqlite_Blob*)blob); + } + } +} + PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) { int rc; @@ -337,6 +420,8 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) pysqlite_do_all_statements(self, ACTION_FINALIZE, 1); + pysqlite_close_all_blobs(self); + if (self->db) { rc = SQLITE3_CLOSE(self->db); @@ -1768,6 +1853,8 @@ static PyGetSetDef connection_getset[] = { static PyMethodDef connection_methods[] = { {"cursor", (PyCFunction)(void(*)(void))pysqlite_connection_cursor, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("Return a cursor for the connection.")}, + {"open_blob", (PyCFunction)pysqlite_connection_blob, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("return a blob object")}, {"close", (PyCFunction)pysqlite_connection_close, METH_NOARGS, PyDoc_STR("Closes the connection.")}, {"commit", (PyCFunction)pysqlite_connection_commit, METH_NOARGS, diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 206085e00a00c7..708a199087c002 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,9 +66,10 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements and cursors used within this connection */ + /* Lists of weak references to statements, blobs and cursors used within this connection */ PyObject* statements; PyObject* cursors; + PyObject* blobs; /* Counters for how many statements/cursors were created in the connection. May be * reset to 0 at certain intervals */ diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 71d951ee887e47..f138e02357753f 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -28,6 +28,7 @@ #include "prepare_protocol.h" #include "microprotocols.h" #include "row.h" +#include "blob.h" #if SQLITE_VERSION_NUMBER >= 3003003 #define HAVE_SHARED_CACHE @@ -368,7 +369,8 @@ PyMODINIT_FUNC PyInit__sqlite3(void) (pysqlite_connection_setup_types() < 0) || (pysqlite_cache_setup_types() < 0) || (pysqlite_statement_setup_types() < 0) || - (pysqlite_prepare_protocol_setup_types() < 0) + (pysqlite_prepare_protocol_setup_types() < 0) || + (pysqlite_blob_setup_types() < 0) ) { Py_XDECREF(module); return NULL; diff --git a/setup.py b/setup.py index 21a5a58981fc15..8c70025489e181 100644 --- a/setup.py +++ b/setup.py @@ -1523,7 +1523,8 @@ def detect_sqlite(self): '_sqlite/prepare_protocol.c', '_sqlite/row.c', '_sqlite/statement.c', - '_sqlite/util.c', ] + '_sqlite/util.c', + '_sqlite/blob.c' ] sqlite_defines = [] if not MS_WINDOWS: From 3fe91083c9b015ec326279a37e8c297e49747d61 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:11:32 +0200 Subject: [PATCH 02/77] Note that blob size cannot be changed using the blob object --- Doc/library/sqlite3.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ceb92b3fe90fab..e464192e136c13 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -309,6 +309,11 @@ Connection Objects When *readonly* is :const:`True` the BLOB is opened with read permissions. Otherwise the BLOB has read and write permissions. + .. note:: + + The BLOB size cannot be changed using the :class:`Blob` class. Use + `zeroblob` to create the blob in the wanted size in advance. + .. versionadded:: 3.7 .. method:: commit() From 865c1c84be443770f489138dd718f9087a080ac4 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:22:03 +0200 Subject: [PATCH 03/77] Use assertRaises in tests --- Lib/sqlite3/test/dbapi.py | 119 ++++++-------------------------------- 1 file changed, 17 insertions(+), 102 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 85db83ee51f246..8b2fc70bce3abf 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -546,22 +546,12 @@ def CheckSeekFromBlobEnd(self): self.assertEqual(self.blob.tell(), 90) def CheckBlobSeekOverBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(1000) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobSeekUnderBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(-10) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobRead(self): self.assertEqual(self.blob.read(), self.blob_data) @@ -594,81 +584,41 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"a" * 1000) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobReadAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.read() - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteWhenReadOnly(self): read_only_blob = \ self.cx.open_blob("test", "blob_col", 1, readonly=True) - try: + with self.assertRaises(sqlite.OperationalError): read_only_blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") read_only_blob.close() def CheckBlobOpenWithBadDb(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadTable(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("notexisintg", "blob_col", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadColumn(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "notexisting", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadRow(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") @unittest.skipUnless(threading, 'This test requires threading.') @@ -935,13 +885,8 @@ def CheckClosedBlobRead(self): con.execute("insert into test(blob_col) values (zeroblob(100))") blob = con.open_blob("test", "blob_col", 1) con.close() - try: + with self.assertRaises(sqlite.ProgrammingError): blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except: - self.fail("Should have raised a ProgrammingError") def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") @@ -1109,57 +1054,32 @@ def tearDown(self): def CheckClosedRead(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedWrite(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.write(b"aaaaaaaaa") - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedSeek(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.seek(10) - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedTell(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.tell() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedClose(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") class BlobContextManagerTests(unittest.TestCase): @@ -1180,13 +1100,8 @@ def CheckContextExecute(self): def CheckContextCloseBlob(self): with self.cx.open_blob("test", "blob_col", 1) as blob: blob.seek(10) - try: + with self.assertRaises(sqlite.ProgrammingError): blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def suite(): From 788fd54d56e52df55fdfff33b481b032beeb449b Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:38:43 +0200 Subject: [PATCH 04/77] Fix doc error --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e464192e136c13..6afef818e4ad70 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -312,7 +312,7 @@ Connection Objects .. note:: The BLOB size cannot be changed using the :class:`Blob` class. Use - `zeroblob` to create the blob in the wanted size in advance. + ``zeroblob`` to create the blob in the wanted size in advance. .. versionadded:: 3.7 From a1361e5ed71cbd1ef461a5eb09dd695a9e66619d Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:26:39 +0200 Subject: [PATCH 05/77] blob support sequence protocol --- Lib/sqlite3/test/dbapi.py | 56 ++++++ Modules/_sqlite/blob.c | 396 ++++++++++++++++++++++++++++++++++---- 2 files changed, 416 insertions(+), 36 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 8b2fc70bce3abf..24122a0cf58062 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -620,6 +620,62 @@ def CheckBlobOpenWithBadRow(self): with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) + def CheckBlobGetItem(self): + self.assertEqual(self.blob[5], b"a") + + def CheckBlobGetItemIndexOutOfRange(self): + with self.assertRaises(IndexError): + self.blob[105] + with self.assertRaises(IndexError): + self.blob[-105] + + def CheckBlobGetItemNegativeIndex(self): + self.assertEqual(self.blob[-5], b"a") + + def CheckBlobGetItemInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[b"a"] + + def CheckBlobGetSlice(self): + self.assertEqual(self.blob[5:10], b"aaaaa") + + def CheckBlobGetSliceNegativeIndex(self): + self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) + + def CheckBlobGetSliceInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[5:b"a"] + + def CheckBlobGetSliceWithSkip(self): + self.blob.write(b"abcdefghij") + self.assertEqual(self.blob[0:10:2], b"acegi") + + def CheckBlobSetItem(self): + self.blob[0] = b"b" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + + def CheckBlobSetSlice(self): + self.blob[0:5] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + + def CheckBlobSetSliceWithSkip(self): + self.blob[0:10:2] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) + + def CheckBlobGetEmptySlice(self): + self.assertEqual(self.blob[5:5], b"") + + def CheckBlobSetSliceWrongLength(self): + with self.assertRaises(IndexError): + self.blob[5:10] = b"a" + + def CheckBlobConcatNotSupported(self): + with self.assertRaises(SystemError): + self.blob + self.blob + + def CheckBlobRepeateNotSupported(self): + with self.assertRaises(SystemError): + self.blob * 5 @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 11da2b0ad5d691..333c65425a1f0f 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -105,14 +105,43 @@ static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) return blob_length; }; +static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) +{ + PyObject *buffer; + char *raw_buffer; + int rc; + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + return buffer; +} + PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; int blob_length = 0; PyObject *buffer; - char *raw_buffer; - int rc; if (!PyArg_ParseTuple(args, "|i", &read_length)) { return NULL; @@ -138,34 +167,36 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) read_length = blob_length - self->offset; } - buffer = PyBytes_FromStringAndSize(NULL, read_length); - if (!buffer) { - return NULL; + buffer = inner_read(self, read_length, self->offset); + + if (buffer != NULL) { + /* update offset on sucess. */ + self->offset += read_length; } - raw_buffer = PyBytes_AS_STRING(buffer); + + return buffer; +}; + +static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +{ + int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - Py_DECREF(buffer); + if (rc != SQLITE_OK) { /* For some reason after modifying blob the - error is not set on the connection db. */ + error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(pysqlite_OperationalError, "Cannot operate on modified blob"); } else { _pysqlite_seterror(self->connection->db, NULL); } - return NULL; + return -1; } - - /* update offset. */ - self->offset += read_length; - - return buffer; -}; + return 0; +} PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) @@ -191,26 +222,15 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) /* TODO: throw better error on data bigger then blob. */ - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buffer.buf, - data_buffer.len, self->offset); - Py_END_ALLOW_THREADS - if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(pysqlite_OperationalError, - "Cannot operate on modified blob"); - } else { - _pysqlite_seterror(self->connection->db, NULL); - } + rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); + + if (rc == 0) { + self->offset += (int)data_buffer.len; + Py_RETURN_NONE; + } else { PyBuffer_Release(&data_buffer); return NULL; } - - self->offset += (int)data_buffer.len; - PyBuffer_Release(&data_buffer); - Py_RETURN_NONE; } @@ -300,6 +320,300 @@ PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) Py_RETURN_FALSE; } +static PyObject* pysqlite_blob_concat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support concatenation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support repeat operation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return NULL; + } + + return inner_read(self, 1, i); +} + +static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +{ + int blob_length = 0; + const char *buf; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(v); + return write_inner(self, buf, 1, i); +} + + +static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return NULL; + } + // TODO: I am not sure... + return inner_read(self, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + + if (PySlice_GetIndicesEx(item, blob_length, + &start, &stop, &step, &slicelen) < 0) { + return NULL; + } + + if (slicelen <= 0) + return PyBytes_FromStringAndSize("", 0); + else if (step == 1) + return inner_read(self, slicelen, start); + else { + char *result_buf = (char *)PyMem_Malloc(slicelen); + char *data_buff = NULL; + Py_ssize_t cur, i; + PyObject *result; + int rc; + + if (result_buf == NULL) + return PyErr_NoMemory(); + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyMem_Free(result_buf); + return PyErr_NoMemory(); + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return NULL; + } + + for (cur = 0, i = 0; i < slicelen; + cur += step, i++) { + result_buf[i] = data_buff[cur]; + } + result = PyBytes_FromStringAndSize(result_buf, + slicelen); + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "Blob indices must be integers"); + return NULL; + } +} + + +static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + int blob_length = 0; + int rc; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + const char *buf; + + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(value); + return write_inner(self, buf, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + Py_buffer vbuf; + + if (PySlice_GetIndicesEx(item, + blob_length, &start, &stop, + &step, &slicelen) < 0) { + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); + return -1; + } + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) + return -1; + if (vbuf.len != slicelen) { + PyErr_SetString(PyExc_IndexError, + "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; + } + + if (slicelen == 0) { + } + else if (step == 1) { + rc = write_inner(self, vbuf.buf, slicelen, start); + } + else { + Py_ssize_t cur, i; + char *data_buff; + + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyErr_NoMemory(); + return -1; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + + for (cur = 0, i = 0; + i < slicelen; + cur += step, i++) + { + data_buff[cur] = ((char *)vbuf.buf)[i]; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + rc = 0; + + } + PyBuffer_Release(&vbuf); + return rc; + } + else { + PyErr_SetString(PyExc_TypeError, + "mmap indices must be integer"); + return -1; + } +} + static PyMethodDef blob_methods[] = { {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, @@ -320,9 +634,18 @@ static PyMethodDef blob_methods[] = { }; static PySequenceMethods blob_sequence_methods = { - (lenfunc)pysqlite_blob_length, /* sq_length */ + .sq_length = (lenfunc)pysqlite_blob_length, + .sq_concat = (binaryfunc)pysqlite_blob_concat, + .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, + .sq_item = (ssizeargfunc)pysqlite_blob_item, + .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, }; +static PyMappingMethods blob_mapping_methods = { + (lenfunc)pysqlite_blob_length, + (binaryfunc)pysqlite_blob_subscript, + (objobjargproc)pysqlite_blob_ass_subscript, +}; PyTypeObject pysqlite_BlobType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -330,6 +653,7 @@ PyTypeObject pysqlite_BlobType = { .tp_basicsize = sizeof(pysqlite_Blob), .tp_dealloc = (destructor)pysqlite_blob_dealloc, .tp_as_sequence = &blob_sequence_methods, + .tp_as_mapping = &blob_mapping_methods, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), .tp_methods = blob_methods, From 5aedfba9a1541473e72faaaaabf3d63e0b972ad6 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:34:59 +0200 Subject: [PATCH 06/77] Calculate blob length once at creation --- Modules/_sqlite/blob.c | 73 ++++++++++++------------------------------ Modules/_sqlite/blob.h | 1 + 2 files changed, 21 insertions(+), 53 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 333c65425a1f0f..abe8cac1b271bc 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,6 +11,10 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, self->blob = blob; self->in_weakreflist = NULL; + Py_BEGIN_ALLOW_THREADS + self->length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + if (!pysqlite_check_thread(self->connection)) { return -1; } @@ -94,15 +98,11 @@ PyObject* pysqlite_blob_close(pysqlite_Blob *self) static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) { - int blob_length; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - return blob_length; + return self->length; }; static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) @@ -140,7 +140,6 @@ static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; - int blob_length = 0; PyObject *buffer; if (!PyArg_ParseTuple(args, "|i", &read_length)) { @@ -151,20 +150,14 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) return NULL; } - - /* TODO: make this multithreaded and safe! */ - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (read_length < 0) { /* same as file read. */ - read_length = blob_length; + read_length = self->length; } /* making sure we don't read more then blob size */ - if (read_length > blob_length - self->offset) { - read_length = blob_length - self->offset; + if (read_length > self->length - self->offset) { + read_length = self->length - self->offset; } buffer = inner_read(self, read_length, self->offset); @@ -236,7 +229,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) { - int blob_length, offset, from_what = 0; + int offset, from_what = 0; if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { return NULL; @@ -247,10 +240,6 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - switch (from_what) { case 0: // relative to blob begin break; @@ -261,17 +250,17 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->offset + offset; break; case 2: // relative to blob end - if (offset > INT_MAX - blob_length) { + if (offset > INT_MAX - self->length) { goto overflow; } - offset = blob_length + offset; + offset = self->length + offset; break; default: return PyErr_Format(PyExc_ValueError, "from_what should be 0, 1 or 2"); } - if (offset < 0 || offset > blob_length) { + if (offset < 0 || offset > self->length) { return PyErr_Format(PyExc_ValueError, "offset out of blob range"); } @@ -340,17 +329,11 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } @@ -360,18 +343,13 @@ static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) { - int blob_length = 0; const char *buf; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } @@ -393,23 +371,17 @@ static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; @@ -420,7 +392,7 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, blob_length, + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, &slicelen) < 0) { return NULL; } @@ -484,17 +456,12 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int blob_length = 0; int rc; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); const char *buf; @@ -502,8 +469,8 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; @@ -527,7 +494,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb Py_buffer vbuf; if (PySlice_GetIndicesEx(item, - blob_length, &start, &stop, + self->length, &start, &stop, &step, &slicelen) < 0) { return -1; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 204918fd4f01c6..649f09e5ecca24 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -10,6 +10,7 @@ typedef struct pysqlite_Connection* connection; sqlite3_blob *blob; int offset; + int length; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Blob; From db6ef327457f9eb56e448c50309b714322212eed Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:41:20 +0200 Subject: [PATCH 07/77] Add initial Doc for sequence protocol --- Doc/library/sqlite3.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 6afef818e4ad70..068e15bf2534b9 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -878,7 +878,9 @@ Blob Objects .. class:: Blob A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. + :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both + the file and sequence protocol. For example You can read data from the + :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. .. method:: Blob.close() From 219f4cb4a1b3db1783f4e9c272cd3e6de4028908 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:04:22 +0300 Subject: [PATCH 08/77] Don't support blob operation --- Lib/sqlite3/test/dbapi.py | 4 ++++ Modules/_sqlite/blob.c | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 24122a0cf58062..d769d62446b2b7 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -677,6 +677,10 @@ def CheckBlobRepeateNotSupported(self): with self.assertRaises(SystemError): self.blob * 5 + def CheckBlobContainsNotSupported(self): + with self.assertRaises(SystemError): + b"aaaaa" in self.blob + @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index abe8cac1b271bc..7b2432491450a4 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -327,6 +327,15 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) return NULL; } +static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support cotains operation"); + } + return -1; +} + static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { if (!pysqlite_check_blob(self)) { @@ -606,6 +615,7 @@ static PySequenceMethods blob_sequence_methods = { .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, .sq_item = (ssizeargfunc)pysqlite_blob_item, .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, + .sq_contains = (objobjproc)pysqlite_blob_contains, }; static PyMappingMethods blob_mapping_methods = { From 6dafe0e7b5d651f373304f39961c5a5552e05531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:16:26 +0300 Subject: [PATCH 09/77] move news entry to blurb --- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst new file mode 100644 index 00000000000000..4f7b2fefed8039 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -0,0 +1,4 @@ +The :class:`sqlite3.Connection` now has the +:meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` +allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +:issue:`24905`) From ffac9013a89c68561b51443585a799aacad3b6e7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:32:15 +0300 Subject: [PATCH 10/77] Add blob to PCBuild --- PCbuild/_sqlite3.vcxproj | 2 ++ PCbuild/_sqlite3.vcxproj.filters | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 7e0062692b8f83..55f46c963b1e7d 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -107,6 +107,7 @@ + @@ -118,6 +119,7 @@ + diff --git a/PCbuild/_sqlite3.vcxproj.filters b/PCbuild/_sqlite3.vcxproj.filters index 51830f6a4451a4..8d26c9ab6eb43f 100644 --- a/PCbuild/_sqlite3.vcxproj.filters +++ b/PCbuild/_sqlite3.vcxproj.filters @@ -39,6 +39,9 @@ Header Files + + Header Files + @@ -68,6 +71,9 @@ Source Files + + Source Files + From 3475fa1c422a7418dc1a7765ad33479307afa531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:30:45 +0300 Subject: [PATCH 11/77] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 068e15bf2534b9..9b6d0472089063 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.7 + .. versionadded:: 3.8 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.7 +.. versionadded:: 3.8 .. class:: Blob From f6015ff3b39343898066e70c879d17dc5a78f89d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:32:14 +0300 Subject: [PATCH 12/77] Fix memory leak --- Modules/_sqlite/blob.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7b2432491450a4..d560563d3e766d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -219,6 +219,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) if (rc == 0) { self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); Py_RETURN_NONE; } else { PyBuffer_Release(&data_buffer); From 01e526ca5c94b480380919a79e320037bc7f65f0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 29 Jul 2020 19:40:23 +0300 Subject: [PATCH 13/77] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9b6d0472089063..841b2be4fdacf8 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.8 + .. versionadded:: 3.10 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.8 +.. versionadded:: 3.10 .. class:: Blob From 354bebf2dc561a6f079fc430edf8d9eb7cab6496 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:01:26 +0300 Subject: [PATCH 14/77] Fix CR comments in documentation and testing --- Doc/includes/sqlite3/blob.py | 6 +++--- Doc/includes/sqlite3/blob_with.py | 6 +++--- Doc/library/sqlite3.rst | 5 +++-- Lib/sqlite3/test/dbapi.py | 8 +++++--- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index b8d2c78d1a365d..18514876af6191 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -6,8 +6,8 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle blob = con.open_blob("test", "blob_col", 1, 1) -blob.write(b"a" * 5) -blob.write(b"b" * 5) +blob.write(b"Hello") +blob.write(b"World") blob.seek(0) -print(blob.read()) # will print b"aaaaabbbbb" +print(blob.read()) # will print b"HelloWorld" blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 624b680591901a..f220518c09a1ab 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -6,7 +6,7 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle with con.open_blob("test", "blob_col", 1, 1) as blob: - blob.write(b"a" * 5) - blob.write(b"b" * 5) + blob.write(b"Hello") + blob.write(b"World") blob.seek(0) - print(blob.read()) # will print b"aaaaabbbbb" + print(blob.read()) # will print b"HelloWorld" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 841b2be4fdacf8..b754481ad48757 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -879,8 +879,9 @@ Blob Objects A :class:`Blob` instance can read and write the data in the :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both - the file and sequence protocol. For example You can read data from the + the file and sequence protocol. For example, you can read data from the :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. + You can call ``len(obj)`` to get size of the BLOB. .. method:: Blob.close() @@ -911,7 +912,7 @@ Blob Objects Return the current offset of the BLOB. - .. method:: Blob.seek(offset, [whence]) + .. method:: Blob.seek(offset, whence=os.SEEK_SET) Set the BLOB offset. The *whence* argument is optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index d769d62446b2b7..a611feb56414eb 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -1177,9 +1177,11 @@ def suite(): blob_suite = unittest.makeSuite(BlobTests, "Check") closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") - return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, - ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, - blob_suite, closed_blob_suite, blob_context_manager_suite)) + return unittest.TestSuite(( + module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite, + )) def test(): runner = unittest.TextTestRunner() diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index 4f7b2fefed8039..c7d2405d89539e 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,4 +1,4 @@ The :class:`sqlite3.Connection` now has the :meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` -allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +allows incremental I/O operations to blobs. (Patch by Aviv Palivoda in :issue:`24905`) From 9709456b864e48e1cc3a3e838b311806a9ca6acb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:08:58 +0300 Subject: [PATCH 15/77] Fix CR comments in code --- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/connection.h | 2 +- setup.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index b99fcdb24fc25f..7e112b7fdb705a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -336,7 +336,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"table", "column", "row", "readonly", - "dbname", NULL, NULL}; + "dbname", NULL}; int rc; const char *dbname = "main", *table, *column; long long row; diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 708a199087c002..52dc27c8adff2d 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,7 +66,7 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements, blobs and cursors used within this connection */ + /* Lists of weak references to statements, cursors and blobs used within this connection */ PyObject* statements; PyObject* cursors; PyObject* blobs; diff --git a/setup.py b/setup.py index 8c70025489e181..dd6b3509247fd8 100644 --- a/setup.py +++ b/setup.py @@ -1524,7 +1524,8 @@ def detect_sqlite(self): '_sqlite/row.c', '_sqlite/statement.c', '_sqlite/util.c', - '_sqlite/blob.c' ] + '_sqlite/blob.c', + ] sqlite_defines = [] if not MS_WINDOWS: From e6e5099d5a5d9a4e69317208a4a463922a903fd5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:23:21 +0300 Subject: [PATCH 16/77] Make readonly and dbname keyword arguements only --- Doc/includes/sqlite3/blob.py | 2 +- Doc/includes/sqlite3/blob_with.py | 2 +- Doc/library/sqlite3.rst | 2 +- Modules/_sqlite/connection.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index 18514876af6191..afd7812a8b3af9 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -blob = con.open_blob("test", "blob_col", 1, 1) +blob = con.open_blob("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index f220518c09a1ab..fdca9fbc638ea2 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -with con.open_blob("test", "blob_col", 1, 1) as blob: +with con.open_blob("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index b754481ad48757..827c2e0588611a 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,7 +301,7 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, readonly=False, dbname="main") + .. method:: open_blob(table, column, row, *, readonly=False, dbname="main") On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7e112b7fdb705a..1d64bae7ca98f6 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -346,7 +346,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *weakref; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|$ps", kwlist, &table, &column, &row, &readonly, &dbname)) { return NULL; From e9a80800f0883de7e3b9fb34ca1b8172fd745460 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:32:47 +0300 Subject: [PATCH 17/77] Fix more CR comments --- Modules/_sqlite/blob.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index d560563d3e766d..f222d5cd477f22 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -257,19 +257,19 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_Format(PyExc_ValueError, + return PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); } if (offset < 0 || offset > self->length) { - return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); + return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); } @@ -332,7 +332,7 @@ static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) { if (pysqlite_check_blob(self)) { PyErr_SetString(PyExc_SystemError, - "Blob don't support cotains operation"); + "Blob don't support contains operation"); } return -1; } @@ -407,11 +407,11 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) return NULL; } - if (slicelen <= 0) + if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); - else if (step == 1) + } else if (step == 1) { return inner_read(self, slicelen, start); - else { + } else { char *result_buf = (char *)PyMem_Malloc(slicelen); char *data_buff = NULL; Py_ssize_t cur, i; @@ -586,7 +586,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb } else { PyErr_SetString(PyExc_TypeError, - "mmap indices must be integer"); + "Blob indices must be integer"); return -1; } } From d4fb1b5afd50091d99ecaa4022df109626d7325e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:57:07 +0300 Subject: [PATCH 18/77] Add more indicative error on write bigger then blob length --- Lib/sqlite3/test/dbapi.py | 2 +- Modules/_sqlite/blob.c | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index a611feb56414eb..fd88faa7765e02 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -584,7 +584,7 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaises(ValueError): self.blob.write(b"a" * 1000) def CheckBlobReadAfterRowChange(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f222d5cd477f22..49e664631e2774 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -208,12 +208,17 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) return NULL; } - if (!pysqlite_check_blob(self)) { + if (data_buffer.len > self->length - self->offset) { + PyErr_SetString(PyExc_ValueError, + "data longer than blob length"); PyBuffer_Release(&data_buffer); return NULL; } - /* TODO: throw better error on data bigger then blob. */ + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); @@ -257,19 +262,22 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); + return NULL; } if (offset < 0 || offset > self->length) { - return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + return NULL; } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + return NULL; } From 525a9c3e6c7418309054e812fd7fd2bbf358a981 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 11:44:29 +0200 Subject: [PATCH 19/77] Adapt sqlite3.Connection.open_blob() to AC --- Modules/_sqlite/clinic/connection.c.h | 97 ++++++++++++++++++++++++++- Modules/_sqlite/connection.c | 35 +++++----- 2 files changed, 115 insertions(+), 17 deletions(-) diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 315d163dde668f..e14f94a8d04038 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -140,6 +140,101 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ return return_value; } +PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, +"open_blob($self, /, table, column, row, *, readonly=False,\n" +" dbname=\'main\')\n" +"--\n" +"\n" +"Return a blob object. Non-standard."); + +#define PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF \ + {"open_blob", (PyCFunction)(void(*)(void))pysqlite_connection_open_blob, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_open_blob__doc__}, + +static PyObject * +pysqlite_connection_open_blob_impl(pysqlite_Connection *self, + const char *table, const char *column, + int row, int readonly, const char *dbname); + +static PyObject * +pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"table", "column", "row", "readonly", "dbname", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "open_blob", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + const char *table; + const char *column; + int row; + int readonly = 0; + const char *dbname = "main"; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("open_blob", "argument 'table'", "str", args[0]); + goto exit; + } + Py_ssize_t table_length; + table = PyUnicode_AsUTF8AndSize(args[0], &table_length); + if (table == NULL) { + goto exit; + } + if (strlen(table) != (size_t)table_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("open_blob", "argument 'column'", "str", args[1]); + goto exit; + } + Py_ssize_t column_length; + column = PyUnicode_AsUTF8AndSize(args[1], &column_length); + if (column == NULL) { + goto exit; + } + if (strlen(column) != (size_t)column_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + row = _PyLong_AsInt(args[2]); + if (row == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + readonly = _PyLong_AsInt(args[3]); + if (readonly == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (!PyUnicode_Check(args[4])) { + _PyArg_BadArgument("open_blob", "argument 'dbname'", "str", args[4]); + goto exit; + } + Py_ssize_t dbname_length; + dbname = PyUnicode_AsUTF8AndSize(args[4], &dbname_length); + if (dbname == NULL) { + goto exit; + } + if (strlen(dbname) != (size_t)dbname_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_kwonly: + return_value = pysqlite_connection_open_blob_impl(self, table, column, row, readonly, dbname); + +exit: + return return_value; +} + PyDoc_STRVAR(pysqlite_connection_close__doc__, "close($self, /)\n" "--\n" @@ -816,4 +911,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=9c0dfc6c1ebf9039 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c37726d47594c3d input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index b8a951fa2f6ed5..37e320356bc901 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -374,26 +374,30 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return cursor; } -PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, - PyObject *kwargs) +/*[clinic input] +_sqlite3.Connection.open_blob as pysqlite_connection_open_blob + + table: str + column: str + row: int + * + readonly: bool(accept={int}) = False + dbname: str = "main" + +Return a blob object. Non-standard. +[clinic start generated code]*/ + +static PyObject * +pysqlite_connection_open_blob_impl(pysqlite_Connection *self, + const char *table, const char *column, + int row, int readonly, const char *dbname) +/*[clinic end generated code: output=54dfadc6f1283245 input=38c93f0f7d73a1ef]*/ { - static char *kwlist[] = {"table", "column", "row", "readonly", - "dbname", NULL}; int rc; - const char *dbname = "main", *table, *column; - long long row; - int readonly = 0; sqlite3_blob *blob; pysqlite_Blob *pyblob = NULL; PyObject *weakref; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|$ps", kwlist, - &table, &column, &row, &readonly, - &dbname)) { - return NULL; - } - Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_open(self->db, dbname, table, column, row, !readonly, &blob); @@ -1957,8 +1961,6 @@ static PyGetSetDef connection_getset[] = { }; static PyMethodDef connection_methods[] = { - {"open_blob", (PyCFunction)pysqlite_connection_blob, METH_VARARGS|METH_KEYWORDS, - PyDoc_STR("return a blob object")}, PYSQLITE_CONNECTION_BACKUP_METHODDEF PYSQLITE_CONNECTION_CLOSE_METHODDEF PYSQLITE_CONNECTION_COMMIT_METHODDEF @@ -1975,6 +1977,7 @@ static PyMethodDef connection_methods[] = { PYSQLITE_CONNECTION_INTERRUPT_METHODDEF PYSQLITE_CONNECTION_ITERDUMP_METHODDEF PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF + PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF PYSQLITE_CONNECTION_ROLLBACK_METHODDEF PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF From 2f65cc886fd2e5b69e218590d7094767d2b72170 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:05:41 +0200 Subject: [PATCH 20/77] Adapt sqlite.Blob to AC --- Modules/_sqlite/blob.c | 159 +++++++++++++++++-------- Modules/_sqlite/blob.h | 1 + Modules/_sqlite/clinic/blob.c.h | 202 ++++++++++++++++++++++++++++++++ Modules/_sqlite/connection.c | 15 --- 4 files changed, 316 insertions(+), 61 deletions(-) create mode 100644 Modules/_sqlite/clinic/blob.c.h diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7fe768f62b4816..3ab801cc05e26a 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -1,6 +1,15 @@ #include "blob.h" #include "util.h" +#define clinic_state() (pysqlite_get_state(NULL)) +#include "clinic/blob.c.h" +#undef clinic_state + +/*[clinic input] +module _sqlite3 +class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, sqlite3_blob *blob) @@ -85,7 +94,15 @@ int pysqlite_check_blob(pysqlite_Blob *blob) } -PyObject* pysqlite_blob_close(pysqlite_Blob *self) +/*[clinic input] +_sqlite3.Blob.close as blob_close + +Close blob. +[clinic start generated code]*/ + +static PyObject * +blob_close_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=848accc20a138d1b input=56c86df5cab22490]*/ { if (!pysqlite_check_blob(self)) { @@ -96,6 +113,21 @@ PyObject* pysqlite_blob_close(pysqlite_Blob *self) Py_RETURN_NONE; }; +void +pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + int i; + PyObject *weakref; + PyObject *blob; + + for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + weakref = PyList_GET_ITEM(self->blobs, i); + blob = PyWeakref_GetObject(weakref); + if (blob != Py_None) { + blob_close_impl((pysqlite_Blob*)blob); + } + } +} static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) { @@ -138,15 +170,21 @@ static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) } -PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) +/*[clinic input] +_sqlite3.Blob.read as blob_read + + read_length: int = -1 + / + +Read data from blob. +[clinic start generated code]*/ + +static PyObject * +blob_read_impl(pysqlite_Blob *self, int read_length) +/*[clinic end generated code: output=9c4881a77860b216 input=753a766082129348]*/ { - int read_length = -1; PyObject *buffer; - if (!PyArg_ParseTuple(args, "|i", &read_length)) { - return NULL; - } - if (!pysqlite_check_blob(self)) { return NULL; } @@ -193,56 +231,62 @@ static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int } -PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) +/*[clinic input] +_sqlite3.Blob.write as blob_write + + data_buffer: Py_buffer + / + +Write data to blob. +[clinic start generated code]*/ + +static PyObject * +blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) +/*[clinic end generated code: output=dc8da6900b969799 input=8597402caf368add]*/ { - Py_buffer data_buffer; int rc; - if (PyObject_GetBuffer(data, &data_buffer, PyBUF_SIMPLE) < 0) { - return NULL; - } - - if (data_buffer.len > INT_MAX) { + if (data_buffer->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); - PyBuffer_Release(&data_buffer); return NULL; } - if (data_buffer.len > self->length - self->offset) { + if (data_buffer->len > self->length - self->offset) { PyErr_SetString(PyExc_ValueError, "data longer than blob length"); - PyBuffer_Release(&data_buffer); return NULL; } if (!pysqlite_check_blob(self)) { - PyBuffer_Release(&data_buffer); return NULL; } - rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); + rc = write_inner(self, data_buffer->buf, data_buffer->len, self->offset); if (rc == 0) { - self->offset += (int)data_buffer.len; - PyBuffer_Release(&data_buffer); + self->offset += (int)data_buffer->len; Py_RETURN_NONE; } else { - PyBuffer_Release(&data_buffer); return NULL; } } -PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) -{ - int offset, from_what = 0; +/*[clinic input] +_sqlite3.Blob.seek as blob_seek - if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { - return NULL; - } + offset: int + from_what: int = 0 + / +Change the access position for a blob. +[clinic start generated code]*/ +static PyObject * +blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) +/*[clinic end generated code: output=c7f07cd9e1d90bf7 input=f8dae77055129be3]*/ +{ if (!pysqlite_check_blob(self)) { return NULL; } @@ -282,7 +326,15 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) } -PyObject* pysqlite_blob_tell(pysqlite_Blob *self) +/*[clinic input] +_sqlite3.Blob.tell as blob_tell + +Return current access position for a blob. +[clinic start generated code]*/ + +static PyObject * +blob_tell_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=3d3ba484a90b3a99 input=aa1660f9aee18be4]*/ { if (!pysqlite_check_blob(self)) { return NULL; @@ -292,7 +344,15 @@ PyObject* pysqlite_blob_tell(pysqlite_Blob *self) } -PyObject* pysqlite_blob_enter(pysqlite_Blob *self) +/*[clinic input] +_sqlite3.Blob.__enter__ as blob_enter + +Blob context manager enter. +[clinic start generated code]*/ + +static PyObject * +blob_enter_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=4fd32484b071a6cd input=fe4842c3c582d5a7]*/ { if (!pysqlite_check_blob(self)) { return NULL; @@ -303,14 +363,28 @@ PyObject* pysqlite_blob_enter(pysqlite_Blob *self) } -PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) +/*[clinic input] +_sqlite3.Blob.__exit__ as blob_exit + + type: object + val: object + tb: object + / + +Blob context manager exit. +[clinic start generated code]*/ + +static PyObject * +blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, + PyObject *tb) +/*[clinic end generated code: output=fc86ceeb2b68c7b2 input=575d9ecea205f35f]*/ { PyObject *res; if (!pysqlite_check_blob(self)) { return NULL; } - res = pysqlite_blob_close(self); + res = blob_close_impl(self); if (!res) { return NULL; } @@ -602,20 +676,13 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb static PyMethodDef blob_methods[] = { - {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, - PyDoc_STR("read data from blob")}, - {"write", (PyCFunction)pysqlite_blob_write, METH_O, - PyDoc_STR("write data to blob")}, - {"close", (PyCFunction)pysqlite_blob_close, METH_NOARGS, - PyDoc_STR("close blob")}, - {"seek", (PyCFunction)pysqlite_blob_seek, METH_VARARGS, - PyDoc_STR("change blob current offset")}, - {"tell", (PyCFunction)pysqlite_blob_tell, METH_NOARGS, - PyDoc_STR("return blob current offset")}, - {"__enter__", (PyCFunction)pysqlite_blob_enter, METH_NOARGS, - PyDoc_STR("blob context manager enter")}, - {"__exit__", (PyCFunction)pysqlite_blob_exit, METH_VARARGS, - PyDoc_STR("blob context manager exit")}, + BLOB_CLOSE_METHODDEF + BLOB_ENTER_METHODDEF + BLOB_EXIT_METHODDEF + BLOB_READ_METHODDEF + BLOB_SEEK_METHODDEF + BLOB_TELL_METHODDEF + BLOB_WRITE_METHODDEF {NULL, NULL} }; diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 649f09e5ecca24..bae8e4f088b1e0 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -22,5 +22,6 @@ int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, PyObject* pysqlite_blob_close(pysqlite_Blob *self); int pysqlite_blob_setup_types(void); +void pysqlite_close_all_blobs(pysqlite_Connection *self); #endif diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h new file mode 100644 index 00000000000000..84bad030650a79 --- /dev/null +++ b/Modules/_sqlite/clinic/blob.c.h @@ -0,0 +1,202 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(blob_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"Close blob."); + +#define BLOB_CLOSE_METHODDEF \ + {"close", (PyCFunction)blob_close, METH_NOARGS, blob_close__doc__}, + +static PyObject * +blob_close_impl(pysqlite_Blob *self); + +static PyObject * +blob_close(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_close_impl(self); +} + +PyDoc_STRVAR(blob_read__doc__, +"read($self, read_length=-1, /)\n" +"--\n" +"\n" +"Read data from blob."); + +#define BLOB_READ_METHODDEF \ + {"read", (PyCFunction)(void(*)(void))blob_read, METH_FASTCALL, blob_read__doc__}, + +static PyObject * +blob_read_impl(pysqlite_Blob *self, int read_length); + +static PyObject * +blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int read_length = -1; + + if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + read_length = _PyLong_AsInt(args[0]); + if (read_length == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = blob_read_impl(self, read_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(blob_write__doc__, +"write($self, data_buffer, /)\n" +"--\n" +"\n" +"Write data to blob."); + +#define BLOB_WRITE_METHODDEF \ + {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, + +static PyObject * +blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer); + +static PyObject * +blob_write(pysqlite_Blob *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer data_buffer = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &data_buffer, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!PyBuffer_IsContiguous(&data_buffer, 'C')) { + _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); + goto exit; + } + return_value = blob_write_impl(self, &data_buffer); + +exit: + /* Cleanup for data_buffer */ + if (data_buffer.obj) { + PyBuffer_Release(&data_buffer); + } + + return return_value; +} + +PyDoc_STRVAR(blob_seek__doc__, +"seek($self, offset, from_what=0, /)\n" +"--\n" +"\n" +"Change the access position for a blob."); + +#define BLOB_SEEK_METHODDEF \ + {"seek", (PyCFunction)(void(*)(void))blob_seek, METH_FASTCALL, blob_seek__doc__}, + +static PyObject * +blob_seek_impl(pysqlite_Blob *self, int offset, int from_what); + +static PyObject * +blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int offset; + int from_what = 0; + + if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { + goto exit; + } + offset = _PyLong_AsInt(args[0]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + from_what = _PyLong_AsInt(args[1]); + if (from_what == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = blob_seek_impl(self, offset, from_what); + +exit: + return return_value; +} + +PyDoc_STRVAR(blob_tell__doc__, +"tell($self, /)\n" +"--\n" +"\n" +"Return current access position for a blob."); + +#define BLOB_TELL_METHODDEF \ + {"tell", (PyCFunction)blob_tell, METH_NOARGS, blob_tell__doc__}, + +static PyObject * +blob_tell_impl(pysqlite_Blob *self); + +static PyObject * +blob_tell(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_tell_impl(self); +} + +PyDoc_STRVAR(blob_enter__doc__, +"__enter__($self, /)\n" +"--\n" +"\n" +"Blob context manager enter."); + +#define BLOB_ENTER_METHODDEF \ + {"__enter__", (PyCFunction)blob_enter, METH_NOARGS, blob_enter__doc__}, + +static PyObject * +blob_enter_impl(pysqlite_Blob *self); + +static PyObject * +blob_enter(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_enter_impl(self); +} + +PyDoc_STRVAR(blob_exit__doc__, +"__exit__($self, type, val, tb, /)\n" +"--\n" +"\n" +"Blob context manager exit."); + +#define BLOB_EXIT_METHODDEF \ + {"__exit__", (PyCFunction)(void(*)(void))blob_exit, METH_FASTCALL, blob_exit__doc__}, + +static PyObject * +blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, + PyObject *tb); + +static PyObject * +blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *type; + PyObject *val; + PyObject *tb; + + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { + goto exit; + } + type = args[0]; + val = args[1]; + tb = args[2]; + return_value = blob_exit_impl(self, type, val, tb); + +exit: + return return_value; +} +/*[clinic end generated code: output=5d378130443aa9ce input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 37e320356bc901..789098614ac702 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -441,21 +441,6 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } -static void pysqlite_close_all_blobs(pysqlite_Connection *self) -{ - int i; - PyObject *weakref; - PyObject *blob; - - for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { - weakref = PyList_GET_ITEM(self->blobs, i); - blob = PyWeakref_GetObject(weakref); - if (blob != Py_None) { - pysqlite_blob_close((pysqlite_Blob*)blob); - } - } -} - /*[clinic input] _sqlite3.Connection.close as pysqlite_connection_close From 0dd047fcfbb3868384a35b51f129781a02e59b5c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:31:32 +0200 Subject: [PATCH 21/77] PEP 7 and normalised naming --- Modules/_sqlite/blob.c | 312 +++++++++++++++++------------------ Modules/_sqlite/blob.h | 11 +- Modules/_sqlite/connection.c | 18 +- 3 files changed, 162 insertions(+), 179 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 3ab801cc05e26a..2c5750b5b18bb7 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,8 +11,9 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ -int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, - sqlite3_blob *blob) +int +pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, + sqlite3_blob *blob) { Py_INCREF(connection); self->connection = connection; @@ -30,45 +31,43 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, return 0; } -static void remove_blob_from_connection_blob_list(pysqlite_Blob *self) +static void +remove_blob_from_connection(pysqlite_Blob *self) { - Py_ssize_t i; - PyObject *item; - - for (i = 0; i < PyList_GET_SIZE(self->connection->blobs); i++) { - item = PyList_GET_ITEM(self->connection->blobs, i); + Py_ssize_t size = PyList_GET_SIZE(self->connection->blobs); + for (Py_ssize_t i = 0; i < size; i++) { + PyObject *item = PyList_GET_ITEM(self->connection->blobs, i); if (PyWeakref_GetObject(item) == (PyObject *)self) { - PyList_SetSlice(self->connection->blobs, i, i+1, NULL); + Py_ssize_t low = i; + Py_ssize_t high = low + 1; + PyList_SetSlice(self->connection->blobs, low, high, NULL); break; } } } -static void _close_blob_inner(pysqlite_Blob* self) +static void +close_blob(pysqlite_Blob *self) { - sqlite3_blob *blob; - - /* close the blob */ - blob = self->blob; - self->blob = NULL; - if (blob) { + if (self->blob) { Py_BEGIN_ALLOW_THREADS - sqlite3_blob_close(blob); + sqlite3_blob_close(self->blob); Py_END_ALLOW_THREADS + self->blob = NULL; } - /* remove from connection weaklist */ - remove_blob_from_connection_blob_list(self); + remove_blob_from_connection(self); if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)self); + PyObject_ClearWeakRefs((PyObject *)self); } } -static void pysqlite_blob_dealloc(pysqlite_Blob* self) +static void +blob_dealloc(pysqlite_Blob *self) { - _close_blob_inner(self); + close_blob(self); Py_XDECREF(self->connection); - Py_TYPE(self)->tp_free((PyObject*)self); + Py_TYPE(self)->tp_free((PyObject *)self); } @@ -77,20 +76,20 @@ static void pysqlite_blob_dealloc(pysqlite_Blob* self) * * 0 => error; 1 => ok */ -int pysqlite_check_blob(pysqlite_Blob *blob) +static int +check_blob(pysqlite_Blob *blob) { - - if (!blob->blob) { + if (blob->blob == NULL) { pysqlite_state *state = pysqlite_get_state(NULL); PyErr_SetString(state->ProgrammingError, "Cannot operate on a closed blob."); return 0; - } else if (!pysqlite_check_connection(blob->connection) || - !pysqlite_check_thread(blob->connection)) { + } + else if (!pysqlite_check_connection(blob->connection) || + !pysqlite_check_thread(blob->connection)) { return 0; - } else { - return 1; } + return 1; } @@ -104,64 +103,57 @@ static PyObject * blob_close_impl(pysqlite_Blob *self) /*[clinic end generated code: output=848accc20a138d1b input=56c86df5cab22490]*/ { - - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } - - _close_blob_inner(self); + close_blob(self); Py_RETURN_NONE; }; void pysqlite_close_all_blobs(pysqlite_Connection *self) { - int i; - PyObject *weakref; - PyObject *blob; - - for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { - weakref = PyList_GET_ITEM(self->blobs, i); - blob = PyWeakref_GetObject(weakref); + for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + PyObject *weakref = PyList_GET_ITEM(self->blobs, i); + PyObject *blob = PyWeakref_GetObject(weakref); if (blob != Py_None) { - blob_close_impl((pysqlite_Blob*)blob); + blob_close_impl((pysqlite_Blob *)blob); } } } -static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) +static Py_ssize_t +blob_length(pysqlite_Blob *self) { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return -1; } - return self->length; }; -static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) +static PyObject * +inner_read(pysqlite_Blob *self, int read_length, int offset) { - PyObject *buffer; - char *raw_buffer; - int rc; - - buffer = PyBytes_FromStringAndSize(NULL, read_length); - if (!buffer) { + PyObject *buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (buffer == NULL) { return NULL; } - raw_buffer = PyBytes_AS_STRING(buffer); + char *raw_buffer = PyBytes_AS_STRING(buffer); + int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); Py_END_ALLOW_THREADS - if (rc != SQLITE_OK){ + if (rc != SQLITE_OK) { Py_DECREF(buffer); /* For some reason after modifying blob the error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } return NULL; @@ -183,9 +175,7 @@ static PyObject * blob_read_impl(pysqlite_Blob *self, int read_length) /*[clinic end generated code: output=9c4881a77860b216 input=753a766082129348]*/ { - PyObject *buffer; - - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -199,8 +189,7 @@ blob_read_impl(pysqlite_Blob *self, int read_length) read_length = self->length - self->offset; } - buffer = inner_read(self, read_length, self->offset); - + PyObject *buffer = inner_read(self, read_length, self->offset); if (buffer != NULL) { /* update offset on sucess. */ self->offset += read_length; @@ -209,20 +198,23 @@ blob_read_impl(pysqlite_Blob *self, int read_length) return buffer; }; -static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +static int +write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { /* For some reason after modifying blob the error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } return -1; @@ -247,18 +239,16 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) int rc; if (data_buffer->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "data longer than INT_MAX bytes"); + PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); return NULL; } if (data_buffer->len > self->length - self->offset) { - PyErr_SetString(PyExc_ValueError, - "data longer than blob length"); + PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return NULL; } - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -287,7 +277,7 @@ static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) /*[clinic end generated code: output=c7f07cd9e1d90bf7 input=f8dae77055129be3]*/ { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -307,8 +297,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) offset = self->length + offset; break; default: - PyErr_SetString(PyExc_ValueError, - "from_what should be 0, 1 or 2"); + PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); return NULL; } @@ -336,10 +325,9 @@ static PyObject * blob_tell_impl(pysqlite_Blob *self) /*[clinic end generated code: output=3d3ba484a90b3a99 input=aa1660f9aee18be4]*/ { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } - return PyLong_FromLong(self->offset); } @@ -354,7 +342,7 @@ static PyObject * blob_enter_impl(pysqlite_Blob *self) /*[clinic end generated code: output=4fd32484b071a6cd input=fe4842c3c582d5a7]*/ { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -380,7 +368,7 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, /*[clinic end generated code: output=fc86ceeb2b68c7b2 input=575d9ecea205f35f]*/ { PyObject *res; - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -393,36 +381,39 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, Py_RETURN_FALSE; } -static PyObject* pysqlite_blob_concat(pysqlite_Blob *self, PyObject *args) +static PyObject * +blob_concat(pysqlite_Blob *self, PyObject *args) { - if (pysqlite_check_blob(self)) { - PyErr_SetString(PyExc_SystemError, - "Blob don't support concatenation"); + if (check_blob(self)) { + PyErr_SetString(PyExc_SystemError, "Blob don't support concatenation"); } return NULL; } -static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) +static PyObject * +blob_repeat(pysqlite_Blob *self, PyObject *args) { - if (pysqlite_check_blob(self)) { + if (check_blob(self)) { PyErr_SetString(PyExc_SystemError, "Blob don't support repeat operation"); } return NULL; } -static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) +static int +blob_contains(pysqlite_Blob *self, PyObject *args) { - if (pysqlite_check_blob(self)) { + if (check_blob(self)) { PyErr_SetString(PyExc_SystemError, "Blob don't support contains operation"); } return -1; } -static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) +static PyObject * +blob_item(pysqlite_Blob *self, Py_ssize_t i) { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } @@ -434,11 +425,10 @@ static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) return inner_read(self, 1, i); } -static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +static int +blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) { - const char *buf; - - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return -1; } @@ -457,26 +447,28 @@ static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v return -1; } - buf = PyBytes_AsString(v); + const char *buf = PyBytes_AsString(v); return write_inner(self, buf, 1, i); } -static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) +static PyObject * +blob_subscript(pysqlite_Blob *self, PyObject *item) { - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return NULL; } if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) + if (i == -1 && PyErr_Occurred()) { return NULL; - if (i < 0) + } + if (i < 0) { i += self->length; + } if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, - "Blob index out of range"); + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } // TODO: I am not sure... @@ -485,87 +477,85 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->length, - &start, &stop, &step, &slicelen) < 0) { + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, + &slicelen) < 0) { return NULL; } if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); - } else if (step == 1) { + } + else if (step == 1) { return inner_read(self, slicelen, start); - } else { + } + else { char *result_buf = (char *)PyMem_Malloc(slicelen); - char *data_buff = NULL; - Py_ssize_t cur, i; - PyObject *result; - int rc; - - if (result_buf == NULL) + if (result_buf == NULL) { return PyErr_NoMemory(); + } - data_buff = (char *)PyMem_Malloc(stop - start); + char *data_buff = (char *)PyMem_Malloc(stop - start); if (data_buff == NULL) { PyMem_Free(result_buf); return PyErr_NoMemory(); } + int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); Py_END_ALLOW_THREADS - if (rc != SQLITE_OK){ + if (rc != SQLITE_OK) { /* For some reason after modifying blob the error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { - _pysqlite_seterror(self->connection->state, self->connection->db); + } + else { + _pysqlite_seterror(self->connection->state, + self->connection->db); } PyMem_Free(result_buf); PyMem_Free(data_buff); return NULL; } - for (cur = 0, i = 0; i < slicelen; - cur += step, i++) { + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { result_buf[i] = data_buff[cur]; } - result = PyBytes_FromStringAndSize(result_buf, - slicelen); + PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); PyMem_Free(result_buf); PyMem_Free(data_buff); return result; } } else { - PyErr_SetString(PyExc_TypeError, - "Blob indices must be integers"); + PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); return NULL; } } -static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +static int +blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { int rc; - if (!pysqlite_check_blob(self)) { + if (!check_blob(self)) { return -1; } if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - const char *buf; - - if (i == -1 && PyErr_Occurred()) + if (i == -1 && PyErr_Occurred()) { return -1; - if (i < 0) + } + if (i < 0) { i += self->length; + } if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, - "Blob index out of range"); + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } if (value == NULL) { @@ -579,43 +569,39 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb return -1; } - buf = PyBytes_AsString(value); + const char *buf = PyBytes_AsString(value); return write_inner(self, buf, 1, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - Py_buffer vbuf; - - if (PySlice_GetIndicesEx(item, - self->length, &start, &stop, - &step, &slicelen) < 0) { + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, + &slicelen) < 0) { return -1; } if (value == NULL) { PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); + "Blob object doesn't support slice deletion"); return -1; } - if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) + + Py_buffer vbuf; + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { return -1; + } if (vbuf.len != slicelen) { PyErr_SetString(PyExc_IndexError, - "Blob slice assignment is wrong size"); + "Blob slice assignment is wrong size"); PyBuffer_Release(&vbuf); return -1; } - if (slicelen == 0) { + if (slicelen == 0) { // FIXME } else if (step == 1) { rc = write_inner(self, vbuf.buf, slicelen, start); } else { - Py_ssize_t cur, i; - char *data_buff; - - - data_buff = (char *)PyMem_Malloc(stop - start); + char *data_buff = (char *)PyMem_Malloc(stop - start); if (data_buff == NULL) { PyErr_NoMemory(); return -1; @@ -631,17 +617,15 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } PyMem_Free(data_buff); rc = -1; } - for (cur = 0, i = 0; - i < slicelen; - cur += step, i++) - { + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { data_buff[cur] = ((char *)vbuf.buf)[i]; } @@ -655,7 +639,8 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on modified blob"); - } else { + } + else { _pysqlite_seterror(self->connection->state, self->connection->db); } PyMem_Free(data_buff); @@ -687,33 +672,34 @@ static PyMethodDef blob_methods[] = { }; static PySequenceMethods blob_sequence_methods = { - .sq_length = (lenfunc)pysqlite_blob_length, - .sq_concat = (binaryfunc)pysqlite_blob_concat, - .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, - .sq_item = (ssizeargfunc)pysqlite_blob_item, - .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, - .sq_contains = (objobjproc)pysqlite_blob_contains, + .sq_length = (lenfunc)blob_length, + .sq_concat = (binaryfunc)blob_concat, + .sq_repeat = (ssizeargfunc)blob_repeat, + .sq_item = (ssizeargfunc)blob_item, + .sq_ass_item = (ssizeobjargproc)blob_ass_item, + .sq_contains = (objobjproc)blob_contains, }; static PyMappingMethods blob_mapping_methods = { - (lenfunc)pysqlite_blob_length, - (binaryfunc)pysqlite_blob_subscript, - (objobjargproc)pysqlite_blob_ass_subscript, + (lenfunc)blob_length, + (binaryfunc)blob_subscript, + (objobjargproc)blob_ass_subscript, }; PyTypeObject pysqlite_BlobType = { - PyVarObject_HEAD_INIT(NULL, 0) - MODULE_NAME ".Blob", - .tp_basicsize = sizeof(pysqlite_Blob), - .tp_dealloc = (destructor)pysqlite_blob_dealloc, - .tp_as_sequence = &blob_sequence_methods, - .tp_as_mapping = &blob_mapping_methods, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), - .tp_methods = blob_methods, + PyVarObject_HEAD_INIT(NULL, 0) + MODULE_NAME ".Blob", + .tp_basicsize = sizeof(pysqlite_Blob), + .tp_dealloc = (destructor)blob_dealloc, + .tp_as_sequence = &blob_sequence_methods, + .tp_as_mapping = &blob_mapping_methods, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), + .tp_methods = blob_methods, }; -extern int pysqlite_blob_setup_types(void) +extern int +pysqlite_blob_setup_types(void) { pysqlite_BlobType.tp_new = PyType_GenericNew; return PyType_Ready(&pysqlite_BlobType); diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index bae8e4f088b1e0..b55dcc9dc7b5bb 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -4,22 +4,21 @@ #include "sqlite3.h" #include "connection.h" -typedef struct -{ +typedef struct { PyObject_HEAD - pysqlite_Connection* connection; + pysqlite_Connection *connection; sqlite3_blob *blob; int offset; int length; - PyObject* in_weakreflist; /* List of weak references */ + PyObject *in_weakreflist; } pysqlite_Blob; extern PyTypeObject pysqlite_BlobType; -int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, +int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, sqlite3_blob *blob); -PyObject* pysqlite_blob_close(pysqlite_Blob *self); +PyObject *pysqlite_blob_close(pysqlite_Blob *self); int pysqlite_blob_setup_types(void); void pysqlite_close_all_blobs(pysqlite_Connection *self); diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 789098614ac702..4ebecb9fc639b7 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -395,12 +395,10 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, { int rc; sqlite3_blob *blob; - pysqlite_Blob *pyblob = NULL; - PyObject *weakref; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_open(self->db, dbname, table, column, row, - !readonly, &blob); + rc = sqlite3_blob_open(self->db, dbname, table, column, row, !readonly, + &blob); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -408,24 +406,24 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); - if (!pyblob) { + pysqlite_Blob *pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (pyblob == NULL) { goto error; } rc = pysqlite_blob_init(pyblob, self, blob); - if (rc) { + if (rc < 0) { Py_CLEAR(pyblob); goto error; } // Add our blob to connection blobs list - weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); - if (!weakref) { + PyObject *weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + if (weakref == NULL) { Py_CLEAR(pyblob); goto error; } - if (PyList_Append(self->blobs, weakref) != 0) { + if (PyList_Append(self->blobs, weakref) < 0) { Py_CLEAR(weakref); Py_CLEAR(pyblob); goto error; From d34a77bb4bde0c71cd3564ccbd101fb99d72a9f1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:33:16 +0200 Subject: [PATCH 22/77] Use Py_NewRef and Py_IsNone --- Modules/_sqlite/blob.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 2c5750b5b18bb7..9d9bde5e303672 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -15,8 +15,7 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, sqlite3_blob *blob) { - Py_INCREF(connection); - self->connection = connection; + self->connection = (pysqlite_Connection *)Py_NewRef(connection); self->offset = 0; self->blob = blob; self->in_weakreflist = NULL; @@ -116,7 +115,7 @@ pysqlite_close_all_blobs(pysqlite_Connection *self) for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { PyObject *weakref = PyList_GET_ITEM(self->blobs, i); PyObject *blob = PyWeakref_GetObject(weakref); - if (blob != Py_None) { + if (!Py_IsNone(blob)) { blob_close_impl((pysqlite_Blob *)blob); } } @@ -345,9 +344,7 @@ blob_enter_impl(pysqlite_Blob *self) if (!check_blob(self)) { return NULL; } - - Py_INCREF(self); - return (PyObject *)self; + return Py_NewRef(self); } From 9d557054307581bbbf202077eeee07f2470cb713 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:35:25 +0200 Subject: [PATCH 23/77] Harden blob_open() --- Modules/_sqlite/connection.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 4ebecb9fc639b7..eadff8ac8b8845 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -413,26 +413,24 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, rc = pysqlite_blob_init(pyblob, self, blob); if (rc < 0) { - Py_CLEAR(pyblob); goto error; } // Add our blob to connection blobs list - PyObject *weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + PyObject *weakref = PyWeakref_NewRef((PyObject *)pyblob, NULL); if (weakref == NULL) { - Py_CLEAR(pyblob); goto error; } - if (PyList_Append(self->blobs, weakref) < 0) { - Py_CLEAR(weakref); - Py_CLEAR(pyblob); + rc = PyList_Append(self->blobs, weakref); + Py_DECREF(weakref); + if (rc < 0) { goto error; } - Py_DECREF(weakref); - return (PyObject*)pyblob; + return (PyObject *)pyblob; error: + Py_XDECREF(pyblob); Py_BEGIN_ALLOW_THREADS sqlite3_blob_close(blob); Py_END_ALLOW_THREADS From d47ea17c573527c6e40aa772c45f57487208f604 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:52:40 +0200 Subject: [PATCH 24/77] Move blob init to blob_open() --- Lib/sqlite3/test/dbapi.py | 2 +- Modules/_sqlite/blob.c | 18 ----------- Modules/_sqlite/blob.h | 2 -- Modules/_sqlite/clinic/connection.c.h | 39 ++++++++++++------------ Modules/_sqlite/connection.c | 43 ++++++++++++++++----------- 5 files changed, 46 insertions(+), 58 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 51ddc8564dc109..f2f51a7891d760 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -824,7 +824,7 @@ def test_blob_write_when_readonly(self): def test_blob_open_with_bad_db(self): with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") + self.cx.open_blob("test", "blob_col", 1, name="notexisintg") def test_blob_open_with_bad_table(self): with self.assertRaises(sqlite.OperationalError): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 9d9bde5e303672..c11f27d047d529 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,24 +11,6 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ -int -pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, - sqlite3_blob *blob) -{ - self->connection = (pysqlite_Connection *)Py_NewRef(connection); - self->offset = 0; - self->blob = blob; - self->in_weakreflist = NULL; - - Py_BEGIN_ALLOW_THREADS - self->length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (!pysqlite_check_thread(self->connection)) { - return -1; - } - return 0; -} static void remove_blob_from_connection(pysqlite_Blob *self) diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index b55dcc9dc7b5bb..462cd1386e8a92 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -16,8 +16,6 @@ typedef struct { extern PyTypeObject pysqlite_BlobType; -int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection *connection, - sqlite3_blob *blob); PyObject *pysqlite_blob_close(pysqlite_Blob *self); int pysqlite_blob_setup_types(void); diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index e14f94a8d04038..5e0c2157c72df9 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -141,8 +141,7 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ } PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, -"open_blob($self, /, table, column, row, *, readonly=False,\n" -" dbname=\'main\')\n" +"open_blob($self, table, column, row, /, *, readonly=False, name=\'main\')\n" "--\n" "\n" "Return a blob object. Non-standard."); @@ -152,29 +151,29 @@ PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, static PyObject * pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *column, - int row, int readonly, const char *dbname); + const char *table, const char *col, + int row, int readonly, const char *name); static PyObject * pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"table", "column", "row", "readonly", "dbname", NULL}; + static const char * const _keywords[] = {"", "", "", "readonly", "name", NULL}; static _PyArg_Parser _parser = {NULL, _keywords, "open_blob", 0}; PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; const char *table; - const char *column; + const char *col; int row; int readonly = 0; - const char *dbname = "main"; + const char *name = "main"; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); if (!args) { goto exit; } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("open_blob", "argument 'table'", "str", args[0]); + _PyArg_BadArgument("open_blob", "argument 1", "str", args[0]); goto exit; } Py_ssize_t table_length; @@ -187,15 +186,15 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } if (!PyUnicode_Check(args[1])) { - _PyArg_BadArgument("open_blob", "argument 'column'", "str", args[1]); + _PyArg_BadArgument("open_blob", "argument 2", "str", args[1]); goto exit; } - Py_ssize_t column_length; - column = PyUnicode_AsUTF8AndSize(args[1], &column_length); - if (column == NULL) { + Py_ssize_t col_length; + col = PyUnicode_AsUTF8AndSize(args[1], &col_length); + if (col == NULL) { goto exit; } - if (strlen(column) != (size_t)column_length) { + if (strlen(col) != (size_t)col_length) { PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } @@ -216,20 +215,20 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, } } if (!PyUnicode_Check(args[4])) { - _PyArg_BadArgument("open_blob", "argument 'dbname'", "str", args[4]); + _PyArg_BadArgument("open_blob", "argument 'name'", "str", args[4]); goto exit; } - Py_ssize_t dbname_length; - dbname = PyUnicode_AsUTF8AndSize(args[4], &dbname_length); - if (dbname == NULL) { + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[4], &name_length); + if (name == NULL) { goto exit; } - if (strlen(dbname) != (size_t)dbname_length) { + if (strlen(name) != (size_t)name_length) { PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } skip_optional_kwonly: - return_value = pysqlite_connection_open_blob_impl(self, table, column, row, readonly, dbname); + return_value = pysqlite_connection_open_blob_impl(self, table, col, row, readonly, name); exit: return return_value; @@ -911,4 +910,4 @@ pysqlite_connection_exit(pysqlite_Connection *self, PyObject *const *args, Py_ss #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=2c37726d47594c3d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a392a91a234a8fc8 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index eadff8ac8b8845..ee7298090a91cb 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -378,27 +378,34 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) _sqlite3.Connection.open_blob as pysqlite_connection_open_blob table: str - column: str + column as col: str row: int + / * readonly: bool(accept={int}) = False - dbname: str = "main" + name: str = "main" Return a blob object. Non-standard. [clinic start generated code]*/ static PyObject * pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *column, - int row, int readonly, const char *dbname) -/*[clinic end generated code: output=54dfadc6f1283245 input=38c93f0f7d73a1ef]*/ + const char *table, const char *col, + int row, int readonly, const char *name) +/*[clinic end generated code: output=5c11d18e7eb629ef input=9192dd28146d4e14]*/ { - int rc; + if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { + return NULL; + } + + int rc, len; sqlite3_blob *blob; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_open(self->db, dbname, table, column, row, !readonly, - &blob); + rc = sqlite3_blob_open(self->db, name, table, col, row, !readonly, &blob); + if (rc == SQLITE_OK) { + len = sqlite3_blob_bytes(blob); + } Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -406,18 +413,19 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - pysqlite_Blob *pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); - if (pyblob == NULL) { + pysqlite_Blob *obj = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (obj == NULL) { goto error; } - rc = pysqlite_blob_init(pyblob, self, blob); - if (rc < 0) { - goto error; - } + obj->connection = (pysqlite_Connection *)Py_NewRef(self); + obj->blob = blob; + obj->offset = 0; + obj->length = len; + obj->in_weakreflist = NULL; // Add our blob to connection blobs list - PyObject *weakref = PyWeakref_NewRef((PyObject *)pyblob, NULL); + PyObject *weakref = PyWeakref_NewRef((PyObject *)obj, NULL); if (weakref == NULL) { goto error; } @@ -427,10 +435,11 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, goto error; } - return (PyObject *)pyblob; + return (PyObject *)obj; error: - Py_XDECREF(pyblob); + Py_XDECREF(obj); + Py_BEGIN_ALLOW_THREADS sqlite3_blob_close(blob); Py_END_ALLOW_THREADS From e07a116a5ed86c97763e0a91d641870646cd4be1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 12:54:25 +0200 Subject: [PATCH 25/77] initialise rc in blob_ass_subscript() --- Modules/_sqlite/blob.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index c11f27d047d529..7fd00ad1affca3 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -519,7 +519,7 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) static int blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int rc; + int rc = -1; if (!check_blob(self)) { return -1; @@ -574,7 +574,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (slicelen == 0) { // FIXME + if (slicelen == 0) { // FIXME } else if (step == 1) { rc = write_inner(self, vbuf.buf, slicelen, start); From 982c81269193abdf09b7eef86eb77546ec3630b7 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:04:54 +0200 Subject: [PATCH 26/77] Improve blob.seek() parameter naming --- Modules/_sqlite/blob.c | 10 +++++----- Modules/_sqlite/clinic/blob.c.h | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7fd00ad1affca3..79231385e16396 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -248,21 +248,21 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) _sqlite3.Blob.seek as blob_seek offset: int - from_what: int = 0 + origin: int = 0 / Change the access position for a blob. [clinic start generated code]*/ static PyObject * -blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) -/*[clinic end generated code: output=c7f07cd9e1d90bf7 input=f8dae77055129be3]*/ +blob_seek_impl(pysqlite_Blob *self, int offset, int origin) +/*[clinic end generated code: output=854c5a0e208547a5 input=cc33da6f28af0561]*/ { if (!check_blob(self)) { return NULL; } - switch (from_what) { + switch (origin) { case 0: // relative to blob begin break; case 1: // relative to current position @@ -278,7 +278,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int from_what) offset = self->length + offset; break; default: - PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); + PyErr_SetString(PyExc_ValueError, "'origin' should be 0, 1 or 2"); return NULL; } diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index 84bad030650a79..fa995a4938389b 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -92,7 +92,7 @@ blob_write(pysqlite_Blob *self, PyObject *arg) } PyDoc_STRVAR(blob_seek__doc__, -"seek($self, offset, from_what=0, /)\n" +"seek($self, offset, origin=0, /)\n" "--\n" "\n" "Change the access position for a blob."); @@ -101,14 +101,14 @@ PyDoc_STRVAR(blob_seek__doc__, {"seek", (PyCFunction)(void(*)(void))blob_seek, METH_FASTCALL, blob_seek__doc__}, static PyObject * -blob_seek_impl(pysqlite_Blob *self, int offset, int from_what); +blob_seek_impl(pysqlite_Blob *self, int offset, int origin); static PyObject * blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int offset; - int from_what = 0; + int origin = 0; if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { goto exit; @@ -120,12 +120,12 @@ blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 2) { goto skip_optional; } - from_what = _PyLong_AsInt(args[1]); - if (from_what == -1 && PyErr_Occurred()) { + origin = _PyLong_AsInt(args[1]); + if (origin == -1 && PyErr_Occurred()) { goto exit; } skip_optional: - return_value = blob_seek_impl(self, offset, from_what); + return_value = blob_seek_impl(self, offset, origin); exit: return return_value; @@ -199,4 +199,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5d378130443aa9ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=76c066429020440e input=a9049054013a1b77]*/ From e92495b2b22c5d6d225e1b311c17c395fd15cf72 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:40:04 +0200 Subject: [PATCH 27/77] Adapt to heap types and allow calling close multiple times --- Lib/sqlite3/test/dbapi.py | 12 ---- Modules/_sqlite/blob.c | 123 ++++++++++++++++++++--------------- Modules/_sqlite/blob.h | 4 +- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/module.c | 2 +- Modules/_sqlite/module.h | 1 + 6 files changed, 74 insertions(+), 70 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index f2f51a7891d760..3c994c64ae8c33 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -933,12 +933,6 @@ def test_closed_blob_tell(self): with self.assertRaises(sqlite.ProgrammingError): self.blob.tell() - def test_closed_blob_close(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.close() - def test_closed_blob_read(self): con = sqlite.connect(":memory:") con.execute("create table test(id integer primary key, blob_col blob)") @@ -964,12 +958,6 @@ def test_blob_ctx_mgr_execute(self): blob.write(data) self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) - def test_blob_ctx_mgr_close(self): - with self.cx.open_blob("test", "blob_col", 1) as blob: - blob.seek(10) - with self.assertRaises(sqlite.ProgrammingError): - blob.close() - class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 79231385e16396..7a80d7125919af 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -12,21 +12,6 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ -static void -remove_blob_from_connection(pysqlite_Blob *self) -{ - Py_ssize_t size = PyList_GET_SIZE(self->connection->blobs); - for (Py_ssize_t i = 0; i < size; i++) { - PyObject *item = PyList_GET_ITEM(self->connection->blobs, i); - if (PyWeakref_GetObject(item) == (PyObject *)self) { - Py_ssize_t low = i; - Py_ssize_t high = low + 1; - PyList_SetSlice(self->connection->blobs, low, high, NULL); - break; - } - } -} - static void close_blob(pysqlite_Blob *self) { @@ -36,19 +21,37 @@ close_blob(pysqlite_Blob *self) Py_END_ALLOW_THREADS self->blob = NULL; } +} - remove_blob_from_connection(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)self); - } +static int +blob_traverse(pysqlite_Blob *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->connection); + return 0; +} + +static int +blob_clear(pysqlite_Blob *self) +{ + Py_CLEAR(self->connection); + return 0; } static void blob_dealloc(pysqlite_Blob *self) { + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + close_blob(self); - Py_XDECREF(self->connection); - Py_TYPE(self)->tp_free((PyObject *)self); + + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } + tp->tp_clear((PyObject *)self); + tp->tp_free(self); + Py_DECREF(tp); } @@ -58,18 +61,18 @@ blob_dealloc(pysqlite_Blob *self) * 0 => error; 1 => ok */ static int -check_blob(pysqlite_Blob *blob) +check_blob(pysqlite_Blob *self) { - if (blob->blob == NULL) { + if (!pysqlite_check_connection(self->connection) || + !pysqlite_check_thread(self->connection)) { + return 0; + } + if (self->blob == NULL) { pysqlite_state *state = pysqlite_get_state(NULL); PyErr_SetString(state->ProgrammingError, "Cannot operate on a closed blob."); return 0; } - else if (!pysqlite_check_connection(blob->connection) || - !pysqlite_check_thread(blob->connection)) { - return 0; - } return 1; } @@ -84,7 +87,9 @@ static PyObject * blob_close_impl(pysqlite_Blob *self) /*[clinic end generated code: output=848accc20a138d1b input=56c86df5cab22490]*/ { - if (!check_blob(self)) { + if (!pysqlite_check_connection(self->connection) || + !pysqlite_check_thread(self->connection)) + { return NULL; } close_blob(self); @@ -650,36 +655,48 @@ static PyMethodDef blob_methods[] = { {NULL, NULL} }; -static PySequenceMethods blob_sequence_methods = { - .sq_length = (lenfunc)blob_length, - .sq_concat = (binaryfunc)blob_concat, - .sq_repeat = (ssizeargfunc)blob_repeat, - .sq_item = (ssizeargfunc)blob_item, - .sq_ass_item = (ssizeobjargproc)blob_ass_item, - .sq_contains = (objobjproc)blob_contains, +static struct PyMemberDef blob_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(pysqlite_Blob, in_weakreflist), READONLY}, }; -static PyMappingMethods blob_mapping_methods = { - (lenfunc)blob_length, - (binaryfunc)blob_subscript, - (objobjargproc)blob_ass_subscript, +static PyType_Slot blob_slots[] = { + {Py_tp_dealloc, blob_dealloc}, + {Py_tp_traverse, blob_traverse}, + {Py_tp_clear, blob_clear}, + {Py_tp_methods, blob_methods}, + {Py_tp_members, blob_members}, + + // Sequence protocol + {Py_sq_length, blob_length}, + {Py_sq_concat, blob_concat}, + {Py_sq_repeat, blob_repeat}, + {Py_sq_item, blob_item}, + {Py_sq_ass_item, blob_ass_item}, + {Py_sq_contains, blob_contains}, + + // Mapping protocol + {Py_mp_length, blob_length}, + {Py_mp_subscript, blob_subscript}, + {Py_mp_ass_subscript, blob_ass_subscript}, + {0, NULL}, }; -PyTypeObject pysqlite_BlobType = { - PyVarObject_HEAD_INIT(NULL, 0) - MODULE_NAME ".Blob", - .tp_basicsize = sizeof(pysqlite_Blob), - .tp_dealloc = (destructor)blob_dealloc, - .tp_as_sequence = &blob_sequence_methods, - .tp_as_mapping = &blob_mapping_methods, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), - .tp_methods = blob_methods, +static PyType_Spec blob_spec = { + .name = MODULE_NAME ".Blob", + .basicsize = sizeof(pysqlite_Blob), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = blob_slots, }; -extern int -pysqlite_blob_setup_types(void) +int +pysqlite_blob_setup_types(PyObject *module) { - pysqlite_BlobType.tp_new = PyType_GenericNew; - return PyType_Ready(&pysqlite_BlobType); + PyObject *type = PyType_FromModuleAndSpec(module, &blob_spec, NULL); + if (type == NULL) { + return -1; + } + pysqlite_state *state = pysqlite_get_state(module); + state->BlobType = (PyTypeObject *)type; + return 0; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 462cd1386e8a92..cb8a5ebba191b7 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -14,11 +14,9 @@ typedef struct { PyObject *in_weakreflist; } pysqlite_Blob; -extern PyTypeObject pysqlite_BlobType; - PyObject *pysqlite_blob_close(pysqlite_Blob *self); -int pysqlite_blob_setup_types(void); +int pysqlite_blob_setup_types(PyObject *module); void pysqlite_close_all_blobs(pysqlite_Connection *self); #endif diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index ee7298090a91cb..5587037fc6939f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -413,7 +413,7 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - pysqlite_Blob *obj = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + pysqlite_Blob *obj = PyObject_GC_New(pysqlite_Blob, self->state->BlobType); if (obj == NULL) { goto error; } diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 5a58b47f24637b..2207cb4e086bf7 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -448,7 +448,7 @@ PyMODINIT_FUNC PyInit__sqlite3(void) (pysqlite_connection_setup_types(module) < 0) || (pysqlite_statement_setup_types(module) < 0) || (pysqlite_prepare_protocol_setup_types(module) < 0) || - (pysqlite_blob_setup_types() < 0) + (pysqlite_blob_setup_types(module) < 0) ) { goto error; } diff --git a/Modules/_sqlite/module.h b/Modules/_sqlite/module.h index c273c1f9ed9f29..4c684b2b6b9ea8 100644 --- a/Modules/_sqlite/module.h +++ b/Modules/_sqlite/module.h @@ -53,6 +53,7 @@ typedef struct { int BaseTypeAdapted; int enable_callback_tracebacks; + PyTypeObject *BlobType; PyTypeObject *ConnectionType; PyTypeObject *CursorType; PyTypeObject *PrepareProtocolType; From 8f2ce8a33cf351c33dbc5613c10ef99bdd9a24c4 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:41:15 +0200 Subject: [PATCH 28/77] Consolidate tests --- Lib/sqlite3/test/dbapi.py | 83 ++++++++++++--------------------------- 1 file changed, 26 insertions(+), 57 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 3c994c64ae8c33..5615aee476ab0c 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -763,13 +763,10 @@ def test_blob_seek_from_end(self): self.blob.seek(-10, 2) self.assertEqual(self.blob.tell(), 90) - def test_blob_seek_over_size(self): - with self.assertRaises(ValueError): - self.blob.seek(1000) - - def test_blob_seek_under_size(self): - with self.assertRaises(ValueError): - self.blob.seek(-10) + def test_blob_seek_error(self): + for pos in 1000, -10: + with self.subTest(pos=pos): + self.assertRaises(ValueError, self.blob.seek, pos) def test_blob_read(self): self.assertEqual(self.blob.read(), self.blob_data) @@ -899,39 +896,29 @@ def test_blob_contains_not_supported(self): with self.assertRaises(SystemError): b"aaaaa" in self.blob + def test_blob_context_manager(self): + data = b"a" * 100 + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.write(data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) -class ClosedBlobTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(id integer primary key, blob_col blob)") - self.cx.execute("insert into test(blob_col) values (zeroblob(100))") - - def tearDown(self): - self.cx.close() - - def test_closed_blob_read(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.read() - - def test_closed_blob_write(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.write(b"aaaaaaaaa") - - def test_closed_blob_seek(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.seek(10) - - def test_closed_blob_tell(self): - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.blob.close() - with self.assertRaises(sqlite.ProgrammingError): - self.blob.tell() + def test_blob_closed(self): + cx = sqlite.connect(":memory:") + cx.execute("create table test(b blob)") + cx.execute("insert into test values (zeroblob(100))") + blob = cx.open_blob("test", "b", 1) + blob.close() + ops = [ + lambda: blob.read(), + lambda: blob.write(b""), + lambda: blob.seek(0), + lambda: blob.tell(), + ] + msg = "Cannot operate on a closed blob" + for op in ops: + with self.subTest(op=op): + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + op() def test_closed_blob_read(self): con = sqlite.connect(":memory:") @@ -943,22 +930,6 @@ def test_closed_blob_read(self): blob.read() -class BlobContextManagerTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(id integer primary key, blob_col blob)") - self.cx.execute("insert into test(blob_col) values (zeroblob(100))") - - def tearDown(self): - self.cx.close() - - def test_blob_ctx_mgr_execute(self): - data = b"a" * 100 - with self.cx.open_blob("test", "blob_col", 1) as blob: - blob.write(data) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) - - class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -1388,9 +1359,7 @@ def wait(): def suite(): tests = [ - BlobContextManagerTests, BlobTests, - ClosedBlobTests, ClosedConTests, ClosedCurTests, ConnectionTests, From 0a87520c85592cfea15bb8b75fa6d2a27794f8f7 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 13:59:38 +0200 Subject: [PATCH 29/77] Naming: read_length => length --- Modules/_sqlite/blob.c | 24 ++++++++++++------------ Modules/_sqlite/clinic/blob.c.h | 14 +++++++------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7a80d7125919af..a812f6f8fd3712 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -118,9 +118,9 @@ blob_length(pysqlite_Blob *self) }; static PyObject * -inner_read(pysqlite_Blob *self, int read_length, int offset) +inner_read(pysqlite_Blob *self, int length, int offset) { - PyObject *buffer = PyBytes_FromStringAndSize(NULL, read_length); + PyObject *buffer = PyBytes_FromStringAndSize(NULL, length); if (buffer == NULL) { return NULL; } @@ -128,7 +128,7 @@ inner_read(pysqlite_Blob *self, int read_length, int offset) int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + rc = sqlite3_blob_read(self->blob, raw_buffer, length, self->offset); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -151,34 +151,34 @@ inner_read(pysqlite_Blob *self, int read_length, int offset) /*[clinic input] _sqlite3.Blob.read as blob_read - read_length: int = -1 + length: int = -1 / Read data from blob. [clinic start generated code]*/ static PyObject * -blob_read_impl(pysqlite_Blob *self, int read_length) -/*[clinic end generated code: output=9c4881a77860b216 input=753a766082129348]*/ +blob_read_impl(pysqlite_Blob *self, int length) +/*[clinic end generated code: output=1fc99b2541360dde input=b4b443e99af5548f]*/ { if (!check_blob(self)) { return NULL; } - if (read_length < 0) { + if (length < 0) { /* same as file read. */ - read_length = self->length; + length = self->length; } /* making sure we don't read more then blob size */ - if (read_length > self->length - self->offset) { - read_length = self->length - self->offset; + if (length > self->length - self->offset) { + length = self->length - self->offset; } - PyObject *buffer = inner_read(self, read_length, self->offset); + PyObject *buffer = inner_read(self, length, self->offset); if (buffer != NULL) { /* update offset on sucess. */ - self->offset += read_length; + self->offset += length; } return buffer; diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index fa995a4938389b..ef4b2b52ae8239 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -21,7 +21,7 @@ blob_close(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(blob_read__doc__, -"read($self, read_length=-1, /)\n" +"read($self, length=-1, /)\n" "--\n" "\n" "Read data from blob."); @@ -30,13 +30,13 @@ PyDoc_STRVAR(blob_read__doc__, {"read", (PyCFunction)(void(*)(void))blob_read, METH_FASTCALL, blob_read__doc__}, static PyObject * -blob_read_impl(pysqlite_Blob *self, int read_length); +blob_read_impl(pysqlite_Blob *self, int length); static PyObject * blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - int read_length = -1; + int length = -1; if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { goto exit; @@ -44,12 +44,12 @@ blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - read_length = _PyLong_AsInt(args[0]); - if (read_length == -1 && PyErr_Occurred()) { + length = _PyLong_AsInt(args[0]); + if (length == -1 && PyErr_Occurred()) { goto exit; } skip_optional: - return_value = blob_read_impl(self, read_length); + return_value = blob_read_impl(self, length); exit: return return_value; @@ -199,4 +199,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=76c066429020440e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=755e33bbf7642839 input=a9049054013a1b77]*/ From 32132cf53b751147f333bebf3952c0dcbdbab276 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 14:28:16 +0200 Subject: [PATCH 30/77] Wrap error handling in support function --- Modules/_sqlite/blob.c | 65 ++++++++++++------------------------------ 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index a812f6f8fd3712..852e705eb58747 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -54,6 +54,20 @@ blob_dealloc(pysqlite_Blob *self) Py_DECREF(tp); } +static void +blob_seterror(pysqlite_Blob *self, int rc) +{ + assert(self->connection != NULL); +#if SQLITE_VERSION_NUMBER < 3008008 + // SQLite pre 3.8.8 does not set errors on the connection + if (rc == SQLITE_ABORT) { + PyErr_SetString(self->connection->OperationalError, + "Cannot operate on modified blob"); + return; + } +#endif + _pysqlite_seterror(self->connection->state, self->connection->db); +} /* * Checks if a blob object is usable (i. e. not closed). @@ -133,15 +147,7 @@ inner_read(pysqlite_Blob *self, int length, int offset) if (rc != SQLITE_OK) { Py_DECREF(buffer); - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); return NULL; } return buffer; @@ -194,15 +200,7 @@ write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); return -1; } return 0; @@ -490,16 +488,7 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, - self->connection->db); - } + blob_seterror(self, rc); PyMem_Free(result_buf); PyMem_Free(data_buff); return NULL; @@ -596,15 +585,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) Py_END_ALLOW_THREADS if (rc != SQLITE_OK){ - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); PyMem_Free(data_buff); rc = -1; } @@ -618,15 +599,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) Py_END_ALLOW_THREADS if (rc != SQLITE_OK){ - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); - } - else { - _pysqlite_seterror(self->connection->state, self->connection->db); - } + blob_seterror(self, rc); PyMem_Free(data_buff); rc = -1; } From 287803e68f130a255a3dea6f74de75e667518870 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 14:38:10 +0200 Subject: [PATCH 31/77] Add blob seek position markers as constants --- Lib/sqlite3/test/dbapi.py | 13 +++++++++---- Modules/_sqlite/blob.c | 10 ++++++---- Modules/_sqlite/blob.h | 4 ++++ Modules/_sqlite/module.c | 3 +++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 5615aee476ab0c..ffb6188bc10681 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -172,6 +172,11 @@ def test_module_constants(self): if sqlite.version_info >= (3, 8, 3): consts.append("SQLITE_RECURSIVE") consts += ["PARSE_DECLTYPES", "PARSE_COLNAMES"] + consts += [ + "BLOB_SEEK_START", + "BLOB_SEEK_CUR", + "BLOB_SEEK_END", + ] for const in consts: with self.subTest(const=const): self.assertTrue(hasattr(sqlite, const)) @@ -751,16 +756,16 @@ def test_blob_tell(self): def test_blob_seek_from_start(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) - self.blob.seek(10, 0) + self.blob.seek(10, sqlite.BLOB_SEEK_START) self.assertEqual(self.blob.tell(), 10) def test_blob_seek_from_current_pos(self): - self.blob.seek(10, 1) - self.blob.seek(10, 1) + self.blob.seek(10, sqlite.BLOB_SEEK_CUR) + self.blob.seek(10, sqlite.BLOB_SEEK_CUR) self.assertEqual(self.blob.tell(), 20) def test_blob_seek_from_end(self): - self.blob.seek(-10, 2) + self.blob.seek(-10, sqlite.BLOB_SEEK_END) self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 852e705eb58747..32941f966655b0 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -266,22 +266,24 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) } switch (origin) { - case 0: // relative to blob begin + case BLOB_SEEK_START: break; - case 1: // relative to current position + case BLOB_SEEK_CUR: if (offset > INT_MAX - self->offset) { goto overflow; } offset = self->offset + offset; break; - case 2: // relative to blob end + case BLOB_SEEK_END: if (offset > INT_MAX - self->length) { goto overflow; } offset = self->length + offset; break; default: - PyErr_SetString(PyExc_ValueError, "'origin' should be 0, 1 or 2"); + PyErr_SetString(PyExc_ValueError, + "'origin' should be 'BLOB_SEEK_START', " + "'BLOB_SEEK_CUR', or 'BLOB_SEEK_END'"); return NULL; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index cb8a5ebba191b7..9e768d80974b32 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -4,6 +4,10 @@ #include "sqlite3.h" #include "connection.h" +#define BLOB_SEEK_START 0 +#define BLOB_SEEK_CUR 1 +#define BLOB_SEEK_END 2 + typedef struct { PyObject_HEAD pysqlite_Connection *connection; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 2207cb4e086bf7..7865df99aa160f 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -393,6 +393,9 @@ static int add_integer_constants(PyObject *module) { #if SQLITE_VERSION_NUMBER >= 3008003 ret += PyModule_AddIntMacro(module, SQLITE_RECURSIVE); #endif + ret += PyModule_AddIntMacro(module, BLOB_SEEK_START); + ret += PyModule_AddIntMacro(module, BLOB_SEEK_CUR); + ret += PyModule_AddIntMacro(module, BLOB_SEEK_END); return ret; } From 0fc5a3948b7061a810282db06c9ef28c0688a114 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 21:17:25 +0200 Subject: [PATCH 32/77] Adjust SQLITE_ABORT error message --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 32941f966655b0..554966a18dba58 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -62,7 +62,7 @@ blob_seterror(pysqlite_Blob *self, int rc) // SQLite pre 3.8.8 does not set errors on the connection if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, - "Cannot operate on modified blob"); + "Cannot operate on an expired blob handle"); return; } #endif From cd0bde10dad3c2bb2a1bc85d455f3a57f32f3de2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 22:48:58 +0200 Subject: [PATCH 33/77] Remove unneeded sqlite3_blob_close() and fix GC tracking --- Modules/_sqlite/connection.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 5587037fc6939f..958ade836dc630 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -424,6 +424,8 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, obj->length = len; obj->in_weakreflist = NULL; + PyObject_GC_Track(obj); + // Add our blob to connection blobs list PyObject *weakref = PyWeakref_NewRef((PyObject *)obj, NULL); if (weakref == NULL) { @@ -439,10 +441,6 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, error: Py_XDECREF(obj); - - Py_BEGIN_ALLOW_THREADS - sqlite3_blob_close(blob); - Py_END_ALLOW_THREADS return NULL; } From cf7e15edfc58bf209007bc51312bb1c8b34a3efe Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 22:53:11 +0200 Subject: [PATCH 34/77] Use close_blob() in pysqlite_close_all_blobs() --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 554966a18dba58..0a6f92467ba41c 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -117,7 +117,7 @@ pysqlite_close_all_blobs(pysqlite_Connection *self) PyObject *weakref = PyList_GET_ITEM(self->blobs, i); PyObject *blob = PyWeakref_GetObject(weakref); if (!Py_IsNone(blob)) { - blob_close_impl((pysqlite_Blob *)blob); + close_blob((pysqlite_Blob *)blob); } } } From 7e77217c4b0f4e151d8cb57358083dd8b8ba6e16 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 23:04:19 +0200 Subject: [PATCH 35/77] Refactor write/read --- Modules/_sqlite/blob.c | 35 ++++++++++++++------------------- Modules/_sqlite/clinic/blob.c.h | 20 +++++++++---------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 0a6f92467ba41c..b4dc07a8fbf242 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -182,11 +182,10 @@ blob_read_impl(pysqlite_Blob *self, int length) } PyObject *buffer = inner_read(self, length, self->offset); - if (buffer != NULL) { - /* update offset on sucess. */ - self->offset += length; + if (buffer == NULL) { + return NULL; } - + self->offset += length; return buffer; }; @@ -210,40 +209,36 @@ write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) /*[clinic input] _sqlite3.Blob.write as blob_write - data_buffer: Py_buffer + data: Py_buffer / Write data to blob. [clinic start generated code]*/ static PyObject * -blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer) -/*[clinic end generated code: output=dc8da6900b969799 input=8597402caf368add]*/ +blob_write_impl(pysqlite_Blob *self, Py_buffer *data) +/*[clinic end generated code: output=b34cf22601b570b2 input=0dcf4018286f55d2]*/ { - int rc; - - if (data_buffer->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); + if (!check_blob(self)) { return NULL; } - if (data_buffer->len > self->length - self->offset) { - PyErr_SetString(PyExc_ValueError, "data longer than blob length"); + if (data->len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); return NULL; } - if (!check_blob(self)) { + if (data->len > self->length - self->offset) { + PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return NULL; } - rc = write_inner(self, data_buffer->buf, data_buffer->len, self->offset); - - if (rc == 0) { - self->offset += (int)data_buffer->len; - Py_RETURN_NONE; - } else { + int rc = write_inner(self, data->buf, data->len, self->offset); + if (rc < 0) { return NULL; } + self->offset += (int)data->len; + Py_RETURN_NONE; } diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index ef4b2b52ae8239..8276f8e140c2e8 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -56,7 +56,7 @@ blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(blob_write__doc__, -"write($self, data_buffer, /)\n" +"write($self, data, /)\n" "--\n" "\n" "Write data to blob."); @@ -65,27 +65,27 @@ PyDoc_STRVAR(blob_write__doc__, {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, static PyObject * -blob_write_impl(pysqlite_Blob *self, Py_buffer *data_buffer); +blob_write_impl(pysqlite_Blob *self, Py_buffer *data); static PyObject * blob_write(pysqlite_Blob *self, PyObject *arg) { PyObject *return_value = NULL; - Py_buffer data_buffer = {NULL, NULL}; + Py_buffer data = {NULL, NULL}; - if (PyObject_GetBuffer(arg, &data_buffer, PyBUF_SIMPLE) != 0) { + if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { goto exit; } - if (!PyBuffer_IsContiguous(&data_buffer, 'C')) { + if (!PyBuffer_IsContiguous(&data, 'C')) { _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); goto exit; } - return_value = blob_write_impl(self, &data_buffer); + return_value = blob_write_impl(self, &data); exit: - /* Cleanup for data_buffer */ - if (data_buffer.obj) { - PyBuffer_Release(&data_buffer); + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); } return return_value; @@ -199,4 +199,4 @@ blob_exit(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=755e33bbf7642839 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=235d02d1bfa39b2a input=a9049054013a1b77]*/ From c57e45ae7c9d16d6c0cb5576c71e4cda669bf30a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 23:09:52 +0200 Subject: [PATCH 36/77] Sipmlify __exit__ --- Modules/_sqlite/blob.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index b4dc07a8fbf242..fd6d0753c2e327 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -346,17 +346,10 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, PyObject *tb) /*[clinic end generated code: output=fc86ceeb2b68c7b2 input=575d9ecea205f35f]*/ { - PyObject *res; if (!check_blob(self)) { return NULL; } - - res = blob_close_impl(self); - if (!res) { - return NULL; - } - Py_XDECREF(res); - + close_blob(self); Py_RETURN_FALSE; } From dd76f727f1b87a9b622734afeac8d2dbcc862865 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 10 Sep 2021 23:45:22 +0200 Subject: [PATCH 37/77] Use new slice API --- Modules/_sqlite/blob.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index fd6d0753c2e327..d14d445918b951 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -448,11 +448,10 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - - if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, - &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); @@ -537,10 +536,10 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, - &slicelen) < 0) { + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Blob object doesn't support slice deletion"); From 237684ed84111ab2b691175fb28ff71417e2e8b9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 21:54:09 +0200 Subject: [PATCH 38/77] Refactor very large functions --- Modules/_sqlite/blob.c | 293 ++++++++++++++++++++++------------------- 1 file changed, 156 insertions(+), 137 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index d14d445918b951..60a006f23a8020 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -423,186 +423,205 @@ blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) return write_inner(self, buf, 1, i); } +static PyObject * +subscript_index(pysqlite_Blob *self, PyObject *item) +{ + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) { + return NULL; + } + if (i < 0) { + i += self->length; + } + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return NULL; + } + // TODO: I am not sure... + return inner_read(self, 1, i); +} static PyObject * -blob_subscript(pysqlite_Blob *self, PyObject *item) +subscript_slice(pysqlite_Blob *self, PyObject *item) { - if (!check_blob(self)) { + Py_ssize_t start, stop, step, slicelen; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) { - return NULL; - } - if (i < 0) { - i += self->length; - } - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return NULL; - } - // TODO: I am not sure... - return inner_read(self, 1, i); + if (slicelen <= 0) { + return PyBytes_FromStringAndSize("", 0); } - else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelen; - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { - return NULL; + else if (step == 1) { + return inner_read(self, slicelen, start); + } + else { + char *result_buf = (char *)PyMem_Malloc(slicelen); + if (result_buf == NULL) { + return PyErr_NoMemory(); } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (slicelen <= 0) { - return PyBytes_FromStringAndSize("", 0); - } - else if (step == 1) { - return inner_read(self, slicelen, start); + char *data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyMem_Free(result_buf); + return PyErr_NoMemory(); } - else { - char *result_buf = (char *)PyMem_Malloc(slicelen); - if (result_buf == NULL) { - return PyErr_NoMemory(); - } - - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyMem_Free(result_buf); - return PyErr_NoMemory(); - } - - int rc; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS - if (rc != SQLITE_OK) { - blob_seterror(self, rc); - PyMem_Free(result_buf); - PyMem_Free(data_buff); - return NULL; - } + int rc; + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - result_buf[i] = data_buff[cur]; - } - PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); + if (rc != SQLITE_OK) { + blob_seterror(self, rc); PyMem_Free(result_buf); PyMem_Free(data_buff); - return result; + return NULL; + } + + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { + result_buf[i] = data_buff[cur]; } + PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return result; } - else { - PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); +} + +static PyObject * +blob_subscript(pysqlite_Blob *self, PyObject *item) +{ + if (!check_blob(self)) { return NULL; } + + if (PyIndex_Check(item)) { + return subscript_index(self, item); + } + if (PySlice_Check(item)) { + return subscript_slice(self, item); + } + + PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); + return NULL; } +static int +ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) { + return -1; + } + if (i < 0) { + i += self->length; + } + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + const char *buf = PyBytes_AsString(value); + return write_inner(self, buf, 1, i); +} static int -blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) { int rc = -1; - if (!check_blob(self)) { + Py_ssize_t start, stop, step, slicelen; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) { + return -1; + } + slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); return -1; } - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) { - return -1; - } - if (i < 0) { - i += self->length; - } - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return -1; - } - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob doesn't support item deletion"); - return -1; - } - if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); - return -1; - } + Py_buffer vbuf; + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { + return -1; + } + if (vbuf.len != slicelen) { + PyErr_SetString(PyExc_IndexError, + "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; + } - const char *buf = PyBytes_AsString(value); - return write_inner(self, buf, 1, i); + if (slicelen == 0) { // FIXME } - else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelen; - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { - return -1; - } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); + else if (step == 1) { + rc = write_inner(self, vbuf.buf, slicelen, start); + } + else { + char *data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyErr_NoMemory(); return -1; } - Py_buffer vbuf; - if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { - return -1; - } - if (vbuf.len != slicelen) { - PyErr_SetString(PyExc_IndexError, - "Blob slice assignment is wrong size"); - PyBuffer_Release(&vbuf); - return -1; - } + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS - if (slicelen == 0) { // FIXME - } - else if (step == 1) { - rc = write_inner(self, vbuf.buf, slicelen, start); + if (rc != SQLITE_OK){ + blob_seterror(self, rc); + PyMem_Free(data_buff); + rc = -1; } - else { - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyErr_NoMemory(); - return -1; - } - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS + for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { + data_buff[cur] = ((char *)vbuf.buf)[i]; + } - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); - rc = -1; - } + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - data_buff[cur] = ((char *)vbuf.buf)[i]; - } + if (rc != SQLITE_OK){ + blob_seterror(self, rc); + PyMem_Free(data_buff); + rc = -1; + } + rc = 0; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS + } + PyBuffer_Release(&vbuf); + return rc; +} - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); - rc = -1; - } - rc = 0; +static int +blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + if (!check_blob(self)) { + return -1; + } - } - PyBuffer_Release(&vbuf); - return rc; + if (PyIndex_Check(item)) { + return ass_subscript_index(self, item, value); } - else { - PyErr_SetString(PyExc_TypeError, - "Blob indices must be integer"); - return -1; + if (PySlice_Check(item)) { + return ass_subscript_slice(self, item, value); } + + PyErr_SetString(PyExc_TypeError, "Blob indices must be integer"); + return -1; } From 58905e81b86c17d7d8c0ab19b0fef8af7ec1dbe3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:06:20 +0200 Subject: [PATCH 39/77] Simplify subscript slice --- Modules/_sqlite/blob.c | 46 ++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 60a006f23a8020..60b917494ad454 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -138,8 +138,8 @@ inner_read(pysqlite_Blob *self, int length, int offset) if (buffer == NULL) { return NULL; } - char *raw_buffer = PyBytes_AS_STRING(buffer); + char *raw_buffer = PyBytes_AS_STRING(buffer); int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_read(self->blob, raw_buffer, length, self->offset); @@ -456,38 +456,26 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) else if (step == 1) { return inner_read(self, slicelen, start); } - else { - char *result_buf = (char *)PyMem_Malloc(slicelen); - if (result_buf == NULL) { - return PyErr_NoMemory(); - } - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyMem_Free(result_buf); - return PyErr_NoMemory(); - } - - int rc; - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS + PyObject *blob = inner_read(self, stop - start, start); + if (blob == NULL) { + return NULL; + } - if (rc != SQLITE_OK) { - blob_seterror(self, rc); - PyMem_Free(result_buf); - PyMem_Free(data_buff); - return NULL; - } + PyObject *result = PyBytes_FromStringAndSize(NULL, slicelen); + if (result == NULL) { + goto exit; + } - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - result_buf[i] = data_buff[cur]; - } - PyObject *result = PyBytes_FromStringAndSize(result_buf, slicelen); - PyMem_Free(result_buf); - PyMem_Free(data_buff); - return result; + char *blob_buf = PyBytes_AS_STRING(blob); + char *res_buf = PyBytes_AS_STRING(result); + for (Py_ssize_t i = 0, j = 0; i < slicelen; i++, j += step) { + res_buf[i] = blob_buf[j]; } + +exit: + Py_DECREF(blob); + return result; } static PyObject * From 9d69fcade2f56c6f00c7dd2d9cb99145dd80c384 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:24:47 +0200 Subject: [PATCH 40/77] Consolidate more tests --- Lib/sqlite3/test/dbapi.py | 55 +++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index ffb6188bc10681..d752b2c76dd6ae 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -750,21 +750,16 @@ def tearDown(self): def test_blob_length(self): self.assertEqual(len(self.blob), 100) - def test_blob_tell(self): - self.assertEqual(self.blob.tell(), 0) - - def test_blob_seek_from_start(self): + def test_blob_seek_and_tell(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) + self.blob.seek(10, sqlite.BLOB_SEEK_START) self.assertEqual(self.blob.tell(), 10) - def test_blob_seek_from_current_pos(self): - self.blob.seek(10, sqlite.BLOB_SEEK_CUR) self.blob.seek(10, sqlite.BLOB_SEEK_CUR) self.assertEqual(self.blob.tell(), 20) - def test_blob_seek_from_end(self): self.blob.seek(-10, sqlite.BLOB_SEEK_END) self.assertEqual(self.blob.tell(), 90) @@ -824,37 +819,35 @@ def test_blob_write_when_readonly(self): read_only_blob.write(b"aaa") read_only_blob.close() - def test_blob_open_with_bad_db(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "blob_col", 1, name="notexisintg") - - def test_blob_open_with_bad_table(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("notexisintg", "blob_col", 1) - - def test_blob_open_with_bad_column(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "notexisting", 1) - - def test_blob_open_with_bad_row(self): - with self.assertRaises(sqlite.OperationalError): - self.cx.open_blob("test", "blob_col", 2) + def test_blob_open_error(self): + dataset = ( + (("test", "blob_col", 1), {"name": "notexisting"}), + (("notexisting", "blob_col", 1), {}), + (("test", "notexisting", 1), {}), + (("test", "blob_col", 2), {}), + ) + for args, kwds in dataset: + with self.subTest(args=args, kwds=kwds): + with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob(*args, **kwds) def test_blob_get_item(self): self.assertEqual(self.blob[5], b"a") - def test_blob_get_item_index_out_of_range(self): - with self.assertRaises(IndexError): - self.blob[105] - with self.assertRaises(IndexError): - self.blob[-105] - def test_blob_get_item_negative_index(self): self.assertEqual(self.blob[-5], b"a") - def test_blob_get_item_invalid_index(self): - with self.assertRaises(TypeError): - self.blob[b"a"] + def test_blob_get_item_error(self): + dataset = ( + (b"", TypeError), + (105, IndexError), + (-105, IndexError), + (len(self.blob), IndexError), + ) + for idx, exc in dataset: + with self.subTest(idx=idx, exc=exc): + with self.assertRaises(exc): + self.blob[idx] def test_blob_get_slice(self): self.assertEqual(self.blob[5:10], b"aaaaa") From 285bb3d2d57bd10a12e57e22ff67cf86d009f8f6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:39:04 +0200 Subject: [PATCH 41/77] Use supplied offset in inner_read() --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 60b917494ad454..6916380cbfba3b 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -142,7 +142,7 @@ inner_read(pysqlite_Blob *self, int length, int offset) char *raw_buffer = PyBytes_AS_STRING(buffer); int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, length, self->offset); + rc = sqlite3_blob_read(self->blob, raw_buffer, length, offset); Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { From 2f3051a6705f76d0039fa44ec36ec7f6168cacaa Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 22:45:50 +0200 Subject: [PATCH 42/77] Simplify test --- Lib/sqlite3/test/dbapi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index d752b2c76dd6ae..9d1eae194076a9 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -737,9 +737,9 @@ def test_same_query_in_multiple_cursors(self): class BlobTests(unittest.TestCase): def setUp(self): self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("create table test(blob_col blob)") self.blob_data = b"a" * 100 - self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data, )) + self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data,)) self.blob = self.cx.open_blob("test", "blob_col", 1) self.second_data = b"b" * 100 @@ -803,12 +803,12 @@ def test_blob_write_more_then_blob_size(self): self.blob.write(b"a" * 1000) def test_blob_read_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.read() def test_blob_write_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") @@ -920,7 +920,7 @@ def test_blob_closed(self): def test_closed_blob_read(self): con = sqlite.connect(":memory:") - con.execute("create table test(id integer primary key, blob_col blob)") + con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(100))") blob = con.open_blob("test", "blob_col", 1) con.close() From fd7c31173b1ec3d5dd6242ad9f1a12783ef5dfec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:13:03 +0200 Subject: [PATCH 43/77] Simplify assign subscript slice --- Modules/_sqlite/blob.c | 52 ++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 6916380cbfba3b..9394dbe3bea5f4 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -528,18 +528,17 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) static int ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int rc = -1; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); + return -1; + } Py_ssize_t start, stop, step, slicelen; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); - return -1; - } Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { @@ -552,43 +551,26 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (slicelen == 0) { // FIXME + int rc; + if (slicelen == 0) { + rc = 0; } else if (step == 1) { rc = write_inner(self, vbuf.buf, slicelen, start); } else { - char *data_buff = (char *)PyMem_Malloc(stop - start); - if (data_buff == NULL) { - PyErr_NoMemory(); - return -1; - } - - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); + PyObject *read_blob = inner_read(self, stop - start, start); + if (read_blob == NULL) { rc = -1; } - - for (Py_ssize_t cur = 0, i = 0; i < slicelen; cur += step, i++) { - data_buff[cur] = ((char *)vbuf.buf)[i]; - } - - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); - Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - blob_seterror(self, rc); - PyMem_Free(data_buff); - rc = -1; + else { + char *blob_buf = PyBytes_AS_STRING(read_blob); + for (Py_ssize_t i = 0, j = 0; i < slicelen; i++, j += step) { + blob_buf[j] = ((char *)vbuf.buf)[i]; + } + rc = write_inner(self, blob_buf, stop - start, start); + Py_DECREF(read_blob); } - rc = 0; - } PyBuffer_Release(&vbuf); return rc; From 0644e8794a51147304d6d72cfac8bc0e6f25e9d6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:21:58 +0200 Subject: [PATCH 44/77] Early error checking, and use PyBytes_AS_STRING() when possible --- Modules/_sqlite/blob.c | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 9394dbe3bea5f4..5e69812e45ceb5 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -398,28 +398,29 @@ blob_item(pysqlite_Blob *self, Py_ssize_t i) } static int -blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *value) { if (!check_blob(self)) { return -1; } - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return -1; - } - if (v == NULL) { + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Blob object doesn't support item deletion"); return -1; } - if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) { + if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { PyErr_SetString(PyExc_IndexError, "Blob assignment must be length-1 bytes()"); return -1; } - const char *buf = PyBytes_AsString(v); + if (i < 0 || i >= self->length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + + const char *buf = PyBytes_AS_STRING(value); return write_inner(self, buf, 1, i); } @@ -499,6 +500,17 @@ blob_subscript(pysqlite_Blob *self, PyObject *item) static int ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) { + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) { return -1; @@ -510,18 +522,8 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob doesn't support item deletion"); - return -1; - } - if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); - return -1; - } - const char *buf = PyBytes_AsString(value); + const char *buf = PyBytes_AS_STRING(value); return write_inner(self, buf, 1, i); } From ced431a7873a871d761f4274e0594ebd860b17b8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:24:54 +0200 Subject: [PATCH 45/77] Expand test suite --- Lib/sqlite3/test/dbapi.py | 50 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 9d1eae194076a9..119272e8429b4a 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -764,15 +764,24 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): - for pos in 1000, -10: - with self.subTest(pos=pos): - self.assertRaises(ValueError, self.blob.seek, pos) + ops = ( + lambda: self.blob.seek(1000), + lambda: self.blob.seek(-10), + lambda: self.blob.seek(10, -1), + ) + for op in ops: + with self.subTest(op=op): + self.assertRaises(ValueError, op) def test_blob_read(self): - self.assertEqual(self.blob.read(), self.blob_data) + buf = self.blob.read() + self.assertEqual(buf, self.blob_data) + self.assertEqual(len(buf), len(self.blob_data)) - def test_blob_read_size(self): - self.assertEqual(len(self.blob.read(10)), 10) + def test_blob_read_too_much(self): + buf = self.blob.read(len(self.blob_data) * 2) + self.assertEqual(buf, self.blob_data) + self.assertEqual(len(buf), len(self.blob_data)) def test_blob_read_advance_offset(self): self.blob.read(10) @@ -867,10 +876,25 @@ def test_blob_set_item(self): self.blob[0] = b"b" self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + def test_blob_set_item(self): + self.blob[-1] = b"z" + self.assertEqual(self.blob[-1], b"z") + + def test_blob_set_item_error(self): + with self.assertRaises(TypeError): + self.blob["a"] = b"b" + with self.assertRaises(TypeError): + del self.blob[0] + with self.assertRaises(IndexError): + self.blob[1000] = b"a" + def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + def test_blob_set_empty_slice(self): + self.blob[0:0] = b"" + def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"bbbbb" self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) @@ -878,9 +902,13 @@ def test_blob_set_slice_with_skip(self): def test_blob_get_empty_slice(self): self.assertEqual(self.blob[5:5], b"") - def test_blob_set_slice_wrong_length(self): + def test_blob_set_slice_error(self): with self.assertRaises(IndexError): self.blob[5:10] = b"a" + with self.assertRaises(IndexError): + self.blob[5:10] = b"a" * 1000 + with self.assertRaises(TypeError): + del self.blob[5:10] def test_blob_concat_not_supported(self): with self.assertRaises(SystemError): @@ -906,11 +934,19 @@ def test_blob_closed(self): cx.execute("insert into test values (zeroblob(100))") blob = cx.open_blob("test", "b", 1) blob.close() + + def assign(): blob[0] = b"" ops = [ lambda: blob.read(), lambda: blob.write(b""), lambda: blob.seek(0), lambda: blob.tell(), + lambda: blob.__enter__(), + lambda: blob.__exit__(None, None, None), + lambda: len(blob), + lambda: blob[0], + lambda: blob[0:1], + assign, ] msg = "Cannot operate on a closed blob" for op in ops: From 2dae1b9efabeaed98d3d3c765290c8d3a2b32a96 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 11 Sep 2021 23:40:34 +0200 Subject: [PATCH 46/77] Remove unneeded parts of sequence protocol --- Lib/sqlite3/test/dbapi.py | 20 +++++----- Modules/_sqlite/blob.c | 80 --------------------------------------- 2 files changed, 9 insertions(+), 91 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 119272e8429b4a..c947bdb222251a 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -910,17 +910,15 @@ def test_blob_set_slice_error(self): with self.assertRaises(TypeError): del self.blob[5:10] - def test_blob_concat_not_supported(self): - with self.assertRaises(SystemError): - self.blob + self.blob - - def test_blob_repeat_not_supported(self): - with self.assertRaises(SystemError): - self.blob * 5 - - def test_blob_contains_not_supported(self): - with self.assertRaises(SystemError): - b"aaaaa" in self.blob + def test_blob_sequence_not_supported(self): + ops = ( + lambda: self.blob + self.blob, + lambda: self.blob * 5, + lambda: b"a" in self.blob, + ) + for op in ops: + with self.subTest(op=op): + self.assertRaises(TypeError, op) def test_blob_context_manager(self): data = b"a" * 100 diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 5e69812e45ceb5..4c01a1fb63e690 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -353,77 +353,6 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, Py_RETURN_FALSE; } -static PyObject * -blob_concat(pysqlite_Blob *self, PyObject *args) -{ - if (check_blob(self)) { - PyErr_SetString(PyExc_SystemError, "Blob don't support concatenation"); - } - return NULL; -} - -static PyObject * -blob_repeat(pysqlite_Blob *self, PyObject *args) -{ - if (check_blob(self)) { - PyErr_SetString(PyExc_SystemError, - "Blob don't support repeat operation"); - } - return NULL; -} - -static int -blob_contains(pysqlite_Blob *self, PyObject *args) -{ - if (check_blob(self)) { - PyErr_SetString(PyExc_SystemError, - "Blob don't support contains operation"); - } - return -1; -} - -static PyObject * -blob_item(pysqlite_Blob *self, Py_ssize_t i) -{ - if (!check_blob(self)) { - return NULL; - } - - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return NULL; - } - - return inner_read(self, 1, i); -} - -static int -blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *value) -{ - if (!check_blob(self)) { - return -1; - } - - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support item deletion"); - return -1; - } - if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); - return -1; - } - - if (i < 0 || i >= self->length) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); - return -1; - } - - const char *buf = PyBytes_AS_STRING(value); - return write_inner(self, buf, 1, i); -} - static PyObject * subscript_index(pysqlite_Blob *self, PyObject *item) { @@ -438,7 +367,6 @@ subscript_index(pysqlite_Blob *self, PyObject *item) PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } - // TODO: I am not sure... return inner_read(self, 1, i); } @@ -619,14 +547,6 @@ static PyType_Slot blob_slots[] = { {Py_tp_methods, blob_methods}, {Py_tp_members, blob_members}, - // Sequence protocol - {Py_sq_length, blob_length}, - {Py_sq_concat, blob_concat}, - {Py_sq_repeat, blob_repeat}, - {Py_sq_item, blob_item}, - {Py_sq_ass_item, blob_ass_item}, - {Py_sq_contains, blob_contains}, - // Mapping protocol {Py_mp_length, blob_length}, {Py_mp_subscript, blob_subscript}, From e8fa47e5665709a4c8f998ce5343116a26f3ba89 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:12:46 +0200 Subject: [PATCH 47/77] Normalise error messages --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 4c01a1fb63e690..1161da229e75af 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -460,7 +460,7 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) { if (value == NULL) { PyErr_SetString(PyExc_TypeError, - "Blob object doesn't support slice deletion"); + "Blob doesn't support slice deletion"); return -1; } From 644166845a44f79052b2ec4955f0238a284f70e1 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:18:49 +0200 Subject: [PATCH 48/77] Improve error message/type for to large subscript assignment --- Lib/sqlite3/test/dbapi.py | 2 ++ Modules/_sqlite/blob.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index c947bdb222251a..44f44d85a6b67c 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -883,6 +883,8 @@ def test_blob_set_item(self): def test_blob_set_item_error(self): with self.assertRaises(TypeError): self.blob["a"] = b"b" + with self.assertRaises(ValueError): + self.blob[0] = b"abc" with self.assertRaises(TypeError): del self.blob[0] with self.assertRaises(IndexError): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 1161da229e75af..f8e879acd156a3 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -434,8 +434,8 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } if (!PyBytes_Check(value) || PyBytes_Size(value) != 1) { - PyErr_SetString(PyExc_IndexError, - "Blob assignment must be length-1 bytes()"); + PyErr_SetString(PyExc_ValueError, + "Blob assignment must be a single byte"); return -1; } From 03d2152dbcf96966abe0529c140f223c66232b41 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:28:03 +0200 Subject: [PATCH 49/77] Normalise naming: write_inner => inner_write --- Modules/_sqlite/blob.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f8e879acd156a3..a4a524d2edfb0d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -190,7 +190,7 @@ blob_read_impl(pysqlite_Blob *self, int length) }; static int -write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { int rc; @@ -233,7 +233,7 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - int rc = write_inner(self, data->buf, data->len, self->offset); + int rc = inner_write(self, data->buf, data->len, self->offset); if (rc < 0) { return NULL; } @@ -452,7 +452,7 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) } const char *buf = PyBytes_AS_STRING(value); - return write_inner(self, buf, 1, i); + return inner_write(self, buf, 1, i); } static int @@ -486,7 +486,7 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) rc = 0; } else if (step == 1) { - rc = write_inner(self, vbuf.buf, slicelen, start); + rc = inner_write(self, vbuf.buf, slicelen, start); } else { PyObject *read_blob = inner_read(self, stop - start, start); @@ -498,7 +498,7 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) for (Py_ssize_t i = 0, j = 0; i < slicelen; i++, j += step) { blob_buf[j] = ((char *)vbuf.buf)[i]; } - rc = write_inner(self, blob_buf, stop - start, start); + rc = inner_write(self, blob_buf, stop - start, start); Py_DECREF(read_blob); } } From 9360e62cf3bba7a8a20f2a6948ce748f901be2ec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:29:09 +0200 Subject: [PATCH 50/77] Adjust comment --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index a4a524d2edfb0d..92c38d3754aad2 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -59,7 +59,7 @@ blob_seterror(pysqlite_Blob *self, int rc) { assert(self->connection != NULL); #if SQLITE_VERSION_NUMBER < 3008008 - // SQLite pre 3.8.8 does not set errors on the connection + // SQLite pre 3.8.8 does not set this blob error on the connection if (rc == SQLITE_ABORT) { PyErr_SetString(self->connection->OperationalError, "Cannot operate on an expired blob handle"); From 8eb16f7905eb07ac34c1df52d85b1b0a597affaa Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:30:43 +0200 Subject: [PATCH 51/77] Move blob_seterror() closer to where it's first used --- Modules/_sqlite/blob.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 92c38d3754aad2..6bc0a3ba3923ae 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -54,21 +54,6 @@ blob_dealloc(pysqlite_Blob *self) Py_DECREF(tp); } -static void -blob_seterror(pysqlite_Blob *self, int rc) -{ - assert(self->connection != NULL); -#if SQLITE_VERSION_NUMBER < 3008008 - // SQLite pre 3.8.8 does not set this blob error on the connection - if (rc == SQLITE_ABORT) { - PyErr_SetString(self->connection->OperationalError, - "Cannot operate on an expired blob handle"); - return; - } -#endif - _pysqlite_seterror(self->connection->state, self->connection->db); -} - /* * Checks if a blob object is usable (i. e. not closed). * @@ -131,6 +116,21 @@ blob_length(pysqlite_Blob *self) return self->length; }; +static void +blob_seterror(pysqlite_Blob *self, int rc) +{ + assert(self->connection != NULL); +#if SQLITE_VERSION_NUMBER < 3008008 + // SQLite pre 3.8.8 does not set this blob error on the connection + if (rc == SQLITE_ABORT) { + PyErr_SetString(self->connection->OperationalError, + "Cannot operate on an expired blob handle"); + return; + } +#endif + _pysqlite_seterror(self->connection->state, self->connection->db); +} + static PyObject * inner_read(pysqlite_Blob *self, int length, int offset) { From 411d07e6e9ebd5779d9d49f7b3a0b31ba9e49616 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:32:24 +0200 Subject: [PATCH 52/77] Fetch state from connection in check_blob() --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 6bc0a3ba3923ae..f8b8bc2938bbc7 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -67,7 +67,7 @@ check_blob(pysqlite_Blob *self) return 0; } if (self->blob == NULL) { - pysqlite_state *state = pysqlite_get_state(NULL); + pysqlite_state *state = self->connection->state; PyErr_SetString(state->ProgrammingError, "Cannot operate on a closed blob."); return 0; From 8149e36f642f6a5fb6e56187eea6a5d7dbb02bb3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:33:04 +0200 Subject: [PATCH 53/77] Remove unused declaration --- Modules/_sqlite/blob.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 9e768d80974b32..737df70ff205da 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -1,5 +1,6 @@ #ifndef PYSQLITE_BLOB_H #define PYSQLITE_BLOB_H + #include "Python.h" #include "sqlite3.h" #include "connection.h" @@ -18,8 +19,6 @@ typedef struct { PyObject *in_weakreflist; } pysqlite_Blob; -PyObject *pysqlite_blob_close(pysqlite_Blob *self); - int pysqlite_blob_setup_types(PyObject *module); void pysqlite_close_all_blobs(pysqlite_Connection *self); From 56b4caabc8746233b7d717beca7e66aa613d5ce3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:53:18 +0200 Subject: [PATCH 54/77] Doc adjustments --- Doc/includes/sqlite3/blob.py | 7 +++-- Doc/includes/sqlite3/blob_with.py | 6 ++--- Doc/library/sqlite3.rst | 45 ++++++++++++++++--------------- Lib/sqlite3/test/dbapi.py | 12 +++------ Modules/_sqlite/blob.c | 9 +++---- Modules/_sqlite/module.c | 3 --- 6 files changed, 37 insertions(+), 45 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index afd7812a8b3af9..f928349fd0e981 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -1,13 +1,12 @@ import sqlite3 con = sqlite3.connect(":memory:") -# creating the table -con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -# opening blob handle + blob = con.open_blob("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) -print(blob.read()) # will print b"HelloWorld" +print(blob.read()) # will print b"HelloWorld" blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index fdca9fbc638ea2..2e48d568830820 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -1,12 +1,12 @@ import sqlite3 con = sqlite3.connect(":memory:") -# creating the table + con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -# opening blob handle + with con.open_blob("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) - print(blob.read()) # will print b"HelloWorld" + print(blob.read()) # will print b"HelloWorld" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 4c6b24ec4ea5ce..ad621c5ae70fd3 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -304,20 +304,20 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, *, readonly=False, dbname="main") + .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, - column *column*, table *table* in database *dbname* will be returned. - When *readonly* is :const:`True` the BLOB is opened with read - permissions. Otherwise the BLOB has read and write permissions. + column *column*, table *table* in database *name* will be returned. + When *readonly* is :const:`True` the BLOB is opened without write + permissions. .. note:: The BLOB size cannot be changed using the :class:`Blob` class. Use - ``zeroblob`` to create the blob in the wanted size in advance. + the SQL function ``zeroblob`` to create a blob with a fixed size. - .. versionadded:: 3.10 + .. versionadded:: 3.11 .. method:: commit() @@ -900,21 +900,21 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.10 +.. versionadded:: 3.11 .. class:: Blob A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both - the file and sequence protocol. For example, you can read data from the + :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` class implements + the file and mapping protocols. For example, you can read data from the :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. You can call ``len(obj)`` to get size of the BLOB. .. method:: Blob.close() - Close the BLOB now (rather than whenever __del__ is called). + Close the BLOB. - The BLOB will be unusable from this point forward; an + The BLOB will be unusable from this point forward. An :class:`~sqlite3.Error` (or subclass) exception will be raised if any operation is attempted with the BLOB. @@ -926,25 +926,26 @@ Blob Objects Read *size* bytes of data from the BLOB at the current offset position. If the end of the BLOB is reached we will return the data up to end of - file. When *size* is not specified or negative we will read up to end - of BLOB. + file. When *size* is not specified or is negative, :meth:`~Blob.read` + will read till the end of the BLOB. - .. method:: Blob.write(data) + .. method:: Blob.write(data, /) Write *data* to the BLOB at the current offset. This function cannot - changed BLOB length. If data write will result in writing to more - then BLOB current size an error will be raised. + change the BLOB length. Writing beyond the end of the blob will result in + an exception being raised. .. method:: Blob.tell() - Return the current offset of the BLOB. + Return the current access position of the BLOB. - .. method:: Blob.seek(offset, whence=os.SEEK_SET) + .. method:: Blob.seek(offset, /, origin=sqlite3.BLOB_SEEK_START) - Set the BLOB offset. The *whence* argument is optional and defaults to - :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values - are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and - :data:`os.SEEK_END` or 2 (seek relative to the BLOB’s end). + Set the current access position of the BLOB. The *origin* argument is + optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB + positioning); other values are :data:`os.SEEK_CUR` or 1 (seek relative to + the current position) and :data:`os.SEEK_END` or 2 (seek relative to the + BLOB’s end). :class:`Blob` example: diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 44f44d85a6b67c..183479f18ad341 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -172,11 +172,6 @@ def test_module_constants(self): if sqlite.version_info >= (3, 8, 3): consts.append("SQLITE_RECURSIVE") consts += ["PARSE_DECLTYPES", "PARSE_COLNAMES"] - consts += [ - "BLOB_SEEK_START", - "BLOB_SEEK_CUR", - "BLOB_SEEK_END", - ] for const in consts: with self.subTest(const=const): self.assertTrue(hasattr(sqlite, const)) @@ -754,13 +749,14 @@ def test_blob_seek_and_tell(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) - self.blob.seek(10, sqlite.BLOB_SEEK_START) + import os + self.blob.seek(10, os.SEEK_SET) self.assertEqual(self.blob.tell(), 10) - self.blob.seek(10, sqlite.BLOB_SEEK_CUR) + self.blob.seek(10, os.SEEK_CUR) self.assertEqual(self.blob.tell(), 20) - self.blob.seek(-10, sqlite.BLOB_SEEK_END) + self.blob.seek(-10, os.SEEK_END) self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f8b8bc2938bbc7..9c840e90b97eea 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -261,15 +261,15 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) } switch (origin) { - case BLOB_SEEK_START: + case 0: break; - case BLOB_SEEK_CUR: + case 1: if (offset > INT_MAX - self->offset) { goto overflow; } offset = self->offset + offset; break; - case BLOB_SEEK_END: + case 2: if (offset > INT_MAX - self->length) { goto overflow; } @@ -277,8 +277,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) break; default: PyErr_SetString(PyExc_ValueError, - "'origin' should be 'BLOB_SEEK_START', " - "'BLOB_SEEK_CUR', or 'BLOB_SEEK_END'"); + "'origin' should be 0, 1, or 2"); return NULL; } diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 7865df99aa160f..2207cb4e086bf7 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -393,9 +393,6 @@ static int add_integer_constants(PyObject *module) { #if SQLITE_VERSION_NUMBER >= 3008003 ret += PyModule_AddIntMacro(module, SQLITE_RECURSIVE); #endif - ret += PyModule_AddIntMacro(module, BLOB_SEEK_START); - ret += PyModule_AddIntMacro(module, BLOB_SEEK_CUR); - ret += PyModule_AddIntMacro(module, BLOB_SEEK_END); return ret; } From afdeb2e601eb7b646ece4413e363787ba5ce5a60 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 00:59:22 +0200 Subject: [PATCH 55/77] Add What's New and adjust NEWS --- Doc/whatsnew/3.11.rst | 4 ++++ .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 7 +++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 9befe8f2732e70..f3766efefc58f5 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -232,6 +232,10 @@ sqlite3 (Contributed by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland in :issue:`16379`.) +* Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. + :class:`sqlite3.Blob` allows incremental I/O operations on blobs. + (Contributed by Aviv Palivoda and Erlend E. Aasland in :issue:`24905`) + Removed ======= diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index c7d2405d89539e..c9571f374862f0 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,4 +1,3 @@ -The :class:`sqlite3.Connection` now has the -:meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` -allows incremental I/O operations to blobs. (Patch by Aviv Palivoda in -:issue:`24905`) +Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. +:class:`sqlite3.Blob` allows incremental I/O operations on blobs. +Patch by Aviv Palivoda and Erlend E. Aasland. From b12219fca30def125364aa1280c4b23ed88a023e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 21:52:22 +0200 Subject: [PATCH 56/77] Use sqlite3_blob_bytes() iso. storing length on blob object Prepare for adding reopen() support --- Modules/_sqlite/blob.c | 44 +++++++++++++++++++++++------------------- Modules/_sqlite/blob.h | 1 - 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 9c840e90b97eea..c67e17c7090c94 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -113,7 +113,7 @@ blob_length(pysqlite_Blob *self) if (!check_blob(self)) { return -1; } - return self->length; + return sqlite3_blob_bytes(self->blob); }; static void @@ -171,14 +171,12 @@ blob_read_impl(pysqlite_Blob *self, int length) return NULL; } - if (length < 0) { - /* same as file read. */ - length = self->length; - } - - /* making sure we don't read more then blob size */ - if (length > self->length - self->offset) { - length = self->length - self->offset; + /* Make sure we never read past "EOB". Also read the rest of the blob if a + * negative length is specified. */ + int blob_len = sqlite3_blob_bytes(self->blob); + int max_read_len = blob_len - self->offset; + if (length < 0 || length > max_read_len) { + length = max_read_len; } PyObject *buffer = inner_read(self, length, self->offset); @@ -228,7 +226,8 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - if (data->len > self->length - self->offset) { + int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; + if (data->len > remaining_len) { PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return NULL; } @@ -260,6 +259,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) return NULL; } + int blob_len = sqlite3_blob_bytes(self->blob); switch (origin) { case 0: break; @@ -267,13 +267,13 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) if (offset > INT_MAX - self->offset) { goto overflow; } - offset = self->offset + offset; + offset += self->offset; break; case 2: - if (offset > INT_MAX - self->length) { + if (offset > INT_MAX - blob_len) { goto overflow; } - offset = self->length + offset; + offset += blob_len; break; default: PyErr_SetString(PyExc_ValueError, @@ -281,7 +281,7 @@ blob_seek_impl(pysqlite_Blob *self, int offset, int origin) return NULL; } - if (offset < 0 || offset > self->length) { + if (offset < 0 || offset > blob_len) { PyErr_SetString(PyExc_ValueError, "offset out of blob range"); return NULL; } @@ -359,10 +359,11 @@ subscript_index(pysqlite_Blob *self, PyObject *item) if (i == -1 && PyErr_Occurred()) { return NULL; } + int blob_len = sqlite3_blob_bytes(self->blob); if (i < 0) { - i += self->length; + i += blob_len; } - if (i < 0 || i >= self->length) { + if (i < 0 || i >= blob_len) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } @@ -376,7 +377,8 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); + int blob_len = sqlite3_blob_bytes(self->blob); + slicelen = PySlice_AdjustIndices(blob_len, &start, &stop, step); if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); @@ -442,10 +444,11 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) if (i == -1 && PyErr_Occurred()) { return -1; } + int blob_len = sqlite3_blob_bytes(self->blob); if (i < 0) { - i += self->length; + i += blob_len; } - if (i < 0 || i >= self->length) { + if (i < 0 || i >= blob_len) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } @@ -467,7 +470,8 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelen = PySlice_AdjustIndices(self->length, &start, &stop, step); + int blob_len = sqlite3_blob_bytes(self->blob); + slicelen = PySlice_AdjustIndices(blob_len, &start, &stop, step); Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 737df70ff205da..b4ac4ae0e6c7dc 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -14,7 +14,6 @@ typedef struct { pysqlite_Connection *connection; sqlite3_blob *blob; int offset; - int length; PyObject *in_weakreflist; } pysqlite_Blob; From 36b0ca196f53a50f59a0392c7cba6d157636503a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 21:57:17 +0200 Subject: [PATCH 57/77] Also remove length from blob_open() --- Modules/_sqlite/connection.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 45276ef73f46bd..e9ce33fa19b2bc 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -431,14 +431,11 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, return NULL; } - int rc, len; + int rc; sqlite3_blob *blob; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_open(self->db, name, table, col, row, !readonly, &blob); - if (rc == SQLITE_OK) { - len = sqlite3_blob_bytes(blob); - } Py_END_ALLOW_THREADS if (rc != SQLITE_OK) { @@ -454,7 +451,6 @@ pysqlite_connection_open_blob_impl(pysqlite_Connection *self, obj->connection = (pysqlite_Connection *)Py_NewRef(self); obj->blob = blob; obj->offset = 0; - obj->length = len; obj->in_weakreflist = NULL; PyObject_GC_Track(obj); From 3d91705f3bc2cc5154e5d4f437dd1c805c2a59b6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 22:07:08 +0200 Subject: [PATCH 58/77] Add get subscript index helper --- Modules/_sqlite/blob.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index c67e17c7090c94..8dd3678c9f0ef7 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -352,12 +352,12 @@ blob_exit_impl(pysqlite_Blob *self, PyObject *type, PyObject *val, Py_RETURN_FALSE; } -static PyObject * -subscript_index(pysqlite_Blob *self, PyObject *item) +static int +get_subscript_index(pysqlite_Blob *self, PyObject *item) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) { - return NULL; + return -1; } int blob_len = sqlite3_blob_bytes(self->blob); if (i < 0) { @@ -365,6 +365,16 @@ subscript_index(pysqlite_Blob *self, PyObject *item) } if (i < 0 || i >= blob_len) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + return i; +} + +static PyObject * +subscript_index(pysqlite_Blob *self, PyObject *item) +{ + int i = get_subscript_index(self, item); + if (i < 0) { return NULL; } return inner_read(self, 1, i); @@ -439,20 +449,10 @@ ass_subscript_index(pysqlite_Blob *self, PyObject *item, PyObject *value) "Blob assignment must be a single byte"); return -1; } - - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) { - return -1; - } - int blob_len = sqlite3_blob_bytes(self->blob); + int i = get_subscript_index(self, item); if (i < 0) { - i += blob_len; - } - if (i < 0 || i >= blob_len) { - PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } - const char *buf = PyBytes_AS_STRING(value); return inner_write(self, buf, 1, i); } From ceee315b56684644ec6f02cdca30dde1ed44a13d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 12 Sep 2021 22:13:06 +0200 Subject: [PATCH 59/77] Expand test suite --- Lib/sqlite3/test/dbapi.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 183479f18ad341..aab7044955e762 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -21,6 +21,7 @@ # 3. This notice may not be removed or altered from any source distribution. import contextlib +import os import sqlite3 as sqlite import subprocess import sys @@ -749,7 +750,6 @@ def test_blob_seek_and_tell(self): self.blob.seek(10) self.assertEqual(self.blob.tell(), 10) - import os self.blob.seek(10, os.SEEK_SET) self.assertEqual(self.blob.tell(), 10) @@ -760,14 +760,16 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 90) def test_blob_seek_error(self): - ops = ( - lambda: self.blob.seek(1000), - lambda: self.blob.seek(-10), - lambda: self.blob.seek(10, -1), + dataset = ( + (ValueError, lambda: self.blob.seek(1000)), + (ValueError, lambda: self.blob.seek(-10)), + (ValueError, lambda: self.blob.seek(10, -1)), + (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_CUR)), + (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_END)), ) - for op in ops: - with self.subTest(op=op): - self.assertRaises(ValueError, op) + for exc, fn in dataset: + with self.subTest(exc=exc, fn=fn): + self.assertRaises(exc, fn) def test_blob_read(self): buf = self.blob.read() @@ -907,6 +909,10 @@ def test_blob_set_slice_error(self): self.blob[5:10] = b"a" * 1000 with self.assertRaises(TypeError): del self.blob[5:10] + with self.assertRaises(BufferError): + self.blob[5:10] = memoryview(b"abcde")[::2] + with self.assertRaises(ValueError): + self.blob[5:10:0] = b"12345" def test_blob_sequence_not_supported(self): ops = ( @@ -950,6 +956,16 @@ def assign(): blob[0] = b"" with self.assertRaisesRegex(sqlite.ProgrammingError, msg): op() + def test_blob_close_bad_connection(self): + cx = sqlite.connect(":memory:") + cx.execute("create table test(b blob)") + cx.execute("insert into test values(zeroblob(1))") + blob = cx.open_blob("test", "b", 1) + cx.close() + self.assertRaisesRegex(sqlite.ProgrammingError, + "Cannot operate on a closed database", + blob.close) + def test_closed_blob_read(self): con = sqlite.connect(":memory:") con.execute("create table test(blob_col blob)") @@ -964,7 +980,8 @@ class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") self.cur = self.con.cursor() - self.cur.execute("create table test(name text)") + self.cur.execute("create table test(name text, b blob)") + self.cur.execute("insert into test values('blob', zeroblob(1))") def tearDown(self): self.cur.close() @@ -997,6 +1014,7 @@ def test_check_connection_thread(self): lambda: self.con.set_trace_callback(None), lambda: self.con.set_authorizer(None), lambda: self.con.create_collation("foo", None), + lambda: self.con.open_blob("test", "b", 1) ] for fn in fns: with self.subTest(fn=fn): From d9b5cdf9c979ad487a59f6cd408b522efc928642 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 17 Nov 2021 22:00:37 +0100 Subject: [PATCH 60/77] Format docs with linewidth 80 --- Doc/library/sqlite3.rst | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 4424f3309f78b1..54490b1c9de4d8 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -364,16 +364,15 @@ Connection Objects .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") - On success a :class:`Blob` handle to the - :abbr:`BLOB (Binary Large OBject)` located in row *row*, - column *column*, table *table* in database *name* will be returned. - When *readonly* is :const:`True` the BLOB is opened without write - permissions. + On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large + OBject)` located in row *row*, column *column*, table *table* in database + *name* will be returned. When *readonly* is :const:`True` the BLOB is + opened without write permissions. .. note:: - The BLOB size cannot be changed using the :class:`Blob` class. Use - the SQL function ``zeroblob`` to create a blob with a fixed size. + The BLOB size cannot be changed using the :class:`Blob` class. Use the + SQL function ``zeroblob`` to create a blob with a fixed size. .. versionadded:: 3.11 @@ -1012,19 +1011,19 @@ Blob Objects .. class:: Blob - A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` class implements - the file and mapping protocols. For example, you can read data from the - :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. - You can call ``len(obj)`` to get size of the BLOB. + A :class:`Blob` instance can read and write the data in the :abbr:`BLOB + (Binary Large OBject)`. The :class:`Blob` class implements the file and + mapping protocols. For example, you can read data from the :class:`Blob` by + doing ``obj.read(5)`` or by doing ``obj[:5]``. Call ``len(obj)`` to get size + of the BLOB. .. method:: Blob.close() Close the BLOB. - The BLOB will be unusable from this point forward. An - :class:`~sqlite3.Error` (or subclass) exception will be - raised if any operation is attempted with the BLOB. + The BLOB will be unusable from this point forward. An + :class:`~sqlite3.Error` (or subclass) exception will be raised if any + operation is attempted with the BLOB. .. method:: Blob.__len__() @@ -1034,13 +1033,13 @@ Blob Objects Read *size* bytes of data from the BLOB at the current offset position. If the end of the BLOB is reached we will return the data up to end of - file. When *size* is not specified or is negative, :meth:`~Blob.read` + file. When *size* is not specified or is negative, :meth:`~Blob.read` will read till the end of the BLOB. .. method:: Blob.write(data, /) - Write *data* to the BLOB at the current offset. This function cannot - change the BLOB length. Writing beyond the end of the blob will result in + Write *data* to the BLOB at the current offset. This function cannot + change the BLOB length. Writing beyond the end of the blob will result in an exception being raised. .. method:: Blob.tell() @@ -1049,7 +1048,7 @@ Blob Objects .. method:: Blob.seek(offset, /, origin=sqlite3.BLOB_SEEK_START) - Set the current access position of the BLOB. The *origin* argument is + Set the current access position of the BLOB. The *origin* argument is optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and :data:`os.SEEK_END` or 2 (seek relative to the @@ -1059,7 +1058,7 @@ Blob Objects .. literalinclude:: ../includes/sqlite3/blob.py - A :class:`Blob` can also be used with :term:`context manager`: + A :class:`Blob` can also be used as a :term:`context manager`: .. literalinclude:: ../includes/sqlite3/blob_with.py From aaa2721add9cb5ebb0927c52685189b3e7862741 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 17 Nov 2021 22:14:28 +0100 Subject: [PATCH 61/77] Improve tests --- Lib/test/test_sqlite3/test_dbapi.py | 42 +++++++++++++++++------------ Modules/_sqlite/blob.c | 2 +- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 992c196a74bce2..1aa7acb50a1cb1 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1093,9 +1093,10 @@ def test_blob_open_error(self): (("test", "notexisting", 1), {}), (("test", "blob_col", 2), {}), ) + regex = "no such" for args, kwds in dataset: with self.subTest(args=args, kwds=kwds): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaisesRegex(sqlite.OperationalError, regex): self.cx.open_blob(*args, **kwds) def test_blob_get_item(self): @@ -1106,14 +1107,14 @@ def test_blob_get_item_negative_index(self): def test_blob_get_item_error(self): dataset = ( - (b"", TypeError), - (105, IndexError), - (-105, IndexError), - (len(self.blob), IndexError), + (b"", TypeError, "Blob indices must be integers"), + (105, IndexError, "Blob index out of range"), + (-105, IndexError, "Blob index out of range"), + (len(self.blob), IndexError, "Blob index out of range"), ) - for idx, exc in dataset: - with self.subTest(idx=idx, exc=exc): - with self.assertRaises(exc): + for idx, exc, regex in dataset: + with self.subTest(idx=idx, exc=exc, regex=regex): + with self.assertRaisesRegex(exc, regex): self.blob[idx] def test_blob_get_slice(self): @@ -1123,7 +1124,7 @@ def test_blob_get_slice_negative_index(self): self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) def test_blob_get_slice_invalid_index(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "indices must be integers"): self.blob[5:b"a"] def test_blob_get_slice_with_skip(self): @@ -1132,32 +1133,38 @@ def test_blob_get_slice_with_skip(self): def test_blob_set_item(self): self.blob[0] = b"b" - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + expected = b"b" + self.blob_data[1:] + self.assertEqual(actual, expected) def test_blob_set_item(self): self.blob[-1] = b"z" self.assertEqual(self.blob[-1], b"z") def test_blob_set_item_error(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "indices must be integers"): self.blob["a"] = b"b" - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, "must be a single byte"): self.blob[0] = b"abc" - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, "doesn't support.*deletion"): del self.blob[0] - with self.assertRaises(IndexError): + with self.assertRaisesRegex(IndexError, "Blob index out of range"): self.blob[1000] = b"a" def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + expected = b"bbbbb" + self.blob_data[5:] + self.assertEqual(actual, expected) def test_blob_set_empty_slice(self): self.blob[0:0] = b"" def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"bbbbb" - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + expected = b"bababababa" + self.blob_data[10:] + self.assertEqual(actual, expected) def test_blob_get_empty_slice(self): self.assertEqual(self.blob[5:5], b"") @@ -1188,7 +1195,8 @@ def test_blob_context_manager(self): data = b"a" * 100 with self.cx.open_blob("test", "blob_col", 1) as blob: blob.write(data) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) + actual = self.cx.execute("select blob_col from test").fetchone()[0] + self.assertEqual(actual, data) def test_blob_closed(self): cx = sqlite.connect(":memory:") diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 8dd3678c9f0ef7..57b66c82022eee 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -523,7 +523,7 @@ blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) return ass_subscript_slice(self, item, value); } - PyErr_SetString(PyExc_TypeError, "Blob indices must be integer"); + PyErr_SetString(PyExc_TypeError, "Blob indices must be integers"); return -1; } From 8f685ba2c1f42c853aa67e2e2bd622262563d30f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 2 Jan 2022 23:38:56 +0100 Subject: [PATCH 62/77] Add clinic_state() stub. (Currently not used) --- Modules/_sqlite/blob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 57b66c82022eee..713aaec93f4b59 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -1,7 +1,7 @@ #include "blob.h" #include "util.h" -#define clinic_state() (pysqlite_get_state(NULL)) +#define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" #undef clinic_state From 96df661180458fc4ad218c926903bb23e967424a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 2 Jan 2022 23:39:20 +0100 Subject: [PATCH 63/77] Visit/clear blob type during module traverse/clear --- Modules/_sqlite/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index c2444b9fdfbf6e..0eb64bfb12d64a 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -581,6 +581,7 @@ module_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->Warning); // Types + Py_VISIT(state->BlobType); Py_VISIT(state->ConnectionType); Py_VISIT(state->CursorType); Py_VISIT(state->PrepareProtocolType); @@ -613,6 +614,7 @@ module_clear(PyObject *module) Py_CLEAR(state->Warning); // Types + Py_CLEAR(state->BlobType); Py_CLEAR(state->ConnectionType); Py_CLEAR(state->CursorType); Py_CLEAR(state->PrepareProtocolType); From 97d12a8137c4dd02e96af571e7e25f76ee428fa8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 00:06:50 +0100 Subject: [PATCH 64/77] Harden some tests, simplify others --- Lib/test/test_sqlite3/test_dbapi.py | 176 +++++++++++++++------------- 1 file changed, 92 insertions(+), 84 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1aa7acb50a1cb1..055bc8d4c13d18 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -993,18 +993,17 @@ def test_same_query_in_multiple_cursors(self): class BlobTests(unittest.TestCase): def setUp(self): self.cx = sqlite.connect(":memory:") - self.cx.execute("create table test(blob_col blob)") - self.blob_data = b"a" * 100 - self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data,)) - self.blob = self.cx.open_blob("test", "blob_col", 1) - self.second_data = b"b" * 100 + self.cx.execute("create table test(b blob)") + self.data = b"this blob data string is exactly fifty bytes long!" + self.cx.execute("insert into test(b) values (?)", (self.data,)) + self.blob = self.cx.open_blob("test", "b", 1) def tearDown(self): self.blob.close() self.cx.close() def test_blob_length(self): - self.assertEqual(len(self.blob), 100) + self.assertEqual(len(self.blob), 50) def test_blob_seek_and_tell(self): self.blob.seek(10) @@ -1017,7 +1016,7 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 20) self.blob.seek(-10, os.SEEK_END) - self.assertEqual(self.blob.tell(), 90) + self.assertEqual(self.blob.tell(), 40) def test_blob_seek_error(self): dataset = ( @@ -1033,65 +1032,71 @@ def test_blob_seek_error(self): def test_blob_read(self): buf = self.blob.read() - self.assertEqual(buf, self.blob_data) - self.assertEqual(len(buf), len(self.blob_data)) + self.assertEqual(buf, self.data) + self.assertEqual(len(buf), len(self.data)) def test_blob_read_too_much(self): - buf = self.blob.read(len(self.blob_data) * 2) - self.assertEqual(buf, self.blob_data) - self.assertEqual(len(buf), len(self.blob_data)) + buf = self.blob.read(len(self.data) * 2) + self.assertEqual(buf, self.data) + self.assertEqual(len(buf), len(self.data)) def test_blob_read_advance_offset(self): self.blob.read(10) self.assertEqual(self.blob.tell(), 10) def test_blob_read_start_at_offset(self): + new_data = b"b" * 50 self.blob.seek(10) - self.blob.write(self.second_data[:10]) + self.blob.write(new_data[:10]) self.blob.seek(10) - self.assertEqual(self.blob.read(10), self.second_data[:10]) + self.assertEqual(self.blob.read(10), new_data[:10]) def test_blob_write(self): - self.blob.write(self.second_data) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], self.second_data) + new_data = b"new data".ljust(50) + self.blob.write(new_data) + row = self.cx.execute("select b from test").fetchone() + self.assertEqual(row[0], new_data) def test_blob_write_at_offset(self): - self.blob.seek(50) - self.blob.write(self.second_data[:50]) - self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], - self.blob_data[:50] + self.second_data[:50]) + new_data = b"c" * 50 + self.blob.seek(25) + self.blob.write(new_data[:25]) + row = self.cx.execute("select b from test").fetchone() + self.assertEqual(row[0], self.data[:25] + new_data[:25]) def test_blob_write_advance_offset(self): - self.blob.write(self.second_data[:50]) - self.assertEqual(self.blob.tell(), 50) + new_data = b"d" * 50 + self.blob.write(new_data[:25]) + self.assertEqual(self.blob.tell(), 25) def test_blob_write_more_then_blob_size(self): - with self.assertRaises(ValueError): + msg = "data longer than blob length" + with self.assertRaisesRegex(ValueError, msg): self.blob.write(b"a" * 1000) def test_blob_read_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") + self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.read() def test_blob_write_after_row_change(self): - self.cx.execute("UPDATE test SET blob_col='aaaa' where rowid=1") + self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") def test_blob_write_when_readonly(self): read_only_blob = \ - self.cx.open_blob("test", "blob_col", 1, readonly=True) - with self.assertRaises(sqlite.OperationalError): + self.cx.open_blob("test", "b", 1, readonly=True) + with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): read_only_blob.write(b"aaa") read_only_blob.close() def test_blob_open_error(self): dataset = ( - (("test", "blob_col", 1), {"name": "notexisting"}), - (("notexisting", "blob_col", 1), {}), + (("test", "b", 1), {"name": "notexisting"}), + (("notexisting", "b", 1), {}), (("test", "notexisting", 1), {}), - (("test", "blob_col", 2), {}), + (("test", "b", 2), {}), ) regex = "no such" for args, kwds in dataset: @@ -1100,10 +1105,11 @@ def test_blob_open_error(self): self.cx.open_blob(*args, **kwds) def test_blob_get_item(self): - self.assertEqual(self.blob[5], b"a") - - def test_blob_get_item_negative_index(self): - self.assertEqual(self.blob[-5], b"a") + self.assertEqual(self.blob[5], b"b") + self.assertEqual(self.blob[6], b"l") + self.assertEqual(self.blob[7], b"o") + self.assertEqual(self.blob[8], b"b") + self.assertEqual(self.blob[-1], b"!") def test_blob_get_item_error(self): dataset = ( @@ -1118,10 +1124,10 @@ def test_blob_get_item_error(self): self.blob[idx] def test_blob_get_slice(self): - self.assertEqual(self.blob[5:10], b"aaaaa") + self.assertEqual(self.blob[5:14], b"blob data") def test_blob_get_slice_negative_index(self): - self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) + self.assertEqual(self.blob[5:-5], self.data[5:-5]) def test_blob_get_slice_invalid_index(self): with self.assertRaisesRegex(TypeError, "indices must be integers"): @@ -1133,8 +1139,8 @@ def test_blob_get_slice_with_skip(self): def test_blob_set_item(self): self.blob[0] = b"b" - actual = self.cx.execute("select blob_col from test").fetchone()[0] - expected = b"b" + self.blob_data[1:] + actual = self.cx.execute("select b from test").fetchone()[0] + expected = b"b" + self.data[1:] self.assertEqual(actual, expected) def test_blob_set_item(self): @@ -1153,17 +1159,18 @@ def test_blob_set_item_error(self): def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" - actual = self.cx.execute("select blob_col from test").fetchone()[0] - expected = b"bbbbb" + self.blob_data[5:] + actual = self.cx.execute("select b from test").fetchone()[0] + expected = b"bbbbb" + self.data[5:] self.assertEqual(actual, expected) def test_blob_set_empty_slice(self): self.blob[0:0] = b"" + self.assertEqual(self.blob[:], self.data) def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"bbbbb" - actual = self.cx.execute("select blob_col from test").fetchone()[0] - expected = b"bababababa" + self.blob_data[10:] + actual = self.cx.execute("select b from test").fetchone()[0] + expected = b"bhbsbbbob " + self.data[10:] self.assertEqual(actual, expected) def test_blob_get_empty_slice(self): @@ -1192,56 +1199,57 @@ def test_blob_sequence_not_supported(self): self.assertRaises(TypeError, op) def test_blob_context_manager(self): - data = b"a" * 100 - with self.cx.open_blob("test", "blob_col", 1) as blob: + data = b"a" * 50 + with self.cx.open_blob("test", "b", 1) as blob: blob.write(data) - actual = self.cx.execute("select blob_col from test").fetchone()[0] + actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, data) def test_blob_closed(self): - cx = sqlite.connect(":memory:") - cx.execute("create table test(b blob)") - cx.execute("insert into test values (zeroblob(100))") - blob = cx.open_blob("test", "b", 1) - blob.close() - - def assign(): blob[0] = b"" - ops = [ - lambda: blob.read(), - lambda: blob.write(b""), - lambda: blob.seek(0), - lambda: blob.tell(), - lambda: blob.__enter__(), - lambda: blob.__exit__(None, None, None), - lambda: len(blob), - lambda: blob[0], - lambda: blob[0:1], - assign, - ] - msg = "Cannot operate on a closed blob" - for op in ops: - with self.subTest(op=op): - with self.assertRaisesRegex(sqlite.ProgrammingError, msg): - op() + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test values (zeroblob(100))") + blob = cx.open_blob("test", "b", 1) + blob.close() + + def assign(): blob[0] = b"" + ops = [ + lambda: blob.read(), + lambda: blob.write(b""), + lambda: blob.seek(0), + lambda: blob.tell(), + lambda: blob.__enter__(), + lambda: blob.__exit__(None, None, None), + lambda: len(blob), + lambda: blob[0], + lambda: blob[0:1], + assign, + ] + msg = "Cannot operate on a closed blob" + for op in ops: + with self.subTest(op=op): + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + op() def test_blob_close_bad_connection(self): - cx = sqlite.connect(":memory:") - cx.execute("create table test(b blob)") - cx.execute("insert into test values(zeroblob(1))") - blob = cx.open_blob("test", "b", 1) - cx.close() - self.assertRaisesRegex(sqlite.ProgrammingError, - "Cannot operate on a closed database", - blob.close) + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test values(zeroblob(1))") + blob = cx.open_blob("test", "b", 1) + cx.close() + self.assertRaisesRegex(sqlite.ProgrammingError, + "Cannot operate on a closed database", + blob.close) def test_closed_blob_read(self): - con = sqlite.connect(":memory:") - con.execute("create table test(blob_col blob)") - con.execute("insert into test(blob_col) values (zeroblob(100))") - blob = con.open_blob("test", "blob_col", 1) - con.close() - with self.assertRaises(sqlite.ProgrammingError): - blob.read() + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test(b) values (zeroblob(100))") + blob = cx.open_blob("test", "b", 1) + cx.close() + self.assertRaisesRegex(sqlite.ProgrammingError, + "Cannot operate on a closed database", + blob.read) class ThreadTests(unittest.TestCase): From 2e63b3ef0002caf40c43996e69dea7b7a83b387a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 13:49:52 +0100 Subject: [PATCH 65/77] Simplify example --- Doc/includes/sqlite3/blob_with.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 2e48d568830820..67eef36cb4f6e2 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -2,7 +2,7 @@ con = sqlite3.connect(":memory:") -con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") with con.open_blob("test", "blob_col", 1) as blob: From be2774789d6dfab1a18b0e1642fdef2df6cf6edf Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:27:22 +0100 Subject: [PATCH 66/77] Update docs - sync with docstrings - simplify some sections - clarify some parameters --- Doc/library/sqlite3.rst | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e60ffa08dafe34..32ec855cdcf791 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -382,9 +382,9 @@ Connection Objects .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") - On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large + On success, a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, column *column*, table *table* in database - *name* will be returned. When *readonly* is :const:`True` the BLOB is + *name* will be returned. When *readonly* is :const:`True` the blob is opened without write permissions. .. note:: @@ -1029,11 +1029,9 @@ Blob Objects .. class:: Blob - A :class:`Blob` instance can read and write the data in the :abbr:`BLOB + A :class:`Blob` instance can read and write the data in a :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` class implements the file and - mapping protocols. For example, you can read data from the :class:`Blob` by - doing ``obj.read(5)`` or by doing ``obj[:5]``. Call ``len(obj)`` to get size - of the BLOB. + mapping protocols. .. method:: Blob.close() @@ -1041,15 +1039,15 @@ Blob Objects The BLOB will be unusable from this point forward. An :class:`~sqlite3.Error` (or subclass) exception will be raised if any - operation is attempted with the BLOB. + further operation is attempted with the BLOB. .. method:: Blob.__len__() - Return the BLOB size. + Return the BLOB size as length in bytes. - .. method:: Blob.read([size]) + .. method:: Blob.read(length=-1, /) - Read *size* bytes of data from the BLOB at the current offset position. + Read *length* bytes of data from the BLOB at the current offset position. If the end of the BLOB is reached we will return the data up to end of file. When *size* is not specified or is negative, :meth:`~Blob.read` will read till the end of the BLOB. @@ -1064,13 +1062,13 @@ Blob Objects Return the current access position of the BLOB. - .. method:: Blob.seek(offset, /, origin=sqlite3.BLOB_SEEK_START) + .. method:: Blob.seek(offset, origin=sqlite3.BLOB_SEEK_START, /) - Set the current access position of the BLOB. The *origin* argument is - optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB - positioning); other values are :data:`os.SEEK_CUR` or 1 (seek relative to - the current position) and :data:`os.SEEK_END` or 2 (seek relative to the - BLOB’s end). + Set the current access position of the BLOB to *offset*. The *origin* + argument defaults to :data:`os.SEEK_SET` (absolute BLOB positioning). + Other values for *origin* are :data:`os.SEEK_CUR` (seek relative to the + current position) and :data:`os.SEEK_END` (seek relative to the BLOB’s + end). :class:`Blob` example: From 5add365e2026aa79240180a911e75ac7a2b0194c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:37:26 +0100 Subject: [PATCH 67/77] Simplify tests --- Lib/test/test_sqlite3/test_dbapi.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 055bc8d4c13d18..824557b6d96efc 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1085,11 +1085,10 @@ def test_blob_write_after_row_change(self): self.blob.write(b"aaa") def test_blob_write_when_readonly(self): - read_only_blob = \ - self.cx.open_blob("test", "b", 1, readonly=True) + ro_blob = self.cx.open_blob("test", "b", 1, readonly=True) with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): - read_only_blob.write(b"aaa") - read_only_blob.close() + ro_blob.write(b"aaa") + ro_blob.close() def test_blob_open_error(self): dataset = ( @@ -1134,13 +1133,12 @@ def test_blob_get_slice_invalid_index(self): self.blob[5:b"a"] def test_blob_get_slice_with_skip(self): - self.blob.write(b"abcdefghij") - self.assertEqual(self.blob[0:10:2], b"acegi") + self.assertEqual(self.blob[0:10:2], b"ti lb") def test_blob_set_item(self): self.blob[0] = b"b" - actual = self.cx.execute("select b from test").fetchone()[0] expected = b"b" + self.data[1:] + actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, expected) def test_blob_set_item(self): @@ -1159,8 +1157,8 @@ def test_blob_set_item_error(self): def test_blob_set_slice(self): self.blob[0:5] = b"bbbbb" - actual = self.cx.execute("select b from test").fetchone()[0] expected = b"bbbbb" + self.data[5:] + actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, expected) def test_blob_set_empty_slice(self): From bacf08726483b7fcddb0729f9b51a1041d393d90 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:41:26 +0100 Subject: [PATCH 68/77] Safe close --- Modules/_sqlite/blob.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 713aaec93f4b59..e2d2a4c9e3ab1c 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -16,10 +16,12 @@ static void close_blob(pysqlite_Blob *self) { if (self->blob) { + sqlite3_blob *blob = self->blob; + self->blob = NULL; + Py_BEGIN_ALLOW_THREADS - sqlite3_blob_close(self->blob); + sqlite3_blob_close(blob); Py_END_ALLOW_THREADS - self->blob = NULL; } } From 5ff202d3189c87ac7b8ebcd4010e378caa7afa28 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:50:39 +0100 Subject: [PATCH 69/77] Condense a comment --- Modules/_sqlite/blob.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index e2d2a4c9e3ab1c..0a070388e753dd 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,7 +11,6 @@ class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ - static void close_blob(pysqlite_Blob *self) { @@ -56,11 +55,7 @@ blob_dealloc(pysqlite_Blob *self) Py_DECREF(tp); } -/* - * Checks if a blob object is usable (i. e. not closed). - * - * 0 => error; 1 => ok - */ +// Return 1 if the blob object is usable, 0 if not. static int check_blob(pysqlite_Blob *self) { From 1fa5901dbeddf003a07fdd2a069742d260d58605 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Jan 2022 14:56:10 +0100 Subject: [PATCH 70/77] Make sure we catch int overflow for all types of writes --- Modules/_sqlite/blob.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 0a070388e753dd..e93e606e962290 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -187,8 +187,12 @@ blob_read_impl(pysqlite_Blob *self, int length) static int inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { - int rc; + if (len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); + return -1; + } + int rc; Py_BEGIN_ALLOW_THREADS rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS @@ -218,11 +222,6 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - if (data->len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); - return NULL; - } - int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; if (data->len > remaining_len) { PyErr_SetString(PyExc_ValueError, "data longer than blob length"); From 7aec288c269d70af04ea28fdda305e6f4cfb7cf9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 18 Jan 2022 23:22:49 +0100 Subject: [PATCH 71/77] Fix typo in test name --- Lib/test/test_sqlite3/test_dbapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 824557b6d96efc..40fd7147407a8a 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1069,7 +1069,7 @@ def test_blob_write_advance_offset(self): self.blob.write(new_data[:25]) self.assertEqual(self.blob.tell(), 25) - def test_blob_write_more_then_blob_size(self): + def test_blob_write_more_than_blob_size(self): msg = "data longer than blob length" with self.assertRaisesRegex(ValueError, msg): self.blob.write(b"a" * 1000) From aaaf7abca33d916be6b8ebe8e171a9b96bf047a2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 18 Jan 2022 23:31:21 +0100 Subject: [PATCH 72/77] Improve test --- Lib/test/test_sqlite3/test_dbapi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 40fd7147407a8a..30c5ee44776c4d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1041,8 +1041,10 @@ def test_blob_read_too_much(self): self.assertEqual(len(buf), len(self.data)) def test_blob_read_advance_offset(self): - self.blob.read(10) - self.assertEqual(self.blob.tell(), 10) + n = 10 + buf = self.blob.read(n) + self.assertEqual(buf, self.data[:n]) + self.assertEqual(self.blob.tell(), n) def test_blob_read_start_at_offset(self): new_data = b"b" * 50 From f9e65c011d252b5f360ecad1fdb256a517654b94 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 00:30:56 +0100 Subject: [PATCH 73/77] Improve tests --- Lib/test/test_sqlite3/test_dbapi.py | 49 ++++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 30c5ee44776c4d..9217d278c2c276 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -20,6 +20,7 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. +import _testcapi import contextlib import os import sqlite3 as sqlite @@ -1019,16 +1020,26 @@ def test_blob_seek_and_tell(self): self.assertEqual(self.blob.tell(), 40) def test_blob_seek_error(self): + msg_oor = "offset out of blob range" + msg_orig = "'origin' should be 0, 1, or 2" + msg_of = "seek offset result in overflow" + dataset = ( - (ValueError, lambda: self.blob.seek(1000)), - (ValueError, lambda: self.blob.seek(-10)), - (ValueError, lambda: self.blob.seek(10, -1)), - (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_CUR)), - (OverflowError, lambda: self.blob.seek(2**65, os.SEEK_END)), + (ValueError, msg_oor, lambda: self.blob.seek(1000)), + (ValueError, msg_oor, lambda: self.blob.seek(-10)), + (ValueError, msg_orig, lambda: self.blob.seek(10, -1)), + (ValueError, msg_orig, lambda: self.blob.seek(10, 3)), ) - for exc, fn in dataset: - with self.subTest(exc=exc, fn=fn): - self.assertRaises(exc, fn) + for exc, msg, fn in dataset: + with self.subTest(exc=exc, msg=msg, fn=fn): + self.assertRaisesRegex(exc, msg, fn) + + n = len(self.data) // 2 + self.blob.seek(n, os.SEEK_SET) + with self.assertRaisesRegex(OverflowError, msg_of): + self.blob.seek(_testcapi.INT_MAX, os.SEEK_CUR) + with self.assertRaisesRegex(OverflowError, msg_of): + self.blob.seek( _testcapi.INT_MAX, os.SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1053,6 +1064,11 @@ def test_blob_read_start_at_offset(self): self.blob.seek(10) self.assertEqual(self.blob.read(10), new_data[:10]) + def test_blob_read_after_row_change(self): + self.cx.execute("update test set b='aaaa' where rowid=1") + with self.assertRaises(sqlite.OperationalError): + self.blob.read() + def test_blob_write(self): new_data = b"new data".ljust(50) self.blob.write(new_data) @@ -1071,22 +1087,16 @@ def test_blob_write_advance_offset(self): self.blob.write(new_data[:25]) self.assertEqual(self.blob.tell(), 25) - def test_blob_write_more_than_blob_size(self): - msg = "data longer than blob length" - with self.assertRaisesRegex(ValueError, msg): + def test_blob_write_error_length(self): + with self.assertRaisesRegex(ValueError, "data longer than blob"): self.blob.write(b"a" * 1000) - def test_blob_read_after_row_change(self): - self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") - with self.assertRaises(sqlite.OperationalError): - self.blob.read() - - def test_blob_write_after_row_change(self): - self.cx.execute("UPDATE test SET b='aaaa' where rowid=1") + def test_blob_write_error_row_changed(self): + self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") - def test_blob_write_when_readonly(self): + def test_blob_write_error_readonly(self): ro_blob = self.cx.open_blob("test", "b", 1, readonly=True) with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): ro_blob.write(b"aaa") @@ -1117,6 +1127,7 @@ def test_blob_get_item_error(self): (b"", TypeError, "Blob indices must be integers"), (105, IndexError, "Blob index out of range"), (-105, IndexError, "Blob index out of range"), + (_testcapi.ULONG_MAX, IndexError, "cannot fit 'int'"), (len(self.blob), IndexError, "Blob index out of range"), ) for idx, exc, regex in dataset: From 6a5c86432aa74c460bbc146d34bbb51e0523ba37 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 00:31:13 +0100 Subject: [PATCH 74/77] Always check length in inner write --- Modules/_sqlite/blob.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index e93e606e962290..34f13e1ad77d43 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -187,8 +187,9 @@ blob_read_impl(pysqlite_Blob *self, int length) static int inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) { - if (len > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "data longer than INT_MAX bytes"); + int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; + if (len > remaining_len) { + PyErr_SetString(PyExc_ValueError, "data longer than blob length"); return -1; } @@ -222,12 +223,6 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) return NULL; } - int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; - if (data->len > remaining_len) { - PyErr_SetString(PyExc_ValueError, "data longer than blob length"); - return NULL; - } - int rc = inner_write(self, data->buf, data->len, self->offset); if (rc < 0) { return NULL; From ea1045ca7ec82ae8fa7784a806ae0416a557d75a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 00:42:03 +0100 Subject: [PATCH 75/77] Fix missing array sentinel --- Modules/_sqlite/blob.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 34f13e1ad77d43..5fa4cd3f98b63d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -532,6 +532,7 @@ static PyMethodDef blob_methods[] = { static struct PyMemberDef blob_members[] = { {"__weaklistoffset__", T_PYSSIZET, offsetof(pysqlite_Blob, in_weakreflist), READONLY}, + {NULL}, }; static PyType_Slot blob_slots[] = { From 2f848d936e6bde7bfe8e795bd18e022b605c9700 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 01:10:56 +0100 Subject: [PATCH 76/77] Match apsw's API open_blob => blobopen --- Doc/includes/sqlite3/blob.py | 2 +- Doc/includes/sqlite3/blob_with.py | 2 +- Doc/library/sqlite3.rst | 2 +- Doc/whatsnew/3.11.rst | 2 +- Lib/test/test_sqlite3/test_dbapi.py | 16 +++++------ .../2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 2 +- Modules/_sqlite/clinic/connection.c.h | 27 +++++++++---------- Modules/_sqlite/connection.c | 11 ++++---- 8 files changed, 31 insertions(+), 33 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index f928349fd0e981..61994fb82dd72a 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -4,7 +4,7 @@ con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -blob = con.open_blob("test", "blob_col", 1) +blob = con.blobopen("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 67eef36cb4f6e2..d489bd632d867d 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -5,7 +5,7 @@ con.execute("create table test(blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") -with con.open_blob("test", "blob_col", 1) as blob: +with con.blobopen("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index a332d4841b4726..8c860e15a618b4 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -394,7 +394,7 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, /, *, readonly=False, name="main") + .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") On success, a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, column *column*, table *table* in database diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index c193d4542c7996..01f03bfbdf4011 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -301,7 +301,7 @@ sqlite3 Instead we leave it to the SQLite library to handle these cases. (Contributed by Erlend E. Aasland in :issue:`44092`.) -* Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. +* Add :meth:`~sqlite3.Connection.blobopen` to :class:`sqlite3.Connection`. :class:`sqlite3.Blob` allows incremental I/O operations on blobs. (Contributed by Aviv Palivoda and Erlend E. Aasland in :issue:`24905`) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 9217d278c2c276..9bf2e54e6dcb06 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -997,7 +997,7 @@ def setUp(self): self.cx.execute("create table test(b blob)") self.data = b"this blob data string is exactly fifty bytes long!" self.cx.execute("insert into test(b) values (?)", (self.data,)) - self.blob = self.cx.open_blob("test", "b", 1) + self.blob = self.cx.blobopen("test", "b", 1) def tearDown(self): self.blob.close() @@ -1097,7 +1097,7 @@ def test_blob_write_error_row_changed(self): self.blob.write(b"aaa") def test_blob_write_error_readonly(self): - ro_blob = self.cx.open_blob("test", "b", 1, readonly=True) + ro_blob = self.cx.blobopen("test", "b", 1, readonly=True) with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): ro_blob.write(b"aaa") ro_blob.close() @@ -1113,7 +1113,7 @@ def test_blob_open_error(self): for args, kwds in dataset: with self.subTest(args=args, kwds=kwds): with self.assertRaisesRegex(sqlite.OperationalError, regex): - self.cx.open_blob(*args, **kwds) + self.cx.blobopen(*args, **kwds) def test_blob_get_item(self): self.assertEqual(self.blob[5], b"b") @@ -1211,7 +1211,7 @@ def test_blob_sequence_not_supported(self): def test_blob_context_manager(self): data = b"a" * 50 - with self.cx.open_blob("test", "b", 1) as blob: + with self.cx.blobopen("test", "b", 1) as blob: blob.write(data) actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, data) @@ -1220,7 +1220,7 @@ def test_blob_closed(self): with memory_database() as cx: cx.execute("create table test(b blob)") cx.execute("insert into test values (zeroblob(100))") - blob = cx.open_blob("test", "b", 1) + blob = cx.blobopen("test", "b", 1) blob.close() def assign(): blob[0] = b"" @@ -1246,7 +1246,7 @@ def test_blob_close_bad_connection(self): with memory_database() as cx: cx.execute("create table test(b blob)") cx.execute("insert into test values(zeroblob(1))") - blob = cx.open_blob("test", "b", 1) + blob = cx.blobopen("test", "b", 1) cx.close() self.assertRaisesRegex(sqlite.ProgrammingError, "Cannot operate on a closed database", @@ -1256,7 +1256,7 @@ def test_closed_blob_read(self): with memory_database() as cx: cx.execute("create table test(b blob)") cx.execute("insert into test(b) values (zeroblob(100))") - blob = cx.open_blob("test", "b", 1) + blob = cx.blobopen("test", "b", 1) cx.close() self.assertRaisesRegex(sqlite.ProgrammingError, "Cannot operate on a closed database", @@ -1303,7 +1303,7 @@ def test_check_connection_thread(self): lambda: self.con.create_collation("foo", None), lambda: self.con.setlimit(sqlite.SQLITE_LIMIT_LENGTH, -1), lambda: self.con.getlimit(sqlite.SQLITE_LIMIT_LENGTH), - lambda: self.con.open_blob("test", "b", 1), + lambda: self.con.blobopen("test", "b", 1), ] for fn in fns: with self.subTest(fn=fn): diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index c9571f374862f0..0a57f90c12378f 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,3 +1,3 @@ -Add :meth:`~sqlite3.Connection.open_blob` to :class:`sqlite3.Connection`. +Add :meth:`~sqlite3.Connection.blobopen` to :class:`sqlite3.Connection`. :class:`sqlite3.Blob` allows incremental I/O operations on blobs. Patch by Aviv Palivoda and Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index acadbe3b7eae49..c2a45594ca907b 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -145,26 +145,25 @@ pysqlite_connection_cursor(pysqlite_Connection *self, PyObject *const *args, Py_ return return_value; } -PyDoc_STRVAR(pysqlite_connection_open_blob__doc__, -"open_blob($self, table, column, row, /, *, readonly=False, name=\'main\')\n" +PyDoc_STRVAR(blobopen__doc__, +"blobopen($self, table, column, row, /, *, readonly=False, name=\'main\')\n" "--\n" "\n" "Return a blob object. Non-standard."); -#define PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF \ - {"open_blob", (PyCFunction)(void(*)(void))pysqlite_connection_open_blob, METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_open_blob__doc__}, +#define BLOBOPEN_METHODDEF \ + {"blobopen", (PyCFunction)(void(*)(void))blobopen, METH_FASTCALL|METH_KEYWORDS, blobopen__doc__}, static PyObject * -pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *col, - int row, int readonly, const char *name); +blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, + int row, int readonly, const char *name); static PyObject * -pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"", "", "", "readonly", "name", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "open_blob", 0}; + static _PyArg_Parser _parser = {NULL, _keywords, "blobopen", 0}; PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; const char *table; @@ -178,7 +177,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("open_blob", "argument 1", "str", args[0]); + _PyArg_BadArgument("blobopen", "argument 1", "str", args[0]); goto exit; } Py_ssize_t table_length; @@ -191,7 +190,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } if (!PyUnicode_Check(args[1])) { - _PyArg_BadArgument("open_blob", "argument 2", "str", args[1]); + _PyArg_BadArgument("blobopen", "argument 2", "str", args[1]); goto exit; } Py_ssize_t col_length; @@ -220,7 +219,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, } } if (!PyUnicode_Check(args[4])) { - _PyArg_BadArgument("open_blob", "argument 'name'", "str", args[4]); + _PyArg_BadArgument("blobopen", "argument 'name'", "str", args[4]); goto exit; } Py_ssize_t name_length; @@ -233,7 +232,7 @@ pysqlite_connection_open_blob(pysqlite_Connection *self, PyObject *const *args, goto exit; } skip_optional_kwonly: - return_value = pysqlite_connection_open_blob_impl(self, table, col, row, readonly, name); + return_value = blobopen_impl(self, table, col, row, readonly, name); exit: return return_value; @@ -930,4 +929,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg) #ifndef PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #define PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF #endif /* !defined(PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF) */ -/*[clinic end generated code: output=ecc1c000c7217a90 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=959de9d109b85d3f input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3d7b9227ebc256..bf6b5c78d0b1b8 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -438,7 +438,7 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) } /*[clinic input] -_sqlite3.Connection.open_blob as pysqlite_connection_open_blob +_sqlite3.Connection.blobopen as blobopen table: str column as col: str @@ -452,10 +452,9 @@ Return a blob object. Non-standard. [clinic start generated code]*/ static PyObject * -pysqlite_connection_open_blob_impl(pysqlite_Connection *self, - const char *table, const char *col, - int row, int readonly, const char *name) -/*[clinic end generated code: output=5c11d18e7eb629ef input=9192dd28146d4e14]*/ +blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, + int row, int readonly, const char *name) +/*[clinic end generated code: output=0c8e2e58516d0b5c input=1eec15d2b87bf09e]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2041,13 +2040,13 @@ static PyMethodDef connection_methods[] = { PYSQLITE_CONNECTION_INTERRUPT_METHODDEF PYSQLITE_CONNECTION_ITERDUMP_METHODDEF PYSQLITE_CONNECTION_LOAD_EXTENSION_METHODDEF - PYSQLITE_CONNECTION_OPEN_BLOB_METHODDEF PYSQLITE_CONNECTION_ROLLBACK_METHODDEF PYSQLITE_CONNECTION_SET_AUTHORIZER_METHODDEF PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF PYSQLITE_CONNECTION_SET_TRACE_CALLBACK_METHODDEF SETLIMIT_METHODDEF GETLIMIT_METHODDEF + BLOBOPEN_METHODDEF {NULL, NULL} }; From 8d906bca4574d2adb0862b86e9e537d73498188c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 19 Jan 2022 01:33:24 +0100 Subject: [PATCH 77/77] Fix overflow test on win x64 --- Lib/test/test_sqlite3/test_dbapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 9bf2e54e6dcb06..b25e6b125f9095 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1127,7 +1127,7 @@ def test_blob_get_item_error(self): (b"", TypeError, "Blob indices must be integers"), (105, IndexError, "Blob index out of range"), (-105, IndexError, "Blob index out of range"), - (_testcapi.ULONG_MAX, IndexError, "cannot fit 'int'"), + (_testcapi.ULLONG_MAX, IndexError, "cannot fit 'int'"), (len(self.blob), IndexError, "Blob index out of range"), ) for idx, exc, regex in dataset: