From 4fa0e91bd77290614e721718691bb76ac2dc7f0e Mon Sep 17 00:00:00 2001 From: palaviv Date: Fri, 24 Feb 2017 12:23:19 +0200 Subject: [PATCH 01/18] support BLOB incremental I/O in sqlite module --- Doc/includes/sqlite3/blob.py | 13 ++ Doc/includes/sqlite3/blob_with.py | 12 ++ Doc/library/sqlite3.rst | 67 ++++++ Lib/sqlite3/test/dbapi.py | 279 +++++++++++++++++++++++- Modules/_sqlite/blob.c | 342 ++++++++++++++++++++++++++++++ Modules/_sqlite/blob.h | 25 +++ Modules/_sqlite/connection.c | 93 +++++++- Modules/_sqlite/connection.h | 3 +- Modules/_sqlite/module.c | 4 +- setup.py | 3 +- 10 files changed, 830 insertions(+), 11 deletions(-) create mode 100644 Doc/includes/sqlite3/blob.py create mode 100644 Doc/includes/sqlite3/blob_with.py create mode 100644 Modules/_sqlite/blob.c create mode 100644 Modules/_sqlite/blob.h diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py new file mode 100644 index 00000000000000..b8d2c78d1a365d --- /dev/null +++ b/Doc/includes/sqlite3/blob.py @@ -0,0 +1,13 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +blob = con.open_blob("test", "blob_col", 1, 1) +blob.write(b"a" * 5) +blob.write(b"b" * 5) +blob.seek(0) +print(blob.read()) # will print b"aaaaabbbbb" +blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py new file mode 100644 index 00000000000000..624b680591901a --- /dev/null +++ b/Doc/includes/sqlite3/blob_with.py @@ -0,0 +1,12 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +# creating the table +con.execute("create table test(id integer primary key, blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") +# opening blob handle +with con.open_blob("test", "blob_col", 1, 1) as blob: + blob.write(b"a" * 5) + blob.write(b"b" * 5) + blob.seek(0) + print(blob.read()) # will print b"aaaaabbbbb" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ccb82278bdaa13..ceb92b3fe90fab 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,6 +301,16 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. + .. method:: open_blob(table, column, row, readonly=False, dbname="main") + + On success a :class:`Blob` handle to the + :abbr:`BLOB (Binary Large OBject)` located in row *row*, + column *column*, table *table* in database *dbname* will be returned. + When *readonly* is :const:`True` the BLOB is opened with read + permissions. Otherwise the BLOB has read and write permissions. + + .. versionadded:: 3.7 + .. method:: commit() This method commits the current transaction. If you don't call this method, @@ -853,6 +863,63 @@ Exceptions transactions turned off. It is a subclass of :exc:`DatabaseError`. +.. _sqlite3-blob-objects: + +Blob Objects +------------ + +.. versionadded:: 3.7 + +.. class:: Blob + + A :class:`Blob` instance can read and write the data in the + :abbr:`BLOB (Binary Large OBject)`. + + .. method:: Blob.close() + + Close the BLOB now (rather than whenever __del__ is called). + + The BLOB will be unusable from this point forward; an + :class:`~sqlite3.Error` (or subclass) exception will be + raised if any operation is attempted with the BLOB. + + .. method:: Blob.__len__() + + Return the BLOB size. + + .. method:: Blob.read([size]) + + Read *size* bytes of data from the BLOB at the current offset position. + If the end of the BLOB is reached we will return the data up to end of + file. When *size* is not specified or negative we will read up to end + of BLOB. + + .. method:: Blob.write(data) + + Write *data* to the BLOB at the current offset. This function cannot + changed BLOB length. If data write will result in writing to more + then BLOB current size an error will be raised. + + .. method:: Blob.tell() + + Return the current offset of the BLOB. + + .. method:: Blob.seek(offset, [whence]) + + Set the BLOB offset. The *whence* argument is optional and defaults to + :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values + are :data:`os.SEEK_CUR` or 1 (seek relative to the current position) and + :data:`os.SEEK_END` or 2 (seek relative to the BLOB’s end). + + :class:`Blob` example: + + .. literalinclude:: ../includes/sqlite3/blob.py + + A :class:`Blob` can also be used with :term:`context manager`: + + .. literalinclude:: ../includes/sqlite3/blob_with.py + + .. _sqlite3-types: SQLite and Python types diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index be11337154bdd2..85db83ee51f246 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -511,6 +511,167 @@ def CheckLastRowIDInsertOR(self): self.assertEqual(results, expected) +class BlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.blob_data = b"a" * 100 + self.cx.execute("insert into test(blob_col) values (?)", (self.blob_data, )) + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.second_data = b"b" * 100 + + def tearDown(self): + self.blob.close() + self.cx.close() + + def CheckLength(self): + self.assertEqual(len(self.blob), 100) + + def CheckTell(self): + self.assertEqual(self.blob.tell(), 0) + + def CheckSeekFromBlobStart(self): + self.blob.seek(10) + self.assertEqual(self.blob.tell(), 10) + self.blob.seek(10, 0) + self.assertEqual(self.blob.tell(), 10) + + def CheckSeekFromCurrentPosition(self): + self.blob.seek(10, 1) + self.blob.seek(10, 1) + self.assertEqual(self.blob.tell(), 20) + + def CheckSeekFromBlobEnd(self): + self.blob.seek(-10, 2) + self.assertEqual(self.blob.tell(), 90) + + def CheckBlobSeekOverBlobSize(self): + try: + self.blob.seek(1000) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobSeekUnderBlobSize(self): + try: + self.blob.seek(-10) + self.fail("should have raised a ValueError") + except ValueError: + pass + except Exception: + self.fail("should have raised a ValueError") + + def CheckBlobRead(self): + self.assertEqual(self.blob.read(), self.blob_data) + + def CheckBlobReadSize(self): + self.assertEqual(len(self.blob.read(10)), 10) + + def CheckBlobReadAdvanceOffset(self): + self.blob.read(10) + self.assertEqual(self.blob.tell(), 10) + + def CheckBlobReadStartAtOffset(self): + self.blob.seek(10) + self.blob.write(self.second_data[:10]) + self.blob.seek(10) + self.assertEqual(self.blob.read(10), self.second_data[:10]) + + def CheckBlobWrite(self): + self.blob.write(self.second_data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], self.second_data) + + def CheckBlobWriteAtOffset(self): + self.blob.seek(50) + self.blob.write(self.second_data[:50]) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], + self.blob_data[:50] + self.second_data[:50]) + + def CheckBlobWriteAdvanceOffset(self): + self.blob.write(self.second_data[:50]) + self.assertEqual(self.blob.tell(), 50) + + def CheckBlobWriteMoreThenBlobSize(self): + try: + self.blob.write(b"a" * 1000) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobReadAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.read() + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteAfterRowChange(self): + self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") + try: + self.blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobWriteWhenReadOnly(self): + read_only_blob = \ + self.cx.open_blob("test", "blob_col", 1, readonly=True) + try: + read_only_blob.write(b"aaa") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + read_only_blob.close() + + def CheckBlobOpenWithBadDb(self): + try: + self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadTable(self): + try: + self.cx.open_blob("notexisintg", "blob_col", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadColumn(self): + try: + self.cx.open_blob("test", "notexisting", 1) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + def CheckBlobOpenWithBadRow(self): + try: + self.cx.open_blob("test", "blob_col", 2) + self.fail("should have raised a sqlite.OperationalError") + except sqlite.OperationalError: + pass + except Exception: + self.fail("should have raised a sqlite.OperationalError") + + +@unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -768,6 +929,20 @@ def CheckClosedCurExecute(self): with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4") + def CheckClosedBlobRead(self): + con = sqlite.connect(":memory:") + con.execute("create table test(id integer primary key, blob_col blob)") + con.execute("insert into test(blob_col) values (zeroblob(100))") + blob = con.open_blob("test", "blob_col", 1) + con.close() + try: + blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except: + self.fail("Should have raised a ProgrammingError") + def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") con.close() @@ -921,6 +1096,99 @@ def CheckOnConflictReplace(self): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) + +class ClosedBlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckClosedRead(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.read() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedWrite(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.write(b"aaaaaaaaa") + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedSeek(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.seek(10) + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedTell(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.tell() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def CheckClosedClose(self): + self.blob = self.cx.open_blob("test", "blob_col", 1) + self.blob.close() + try: + self.blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + +class BlobContextManagerTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(id integer primary key, blob_col blob)") + self.cx.execute("insert into test(blob_col) values (zeroblob(100))") + + def tearDown(self): + self.cx.close() + + def CheckContextExecute(self): + data = b"a" * 100 + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.write(data) + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], data) + + def CheckContextCloseBlob(self): + with self.cx.open_blob("test", "blob_col", 1) as blob: + blob.seek(10) + try: + blob.close() + self.fail("Should have raised a ProgrammingError") + except sqlite.ProgrammingError: + pass + except Exception: + self.fail("Should have raised a ProgrammingError") + + def suite(): module_suite = unittest.makeSuite(ModuleTests, "Check") connection_suite = unittest.makeSuite(ConnectionTests, "Check") @@ -931,11 +1199,12 @@ def suite(): closed_con_suite = unittest.makeSuite(ClosedConTests, "Check") closed_cur_suite = unittest.makeSuite(ClosedCurTests, "Check") on_conflict_suite = unittest.makeSuite(SqliteOnConflictTests, "Check") - return unittest.TestSuite(( - module_suite, connection_suite, cursor_suite, thread_suite, - constructor_suite, ext_suite, closed_con_suite, closed_cur_suite, - on_conflict_suite, - )) + blob_suite = unittest.makeSuite(BlobTests, "Check") + closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") + blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") + return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite)) def test(): runner = unittest.TextTestRunner() diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c new file mode 100644 index 00000000000000..11da2b0ad5d691 --- /dev/null +++ b/Modules/_sqlite/blob.c @@ -0,0 +1,342 @@ +#include "blob.h" +#include "util.h" + + +int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, + sqlite3_blob *blob) +{ + Py_INCREF(connection); + self->connection = connection; + self->offset = 0; + self->blob = blob; + self->in_weakreflist = NULL; + + if (!pysqlite_check_thread(self->connection)) { + return -1; + } + return 0; +} + +static void remove_blob_from_connection_blob_list(pysqlite_Blob *self) +{ + Py_ssize_t i; + PyObject *item; + + for (i = 0; i < PyList_GET_SIZE(self->connection->blobs); i++) { + item = PyList_GET_ITEM(self->connection->blobs, i); + if (PyWeakref_GetObject(item) == (PyObject *)self) { + PyList_SetSlice(self->connection->blobs, i, i+1, NULL); + break; + } + } +} + +static void _close_blob_inner(pysqlite_Blob* self) +{ + sqlite3_blob *blob; + + /* close the blob */ + blob = self->blob; + self->blob = NULL; + if (blob) { + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + } + + /* remove from connection weaklist */ + remove_blob_from_connection_blob_list(self); + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } +} + +static void pysqlite_blob_dealloc(pysqlite_Blob* self) +{ + _close_blob_inner(self); + Py_XDECREF(self->connection); + Py_TYPE(self)->tp_free((PyObject*)self); +} + + +/* + * Checks if a blob object is usable (i. e. not closed). + * + * 0 => error; 1 => ok + */ +int pysqlite_check_blob(pysqlite_Blob *blob) +{ + + if (!blob->blob) { + PyErr_SetString(pysqlite_ProgrammingError, + "Cannot operate on a closed blob."); + return 0; + } else if (!pysqlite_check_connection(blob->connection) || + !pysqlite_check_thread(blob->connection)) { + return 0; + } else { + return 1; + } +} + + +PyObject* pysqlite_blob_close(pysqlite_Blob *self) +{ + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + _close_blob_inner(self); + Py_RETURN_NONE; +}; + + +static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) +{ + int blob_length; + if (!pysqlite_check_blob(self)) { + return -1; + } + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + return blob_length; +}; + + +PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) +{ + int read_length = -1; + int blob_length = 0; + PyObject *buffer; + char *raw_buffer; + int rc; + + if (!PyArg_ParseTuple(args, "|i", &read_length)) { + return NULL; + } + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + + /* TODO: make this multithreaded and safe! */ + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (read_length < 0) { + /* same as file read. */ + read_length = blob_length; + } + + /* making sure we don't read more then blob size */ + if (read_length > blob_length - self->offset) { + read_length = blob_length - self->offset; + } + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + + /* update offset. */ + self->offset += read_length; + + return buffer; +}; + + +PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) +{ + Py_buffer data_buffer; + int rc; + + if (PyObject_GetBuffer(data, &data_buffer, PyBUF_SIMPLE) < 0) { + return NULL; + } + + if (data_buffer.len > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "data longer than INT_MAX bytes"); + PyBuffer_Release(&data_buffer); + return NULL; + } + + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } + + /* TODO: throw better error on data bigger then blob. */ + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buffer.buf, + data_buffer.len, self->offset); + Py_END_ALLOW_THREADS + if (rc != SQLITE_OK) { + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyBuffer_Release(&data_buffer); + return NULL; + } + + self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); + Py_RETURN_NONE; +} + + +PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) +{ + int blob_length, offset, from_what = 0; + + if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { + return NULL; + } + + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + switch (from_what) { + case 0: // relative to blob begin + break; + case 1: // relative to current position + if (offset > INT_MAX - self->offset) { + goto overflow; + } + offset = self->offset + offset; + break; + case 2: // relative to blob end + if (offset > INT_MAX - blob_length) { + goto overflow; + } + offset = blob_length + offset; + break; + default: + return PyErr_Format(PyExc_ValueError, + "from_what should be 0, 1 or 2"); + } + + if (offset < 0 || offset > blob_length) { + return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + } + + self->offset = offset; + Py_RETURN_NONE; + +overflow: + return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); +} + + +PyObject* pysqlite_blob_tell(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + return PyLong_FromLong(self->offset); +} + + +PyObject* pysqlite_blob_enter(pysqlite_Blob *self) +{ + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_INCREF(self); + return (PyObject *)self; +} + + +PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) +{ + PyObject *res; + if (!pysqlite_check_blob(self)) { + return NULL; + } + + res = pysqlite_blob_close(self); + if (!res) { + return NULL; + } + Py_XDECREF(res); + + Py_RETURN_FALSE; +} + + +static PyMethodDef blob_methods[] = { + {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, + PyDoc_STR("read data from blob")}, + {"write", (PyCFunction)pysqlite_blob_write, METH_O, + PyDoc_STR("write data to blob")}, + {"close", (PyCFunction)pysqlite_blob_close, METH_NOARGS, + PyDoc_STR("close blob")}, + {"seek", (PyCFunction)pysqlite_blob_seek, METH_VARARGS, + PyDoc_STR("change blob current offset")}, + {"tell", (PyCFunction)pysqlite_blob_tell, METH_NOARGS, + PyDoc_STR("return blob current offset")}, + {"__enter__", (PyCFunction)pysqlite_blob_enter, METH_NOARGS, + PyDoc_STR("blob context manager enter")}, + {"__exit__", (PyCFunction)pysqlite_blob_exit, METH_VARARGS, + PyDoc_STR("blob context manager exit")}, + {NULL, NULL} +}; + +static PySequenceMethods blob_sequence_methods = { + (lenfunc)pysqlite_blob_length, /* sq_length */ +}; + + +PyTypeObject pysqlite_BlobType = { + PyVarObject_HEAD_INIT(NULL, 0) + MODULE_NAME ".Blob", + .tp_basicsize = sizeof(pysqlite_Blob), + .tp_dealloc = (destructor)pysqlite_blob_dealloc, + .tp_as_sequence = &blob_sequence_methods, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), + .tp_methods = blob_methods, +}; + +extern int pysqlite_blob_setup_types(void) +{ + pysqlite_BlobType.tp_new = PyType_GenericNew; + return PyType_Ready(&pysqlite_BlobType); +} diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h new file mode 100644 index 00000000000000..204918fd4f01c6 --- /dev/null +++ b/Modules/_sqlite/blob.h @@ -0,0 +1,25 @@ +#ifndef PYSQLITE_BLOB_H +#define PYSQLITE_BLOB_H +#include "Python.h" +#include "sqlite3.h" +#include "connection.h" + +typedef struct +{ + PyObject_HEAD + pysqlite_Connection* connection; + sqlite3_blob *blob; + int offset; + + PyObject* in_weakreflist; /* List of weak references */ +} pysqlite_Blob; + +extern PyTypeObject pysqlite_BlobType; + +int pysqlite_blob_init(pysqlite_Blob* self, pysqlite_Connection* connection, + sqlite3_blob *blob); +PyObject* pysqlite_blob_close(pysqlite_Blob *self); + +int pysqlite_blob_setup_types(void); + +#endif diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 958be7d869794a..b99fcdb24fc25f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -27,6 +27,7 @@ #include "connection.h" #include "statement.h" #include "cursor.h" +#include "blob.h" #include "prepare_protocol.h" #include "util.h" @@ -105,7 +106,8 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject Py_CLEAR(self->statement_cache); Py_CLEAR(self->statements); Py_CLEAR(self->cursors); - + Py_CLEAR(self->blobs); + Py_INCREF(Py_None); Py_XSETREF(self->row_factory, Py_None); @@ -159,10 +161,11 @@ int pysqlite_connection_init(pysqlite_Connection* self, PyObject* args, PyObject self->created_statements = 0; self->created_cursors = 0; - /* Create lists of weak references to statements/cursors */ + /* Create lists of weak references to statements/cursors/blobs */ self->statements = PyList_New(0); self->cursors = PyList_New(0); - if (!self->statements || !self->cursors) { + self->blobs = PyList_New(0); + if (!self->statements || !self->cursors || !self->blobs) { return -1; } @@ -258,6 +261,8 @@ void pysqlite_connection_dealloc(pysqlite_Connection* self) Py_XDECREF(self->collations); Py_XDECREF(self->statements); Py_XDECREF(self->cursors); + Py_XDECREF(self->blobs); + Py_TYPE(self)->tp_free((PyObject*)self); } @@ -327,6 +332,84 @@ PyObject* pysqlite_connection_cursor(pysqlite_Connection* self, PyObject* args, return cursor; } +PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, + PyObject *kwargs) +{ + static char *kwlist[] = {"table", "column", "row", "readonly", + "dbname", NULL, NULL}; + int rc; + const char *dbname = "main", *table, *column; + long long row; + int readonly = 0; + sqlite3_blob *blob; + pysqlite_Blob *pyblob = NULL; + PyObject *weakref; + + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + &table, &column, &row, &readonly, + &dbname)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_open(self->db, dbname, table, column, row, + !readonly, &blob); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + _pysqlite_seterror(self->db, NULL); + return NULL; + } + + pyblob = PyObject_New(pysqlite_Blob, &pysqlite_BlobType); + if (!pyblob) { + goto error; + } + + rc = pysqlite_blob_init(pyblob, self, blob); + if (rc) { + Py_CLEAR(pyblob); + goto error; + } + + // Add our blob to connection blobs list + weakref = PyWeakref_NewRef((PyObject*)pyblob, NULL); + if (!weakref) { + Py_CLEAR(pyblob); + goto error; + } + if (PyList_Append(self->blobs, weakref) != 0) { + Py_CLEAR(weakref); + Py_CLEAR(pyblob); + goto error; + } + Py_DECREF(weakref); + + return (PyObject*)pyblob; + +error: + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + return NULL; +} + +static void pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + int i; + PyObject *weakref; + PyObject *blob; + + for (i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + weakref = PyList_GET_ITEM(self->blobs, i); + blob = PyWeakref_GetObject(weakref); + if (blob != Py_None) { + pysqlite_blob_close((pysqlite_Blob*)blob); + } + } +} + PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) { int rc; @@ -337,6 +420,8 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) pysqlite_do_all_statements(self, ACTION_FINALIZE, 1); + pysqlite_close_all_blobs(self); + if (self->db) { rc = SQLITE3_CLOSE(self->db); @@ -1768,6 +1853,8 @@ static PyGetSetDef connection_getset[] = { static PyMethodDef connection_methods[] = { {"cursor", (PyCFunction)(void(*)(void))pysqlite_connection_cursor, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("Return a cursor for the connection.")}, + {"open_blob", (PyCFunction)pysqlite_connection_blob, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("return a blob object")}, {"close", (PyCFunction)pysqlite_connection_close, METH_NOARGS, PyDoc_STR("Closes the connection.")}, {"commit", (PyCFunction)pysqlite_connection_commit, METH_NOARGS, diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 206085e00a00c7..708a199087c002 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,9 +66,10 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements and cursors used within this connection */ + /* Lists of weak references to statements, blobs and cursors used within this connection */ PyObject* statements; PyObject* cursors; + PyObject* blobs; /* Counters for how many statements/cursors were created in the connection. May be * reset to 0 at certain intervals */ diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 71d951ee887e47..f138e02357753f 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -28,6 +28,7 @@ #include "prepare_protocol.h" #include "microprotocols.h" #include "row.h" +#include "blob.h" #if SQLITE_VERSION_NUMBER >= 3003003 #define HAVE_SHARED_CACHE @@ -368,7 +369,8 @@ PyMODINIT_FUNC PyInit__sqlite3(void) (pysqlite_connection_setup_types() < 0) || (pysqlite_cache_setup_types() < 0) || (pysqlite_statement_setup_types() < 0) || - (pysqlite_prepare_protocol_setup_types() < 0) + (pysqlite_prepare_protocol_setup_types() < 0) || + (pysqlite_blob_setup_types() < 0) ) { Py_XDECREF(module); return NULL; diff --git a/setup.py b/setup.py index 21a5a58981fc15..8c70025489e181 100644 --- a/setup.py +++ b/setup.py @@ -1523,7 +1523,8 @@ def detect_sqlite(self): '_sqlite/prepare_protocol.c', '_sqlite/row.c', '_sqlite/statement.c', - '_sqlite/util.c', ] + '_sqlite/util.c', + '_sqlite/blob.c' ] sqlite_defines = [] if not MS_WINDOWS: From 3fe91083c9b015ec326279a37e8c297e49747d61 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:11:32 +0200 Subject: [PATCH 02/18] Note that blob size cannot be changed using the blob object --- Doc/library/sqlite3.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ceb92b3fe90fab..e464192e136c13 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -309,6 +309,11 @@ Connection Objects When *readonly* is :const:`True` the BLOB is opened with read permissions. Otherwise the BLOB has read and write permissions. + .. note:: + + The BLOB size cannot be changed using the :class:`Blob` class. Use + `zeroblob` to create the blob in the wanted size in advance. + .. versionadded:: 3.7 .. method:: commit() From 865c1c84be443770f489138dd718f9087a080ac4 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:22:03 +0200 Subject: [PATCH 03/18] Use assertRaises in tests --- Lib/sqlite3/test/dbapi.py | 119 ++++++-------------------------------- 1 file changed, 17 insertions(+), 102 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 85db83ee51f246..8b2fc70bce3abf 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -546,22 +546,12 @@ def CheckSeekFromBlobEnd(self): self.assertEqual(self.blob.tell(), 90) def CheckBlobSeekOverBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(1000) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobSeekUnderBlobSize(self): - try: + with self.assertRaises(ValueError): self.blob.seek(-10) - self.fail("should have raised a ValueError") - except ValueError: - pass - except Exception: - self.fail("should have raised a ValueError") def CheckBlobRead(self): self.assertEqual(self.blob.read(), self.blob_data) @@ -594,81 +584,41 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"a" * 1000) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobReadAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.read() - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteAfterRowChange(self): self.cx.execute("UPDATE test SET blob_col='aaaa' where id=1") - try: + with self.assertRaises(sqlite.OperationalError): self.blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobWriteWhenReadOnly(self): read_only_blob = \ self.cx.open_blob("test", "blob_col", 1, readonly=True) - try: + with self.assertRaises(sqlite.OperationalError): read_only_blob.write(b"aaa") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") read_only_blob.close() def CheckBlobOpenWithBadDb(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 1, dbname="notexisintg") - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadTable(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("notexisintg", "blob_col", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadColumn(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "notexisting", 1) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") def CheckBlobOpenWithBadRow(self): - try: + with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) - self.fail("should have raised a sqlite.OperationalError") - except sqlite.OperationalError: - pass - except Exception: - self.fail("should have raised a sqlite.OperationalError") @unittest.skipUnless(threading, 'This test requires threading.') @@ -935,13 +885,8 @@ def CheckClosedBlobRead(self): con.execute("insert into test(blob_col) values (zeroblob(100))") blob = con.open_blob("test", "blob_col", 1) con.close() - try: + with self.assertRaises(sqlite.ProgrammingError): blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except: - self.fail("Should have raised a ProgrammingError") def CheckClosedCreateFunction(self): con = sqlite.connect(":memory:") @@ -1109,57 +1054,32 @@ def tearDown(self): def CheckClosedRead(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.read() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedWrite(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.write(b"aaaaaaaaa") - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedSeek(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.seek(10) - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedTell(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.tell() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def CheckClosedClose(self): self.blob = self.cx.open_blob("test", "blob_col", 1) self.blob.close() - try: + with self.assertRaises(sqlite.ProgrammingError): self.blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") class BlobContextManagerTests(unittest.TestCase): @@ -1180,13 +1100,8 @@ def CheckContextExecute(self): def CheckContextCloseBlob(self): with self.cx.open_blob("test", "blob_col", 1) as blob: blob.seek(10) - try: + with self.assertRaises(sqlite.ProgrammingError): blob.close() - self.fail("Should have raised a ProgrammingError") - except sqlite.ProgrammingError: - pass - except Exception: - self.fail("Should have raised a ProgrammingError") def suite(): From 788fd54d56e52df55fdfff33b481b032beeb449b Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 25 Feb 2017 11:38:43 +0200 Subject: [PATCH 04/18] Fix doc error --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e464192e136c13..6afef818e4ad70 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -312,7 +312,7 @@ Connection Objects .. note:: The BLOB size cannot be changed using the :class:`Blob` class. Use - `zeroblob` to create the blob in the wanted size in advance. + ``zeroblob`` to create the blob in the wanted size in advance. .. versionadded:: 3.7 From a1361e5ed71cbd1ef461a5eb09dd695a9e66619d Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:26:39 +0200 Subject: [PATCH 05/18] blob support sequence protocol --- Lib/sqlite3/test/dbapi.py | 56 ++++++ Modules/_sqlite/blob.c | 396 ++++++++++++++++++++++++++++++++++---- 2 files changed, 416 insertions(+), 36 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 8b2fc70bce3abf..24122a0cf58062 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -620,6 +620,62 @@ def CheckBlobOpenWithBadRow(self): with self.assertRaises(sqlite.OperationalError): self.cx.open_blob("test", "blob_col", 2) + def CheckBlobGetItem(self): + self.assertEqual(self.blob[5], b"a") + + def CheckBlobGetItemIndexOutOfRange(self): + with self.assertRaises(IndexError): + self.blob[105] + with self.assertRaises(IndexError): + self.blob[-105] + + def CheckBlobGetItemNegativeIndex(self): + self.assertEqual(self.blob[-5], b"a") + + def CheckBlobGetItemInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[b"a"] + + def CheckBlobGetSlice(self): + self.assertEqual(self.blob[5:10], b"aaaaa") + + def CheckBlobGetSliceNegativeIndex(self): + self.assertEqual(self.blob[5:-5], self.blob_data[5:-5]) + + def CheckBlobGetSliceInvalidIndex(self): + with self.assertRaises(TypeError): + self.blob[5:b"a"] + + def CheckBlobGetSliceWithSkip(self): + self.blob.write(b"abcdefghij") + self.assertEqual(self.blob[0:10:2], b"acegi") + + def CheckBlobSetItem(self): + self.blob[0] = b"b" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"b" + self.blob_data[1:]) + + def CheckBlobSetSlice(self): + self.blob[0:5] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bbbbb" + self.blob_data[5:]) + + def CheckBlobSetSliceWithSkip(self): + self.blob[0:10:2] = b"bbbbb" + self.assertEqual(self.cx.execute("select blob_col from test").fetchone()[0], b"bababababa" + self.blob_data[10:]) + + def CheckBlobGetEmptySlice(self): + self.assertEqual(self.blob[5:5], b"") + + def CheckBlobSetSliceWrongLength(self): + with self.assertRaises(IndexError): + self.blob[5:10] = b"a" + + def CheckBlobConcatNotSupported(self): + with self.assertRaises(SystemError): + self.blob + self.blob + + def CheckBlobRepeateNotSupported(self): + with self.assertRaises(SystemError): + self.blob * 5 @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 11da2b0ad5d691..333c65425a1f0f 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -105,14 +105,43 @@ static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) return blob_length; }; +static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) +{ + PyObject *buffer; + char *raw_buffer; + int rc; + + buffer = PyBytes_FromStringAndSize(NULL, read_length); + if (!buffer) { + return NULL; + } + raw_buffer = PyBytes_AS_STRING(buffer); + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + Py_DECREF(buffer); + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + return NULL; + } + return buffer; +} + PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; int blob_length = 0; PyObject *buffer; - char *raw_buffer; - int rc; if (!PyArg_ParseTuple(args, "|i", &read_length)) { return NULL; @@ -138,34 +167,36 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) read_length = blob_length - self->offset; } - buffer = PyBytes_FromStringAndSize(NULL, read_length); - if (!buffer) { - return NULL; + buffer = inner_read(self, read_length, self->offset); + + if (buffer != NULL) { + /* update offset on sucess. */ + self->offset += read_length; } - raw_buffer = PyBytes_AS_STRING(buffer); + + return buffer; +}; + +static int write_inner(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +{ + int rc; Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_read(self->blob, raw_buffer, read_length, self->offset); + rc = sqlite3_blob_write(self->blob, buf, len, offset); Py_END_ALLOW_THREADS - - if (rc != SQLITE_OK){ - Py_DECREF(buffer); + if (rc != SQLITE_OK) { /* For some reason after modifying blob the - error is not set on the connection db. */ + error is not set on the connection db. */ if (rc == SQLITE_ABORT) { PyErr_SetString(pysqlite_OperationalError, "Cannot operate on modified blob"); } else { _pysqlite_seterror(self->connection->db, NULL); } - return NULL; + return -1; } - - /* update offset. */ - self->offset += read_length; - - return buffer; -}; + return 0; +} PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) @@ -191,26 +222,15 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) /* TODO: throw better error on data bigger then blob. */ - Py_BEGIN_ALLOW_THREADS - rc = sqlite3_blob_write(self->blob, data_buffer.buf, - data_buffer.len, self->offset); - Py_END_ALLOW_THREADS - if (rc != SQLITE_OK) { - /* For some reason after modifying blob the - error is not set on the connection db. */ - if (rc == SQLITE_ABORT) { - PyErr_SetString(pysqlite_OperationalError, - "Cannot operate on modified blob"); - } else { - _pysqlite_seterror(self->connection->db, NULL); - } + rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); + + if (rc == 0) { + self->offset += (int)data_buffer.len; + Py_RETURN_NONE; + } else { PyBuffer_Release(&data_buffer); return NULL; } - - self->offset += (int)data_buffer.len; - PyBuffer_Release(&data_buffer); - Py_RETURN_NONE; } @@ -300,6 +320,300 @@ PyObject* pysqlite_blob_exit(pysqlite_Blob *self, PyObject *args) Py_RETURN_FALSE; } +static PyObject* pysqlite_blob_concat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support concatenation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support repeat operation"); + } + return NULL; +} + +static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return NULL; + } + + return inner_read(self, 1, i); +} + +static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) +{ + int blob_length = 0; + const char *buf; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, "Blob index out of range"); + return -1; + } + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(v) && PyBytes_Size(v)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(v); + return write_inner(self, buf, 1, i); +} + + +static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) +{ + int blob_length = 0; + + if (!pysqlite_check_blob(self)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return NULL; + } + // TODO: I am not sure... + return inner_read(self, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + + if (PySlice_GetIndicesEx(item, blob_length, + &start, &stop, &step, &slicelen) < 0) { + return NULL; + } + + if (slicelen <= 0) + return PyBytes_FromStringAndSize("", 0); + else if (step == 1) + return inner_read(self, slicelen, start); + else { + char *result_buf = (char *)PyMem_Malloc(slicelen); + char *data_buff = NULL; + Py_ssize_t cur, i; + PyObject *result; + int rc; + + if (result_buf == NULL) + return PyErr_NoMemory(); + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyMem_Free(result_buf); + return PyErr_NoMemory(); + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return NULL; + } + + for (cur = 0, i = 0; i < slicelen; + cur += step, i++) { + result_buf[i] = data_buff[cur]; + } + result = PyBytes_FromStringAndSize(result_buf, + slicelen); + PyMem_Free(result_buf); + PyMem_Free(data_buff); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, + "Blob indices must be integers"); + return NULL; + } +} + + +static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) +{ + int blob_length = 0; + int rc; + + if (!pysqlite_check_blob(self)) { + return -1; + } + + Py_BEGIN_ALLOW_THREADS + blob_length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + const char *buf; + + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += blob_length; + if (i < 0 || i >= blob_length) { + PyErr_SetString(PyExc_IndexError, + "Blob index out of range"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob doesn't support item deletion"); + return -1; + } + if (! (PyBytes_Check(value) && PyBytes_Size(value)==1) ) { + PyErr_SetString(PyExc_IndexError, + "Blob assignment must be length-1 bytes()"); + return -1; + } + + buf = PyBytes_AsString(value); + return write_inner(self, buf, 1, i); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelen; + Py_buffer vbuf; + + if (PySlice_GetIndicesEx(item, + blob_length, &start, &stop, + &step, &slicelen) < 0) { + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "Blob object doesn't support slice deletion"); + return -1; + } + if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) + return -1; + if (vbuf.len != slicelen) { + PyErr_SetString(PyExc_IndexError, + "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; + } + + if (slicelen == 0) { + } + else if (step == 1) { + rc = write_inner(self, vbuf.buf, slicelen, start); + } + else { + Py_ssize_t cur, i; + char *data_buff; + + + data_buff = (char *)PyMem_Malloc(stop - start); + if (data_buff == NULL) { + PyErr_NoMemory(); + return -1; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + + for (cur = 0, i = 0; + i < slicelen; + cur += step, i++) + { + data_buff[cur] = ((char *)vbuf.buf)[i]; + } + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, data_buff, stop - start, start); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK){ + /* For some reason after modifying blob the + error is not set on the connection db. */ + if (rc == SQLITE_ABORT) { + PyErr_SetString(pysqlite_OperationalError, + "Cannot operate on modified blob"); + } else { + _pysqlite_seterror(self->connection->db, NULL); + } + PyMem_Free(data_buff); + rc = -1; + } + rc = 0; + + } + PyBuffer_Release(&vbuf); + return rc; + } + else { + PyErr_SetString(PyExc_TypeError, + "mmap indices must be integer"); + return -1; + } +} + static PyMethodDef blob_methods[] = { {"read", (PyCFunction)pysqlite_blob_read, METH_VARARGS, @@ -320,9 +634,18 @@ static PyMethodDef blob_methods[] = { }; static PySequenceMethods blob_sequence_methods = { - (lenfunc)pysqlite_blob_length, /* sq_length */ + .sq_length = (lenfunc)pysqlite_blob_length, + .sq_concat = (binaryfunc)pysqlite_blob_concat, + .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, + .sq_item = (ssizeargfunc)pysqlite_blob_item, + .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, }; +static PyMappingMethods blob_mapping_methods = { + (lenfunc)pysqlite_blob_length, + (binaryfunc)pysqlite_blob_subscript, + (objobjargproc)pysqlite_blob_ass_subscript, +}; PyTypeObject pysqlite_BlobType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -330,6 +653,7 @@ PyTypeObject pysqlite_BlobType = { .tp_basicsize = sizeof(pysqlite_Blob), .tp_dealloc = (destructor)pysqlite_blob_dealloc, .tp_as_sequence = &blob_sequence_methods, + .tp_as_mapping = &blob_mapping_methods, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_weaklistoffset = offsetof(pysqlite_Blob, in_weakreflist), .tp_methods = blob_methods, From 5aedfba9a1541473e72faaaaabf3d63e0b972ad6 Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:34:59 +0200 Subject: [PATCH 06/18] Calculate blob length once at creation --- Modules/_sqlite/blob.c | 73 ++++++++++++------------------------------ Modules/_sqlite/blob.h | 1 + 2 files changed, 21 insertions(+), 53 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 333c65425a1f0f..abe8cac1b271bc 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -11,6 +11,10 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection, self->blob = blob; self->in_weakreflist = NULL; + Py_BEGIN_ALLOW_THREADS + self->length = sqlite3_blob_bytes(self->blob); + Py_END_ALLOW_THREADS + if (!pysqlite_check_thread(self->connection)) { return -1; } @@ -94,15 +98,11 @@ PyObject* pysqlite_blob_close(pysqlite_Blob *self) static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self) { - int blob_length; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - return blob_length; + return self->length; }; static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) @@ -140,7 +140,6 @@ static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset) PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) { int read_length = -1; - int blob_length = 0; PyObject *buffer; if (!PyArg_ParseTuple(args, "|i", &read_length)) { @@ -151,20 +150,14 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args) return NULL; } - - /* TODO: make this multithreaded and safe! */ - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (read_length < 0) { /* same as file read. */ - read_length = blob_length; + read_length = self->length; } /* making sure we don't read more then blob size */ - if (read_length > blob_length - self->offset) { - read_length = blob_length - self->offset; + if (read_length > self->length - self->offset) { + read_length = self->length - self->offset; } buffer = inner_read(self, read_length, self->offset); @@ -236,7 +229,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) { - int blob_length, offset, from_what = 0; + int offset, from_what = 0; if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) { return NULL; @@ -247,10 +240,6 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - switch (from_what) { case 0: // relative to blob begin break; @@ -261,17 +250,17 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->offset + offset; break; case 2: // relative to blob end - if (offset > INT_MAX - blob_length) { + if (offset > INT_MAX - self->length) { goto overflow; } - offset = blob_length + offset; + offset = self->length + offset; break; default: return PyErr_Format(PyExc_ValueError, "from_what should be 0, 1 or 2"); } - if (offset < 0 || offset > blob_length) { + if (offset < 0 || offset > self->length) { return PyErr_Format(PyExc_ValueError, "offset out of blob range"); } @@ -340,17 +329,11 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; } @@ -360,18 +343,13 @@ static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v) { - int blob_length = 0; const char *buf; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - - if (i < 0 || i >= blob_length) { + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; } @@ -393,23 +371,17 @@ static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) { - int blob_length = 0; - if (!pysqlite_check_blob(self)) { return NULL; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return NULL; @@ -420,7 +392,7 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelen; - if (PySlice_GetIndicesEx(item, blob_length, + if (PySlice_GetIndicesEx(item, self->length, &start, &stop, &step, &slicelen) < 0) { return NULL; } @@ -484,17 +456,12 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value) { - int blob_length = 0; int rc; if (!pysqlite_check_blob(self)) { return -1; } - Py_BEGIN_ALLOW_THREADS - blob_length = sqlite3_blob_bytes(self->blob); - Py_END_ALLOW_THREADS - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); const char *buf; @@ -502,8 +469,8 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) - i += blob_length; - if (i < 0 || i >= blob_length) { + i += self->length; + if (i < 0 || i >= self->length) { PyErr_SetString(PyExc_IndexError, "Blob index out of range"); return -1; @@ -527,7 +494,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb Py_buffer vbuf; if (PySlice_GetIndicesEx(item, - blob_length, &start, &stop, + self->length, &start, &stop, &step, &slicelen) < 0) { return -1; } diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h index 204918fd4f01c6..649f09e5ecca24 100644 --- a/Modules/_sqlite/blob.h +++ b/Modules/_sqlite/blob.h @@ -10,6 +10,7 @@ typedef struct pysqlite_Connection* connection; sqlite3_blob *blob; int offset; + int length; PyObject* in_weakreflist; /* List of weak references */ } pysqlite_Blob; From db6ef327457f9eb56e448c50309b714322212eed Mon Sep 17 00:00:00 2001 From: palaviv Date: Sat, 4 Mar 2017 16:41:20 +0200 Subject: [PATCH 07/18] Add initial Doc for sequence protocol --- Doc/library/sqlite3.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 6afef818e4ad70..068e15bf2534b9 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -878,7 +878,9 @@ Blob Objects .. class:: Blob A :class:`Blob` instance can read and write the data in the - :abbr:`BLOB (Binary Large OBject)`. + :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both + the file and sequence protocol. For example You can read data from the + :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. .. method:: Blob.close() From 219f4cb4a1b3db1783f4e9c272cd3e6de4028908 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:04:22 +0300 Subject: [PATCH 08/18] Don't support blob operation --- Lib/sqlite3/test/dbapi.py | 4 ++++ Modules/_sqlite/blob.c | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 24122a0cf58062..d769d62446b2b7 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -677,6 +677,10 @@ def CheckBlobRepeateNotSupported(self): with self.assertRaises(SystemError): self.blob * 5 + def CheckBlobContainsNotSupported(self): + with self.assertRaises(SystemError): + b"aaaaa" in self.blob + @unittest.skipUnless(threading, 'This test requires threading.') class ThreadTests(unittest.TestCase): def setUp(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index abe8cac1b271bc..7b2432491450a4 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -327,6 +327,15 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args) return NULL; } +static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) +{ + if (pysqlite_check_blob(self)) { + PyErr_SetString(PyExc_SystemError, + "Blob don't support cotains operation"); + } + return -1; +} + static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i) { if (!pysqlite_check_blob(self)) { @@ -606,6 +615,7 @@ static PySequenceMethods blob_sequence_methods = { .sq_repeat = (ssizeargfunc)pysqlite_blob_repeat, .sq_item = (ssizeargfunc)pysqlite_blob_item, .sq_ass_item = (ssizeobjargproc)pysqlite_blob_ass_item, + .sq_contains = (objobjproc)pysqlite_blob_contains, }; static PyMappingMethods blob_mapping_methods = { From 6dafe0e7b5d651f373304f39961c5a5552e05531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:16:26 +0300 Subject: [PATCH 09/18] move news entry to blurb --- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst new file mode 100644 index 00000000000000..4f7b2fefed8039 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -0,0 +1,4 @@ +The :class:`sqlite3.Connection` now has the +:meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` +allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +:issue:`24905`) From ffac9013a89c68561b51443585a799aacad3b6e7 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 18 Apr 2018 16:32:15 +0300 Subject: [PATCH 10/18] Add blob to PCBuild --- PCbuild/_sqlite3.vcxproj | 2 ++ PCbuild/_sqlite3.vcxproj.filters | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 7e0062692b8f83..55f46c963b1e7d 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -107,6 +107,7 @@ + @@ -118,6 +119,7 @@ + diff --git a/PCbuild/_sqlite3.vcxproj.filters b/PCbuild/_sqlite3.vcxproj.filters index 51830f6a4451a4..8d26c9ab6eb43f 100644 --- a/PCbuild/_sqlite3.vcxproj.filters +++ b/PCbuild/_sqlite3.vcxproj.filters @@ -39,6 +39,9 @@ Header Files + + Header Files + @@ -68,6 +71,9 @@ Source Files + + Source Files + From 3475fa1c422a7418dc1a7765ad33479307afa531 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:30:45 +0300 Subject: [PATCH 11/18] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 068e15bf2534b9..9b6d0472089063 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.7 + .. versionadded:: 3.8 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.7 +.. versionadded:: 3.8 .. class:: Blob From f6015ff3b39343898066e70c879d17dc5a78f89d Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 8 May 2019 15:32:14 +0300 Subject: [PATCH 12/18] Fix memory leak --- Modules/_sqlite/blob.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 7b2432491450a4..d560563d3e766d 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -219,6 +219,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) if (rc == 0) { self->offset += (int)data_buffer.len; + PyBuffer_Release(&data_buffer); Py_RETURN_NONE; } else { PyBuffer_Release(&data_buffer); From 01e526ca5c94b480380919a79e320037bc7f65f0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Wed, 29 Jul 2020 19:40:23 +0300 Subject: [PATCH 13/18] Update version --- Doc/library/sqlite3.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 9b6d0472089063..841b2be4fdacf8 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -314,7 +314,7 @@ Connection Objects The BLOB size cannot be changed using the :class:`Blob` class. Use ``zeroblob`` to create the blob in the wanted size in advance. - .. versionadded:: 3.8 + .. versionadded:: 3.10 .. method:: commit() @@ -873,7 +873,7 @@ Exceptions Blob Objects ------------ -.. versionadded:: 3.8 +.. versionadded:: 3.10 .. class:: Blob From 354bebf2dc561a6f079fc430edf8d9eb7cab6496 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:01:26 +0300 Subject: [PATCH 14/18] Fix CR comments in documentation and testing --- Doc/includes/sqlite3/blob.py | 6 +++--- Doc/includes/sqlite3/blob_with.py | 6 +++--- Doc/library/sqlite3.rst | 5 +++-- Lib/sqlite3/test/dbapi.py | 8 +++++--- .../next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index b8d2c78d1a365d..18514876af6191 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -6,8 +6,8 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle blob = con.open_blob("test", "blob_col", 1, 1) -blob.write(b"a" * 5) -blob.write(b"b" * 5) +blob.write(b"Hello") +blob.write(b"World") blob.seek(0) -print(blob.read()) # will print b"aaaaabbbbb" +print(blob.read()) # will print b"HelloWorld" blob.close() diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index 624b680591901a..f220518c09a1ab 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -6,7 +6,7 @@ con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle with con.open_blob("test", "blob_col", 1, 1) as blob: - blob.write(b"a" * 5) - blob.write(b"b" * 5) + blob.write(b"Hello") + blob.write(b"World") blob.seek(0) - print(blob.read()) # will print b"aaaaabbbbb" + print(blob.read()) # will print b"HelloWorld" diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 841b2be4fdacf8..b754481ad48757 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -879,8 +879,9 @@ Blob Objects A :class:`Blob` instance can read and write the data in the :abbr:`BLOB (Binary Large OBject)`. The :class:`Blob` object implement both - the file and sequence protocol. For example You can read data from the + the file and sequence protocol. For example, you can read data from the :class:`Blob` by doing ``obj.read(5)`` or by doing ``obj[:5]``. + You can call ``len(obj)`` to get size of the BLOB. .. method:: Blob.close() @@ -911,7 +912,7 @@ Blob Objects Return the current offset of the BLOB. - .. method:: Blob.seek(offset, [whence]) + .. method:: Blob.seek(offset, whence=os.SEEK_SET) Set the BLOB offset. The *whence* argument is optional and defaults to :data:`os.SEEK_SET` or 0 (absolute BLOB positioning); other values diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index d769d62446b2b7..a611feb56414eb 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -1177,9 +1177,11 @@ def suite(): blob_suite = unittest.makeSuite(BlobTests, "Check") closed_blob_suite = unittest.makeSuite(ClosedBlobTests, "Check") blob_context_manager_suite = unittest.makeSuite(BlobContextManagerTests, "Check") - return unittest.TestSuite((module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, - ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, - blob_suite, closed_blob_suite, blob_context_manager_suite)) + return unittest.TestSuite(( + module_suite, connection_suite, cursor_suite, thread_suite, constructor_suite, + ext_suite, closed_con_suite, closed_cur_suite, on_conflict_suite, + blob_suite, closed_blob_suite, blob_context_manager_suite, + )) def test(): runner = unittest.TextTestRunner() diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst index 4f7b2fefed8039..c7d2405d89539e 100644 --- a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -1,4 +1,4 @@ The :class:`sqlite3.Connection` now has the :meth:`sqlite3.Connection.open_blob` method. The :class:`sqlite3.Blob` -allows incremental I/O operations to blobs. (Contributed by Aviv Palivoda in +allows incremental I/O operations to blobs. (Patch by Aviv Palivoda in :issue:`24905`) From 9709456b864e48e1cc3a3e838b311806a9ca6acb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:08:58 +0300 Subject: [PATCH 15/18] Fix CR comments in code --- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/connection.h | 2 +- setup.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index b99fcdb24fc25f..7e112b7fdb705a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -336,7 +336,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"table", "column", "row", "readonly", - "dbname", NULL, NULL}; + "dbname", NULL}; int rc; const char *dbname = "main", *table, *column; long long row; diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 708a199087c002..52dc27c8adff2d 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -66,7 +66,7 @@ typedef struct pysqlite_Cache* statement_cache; - /* Lists of weak references to statements, blobs and cursors used within this connection */ + /* Lists of weak references to statements, cursors and blobs used within this connection */ PyObject* statements; PyObject* cursors; PyObject* blobs; diff --git a/setup.py b/setup.py index 8c70025489e181..dd6b3509247fd8 100644 --- a/setup.py +++ b/setup.py @@ -1524,7 +1524,8 @@ def detect_sqlite(self): '_sqlite/row.c', '_sqlite/statement.c', '_sqlite/util.c', - '_sqlite/blob.c' ] + '_sqlite/blob.c', + ] sqlite_defines = [] if not MS_WINDOWS: From e6e5099d5a5d9a4e69317208a4a463922a903fd5 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:23:21 +0300 Subject: [PATCH 16/18] Make readonly and dbname keyword arguements only --- Doc/includes/sqlite3/blob.py | 2 +- Doc/includes/sqlite3/blob_with.py | 2 +- Doc/library/sqlite3.rst | 2 +- Modules/_sqlite/connection.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py index 18514876af6191..afd7812a8b3af9 100644 --- a/Doc/includes/sqlite3/blob.py +++ b/Doc/includes/sqlite3/blob.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -blob = con.open_blob("test", "blob_col", 1, 1) +blob = con.open_blob("test", "blob_col", 1) blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/includes/sqlite3/blob_with.py b/Doc/includes/sqlite3/blob_with.py index f220518c09a1ab..fdca9fbc638ea2 100644 --- a/Doc/includes/sqlite3/blob_with.py +++ b/Doc/includes/sqlite3/blob_with.py @@ -5,7 +5,7 @@ con.execute("create table test(id integer primary key, blob_col blob)") con.execute("insert into test(blob_col) values (zeroblob(10))") # opening blob handle -with con.open_blob("test", "blob_col", 1, 1) as blob: +with con.open_blob("test", "blob_col", 1) as blob: blob.write(b"Hello") blob.write(b"World") blob.seek(0) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index b754481ad48757..827c2e0588611a 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -301,7 +301,7 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. - .. method:: open_blob(table, column, row, readonly=False, dbname="main") + .. method:: open_blob(table, column, row, *, readonly=False, dbname="main") On success a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` located in row *row*, diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7e112b7fdb705a..1d64bae7ca98f6 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -346,7 +346,7 @@ PyObject* pysqlite_connection_blob(pysqlite_Connection *self, PyObject *args, PyObject *weakref; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|ps", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ssL|$ps", kwlist, &table, &column, &row, &readonly, &dbname)) { return NULL; From e9a80800f0883de7e3b9fb34ca1b8172fd745460 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:32:47 +0300 Subject: [PATCH 17/18] Fix more CR comments --- Modules/_sqlite/blob.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index d560563d3e766d..f222d5cd477f22 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -257,19 +257,19 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_Format(PyExc_ValueError, + return PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); } if (offset < 0 || offset > self->length) { - return PyErr_Format(PyExc_ValueError, "offset out of blob range"); + return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_Format(PyExc_OverflowError, "seek offset result in overflow"); + return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); } @@ -332,7 +332,7 @@ static int pysqlite_blob_contains(pysqlite_Blob *self, PyObject *args) { if (pysqlite_check_blob(self)) { PyErr_SetString(PyExc_SystemError, - "Blob don't support cotains operation"); + "Blob don't support contains operation"); } return -1; } @@ -407,11 +407,11 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item) return NULL; } - if (slicelen <= 0) + if (slicelen <= 0) { return PyBytes_FromStringAndSize("", 0); - else if (step == 1) + } else if (step == 1) { return inner_read(self, slicelen, start); - else { + } else { char *result_buf = (char *)PyMem_Malloc(slicelen); char *data_buff = NULL; Py_ssize_t cur, i; @@ -586,7 +586,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb } else { PyErr_SetString(PyExc_TypeError, - "mmap indices must be integer"); + "Blob indices must be integer"); return -1; } } From d4fb1b5afd50091d99ecaa4022df109626d7325e Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 3 Aug 2020 19:57:07 +0300 Subject: [PATCH 18/18] Add more indicative error on write bigger then blob length --- Lib/sqlite3/test/dbapi.py | 2 +- Modules/_sqlite/blob.c | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index a611feb56414eb..fd88faa7765e02 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -584,7 +584,7 @@ def CheckBlobWriteAdvanceOffset(self): self.assertEqual(self.blob.tell(), 50) def CheckBlobWriteMoreThenBlobSize(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaises(ValueError): self.blob.write(b"a" * 1000) def CheckBlobReadAfterRowChange(self): diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index f222d5cd477f22..49e664631e2774 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -208,12 +208,17 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data) return NULL; } - if (!pysqlite_check_blob(self)) { + if (data_buffer.len > self->length - self->offset) { + PyErr_SetString(PyExc_ValueError, + "data longer than blob length"); PyBuffer_Release(&data_buffer); return NULL; } - /* TODO: throw better error on data bigger then blob. */ + if (!pysqlite_check_blob(self)) { + PyBuffer_Release(&data_buffer); + return NULL; + } rc = write_inner(self, data_buffer.buf, data_buffer.len, self->offset); @@ -257,19 +262,22 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args) offset = self->length + offset; break; default: - return PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_ValueError, "from_what should be 0, 1 or 2"); + return NULL; } if (offset < 0 || offset > self->length) { - return PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + return NULL; } self->offset = offset; Py_RETURN_NONE; overflow: - return PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + PyErr_SetString(PyExc_OverflowError, "seek offset result in overflow"); + return NULL; }