From 2bf6d2fc701bc3cb1cfe87bc02166b723950f0c0 Mon Sep 17 00:00:00 2001 From: Anthony Tuininga Date: Mon, 31 Aug 2020 20:22:36 -0600 Subject: [PATCH 1/4] Update to ODPI-C 4.0.2. --- doc/src/conf.py | 2 +- doc/src/release_notes.rst | 9 +++++++++ odpi | 2 +- setup.py | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/src/conf.py b/doc/src/conf.py index ebdc717..696c684 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -42,7 +42,7 @@ # The short X.Y version. version = '8.0' # The full version, including alpha/beta/rc tags. -release = '8.0.0' +release = '8.0.1' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: diff --git a/doc/src/release_notes.rst b/doc/src/release_notes.rst index 0eec917..be7f084 100644 --- a/doc/src/release_notes.rst +++ b/doc/src/release_notes.rst @@ -5,6 +5,15 @@ cx_Oracle Release Notes ======================= +Version 8.0.1 (TBD) +------------------- + +#) Updated embedded ODPI-C to `version 4.0.2 + `__. This includes the fix for + (`issue 459 `__). + + Version 8.0 (June 2020) ----------------------- diff --git a/odpi b/odpi index 9b8206d..3a6a8d2 160000 --- a/odpi +++ b/odpi @@ -1 +1 @@ -Subproject commit 9b8206d2d6baf8add270038c6b425c1ef450d309 +Subproject commit 3a6a8d2efff86c8b0cdfac1436c9bc3a83495878 diff --git a/setup.py b/setup.py index d855654..2105d19 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ from distutils.extension import Extension # define build constants -BUILD_VERSION = "8.0.0" +BUILD_VERSION = "8.0.1" # setup extra link and compile args extraLinkArgs = [] From 7751ccfa3aeca30e119ea845986805f4f753b7fc Mon Sep 17 00:00:00 2001 From: Anthony Tuininga Date: Mon, 31 Aug 2020 20:23:23 -0600 Subject: [PATCH 2/4] Documentation improvements. --- doc/src/api_manual/module.rst | 9 +- doc/src/index.rst | 2 +- doc/src/release_notes.rst | 1 + doc/src/user_guide/connection_handling.rst | 123 ++++++++++++++------- doc/src/user_guide/ha.rst | 2 +- src/cxoConnection.c | 5 +- 6 files changed, 95 insertions(+), 47 deletions(-) diff --git a/doc/src/api_manual/module.rst b/doc/src/api_manual/module.rst index 74facd4..3bc63f9 100644 --- a/doc/src/api_manual/module.rst +++ b/doc/src/api_manual/module.rst @@ -231,9 +231,12 @@ Module Interface number of connections opened when the pool is created. The increment is the number of connections that are opened whenever a connection request exceeds the number of currently open connections. The max parameter is the maximum - number of connections that can be open in the connection pool. Note that - when :ref:`external authentication ` or :ref:`heterogeneous pools - ` are used, the pool growth behavior is different. + number of connections that can be open in the connection pool. + + Note that when using :ref:`external authentication `, + :ref:`heterogeneous pools `, or :ref:`drcp`, then the pool + growth behavior is different. In these cases the number of connections + created at pool startup is always zero, and the increment is always one. If the connectiontype parameter is specified, all calls to :meth:`~SessionPool.acquire()` will create connection objects of that type, diff --git a/doc/src/index.rst b/doc/src/index.rst index 3a1fb87..6e3d71b 100644 --- a/doc/src/index.rst +++ b/doc/src/index.rst @@ -5,7 +5,7 @@ Welcome to cx_Oracle's documentation! **cx_Oracle** is a module that enables access to Oracle Database and conforms to the Python database API specification. This module is currently tested against Oracle Client 19c, 18c, 12c, and 11.2, and Python 3.5, 3.6, 3.7 and -3.8. +3.8. Older versions of cx_Oracle may be used with previous Python releases. **cx_Oracle** is distributed under an open-source :ref:`license ` (the BSD license). A detailed description of cx_Oracle changes can be found in diff --git a/doc/src/release_notes.rst b/doc/src/release_notes.rst index be7f084..90de8b2 100644 --- a/doc/src/release_notes.rst +++ b/doc/src/release_notes.rst @@ -12,6 +12,7 @@ Version 8.0.1 (TBD) `__. This includes the fix for (`issue 459 `__). +#) Documentation improvements. Version 8.0 (June 2020) diff --git a/doc/src/user_guide/connection_handling.rst b/doc/src/user_guide/connection_handling.rst index 63d6902..137fa77 100644 --- a/doc/src/user_guide/connection_handling.rst +++ b/doc/src/user_guide/connection_handling.rst @@ -22,14 +22,14 @@ There are two ways to connect to Oracle Database using cx_Oracle: * **Pooled connections** - Connection pooling is important for performance when applications - frequently connect and disconnect from the database. Oracle high - availability features in the pool implementation mean that small - pools can also be useful for applications that want a few - connections available for infrequent use. Pools are created with - :meth:`cx_Oracle.SessionPool()` and then - :meth:`SessionPool.acquire()` can be called to obtain a connection - from a pool. + :ref:`Connection pooling ` is important for performance when + applications frequently connect and disconnect from the database. Pools + support Oracle's :ref:`high availability ` features and are + recommended for applications that must be reliable. Small pools can also be + useful for applications that want a few connections available for infrequent + use. Pools are created with :meth:`cx_Oracle.SessionPool()` at application + initialization time, and then :meth:`SessionPool.acquire()` can be called to + obtain a connection from a pool. Many connection behaviors can be controlled by cx_Oracle options. Other settings can be configured in :ref:`optnetfiles` or in :ref:`optclientfiles`. @@ -57,8 +57,7 @@ Connections should be released when they are no longer needed by calling :meth:`Connection.close()`. Alternatively, you may prefer to let connections be automatically cleaned up when references to them go out of scope. This lets cx_Oracle close dependent resources in the correct order. One other approach is -the use of a "with" block, which ensures that a connection is closed once the -block is completed. For example: +the use of a "with" block, for example: .. code-block:: python @@ -73,6 +72,9 @@ This code ensures that, once the block is completed, the connection is closed and resources have been reclaimed by the database. In addition, any attempt to use the variable ``connection`` outside of the block will simply fail. +Prompt closing of connections is important when using connection pools so +connections are available for reuse by other pool users. + .. _connstr: Connection Strings @@ -248,25 +250,38 @@ Connection Pooling ================== cx_Oracle's connection pooling lets applications create and maintain a pool of -connections to the database. The internal implementation uses Oracle's -`session pool technology `__. -In general, each connection in a cx_Oracle connection pool corresponds to one -Oracle session. +connections to the database. Connection pooling is important for performance +when applications frequently connect and disconnect from the database. The pool +implementation uses Oracle's `session pool technology +`__ which supports Oracle's +:ref:`high availability ` features and is recommended for +applications that must be reliable. This also means that small pools can be +useful for applications that want a few connections available for infrequent +use. A connection pool is created by calling :meth:`~cx_Oracle.SessionPool()`. This -is generally called during application initialization. Connections can then be -obtained from a pool by calling :meth:`~SessionPool.acquire()`. The initial -pool size and the maximum pool size are provided at the time of pool creation. -When the pool needs to grow, new connections are created automatically. The -pool can shrink back to the minimum size when connections are no longer in use. +is generally called during application initialization. The initial pool size +and the maximum pool size are provided at the time of pool creation. When the +pool needs to grow, new connections are created automatically. The pool can +shrink back to the minimum size when connections are no longer in use. For +pools created with :ref:`external authentication `, with +:ref:`homogeneous ` set to False, or when using :ref:`drcp`, then +the number of connections initially created is zero even if a larger value is +specified for ``min``. Also in these cases the pool increment is always 1, +regardless of the value of ``increment``. + +After a pool has been created, connections can be obtained from it by calling +:meth:`~SessionPool.acquire()`. These connections can be used in the same way +that standalone connections are used. Connections acquired from the pool should be released back to the pool using :meth:`SessionPool.release()` or :meth:`Connection.close()` when they are no longer required. Otherwise, they will be released back to the pool automatically when all of the variables referencing the connection go out of -scope. The session pool can be completely closed using -:meth:`SessionPool.close()`. +scope. This make connections available for other users of the pool. + +The session pool can be completely closed using :meth:`SessionPool.close()`. The example below shows how to connect to Oracle Database using a connection pool: @@ -291,6 +306,17 @@ connection pool: # Close the pool pool.close() +Other :meth:`cx_Oracle.SessionPool()` options can be used at pool creation. For +example the ``getmode`` value can be set so that any ``aquire()`` call will wait +for a connection to become available if all are currently in use, for example: + +.. code-block:: python + + # Create the session pool + pool = cx_Oracle.SessionPool("hr", userpwd, "dbhost.example.com/orclpdb1", + min=2, max=5, increment=1, getmode=cx_Oracle.SPOOL_ATTRVAL_WAIT, encoding="UTF-8") + + Applications that are using connections concurrently in multiple threads should set the ``threaded`` parameter to *True* when creating a connection pool: @@ -300,6 +326,7 @@ set the ``threaded`` parameter to *True* when creating a connection pool: pool = cx_Oracle.SessionPool("hr", userpwd, "dbhost.example.com/orclpdb1", min=2, max=5, increment=1, threaded=True, encoding="UTF-8") + See `ConnectionPool.py `__ for an example. @@ -314,33 +341,49 @@ will also do a full :ref:`round-trip ` ping to the database when it is about to return a connection that was unused in the pool for 60 seconds. If the ping fails, the connection will be discarded and another one obtained before :meth:`~SessionPool.acquire()` returns to the application. Because this full -ping is time based, it won't catch every failure. Also since network timeouts -and session kills may occur after :meth:`~SessionPool.acquire()` and before -:meth:`Cursor.execute()`, applications need to check for errors after each -:meth:`~Cursor.execute()` and make application-specific decisions about retrying -work if there was a connection failure. Note both the lightweight and full ping -connection checks can mask configuration issues, for example firewalls killing -connections, so monitor the connection rate in AWR for an unexpected value. You -can explicitly initiate a full ping to check connection liveness with -:meth:`Connection.ping()` but overuse will impact performance and scalability. +ping is time based, it won't catch every failure. Also network timeouts and +session kills may occur after :meth:`~SessionPool.acquire()` and before +:meth:`Cursor.execute()`. To handle these cases, applications need to check +for errors after each :meth:`~Cursor.execute()` and make application-specific +decisions about retrying work if there was a connection failure. Oracle's +:ref:`Application Continuity ` can do this automatically in +some cases. Note both the lightweight and full ping connection checks can mask +performance-impacting configuration issues, for example firewalls killing +connections, so monitor the connection rate in `AWR +`__ +for an unexpected value. You can explicitly initiate a full ping to check +connection liveness with :meth:`Connection.ping()` but overuse will impact +performance and scalability. + +Connection Pool Sizing +---------------------- The Oracle Real-World Performance Group's recommendation is to use fixed size -connection pools. The values of min and max should be the same (and the -increment equal to zero). The :ref:`firewall `, `resource manager -`__ -or user profile `IDLE_TIME -`__ -should not expire idle sessions. This avoids connection storms which can -decrease throughput. See `Guideline for Preventing Connection Storms: Use -Static Pools +connection pools. The values of ``min`` and ``max`` should be the same (and the +``increment`` equal to zero). This avoids connection storms which can decrease +throughput. See `Guideline for Preventing Connection Storms: Use Static Pools `__, -which contains details about sizing of pools. +which contains more details about sizing of pools. Having a fixed size will +guarantee that the database can handle the upper pool size. For example, if a +pool needs to grow but the database resources are limited, then +:meth:`SessionPool.acquire()` may return errors such as ORA-28547. With a fixed +pool size, this class of error will occur when the pool is created, allowing you +to change the size before users access the application. With a dynamically +growing pool, the error may occur much later after the pool has been in use for +some time. The Real-World Performance Group also recommends keeping pool sizes small, as they may perform better than larger pools. The pool attributes should be adjusted to handle the desired workload within the bounds of available resources in cx_Oracle and the database. +Make sure the :ref:`firewall `, `resource manager +`__ +or user profile `IDLE_TIME +`__ +do not expire idle sessions, since this will require connections be recreated, +which will impact performance and scalability. + .. _sessioncallback: Session CallBacks for Setting Pooled Connection State diff --git a/doc/src/user_guide/ha.rst b/doc/src/user_guide/ha.rst index e5294c1..d2f64db 100644 --- a/doc/src/user_guide/ha.rst +++ b/doc/src/user_guide/ha.rst @@ -119,7 +119,7 @@ Oracle Application Continuity and Transparent Application Continuity are Oracle Database technologies that record application interaction with the database and, in the event of a database instance outage, attempt to replay the interaction on a surviving database instance. If successful, users will be unaware of any -database issue. +database issue. AC and TAC are best suited for OLTP applications. When AC or TAC are configured on the database service, they are transparently available to cx_Oracle applications. diff --git a/src/cxoConnection.c b/src/cxoConnection.c index 6ff99d5..20ea9fd 100644 --- a/src/cxoConnection.c +++ b/src/cxoConnection.c @@ -1853,8 +1853,9 @@ static PyObject *cxoConnection_unsubscribe(cxoConnection *conn, PyObject* args, //----------------------------------------------------------------------------- -// cxoConnection_commit() -// Commit the transaction on the connection. +// cxoConnection_getSodaDatabase() +// Create and return a new SODA database object associated with the +// connection. //----------------------------------------------------------------------------- static PyObject *cxoConnection_getSodaDatabase(cxoConnection *conn, PyObject *args) From 184f4d19e758f302792af2af83d8c4be1db2c7aa Mon Sep 17 00:00:00 2001 From: Anthony Tuininga Date: Mon, 31 Aug 2020 20:25:01 -0600 Subject: [PATCH 3/4] Added metadata (and an exception) specifying that Python 3.5 and higher is required in order to allow pip (and the exception message) to direct those using Python 2 to use version 7.3 instead. --- doc/src/release_notes.rst | 3 +++ setup.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/src/release_notes.rst b/doc/src/release_notes.rst index 90de8b2..6d10d65 100644 --- a/doc/src/release_notes.rst +++ b/doc/src/release_notes.rst @@ -12,6 +12,9 @@ Version 8.0.1 (TBD) `__. This includes the fix for (`issue 459 `__). +#) Added metadata (and an exception) specifying that Python 3.5 and higher is + required in order to allow pip (and the exception message) to direct those + using Python 2 to use version 7.3 instead. #) Documentation improvements. diff --git a/setup.py b/setup.py index 2105d19..ada30d2 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,8 @@ # check minimum supported Python version if sys.version_info[:2] < (3, 5): - raise Exception("Python 3.5 or higher is required.") + raise Exception("Python 3.5 or higher is required. " + + "For python 2, use 'pip install cx_Oracle==7.3'") # if setuptools is detected, use it to add support for eggs try: @@ -122,6 +123,7 @@ def run(self): author = "Anthony Tuininga", author_email = "anthony.tuininga@gmail.com", url = "https://oracle.github.io/python-cx_Oracle", + python_requires = ">=3.5", ext_modules = [extension], keywords = "Oracle", license = "BSD License", From aecf006ba38758389336384e5bdf8506b12e0f7f Mon Sep 17 00:00:00 2001 From: Anthony Tuininga Date: Mon, 31 Aug 2020 20:25:55 -0600 Subject: [PATCH 4/4] Further tweaks to documentation and release notes in preparation for release of 8.0.1. --- doc/src/release_notes.rst | 14 ++-- doc/src/user_guide/tuning.rst | 151 ++++++++++++++++++---------------- 2 files changed, 86 insertions(+), 79 deletions(-) diff --git a/doc/src/release_notes.rst b/doc/src/release_notes.rst index 6d10d65..5eaf630 100644 --- a/doc/src/release_notes.rst +++ b/doc/src/release_notes.rst @@ -5,16 +5,18 @@ cx_Oracle Release Notes ======================= -Version 8.0.1 (TBD) -------------------- +Version 8.0.1 (August 2020) +--------------------------- #) Updated embedded ODPI-C to `version 4.0.2 `__. This includes the fix for + version-4-0-2-august-31-2020>`__. This includes the fix for binding and + fetching numbers with 39 or 40 decimal digits (`issue 459 `__). -#) Added metadata (and an exception) specifying that Python 3.5 and higher is - required in order to allow pip (and the exception message) to direct those - using Python 2 to use version 7.3 instead. +#) Added build metadata specifying that Python 3.5 and higher is required in + order to avoid downloading and failing to install with Python 2. The + exception message when running ``setup.py`` directly was updated to inform + those using Python 2 to use version 7.3 instead. #) Documentation improvements. diff --git a/doc/src/user_guide/tuning.rst b/doc/src/user_guide/tuning.rst index cbfb4ad..882a392 100644 --- a/doc/src/user_guide/tuning.rst +++ b/doc/src/user_guide/tuning.rst @@ -91,93 +91,74 @@ already buffered in the Oracle Client libraries. Reducing round-trips helps performance and scalability. An overhead of prefetching is the need for an additional data copy from Oracle Client's prefetch buffers. -To tune queries that return an unknown number of rows, estimate the number of -rows returned and start with an appropriate :attr:`Cursor.arraysize` value. The -default is 100. Then set :attr:`Cursor.prefetchrows` to the ``arraysize`` -value. Do not make the sizes unnecessarily large. Keep ``arraysize`` as big, -or bigger than, ``prefetchrows``. Adjust the values as needed for performance, -memory and round-trip usage. An example is: +Choosing values for ``arraysize`` and ``prefetchrows`` +++++++++++++++++++++++++++++++++++++++++++++++++++++++ -.. code-block:: python +The best :attr:`Cursor.arraysize` and :attr:`Cursor.prefetchrows` values can be +found by experimenting with your application under the expected load of normal +application use. This is because the cost of the extra memory copy from the +prefetch buffers when fetching a large quantity of rows or very "wide" rows may +outweigh the cost of a round-trip for a single cx_Oracle user on a fast network. +However under production application load, the reduction of round-trips may help +performance and overall system scalability. The documentation in +:ref:`round-trips ` shows how to measure round-trips. - cur = connection.cursor() +Here are some suggestions for the starting point to begin your tuning: - cur.prefetchrows = 1000 - cur.arraysize = 1000 +* To tune queries that return an unknown number of rows, estimate the number of + rows returned and start with an appropriate :attr:`Cursor.arraysize` value. + The default is 100. Then set :attr:`Cursor.prefetchrows` to the ``arraysize`` + value. Do not make the sizes unnecessarily large. For example: - for row in cur.execute("SELECT * FROM very_big_table"): - print(row) + .. code-block:: python -For a large quantity of rows or very "wide" rows on fast networks you may prefer -to leave ``prefetchrows`` at its default value of 2. The documentation in -:ref:`roundtrips` shows how to measure round-trips. + cur = connection.cursor() -If you are fetching a fixed number of rows, start your tuning by setting -``arraysize`` to the number of expected rows, and set ``prefetchrows`` to one -greater than this value. (Adding one removes the need for a round-trip to check -for end-of-fetch). For example, if you are querying 20 rows, perhaps to -:ref:`display a page ` of data, set ``prefetchrows`` to 21 and -``arraysize`` to 20: + cur.prefetchrows = 1000 + cur.arraysize = 1000 -.. code-block:: python + for row in cur.execute("SELECT * FROM very_big_table"): + print(row) - cur = connection.cursor() + Adjust the values as needed for performance, memory and round-trip usage. For + a large quantity of rows or very "wide" rows on fast networks you may prefer + to leave ``prefetchrows`` at its default value of 2. Keep ``arraysize`` as + big, or bigger than, ``prefetchrows``. - cur.prefetchrows = 21 - cur.arraysize = 20 +* If you are fetching a fixed number of rows, start your tuning by setting + ``arraysize`` to the number of expected rows, and set ``prefetchrows`` to one + greater than this value. (Adding one removes the need for a round-trip to check + for end-of-fetch). For example, if you are querying 20 rows, perhaps to + :ref:`display a page ` of data, set ``prefetchrows`` to 21 and + ``arraysize`` to 20: - for row in cur.execute(""" - SELECT last_name - FROM employees - ORDER BY last_name - OFFSET 0 ROWS FETCH NEXT 20 ROWS ONLY"""): - print(row) + .. code-block:: python -This will return all rows for the query in one round-trip. + cur = connection.cursor() -If you know that a query returns just one row then set :attr:`Cursor.arraysize` -to 1 to minimize memory usage. The default prefetch value of 2 allows minimal -round-trips for single-row queries: + cur.prefetchrows = 21 + cur.arraysize = 20 -.. code-block:: python + for row in cur.execute(""" + SELECT last_name + FROM employees + ORDER BY last_name + OFFSET 0 ROWS FETCH NEXT 20 ROWS ONLY"""): + print(row) - cur = connection.cursor() - cur.arraysize = 1 - cur.execute("select * from MyTable where id = 1"): - row = cur.fetchone() - print(row) + This will return all rows for the query in one round-trip. -The best :attr:`Cursor.arraysize` and :attr:`Cursor.prefetchrows` values can be -found by experimenting with your application under the expected load of normal -application use. This is because the cost of the extra memory copy from the -prefetch buffers when fetching a large quantity of rows or very "wide" rows may -outweigh the cost of a round-trip for a single cx_Oracle user on a fast network. -However under production application load, the reduction of round-trips may help -performance and overall system scalability. +* If you know that a query returns just one row then set :attr:`Cursor.arraysize` + to 1 to minimize memory usage. The default prefetch value of 2 allows minimal + round-trips for single-row queries: -Prefetching can also be enabled in an external :ref:`oraaccess.xml -` file, which may be useful for tuning an application when -modifying its code is not feasible. Setting the size in ``oraaccess.xml`` will -affect the whole application, so it should not be the first tuning choice. - -One place where increasing ``arraysize`` is particularly useful is in copying -data from one database to another: + .. code-block:: python -.. code-block:: python - - # setup cursors - sourceCursor = sourceConnection.cursor() - sourceCursor.arraysize = 1000 - targetCursor = targetConnection.cursor() - - # perform fetch and bulk insertion - sourceCursor.execute("select * from MyTable") - while True: - rows = sourceCursor.fetchmany() - if not rows: - break - targetCursor.executemany("insert into MyTable values (:1, :2)", rows) - targetConnection.commit() + cur = connection.cursor() + cur.arraysize = 1 + cur.execute("select * from MyTable where id = 1"): + row = cur.fetchone() + print(row) In cx_Oracle, the ``arraysize`` and ``prefetchrows`` values are only examined when a statement is executed the first time. To change the values, create a new @@ -207,6 +188,30 @@ to 0: function before cx_Oracle can return them to the application. Setting ``prefetchrows`` to 0 helps give a consistent flow of data to the application. +Prefetching can also be enabled in an external :ref:`oraaccess.xml +` file, which may be useful for tuning an application when +modifying its code is not feasible. Setting the size in ``oraaccess.xml`` will +affect the whole application, so it should not be the first tuning choice. + +One place where increasing ``arraysize`` is particularly useful is in copying +data from one database to another: + +.. code-block:: python + + # setup cursors + sourceCursor = sourceConnection.cursor() + sourceCursor.arraysize = 1000 + targetCursor = targetConnection.cursor() + + # perform fetch and bulk insertion + sourceCursor.execute("select * from MyTable") + while True: + rows = sourceCursor.fetchmany() + if not rows: + break + targetCursor.executemany("insert into MyTable values (:1, :2)", rows) + targetConnection.commit() + .. _roundtrips: Database Round-trips @@ -305,8 +310,8 @@ cache. .. _clientresultcache: -Client Result Cache -=================== +Client Result Caching +===================== cx_Oracle applications can use Oracle Database's `Client Result Cache `__. @@ -332,7 +337,7 @@ restarting the database, for example: SQL> STARTUP FORCE CRC can alternatively be configured in an :ref:`oraaccess.xml ` -or :ref:`sqlnet.ora ` file on the Node.js host, see `Client +or :ref:`sqlnet.ora ` file on the Python host, see `Client Configuration Parameters `__.