diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index ae2e324d0abaa4..a9930183f9a400 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1078,8 +1078,9 @@ SSL Sockets (but passing a non-zero ``flags`` argument is not allowed) - :meth:`~socket.socket.send`, :meth:`~socket.socket.sendall` (with the same limitation) - - :meth:`~socket.socket.sendfile` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send` will be used) + - :meth:`~socket.socket.sendfile` (it may be high-performant only when + the kernel TLS is enabled by setting :data:`~ssl.OP_ENABLE_KTLS` or when a + socket is plain-text, else :meth:`~socket.socket.send` will be used) - :meth:`~socket.socket.shutdown` However, since the SSL (and TLS) protocol has its own framing atop @@ -1113,6 +1114,11 @@ SSL Sockets functions support reading and writing of data larger than 2 GB. Writing zero-length data no longer fails with a protocol violation error. + .. versionchanged:: next + Python now uses ``SSL_sendfile`` internally when possible. The + function sends a file more efficiently because it performs TLS encryption + in the kernel to avoid additional context switches. + SSL sockets also have the following additional methods and attributes: .. method:: SSLSocket.read(len=1024, buffer=None) diff --git a/Lib/socket.py b/Lib/socket.py index 727b0e75f03595..3073c012b19877 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -56,6 +56,7 @@ import os import sys from enum import IntEnum, IntFlag +from functools import partial try: import errno @@ -348,75 +349,83 @@ def makefile(self, mode="r", buffering=None, *, text.mode = mode return text - if hasattr(os, 'sendfile'): + def _sendfile_zerocopy(self, zerocopy_func, giveup_exc_type, file, + offset=0, count=None): + """ + Send a file using a zero-copy function. + """ + import selectors - def _sendfile_use_sendfile(self, file, offset=0, count=None): - # Lazy import to improve module import time - import selectors + self._check_sendfile_params(file, offset, count) + sockno = self.fileno() + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise giveup_exc_type(err) # not a regular file + try: + fsize = os.fstat(fileno).st_size + except OSError as err: + raise giveup_exc_type(err) # not a regular file + if not fsize: + return 0 # empty file + # Truncate to 1GiB to avoid OverflowError, see bpo-38319. + blocksize = min(count or fsize, 2 ** 30) + timeout = self.gettimeout() + if timeout == 0: + raise ValueError("non-blocking sockets are not supported") + # poll/select have the advantage of not requiring any + # extra file descriptor, contrarily to epoll/kqueue + # (also, they require a single syscall). + if hasattr(selectors, 'PollSelector'): + selector = selectors.PollSelector() + else: + selector = selectors.SelectSelector() + selector.register(sockno, selectors.EVENT_WRITE) - self._check_sendfile_params(file, offset, count) - sockno = self.fileno() - try: - fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: - raise _GiveupOnSendfile(err) # not a regular file - try: - fsize = os.fstat(fileno).st_size - except OSError as err: - raise _GiveupOnSendfile(err) # not a regular file - if not fsize: - return 0 # empty file - # Truncate to 1GiB to avoid OverflowError, see bpo-38319. - blocksize = min(count or fsize, 2 ** 30) - timeout = self.gettimeout() - if timeout == 0: - raise ValueError("non-blocking sockets are not supported") - # poll/select have the advantage of not requiring any - # extra file descriptor, contrarily to epoll/kqueue - # (also, they require a single syscall). - if hasattr(selectors, 'PollSelector'): - selector = selectors.PollSelector() - else: - selector = selectors.SelectSelector() - selector.register(sockno, selectors.EVENT_WRITE) - - total_sent = 0 - # localize variable access to minimize overhead - selector_select = selector.select - os_sendfile = os.sendfile - try: - while True: - if timeout and not selector_select(timeout): - raise TimeoutError('timed out') - if count: - blocksize = min(count - total_sent, blocksize) - if blocksize <= 0: - break - try: - sent = os_sendfile(sockno, fileno, offset, blocksize) - except BlockingIOError: - if not timeout: - # Block until the socket is ready to send some - # data; avoids hogging CPU resources. - selector_select() - continue - except OSError as err: - if total_sent == 0: - # We can get here for different reasons, the main - # one being 'file' is not a regular mmap(2)-like - # file, in which case we'll fall back on using - # plain send(). - raise _GiveupOnSendfile(err) - raise err from None - else: - if sent == 0: - break # EOF - offset += sent - total_sent += sent - return total_sent - finally: - if total_sent > 0 and hasattr(file, 'seek'): - file.seek(offset) + total_sent = 0 + # localize variable access to minimize overhead + selector_select = selector.select + try: + while True: + if timeout and not selector_select(timeout): + raise TimeoutError('timed out') + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + break + try: + sent = zerocopy_func(fileno, offset, blocksize) + except BlockingIOError: + if not timeout: + # Block until the socket is ready to send some + # data; avoids hogging CPU resources. + selector_select() + continue + except OSError as err: + if total_sent == 0: + # We can get here for different reasons, the main + # one being 'file' is not a regular mmap(2)-like + # file, in which case we'll fall back on using + # plain send(). + raise giveup_exc_type(err) + raise err from None + else: + if sent == 0: + break # EOF + offset += sent + total_sent += sent + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset) + + if hasattr(os, 'sendfile'): + def _sendfile_use_sendfile(self, file, offset=0, count=None): + return self._sendfile_zerocopy( + partial(os.sendfile, self.fileno()), + _GiveupOnSendfile, + file, offset, count, + ) else: def _sendfile_use_sendfile(self, file, offset=0, count=None): raise _GiveupOnSendfile( diff --git a/Lib/ssl.py b/Lib/ssl.py index 7e3c4cbd6bbf8e..86fb8990636692 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -975,6 +975,10 @@ def _sslcopydoc(func): return func +class _GiveupOnSSLSendfile(Exception): + pass + + class SSLSocket(socket): """This class implements a subtype of socket.socket that wraps the underlying OS socket in an SSL context when necessary, and @@ -1266,15 +1270,26 @@ def sendall(self, data, flags=0): return super().sendall(data, flags) def sendfile(self, file, offset=0, count=None): - """Send a file, possibly by using os.sendfile() if this is a - clear-text socket. Return the total number of bytes sent. + """Send a file, possibly by using an efficient sendfile() call if + the system supports it. Return the total number of bytes sent. """ - if self._sslobj is not None: - return self._sendfile_use_send(file, offset, count) - else: - # os.sendfile() works with plain sockets only + if self._sslobj is None: return super().sendfile(file, offset, count) + if not self._sslobj.uses_ktls_for_send(): + return self._sendfile_use_send(file, offset, count) + + sendfile = getattr(self._sslobj, "sendfile", None) + if sendfile is None: + return self._sendfile_use_send(file, offset, count) + + try: + return self._sendfile_zerocopy( + sendfile, _GiveupOnSSLSendfile, file, offset, count, + ) + except _GiveupOnSSLSendfile: + return self._sendfile_use_send(file, offset, count) + def recv(self, buflen=1024, flags=0): self._checkClosed() if self._sslobj is not None: diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f123f6ece40669..9e519537ca5ed3 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4316,19 +4316,30 @@ def test_read_write_after_close_raises_valuerror(self): self.assertRaises(ValueError, s.write, b'hello') def test_sendfile(self): + """Try to send a file using kTLS if possible.""" TEST_DATA = b"x" * 512 with open(os_helper.TESTFN, 'wb') as f: f.write(TEST_DATA) self.addCleanup(os_helper.unlink, os_helper.TESTFN) client_context, server_context, hostname = testing_context() + client_context.options |= getattr(ssl, 'OP_ENABLE_KTLS', 0) server = ThreadedEchoServer(context=server_context, chatty=False) - with server: - with client_context.wrap_socket(socket.socket(), - server_hostname=hostname) as s: - s.connect((HOST, server.port)) + # kTLS seems to work only with a connection created before + # wrapping `sock` by the SSL context in contrast to calling + # `sock.connect()` after the wrapping. + with server, socket.create_connection((HOST, server.port)) as sock: + with client_context.wrap_socket( + sock, server_hostname=hostname + ) as ssock: + if support.verbose: + ktls_used = ssock._sslobj.uses_ktls_for_send() + print( + 'kTLS is', + 'available' if ktls_used else 'unavailable', + ) with open(os_helper.TESTFN, 'rb') as file: - s.sendfile(file) - self.assertEqual(s.recv(1024), TEST_DATA) + ssock.sendfile(file) + self.assertEqual(ssock.recv(1024), TEST_DATA) def test_session(self): client_context, server_context, hostname = testing_context() diff --git a/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst b/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst new file mode 100644 index 00000000000000..c511c63021442b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-13-22-51-40.gh-issue-99813.40TV02.rst @@ -0,0 +1,4 @@ +:mod:`ssl` now uses ``SSL_sendfile`` internally when it is possible (see +:data:`~ssl.OP_ENABLE_KTLS`). The function sends a file more efficiently +because it performs TLS encryption in the kernel to avoid additional context +switches. Patch by Illia Volochii. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 014e624f6c2f00..24c243e330d4bf 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -75,6 +75,33 @@ #endif +#ifdef BIO_get_ktls_send +# ifdef MS_WINDOWS +typedef long long Py_off_t; +# else +typedef off_t Py_off_t; +# endif + +static int +Py_off_t_converter(PyObject *arg, void *addr) +{ +#ifdef HAVE_LARGEFILE_SUPPORT + *((Py_off_t *)addr) = PyLong_AsLongLong(arg); +#else + *((Py_off_t *)addr) = PyLong_AsLong(arg); +#endif + return PyErr_Occurred() ? 0 : 1; +} + +/*[python input] + +class Py_off_t_converter(CConverter): + type = 'Py_off_t' + converter = 'Py_off_t_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=3fd9ca8ca6f0cbb8]*/ +#endif /* BIO_get_ktls_send */ struct py_ssl_error_code { const char *mnemonic; @@ -2442,6 +2469,184 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) return rc == 0 ? SOCKET_HAS_TIMED_OUT : SOCKET_OPERATION_OK; } +/*[clinic input] +@critical_section +_ssl._SSLSocket.uses_ktls_for_send + +Check if the Kernel TLS data-path is used for sending. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self) +/*[clinic end generated code: output=f9d95fbefceb5068 input=8d1ce4a131190e6b]*/ +{ +#ifdef BIO_get_ktls_send + int uses = BIO_get_ktls_send(SSL_get_wbio(self->ssl)); + // BIO_get_ktls_send() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +/*[clinic input] +@critical_section +_ssl._SSLSocket.uses_ktls_for_recv + +Check if the Kernel TLS data-path is used for receiving. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self) +/*[clinic end generated code: output=ce38b00317a1f681 input=a13778a924fc7d44]*/ +{ +#ifdef BIO_get_ktls_recv + int uses = BIO_get_ktls_recv(SSL_get_rbio(self->ssl)); + // BIO_get_ktls_recv() returns 1 if kTLS is used and 0 if not. + // Also, it returns -1 for failure before OpenSSL 3.0.4. + return Py_NewRef(uses == 1 ? Py_True : Py_False); +#else + Py_RETURN_FALSE; +#endif +} + +#ifdef BIO_get_ktls_send +/*[clinic input] +@critical_section +_ssl._SSLSocket.sendfile + fd: int + offset: Py_off_t + size: size_t + flags: int = 0 + / + +Write size bytes from offset in the file descriptor fd to the SSL connection. + +This method uses the zero-copy technique and returns the number of bytes +written. It should be called only when Kernel TLS is used for sending data in +the connection. + +The meaning of flags is platform dependent. +[clinic start generated code]*/ + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags) +/*[clinic end generated code: output=0c6815b0719ca8d5 input=dfc1b162bb020de1]*/ +{ + Py_ssize_t retval; + int sockstate; + _PySSLError err; + PySocketSockObject *sock = GET_SOCKET(self); + PyTime_t timeout, deadline = 0; + int has_timeout; + + if (sock != NULL) { + if ((PyObject *)sock == Py_None) { + _setSSLError(get_state_sock(self), + "Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); + /* just in case the blocking state of the socket has been changed */ + int nonblocking = (sock->sock_timeout >= 0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } + + timeout = GET_SOCKET_TIMEOUT(sock); + has_timeout = (timeout > 0); + if (has_timeout) { + deadline = _PyDeadline_Init(timeout); + } + + sockstate = PySSL_select(sock, 1, timeout); + switch (sockstate) { + case SOCKET_HAS_TIMED_OUT: + PyErr_SetString(PyExc_TimeoutError, + "The write operation timed out"); + goto error; + case SOCKET_HAS_BEEN_CLOSED: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + case SOCKET_TOO_LARGE_FOR_SELECT: + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket too large for select()."); + goto error; + } + + do { + PySSL_BEGIN_ALLOW_THREADS + retval = SSL_sendfile(self->ssl, fd, (off_t)offset, size, flags); + err = _PySSL_errno(retval < 0, self->ssl, (int)retval); + PySSL_END_ALLOW_THREADS + self->err = err; + + if (PyErr_CheckSignals()) { + goto error; + } + + if (has_timeout) { + timeout = _PyDeadline_Get(deadline); + } + + switch (err.ssl) { + case SSL_ERROR_WANT_READ: + sockstate = PySSL_select(sock, 0, timeout); + break; + case SSL_ERROR_WANT_WRITE: + sockstate = PySSL_select(sock, 1, timeout); + break; + default: + sockstate = SOCKET_OPERATION_OK; + break; + } + + if (sockstate == SOCKET_HAS_TIMED_OUT) { + PyErr_SetString(PyExc_TimeoutError, + "The sendfile operation timed out"); + goto error; + } + else if (sockstate == SOCKET_HAS_BEEN_CLOSED) { + PyErr_SetString(get_state_sock(self)->PySSLErrorObject, + "Underlying socket has been closed."); + goto error; + } + else if (sockstate == SOCKET_IS_NONBLOCKING) { + break; + } + } while (err.ssl == SSL_ERROR_WANT_READ + || err.ssl == SSL_ERROR_WANT_WRITE); + + if (err.ssl == SSL_ERROR_SSL + && ERR_GET_REASON(ERR_peek_error()) == SSL_R_UNINITIALIZED) + { + /* OpenSSL fails to return SSL_ERROR_SYSCALL if an error + * happens in sendfile(), and returns SSL_ERROR_SSL with + * SSL_R_UNINITIALIZED reason instead. */ + _setSSLError(get_state_sock(self), + "Some I/O error occurred in sendfile()", + PY_SSL_ERROR_SYSCALL, __FILE__, __LINE__); + goto error; + } + Py_XDECREF(sock); + if (retval < 0) { + return PySSL_SetError(self, __FILE__, __LINE__); + } + if (PySSL_ChainExceptions(self) < 0) { + return NULL; + } + return PyLong_FromSize_t(retval); +error: + Py_XDECREF(sock); + (void)PySSL_ChainExceptions(self); + return NULL; +} +#endif /* BIO_get_ktls_send */ + /*[clinic input] @critical_section _ssl._SSLSocket.write @@ -3017,6 +3222,9 @@ static PyGetSetDef ssl_getsetlist[] = { static PyMethodDef PySSLMethods[] = { _SSL__SSLSOCKET_DO_HANDSHAKE_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF + _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF + _SSL__SSLSOCKET_SENDFILE_METHODDEF _SSL__SSLSOCKET_WRITE_METHODDEF _SSL__SSLSOCKET_READ_METHODDEF _SSL__SSLSOCKET_PENDING_METHODDEF diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index c6e2abd4d93474..7027d87379283d 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_Size_t_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_ssl__SSLSocket_do_handshake__doc__, @@ -442,6 +443,115 @@ _ssl__SSLSocket_owner_set(PyObject *self, PyObject *value, void *Py_UNUSED(conte return return_value; } +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_send__doc__, +"uses_ktls_for_send($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for sending."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_SEND_METHODDEF \ + {"uses_ktls_for_send", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_send, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_send__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_send(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_send_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_ssl__SSLSocket_uses_ktls_for_recv__doc__, +"uses_ktls_for_recv($self, /)\n" +"--\n" +"\n" +"Check if the Kernel TLS data-path is used for receiving."); + +#define _SSL__SSLSOCKET_USES_KTLS_FOR_RECV_METHODDEF \ + {"uses_ktls_for_recv", (PyCFunction)_ssl__SSLSocket_uses_ktls_for_recv, METH_NOARGS, _ssl__SSLSocket_uses_ktls_for_recv__doc__}, + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self); + +static PyObject * +_ssl__SSLSocket_uses_ktls_for_recv(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_uses_ktls_for_recv_impl((PySSLSocket *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#if defined(BIO_get_ktls_send) + +PyDoc_STRVAR(_ssl__SSLSocket_sendfile__doc__, +"sendfile($self, fd, offset, size, flags=0, /)\n" +"--\n" +"\n" +"Write size bytes from offset in the file descriptor fd to the SSL connection.\n" +"\n" +"This method uses the zero-copy technique and returns the number of bytes\n" +"written. It should be called only when Kernel TLS is used for sending data in\n" +"the connection.\n" +"\n" +"The meaning of flags is platform dependent."); + +#define _SSL__SSLSOCKET_SENDFILE_METHODDEF \ + {"sendfile", _PyCFunction_CAST(_ssl__SSLSocket_sendfile), METH_FASTCALL, _ssl__SSLSocket_sendfile__doc__}, + +static PyObject * +_ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, + size_t size, int flags); + +static PyObject * +_ssl__SSLSocket_sendfile(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fd; + Py_off_t offset; + size_t size; + int flags = 0; + + if (!_PyArg_CheckPositional("sendfile", nargs, 3, 4)) { + goto exit; + } + fd = PyLong_AsInt(args[0]); + if (fd == -1 && PyErr_Occurred()) { + goto exit; + } + if (!Py_off_t_converter(args[1], &offset)) { + goto exit; + } + if (!_PyLong_Size_t_Converter(args[2], &size)) { + goto exit; + } + if (nargs < 4) { + goto skip_optional; + } + flags = PyLong_AsInt(args[3]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _ssl__SSLSocket_sendfile_impl((PySSLSocket *)self, fd, offset, size, flags); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* defined(BIO_get_ktls_send) */ + PyDoc_STRVAR(_ssl__SSLSocket_write__doc__, "write($self, b, /)\n" "--\n" @@ -2893,6 +3003,10 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #endif /* defined(_MSC_VER) */ +#ifndef _SSL__SSLSOCKET_SENDFILE_METHODDEF + #define _SSL__SSLSOCKET_SENDFILE_METHODDEF +#endif /* !defined(_SSL__SSLSOCKET_SENDFILE_METHODDEF) */ + #ifndef _SSL_ENUM_CERTIFICATES_METHODDEF #define _SSL_ENUM_CERTIFICATES_METHODDEF #endif /* !defined(_SSL_ENUM_CERTIFICATES_METHODDEF) */ @@ -2900,4 +3014,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=748650909fec8906 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1adc3780d8ca682a input=a9049054013a1b77]*/