From efb76f75766e0af82fed3216d72f0edaee711db9 Mon Sep 17 00:00:00 2001 From: Florian Best Date: Fri, 13 Nov 2020 23:52:51 +0100 Subject: [PATCH 1/2] test: Implement test cases for reconnection handling test_106_reconnect_restore() handles a SERVER_DOWN exception manually and tries to re-use the connection afterwards again. This established the connection again but did not bind(), so it now raises ldap.INSUFFICIENT_ACCESS. test_107_reconnect_restore() restarts the LDAP server during searches, which causes a UNAVAILABLE exception. --- Lib/slapdtest/_slapdtest.py | 10 ++++- Tests/t_ldapobject.py | 77 ++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/Lib/slapdtest/_slapdtest.py b/Lib/slapdtest/_slapdtest.py index 36841110..4110d945 100644 --- a/Lib/slapdtest/_slapdtest.py +++ b/Lib/slapdtest/_slapdtest.py @@ -467,8 +467,16 @@ def restart(self): """ Restarts the slapd server with same data """ - self._proc.terminate() + self.terminate() self.wait() + self.resume() + + def terminate(self): + """Terminate slapd server""" + self._proc.terminate() + + def resume(self): + """Start slapd server""" self._start_slapd() def wait(self): diff --git a/Tests/t_ldapobject.py b/Tests/t_ldapobject.py index ccc7d218..75a07e62 100644 --- a/Tests/t_ldapobject.py +++ b/Tests/t_ldapobject.py @@ -9,9 +9,13 @@ import os import re import socket +import threading +import time +import traceback import unittest import pickle + # Switch off processing .ldaprc or ldap.conf before importing _ldap os.environ['LDAPNOINIT'] = '1' @@ -631,7 +635,7 @@ def test105_reconnect_restore(self): bind_dn = 'cn=user1,'+self.server.suffix l1.simple_bind_s(bind_dn, 'user1_pw') self.assertEqual(l1.whoami_s(), 'dn:'+bind_dn) - self.server._proc.terminate() + self.server.terminate() self.server.wait() try: l1.whoami_s() @@ -640,9 +644,78 @@ def test105_reconnect_restore(self): else: self.assertEqual(True, False) finally: - self.server._start_slapd() + self.server.resume() self.assertEqual(l1.whoami_s(), 'dn:'+bind_dn) + def test106_reconnect_restore(self): + """ + The idea of this test is to stop the LDAP server, make a search and ignore the `SERVER_DOWN` exception which happens after the reconnect timeout + and then re-use the same connection when the LDAP server is available again. + After starting the server the LDAP connection can be re-used again as it will reconnect on the next operation. + Prior to fixing PR !267 the connection was reestablished but no `bind()` was done resulting in a anonymous search which caused `INSUFFICIENT_ACCESS` when anonymous seach is disallowed. + """ + lo = self.ldap_object_class(self.server.ldap_uri, retry_max=2, retry_delay=1) + bind_dn = 'cn=user1,' + self.server.suffix + lo.simple_bind_s(bind_dn, 'user1_pw') + + dn = lo.whoami_s()[3:] + + self.server.terminate() + self.server.wait() + + # do a search, wait for the timeout, ignore SERVER_DOWN + with self.assertRaises(ldap.SERVER_DOWN): + lo.search_s(dn, ldap.SCOPE_BASE, '(objectClass=*)') + + self.server.resume() + + # try to use the connection again + lo.search_s(dn, ldap.SCOPE_BASE, '(objectClass=*)') + + def test107_reconnect_restore(self): + """ + The idea of this test is to restart the LDAP-Server while there are ongoing searches. + This causes :class:`ldap.UNAVAILABLE` to be raised (with |OpenLDAP|) for a short time. + To increase the chance of triggering this bug we are starting multiple threads + with a large number of retry attempts in a short amount of time. + """ + excs = [] + thread_count = 10 + run_time = 10.0 + start_barrier = threading.Barrier(thread_count + 1) # +1 for the main thread + + def _reconnect_search_thread(): + lo = self.ldap_object_class(self.server.ldap_uri) + bind_dn = 'cn=user1,' + self.server.suffix + lo.simple_bind_s(bind_dn, 'user1_pw') + lo._retry_max = 10E4 + lo._retry_delay = 0.001 + lo.search_ext_s(self.server.suffix, ldap.SCOPE_SUBTREE, "cn=user1", attrlist=["cn"]) + start_barrier.wait() + end_time = time.time() + run_time + while time.time() < end_time: + lo.search_ext_s(self.server.suffix, ldap.SCOPE_SUBTREE, filterstr="cn=user1", attrlist=["cn"]) + + def reconnect_search_thread(): + try: + _reconnect_search_thread() + except Exception as exc: + excs.append((str(exc), traceback.format_exc())) + + threads = [threading.Thread(target=reconnect_search_thread) for _ in range(thread_count)] + for t in threads: + t.start() + + start_barrier.wait() # wait until all threads are ready to start + self.server.restart() # restart after all threads have started their search loop + + for t in threads: + t.join() + + for exc, tb in excs[:5]: + print('Exception occurred', exc, tb) + self.assertEqual(excs, []) + @requires_init_fd() class Test03_SimpleLDAPObjectWithFileno(Test00_SimpleLDAPObject): From 3e9320accae9f6371ead2f27ec5fd6ac0c29e5b2 Mon Sep 17 00:00:00 2001 From: Florian Best Date: Sat, 16 Mar 2019 19:00:51 +0100 Subject: [PATCH 2/2] fix(ReconnectLDAPObject): Also reconnect on UNAVILABLE, CONNECT_ERROR and TIMEOUT --- Lib/ldap/ldapobject.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/ldap/ldapobject.py b/Lib/ldap/ldapobject.py index 7a9c17f6..eab2552c 100644 --- a/Lib/ldap/ldapobject.py +++ b/Lib/ldap/ldapobject.py @@ -820,8 +820,7 @@ def get_naming_contexts(self): class ReconnectLDAPObject(SimpleLDAPObject): """ :py:class:`SimpleLDAPObject` subclass whose synchronous request methods - automatically reconnect and re-try in case of server failure - (:exc:`ldap.SERVER_DOWN`). + automatically reconnect and re-try in case of server failure. The first arguments are same as for the :py:func:`~ldap.initialize()` function. @@ -833,6 +832,10 @@ class ReconnectLDAPObject(SimpleLDAPObject): * retry_delay: specifies the time in seconds between reconnect attempts. This class also implements the pickle protocol. + + .. versionadded:: 3.5 + The exceptions :py:exc:`ldap.SERVER_DOWN`, :py:exc:`ldap.UNAVAILABLE`, :py:exc:`ldap.CONNECT_ERROR` and + :py:exc:`ldap.TIMEOUT` (configurable via :py:attr:`_reconnect_exceptions`) now trigger a reconnect. """ __transient_attrs__ = { @@ -842,6 +845,7 @@ class ReconnectLDAPObject(SimpleLDAPObject): '_reconnect_lock', '_last_bind', } + _reconnect_exceptions = (ldap.SERVER_DOWN, ldap.UNAVAILABLE, ldap.CONNECT_ERROR, ldap.TIMEOUT) def __init__( self,uri, @@ -970,7 +974,7 @@ def _apply_method_s(self,func,*args,**kwargs): self.reconnect(self._uri,retry_max=self._retry_max,retry_delay=self._retry_delay,force=False) try: return func(self,*args,**kwargs) - except ldap.SERVER_DOWN: + except self._reconnect_exceptions: # Try to reconnect self.reconnect(self._uri,retry_max=self._retry_max,retry_delay=self._retry_delay,force=True) # Re-try last operation