Skip to content

Commit 672779f

Browse files
mistotebemistotebe
committed
Add OpenLDAPSyncreplCookie
Fixes: #562
1 parent a58282a commit 672779f

File tree

4 files changed

+252
-1
lines changed

4 files changed

+252
-1
lines changed

‎Doc/reference/ldap-syncrepl.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,6 @@ This module defines the following classes:
2020

2121
.. autoclass:: ldap.syncrepl.SyncreplConsumer
2222
:members:
23+
24+
.. autoclass:: ldap.syncrepl.OpenLDAPSyncreplCookie
25+
:members:

‎Lib/ldap/syncrepl.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
See https://www.python-ldap.org/ for project details.
55
"""
66

7+
from typing import AnyStr, Dict, List, Tuple, Union
78
from uuid import UUID
89

910
# Imports from pyasn1
@@ -535,3 +536,72 @@ def syncrepl_refreshdone(self):
535536
follows.
536537
"""
537538
pass
539+
540+
541+
class OpenLDAPSyncreplCookie:
542+
"""
543+
OpenLDAPSyncreplCookie - allows a consumer to track a cookie across a
544+
refreshAndPersist syncrepl session against a multi-provider OpenLDAP cluster
545+
"""
546+
547+
rid: int = 0
548+
sid: int = 0
549+
_csnset: Dict[int, str]
550+
551+
def __init__(self, cookie: AnyStr = "") -> None:
552+
self._csnset = {}
553+
554+
if cookie:
555+
self.update(cookie)
556+
557+
def _parse_csn(self, csn: str) -> Tuple[str, str, str, str]:
558+
time, order, sid, other = csn.split('#', 3)
559+
return (time, order, sid, other)
560+
561+
def _parse_cookie(self, cookie: AnyStr) -> Dict[str, Union[str, List[str]]]:
562+
if isinstance(cookie, bytes):
563+
cookie = cookie.decode()
564+
565+
result = {}
566+
parts = cookie.split(',')
567+
for part in parts:
568+
if part.startswith('rid='):
569+
result['rid'] = part[4:]
570+
elif part.startswith('sid='):
571+
result['sid'] = part[4:]
572+
elif part.startswith('csn='):
573+
result['csn'] = part[4:].split(';')
574+
elif part.startswith('delcsn='):
575+
result['delcsn'] = part[7:]
576+
else:
577+
# Did not recognise this cookie part
578+
pass
579+
return result
580+
581+
def update(self, cookie: str):
582+
"""
583+
Update the CSN set based on a cookie we just received, use in
584+
syncrepl_set_cookie() to track the session state.
585+
"""
586+
components = self._parse_cookie(cookie)
587+
for csn in components.get('csn', []):
588+
_, _, sid, _ = self._parse_csn(csn)
589+
if sid not in self._csnset or self._csnset[sid] < csn:
590+
self._csnset[sid] = csn
591+
592+
return self
593+
594+
def unparse(self) -> str:
595+
"""
596+
Return the cookie as a string, use in syncrepl_get_cookie() or when
597+
storing the state for later use.
598+
"""
599+
cookie = 'rid={:03},sid={:03x}'.format(self.rid or 0, self.sid or 0)
600+
if self._csnset:
601+
cookie += ',csn='
602+
cookie += ';'.join(csn
603+
for sid, csn in sorted(self._csnset.items()))
604+
return cookie
605+
606+
def __str__(self):
607+
return self.unparse()

‎Lib/slapdtest/_slapdtest.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,11 @@
4141
cn: module
4242
olcModuleLoad: back_%(database)s
4343
44+
dn: olcDatabase=config,cn=config
45+
objectClass: olcDatabaseConfig
46+
olcDatabase: config
47+
olcRootDN: %(rootdn)s
48+
4449
dn: olcDatabase=%(database)s,cn=config
4550
objectClass: olcDatabaseConfig
4651
objectClass: olcMdbConfig

‎Tests/t_ldap_syncrepl.py

Lines changed: 174 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313

1414
import ldap
1515
from ldap.ldapobject import SimpleLDAPObject
16-
from ldap.syncrepl import SyncreplConsumer, SyncInfoMessage
16+
from ldap.syncrepl import SyncreplConsumer, SyncInfoMessage, \
17+
OpenLDAPSyncreplCookie
1718

1819
from slapdtest import SlapdObject, SlapdTestCase
1920

@@ -37,6 +38,10 @@
3738
olcModuleLoad: back_%(database)s
3839
olcModuleLoad: syncprov
3940
41+
dn: olcDatabase=config,cn=config
42+
objectClass: olcDatabaseConfig
43+
olcRootDN: %(rootdn)s
44+
4045
dn: olcDatabase=%(database)s,cn=config
4146
objectClass: olcDatabaseConfig
4247
objectClass: olcMdbConfig
@@ -442,6 +447,174 @@ def setUp(self):
442447
self.suffix = self.server.suffix
443448

444449

450+
class TestMPRSyncrepl(BaseSyncreplTests, SlapdTestCase):
451+
class MPRClient(SyncreplClient):
452+
def __init__(self, *args, **kwargs):
453+
super().__init__(*args, **kwargs)
454+
self.cookie = OpenLDAPSyncreplCookie()
455+
456+
def syncrepl_set_cookie(self, cookie):
457+
self.cookie.update(cookie)
458+
super().syncrepl_set_cookie(self.cookie.unparse())
459+
460+
def setUp(self):
461+
super().setUp()
462+
self.tester = self.MPRClient(
463+
self.server.ldap_uri,
464+
self.server.root_dn,
465+
self.server.root_pw,
466+
bytes_mode=False
467+
)
468+
self.suffix = self.server.suffix
469+
470+
# An active MPR should not have a sid=000 server in it
471+
if self.server.server_id == 0:
472+
self.skip("Server got serverid 0 assigned")
473+
474+
def test_mpr_refresh_and_persist(self):
475+
"""
476+
Make sure we process cookie updates from a live MPR cluster correctly
477+
"""
478+
# Assumes that server_id is not used before the call to start()
479+
self.server2 = self.server_class()
480+
if self.server.server_id == self.server2.server_id:
481+
self.server2.server_id += 1
482+
if self.server2.server_id % 4096 == 0:
483+
self.server2.server_id = 1
484+
485+
with self.server2 as server2:
486+
tester2 = self.MPRClient(
487+
self.server2.ldap_uri,
488+
self.server2.root_dn,
489+
self.server2.root_pw,
490+
bytes_mode=False
491+
)
492+
self.addCleanup(tester2.unbind_s)
493+
494+
self.tester.search(
495+
self.suffix,
496+
'refreshAndPersist',
497+
)
498+
499+
# Run a quick refresh, that shouldn't have any changes.
500+
while self.tester.refresh_done is not True:
501+
poll_result = self.tester.poll(
502+
all=0,
503+
timeout=None
504+
)
505+
self.assertTrue(poll_result)
506+
507+
# Again, server data should not have changed.
508+
self.assertEqual(self.tester.dn_attrs, LDAP_ENTRIES)
509+
510+
# set up replication between both
511+
coords = [(1, self.server.ldap_uri, self.suffix,
512+
self.server.root_dn, self.server.root_pw),
513+
(2, self.server2.ldap_uri, self.suffix,
514+
self.server2.root_dn, self.server2.root_pw),
515+
]
516+
modifications = [
517+
(ldap.MOD_ADD, "olcSyncrepl", [
518+
('rid=%d provider=%s searchbase="%s" type=refreshAndPersist '
519+
'bindmethod=simple binddn="%s" credentials="%s" '
520+
'retry="1 +"' % coord).encode() for coord in coords]),
521+
# do we still support 2.4.x? Change to olcMultiProvider if not
522+
(ldap.MOD_REPLACE, "olcMirrorMode", [b"TRUE"]),
523+
]
524+
525+
self.tester.modify_s(
526+
"olcDatabase={1}%s,cn=config" % (self.server.database),
527+
modifications)
528+
tester2.modify_s(
529+
"olcDatabase={1}%s,cn=config" % (self.server.database),
530+
modifications)
531+
532+
tester2.search(
533+
self.suffix,
534+
'refreshAndPersist',
535+
)
536+
537+
# Wait till server2 catches up
538+
while tester2.refresh_done is not True or \
539+
tester2.cookie.unparse() != self.tester.cookie.unparse():
540+
try:
541+
poll_result = tester2.poll(
542+
all=0,
543+
timeout=None
544+
)
545+
self.assertTrue(poll_result)
546+
except ldap.NO_SUCH_OBJECT:
547+
# 2.6+ Allows a refreshAndPersist against an empty DB, but
548+
# with older ones we need to retry until there's at least
549+
# one entry
550+
tester2.search(
551+
self.suffix,
552+
'refreshAndPersist',
553+
)
554+
555+
# Again, server data should not have changed.
556+
self.assertEqual(tester2.dn_attrs, LDAP_ENTRIES)
557+
558+
# From here on, things get little hairy, server1 might not have
559+
# finished its refresh from 2 and we can't easily confirm this
560+
# without cn=monitor. We just read back our CSNs and make sure
561+
# we've seen both.
562+
563+
# send some mods to both
564+
modification = [('objectClass', [b'device'])]
565+
self.tester.add_s("cn=server1,%s" % self.suffix, modification)
566+
567+
csn1 = self.tester.read_s("cn=server1,%s" % self.suffix,
568+
attrlist=['entryCSN']
569+
)['entryCSN'][0].decode('utf8')
570+
571+
tester2.add_s("cn=server2,%s" % self.suffix, modification)
572+
csn2 = tester2.read_s("cn=server2,%s" % self.suffix,
573+
attrlist=['entryCSN']
574+
)['entryCSN'][0].decode('utf8')
575+
576+
new_state = LDAP_ENTRIES.copy()
577+
new_state["cn=server1,%s" % self.suffix] = {
578+
"objectClass": [b"device"],
579+
"cn": [b"server1"],
580+
}
581+
new_state["cn=server2,%s" % self.suffix] = {
582+
"objectClass": [b"device"],
583+
"cn": [b"server2"],
584+
}
585+
586+
# Wait for the cookie to sync up, a failure would be that this
587+
# doesn't happen, so impose a timeout
588+
while csn1 not in self.tester.cookie.unparse() or \
589+
csn2 not in self.tester.cookie.unparse() or \
590+
csn1 not in tester2.cookie.unparse() or \
591+
csn2 not in tester2.cookie.unparse():
592+
if csn1 not in self.tester.cookie.unparse() or \
593+
csn2 not in self.tester.cookie.unparse():
594+
poll_result = self.tester.poll(
595+
all=0,
596+
timeout=5
597+
)
598+
self.assertTrue(poll_result)
599+
if csn1 not in tester2.cookie.unparse() or \
600+
csn2 not in tester2.cookie.unparse():
601+
poll_result = tester2.poll(
602+
all=0,
603+
timeout=5
604+
)
605+
self.assertTrue(poll_result)
606+
607+
self.assertEqual(self.tester.cookie.unparse(),
608+
tester2.cookie.unparse())
609+
self.assertEqual(self.tester.dn_attrs, new_state)
610+
self.assertEqual(tester2.dn_attrs, new_state)
611+
612+
# self.tester seems to have been unbound by the time
613+
# self.addCleanup callbacks get called? Cleanup manually...
614+
self.tester.delete_s("cn=server1,%s" % self.suffix)
615+
self.tester.delete_s("cn=server2,%s" % self.suffix)
616+
617+
445618
class DecodeSyncreplProtoTests(unittest.TestCase):
446619
"""
447620
Tests of the ASN.1 decoder for tricky cases or past issues to ensure that

0 commit comments

Comments
 (0)