Skip to content

Commit b5abd65

Browse files
mattp-mattp-
committed
PillarCache: reimplement using salt.cache
took the liberty of making it a proper subclass in the process. this now uses the salt.cache infrastructure such that it can be driven by the base cache driver or a different one if so desired. functionality should be equivalent, including using the base bank=pillar key=minion_id for merged pillar, such that minion_data_cache can take advantage of the same cache. because we are updating the cache at the source, we no longer need to update the cache in master/masterapi.
1 parent a4afc6b commit b5abd65

File tree

11 files changed

+287
-344
lines changed

11 files changed

+287
-344
lines changed

‎salt/cache/__init__.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,12 @@ def __init__(self, opts, cachedir=None, **kwargs):
6363
self.cachedir = opts.get("cachedir", salt.syspaths.CACHE_DIR)
6464
else:
6565
self.cachedir = cachedir
66-
self.driver = opts.get("cache", salt.config.DEFAULT_MASTER_OPTS["cache"])
66+
67+
if kwargs.get("driver"):
68+
self.driver = kwargs["driver"]
69+
else:
70+
self.driver = opts.get("cache", salt.config.DEFAULT_MASTER_OPTS["cache"])
71+
6772
self._modules = None
6873
self._kwargs = kwargs
6974
self._kwargs["cachedir"] = self.cachedir
@@ -350,7 +355,7 @@ def fetch(self, bank, key):
350355
# update atime and return
351356
record[0] = now
352357
self.storage[(bank, key)] = record
353-
return record[1]
358+
return record[2]
354359

355360
# Have no value for the key or value is expired
356361
data = super().fetch(bank, key)

‎salt/cache/localfs.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ def fetch(bank, key, cachedir):
7676
inkey = False
7777
key_file = salt.utils.path.join(cachedir, os.path.normpath(bank), f"{key}.p")
7878
if not os.path.isfile(key_file):
79+
log.debug('Cache file "%s" does not exist', key_file)
7980
# The bank includes the full filename, and the key is inside the file
8081
key_file = salt.utils.path.join(cachedir, os.path.normpath(bank) + ".p")
8182
inkey = True

‎salt/config/__init__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1016,6 +1016,8 @@ def _gather_buffer_space():
10161016
"signing_algorithm": str,
10171017
# Master publish channel signing
10181018
"publish_signing_algorithm": str,
1019+
# optional cache driver for pillar cache
1020+
"pillar.cache_driver": (type(None), str),
10191021
}
10201022
)
10211023

@@ -1325,6 +1327,7 @@ def _gather_buffer_space():
13251327
"features": {},
13261328
"encryption_algorithm": "OAEP-SHA1",
13271329
"signing_algorithm": "PKCS1v15-SHA1",
1330+
"pillar.cache_driver": None,
13281331
}
13291332
)
13301333

@@ -1679,6 +1682,7 @@ def _gather_buffer_space():
16791682
"cluster_pool_port": 4520,
16801683
"features": {},
16811684
"publish_signing_algorithm": "PKCS1v15-SHA1",
1685+
"pillar.cache_driver": None,
16821686
}
16831687
)
16841688

‎salt/daemons/masterapi.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -768,7 +768,6 @@ def _pillar(self, load):
768768
)
769769
data = pillar.compile_pillar()
770770
if self.opts.get("minion_data_cache", False):
771-
self.cache.store("pillar", load["id"], data)
772771
self.cache.store("grains", load["id"], load["grains"])
773772
if self.opts.get("minion_data_cache_events") is True:
774773
self.event.fire_event(

‎salt/master.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1789,7 +1789,6 @@ def _pillar(self, load):
17891789
self.fs_.update_opts()
17901790
if self.opts.get("minion_data_cache", False):
17911791
self.masterapi.cache.store("grains", load["id"], load["grains"])
1792-
self.masterapi.cache.store("pillar", load["id"], data)
17931792

17941793
if self.opts.get("minion_data_cache_events") is True:
17951794
self.event.fire_event(

‎salt/pillar/__init__.py

Lines changed: 96 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,25 @@
44

55
import collections
66
import copy
7+
import datetime
78
import fnmatch
89
import logging
9-
import os
1010
import sys
1111
import time
1212
import traceback
1313

1414
import tornado.gen
1515

16+
import salt.cache
1617
import salt.channel.client
1718
import salt.fileclient
1819
import salt.loader
1920
import salt.minion
2021
import salt.utils.args
21-
import salt.utils.cache
2222
import salt.utils.crypt
2323
import salt.utils.data
2424
import salt.utils.dictupdate
25+
import salt.utils.master
2526
import salt.utils.url
2627
from salt.exceptions import SaltClientError
2728
from salt.template import compile_template
@@ -393,144 +394,6 @@ def __del__(self):
393394
# pylint: enable=W1701
394395

395396

396-
class PillarCache:
397-
"""
398-
Return a cached pillar if it exists, otherwise cache it.
399-
400-
Pillar caches are structed in two diminensions: minion_id with a dict of
401-
saltenvs. Each saltenv contains a pillar dict
402-
403-
Example data structure:
404-
405-
```
406-
{'minion_1':
407-
{'base': {'pilar_key_1' 'pillar_val_1'}
408-
}
409-
"""
410-
411-
# TODO ABC?
412-
def __init__(
413-
self,
414-
opts,
415-
grains,
416-
minion_id,
417-
saltenv,
418-
ext=None,
419-
functions=None,
420-
pillar_override=None,
421-
pillarenv=None,
422-
extra_minion_data=None,
423-
clean_cache=False,
424-
):
425-
# Yes, we need all of these because we need to route to the Pillar object
426-
# if we have no cache. This is another refactor target.
427-
428-
# Go ahead and assign these because they may be needed later
429-
self.opts = opts
430-
self.grains = grains
431-
self.minion_id = minion_id
432-
self.ext = ext
433-
self.functions = functions
434-
self.pillar_override = pillar_override
435-
self.pillarenv = pillarenv
436-
self.clean_cache = clean_cache
437-
self.extra_minion_data = extra_minion_data
438-
439-
if saltenv is None:
440-
self.saltenv = "base"
441-
else:
442-
self.saltenv = saltenv
443-
444-
# Determine caching backend
445-
self.cache = salt.utils.cache.CacheFactory.factory(
446-
self.opts["pillar_cache_backend"],
447-
self.opts["pillar_cache_ttl"],
448-
minion_cache_path=self._minion_cache_path(minion_id),
449-
)
450-
451-
def _minion_cache_path(self, minion_id):
452-
"""
453-
Return the path to the cache file for the minion.
454-
455-
Used only for disk-based backends
456-
"""
457-
return os.path.join(self.opts["cachedir"], "pillar_cache", minion_id)
458-
459-
def fetch_pillar(self):
460-
"""
461-
In the event of a cache miss, we need to incur the overhead of caching
462-
a new pillar.
463-
"""
464-
log.debug("Pillar cache getting external pillar with ext: %s", self.ext)
465-
fresh_pillar = Pillar(
466-
self.opts,
467-
self.grains,
468-
self.minion_id,
469-
self.saltenv,
470-
ext=self.ext,
471-
functions=self.functions,
472-
pillar_override=None,
473-
pillarenv=self.pillarenv,
474-
extra_minion_data=self.extra_minion_data,
475-
)
476-
return fresh_pillar.compile_pillar()
477-
478-
def clear_pillar(self):
479-
"""
480-
Clear the cache
481-
"""
482-
self.cache.clear()
483-
484-
return True
485-
486-
def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs
487-
if self.clean_cache:
488-
self.clear_pillar()
489-
log.debug(
490-
"Scanning pillar cache for information about minion %s and pillarenv %s",
491-
self.minion_id,
492-
self.pillarenv,
493-
)
494-
if self.opts["pillar_cache_backend"] == "memory":
495-
cache_dict = self.cache
496-
else:
497-
cache_dict = self.cache._dict
498-
499-
log.debug("Scanning cache: %s", cache_dict)
500-
# Check the cache!
501-
if self.minion_id in self.cache: # Keyed by minion_id
502-
# TODO Compare grains, etc?
503-
if self.pillarenv in self.cache[self.minion_id]:
504-
# We have a cache hit! Send it back.
505-
log.debug(
506-
"Pillar cache hit for minion %s and pillarenv %s",
507-
self.minion_id,
508-
self.pillarenv,
509-
)
510-
return self.cache[self.minion_id][self.pillarenv]
511-
else:
512-
# We found the minion but not the env. Store it.
513-
fresh_pillar = self.fetch_pillar()
514-
515-
minion_cache = self.cache[self.minion_id]
516-
minion_cache[self.pillarenv] = fresh_pillar
517-
self.cache[self.minion_id] = minion_cache
518-
519-
log.debug(
520-
"Pillar cache miss for pillarenv %s for minion %s",
521-
self.pillarenv,
522-
self.minion_id,
523-
)
524-
return fresh_pillar
525-
else:
526-
# We haven't seen this minion yet in the cache. Store it.
527-
fresh_pillar = self.fetch_pillar()
528-
self.cache[self.minion_id] = {self.pillarenv: fresh_pillar}
529-
log.debug("Pillar cache miss for minion %s", self.minion_id)
530-
log.debug("Current pillar cache: %s", cache_dict) # FIXME hack!
531-
return fresh_pillar
532-
533-
534397
class Pillar:
535398
"""
536399
Read over the pillar top files and render the pillar data
@@ -1391,3 +1254,96 @@ class AsyncPillar(Pillar):
13911254
def compile_pillar(self, ext=True):
13921255
ret = super().compile_pillar(ext=ext)
13931256
raise tornado.gen.Return(ret)
1257+
1258+
1259+
class PillarCache(Pillar):
1260+
"""
1261+
Return a cached pillar if it exists, otherwise cache it.
1262+
1263+
Pillar caches are structed in two diminensions: minion_id with a dict of
1264+
saltenvs. Each saltenv contains a pillar dict
1265+
1266+
Example data structure:
1267+
1268+
```
1269+
{'minion_1':
1270+
{'base': {'pilar_key_1' 'pillar_val_1'}
1271+
}
1272+
"""
1273+
1274+
def __init__(
1275+
self,
1276+
*args,
1277+
clean_cache=False,
1278+
pillar_override=None,
1279+
**kwargs,
1280+
):
1281+
super().__init__(*args, **kwargs)
1282+
self.clean_cache = clean_cache
1283+
self.pillar_override = pillar_override
1284+
self.cache = salt.cache.factory(
1285+
self.opts,
1286+
driver=self.opts["pillar.cache_driver"],
1287+
expires=self.opts["pillar_cache_ttl"],
1288+
)
1289+
1290+
@property
1291+
def pillar_key(self):
1292+
if not self.opts["pillarenv"]:
1293+
return self.minion_id
1294+
else:
1295+
return f"{self.minion_id}.{self.opts['pillarenv']}"
1296+
1297+
def cached_pillar(self):
1298+
"""
1299+
Return the cached pillar if it exists, or None
1300+
"""
1301+
return self.cache.fetch("pillar", self.pillar_key)
1302+
1303+
def clear_pillar(self):
1304+
"""
1305+
Clea the pillar cache, if it exists
1306+
"""
1307+
return self.cache.flush("pillar", self.pillar_key)
1308+
1309+
def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs
1310+
# matching to consume the same dataset
1311+
if self.clean_cache:
1312+
self.clear_pillar()
1313+
log.debug(
1314+
"Scanning pillar cache for information about minion %s and pillarenv %s",
1315+
self.minion_id,
1316+
self.opts["pillarenv"],
1317+
)
1318+
1319+
# Check the cache!
1320+
pillar_data = self.cached_pillar()
1321+
if pillar_data:
1322+
log.debug(
1323+
"Pillar cache hit for minion %s and pillarenv %s",
1324+
self.minion_id,
1325+
self.opts["pillarenv"],
1326+
)
1327+
else:
1328+
# We found the minion but not the env. Store it.
1329+
log.debug(
1330+
"Pillar cache miss for pillarenv %s for minion %s",
1331+
self.opts["pillarenv"],
1332+
self.minion_id,
1333+
)
1334+
pillar_data = super().compile_pillar(*args, **kwargs)
1335+
1336+
self.cache.store("pillar", self.pillar_key, pillar_data)
1337+
1338+
# we dont want the pillar_override baked into the cached compile_pillar from above
1339+
if self.pillar_override:
1340+
pillar_data = merge(
1341+
pillar_data,
1342+
self.pillar_override,
1343+
self.opts.get("pillar_source_merging_strategy", "smart"),
1344+
self.opts.get("renderer", "yaml"),
1345+
self.opts.get("pillar_merge_lists", False),
1346+
)
1347+
pillar_data.update(self.pillar_override)
1348+
1349+
return pillar_data

0 commit comments

Comments
 (0)