From 84a1d5e89954c78f7d906b9edec336b8cc6bae00 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 19 Jul 2025 18:31:37 -0700 Subject: [PATCH 01/71] bad key logs useful message instead of traceback When the minion's key is overwritten with bad data, log a useful message instead of traceback. Handle the error in a consistant way accross salt.minion, salt.channel.client, and salt.crypt. --- changelog/68190.fixed.md | 1 + salt/channel/client.py | 6 ++++- salt/crypt.py | 13 ++++++--- tests/pytests/unit/channel/test_client.py | 33 +++++++++++++++++++++++ tests/pytests/unit/test_crypt.py | 8 ++++++ 5 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 changelog/68190.fixed.md diff --git a/changelog/68190.fixed.md b/changelog/68190.fixed.md new file mode 100644 index 000000000000..66fc21bc8d62 --- /dev/null +++ b/changelog/68190.fixed.md @@ -0,0 +1 @@ +Log a useful error if the minion's key is overwritten with bad data; instead of a traceback. diff --git a/salt/channel/client.py b/salt/channel/client.py index 432d1e1e33c5..27f71f4d06ed 100644 --- a/salt/channel/client.py +++ b/salt/channel/client.py @@ -426,7 +426,11 @@ def __init__(self, opts, transport, auth, io_loop=None): self.opts = opts self.io_loop = io_loop self.auth = auth - self.token = self.auth.gen_token(b"salt") + try: + # This loads or generates the minion's public key. + self.token = self.auth.gen_token(b"salt") + except salt.exceptions.InvalidKeyError as exc: + raise salt.exceptions.SaltClientError(str(exc)) self.transport = transport self._closing = False self._reconnected = False diff --git a/salt/crypt.py b/salt/crypt.py index 4923069e216c..8ab34951347e 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -332,10 +332,15 @@ def _get_key_with_evict(path, timestamp, passphrase): else: password = None with salt.utils.files.fopen(path, "rb") as f: - return serialization.load_pem_private_key( - f.read(), - password=password, - ) + try: + return serialization.load_pem_private_key( + f.read(), + password=password, + ) + except ValueError: + raise InvalidKeyError("Encountered bad RSA public key") + except cryptography.exceptions.UnsupportedAlgorithm: + raise InvalidKeyError("Unsupported key algorithm") def get_rsa_key(path, passphrase): diff --git a/tests/pytests/unit/channel/test_client.py b/tests/pytests/unit/channel/test_client.py index 783657c4a45e..b276e73a8385 100644 --- a/tests/pytests/unit/channel/test_client.py +++ b/tests/pytests/unit/channel/test_client.py @@ -1,4 +1,7 @@ +import pytest + import salt.channel.client +import salt.exceptions def test_async_methods(): @@ -17,3 +20,33 @@ def test_async_methods(): assert isinstance(getattr(cls, attr), list) for name in getattr(cls, attr): assert hasattr(cls, name) + + +def test_async_pub_channel_key_overwritten_by_bad_data(minion_opts, tmp_path): + """ + Ensure AsyncPubChannel raises a SaltClientError when it encounters a bad key. + + This bug is a bit nuanced because of how the auth module uses singletons. + We're validating an error from salt.crypt.AsyncAuth.gen_token because of + bad key data results in a SaltClientError. This error is handled by + Minion._connect_minion resulting error message explaining the minion + connection failed due to bad key data. + + https://github.com/saltstack/salt/issues/68190 + """ + minion_opts["pki_dir"] = str(tmp_path) + minion_opts["id"] = "minion" + minion_opts["master_ip"] = "127.0.0.1" + + # This will initialize the singleton with a valid key. + salt.channel.client.AsyncPubChannel.factory(minion_opts, crypt="aes") + + # Now we need to overwrite the bad key with the new one. When gen_token + # gets called a SaltClientError will ge traised + key_path = tmp_path / "minion.pem" + key_path.chmod(0o660) + key_path.write_text( + "asdfiosjaoiasdfjooaisjdfo902j0ianosdifn091091jw0edw09jcr89eq79vr" + ) + with pytest.raises(salt.exceptions.SaltClientError): + salt.channel.client.AsyncPubChannel.factory(minion_opts, crypt="aes") diff --git a/tests/pytests/unit/test_crypt.py b/tests/pytests/unit/test_crypt.py index 9a6cfc1e40fe..71dd4dea4a8d 100644 --- a/tests/pytests/unit/test_crypt.py +++ b/tests/pytests/unit/test_crypt.py @@ -1,6 +1,7 @@ import pytest import salt.crypt as crypt +import salt.exceptions @pytest.fixture @@ -180,3 +181,10 @@ def mock_sign_in(*args, **kwargs): assert isinstance(auth._creds, dict) assert auth._creds["aes"] == aes1 assert auth._creds["session"] == session1 + + +def test_get_key_with_evict_bad_key(tmp_path): + key_path = tmp_path / "key" + key_path.write_text("asdfasoiasdofaoiu0923jnoiausbd98sb9") + with pytest.raises(salt.exceptions.InvalidKeyError): + crypt._get_key_with_evict(str(key_path), 1, None) From 11ca7d722cbacfcdf3a8d4a5b44950f7c8a7bf1f Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 22 Jul 2025 09:57:11 -0600 Subject: [PATCH 02/71] Add force option to win_pkg --- changelog/68102.added.md | 2 ++ salt/modules/win_pkg.py | 33 ++++++++++++++-------- tests/pytests/unit/modules/test_win_pkg.py | 23 +++++++++++++++ 3 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 changelog/68102.added.md diff --git a/changelog/68102.added.md b/changelog/68102.added.md new file mode 100644 index 000000000000..d13160706c37 --- /dev/null +++ b/changelog/68102.added.md @@ -0,0 +1,2 @@ +Added the ability to pass for to pkg.install on Windows to force run the +installer diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index d4bee486e0d6..afafd9eb3e4e 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1471,6 +1471,13 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): .. versionadded:: 2016.11.0 + force (bool): + If ``True``, the installation will run whether the package is + already installed or not. If ``False``, the installation will not + run if the correct version of the package is already installed. + + .. versionadded:: 3006.15 + Returns: dict: Return a dict containing the new package names and versions. If the package is already installed, an empty dict is returned. @@ -1603,12 +1610,13 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # If the version was not passed, version_num will be None if not version_num: if pkg_name in old: - log.debug( - "pkg.install: '%s' version '%s' is already installed", - pkg_name, - old[pkg_name][0], - ) - continue + if not kwargs.get("force", False): + log.debug( + "pkg.install: '%s' version '%s' is already installed", + pkg_name, + old[pkg_name][0], + ) + continue # Get the most recent version number available from winrepo.p # May also return `latest` or an empty string version_num = _get_latest_pkg_version(pkginfo) @@ -1621,12 +1629,13 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # Check if the version is already installed if version_num in old.get(pkg_name, []): # Desired version number already installed - log.debug( - "pkg.install: '%s' version '%s' is already installed", - pkg_name, - version_num, - ) - continue + if not kwargs.get("force", False): + log.debug( + "pkg.install: '%s' version '%s' is already installed", + pkg_name, + version_num, + ) + continue # If version number not installed, is the version available? elif version_num != "latest" and version_num not in pkginfo: log.error("Version %s not found for package %s", version_num, pkg_name) diff --git a/tests/pytests/unit/modules/test_win_pkg.py b/tests/pytests/unit/modules/test_win_pkg.py index dd35d8f6521c..b056a13a8ffd 100644 --- a/tests/pytests/unit/modules/test_win_pkg.py +++ b/tests/pytests/unit/modules/test_win_pkg.py @@ -193,6 +193,29 @@ def test_pkg_install_existing(): assert expected == result +def test_pkg_install_existing_force_true(): + """ + test pkg.install when the package is already installed + and force=True + """ + ret_reg = {"Nullsoft Install System": "3.03"} + # The 2nd time it's run, pkg.list_pkgs uses with stringify + se_list_pkgs = {"nsis": ["3.03"]} + with patch.object(win_pkg, "list_pkgs", return_value=se_list_pkgs), patch.object( + win_pkg, "_get_reg_software", return_value=ret_reg + ), patch.dict( + win_pkg.__salt__, + { + "cmd.run_all": MagicMock(return_value={"retcode": 0}), + "cp.cache_file": MagicMock(return_value="C:\\fake\\path.exe"), + "cp.is_cached": MagicMock(return_value=True), + }, + ): + expected = {"nsis": {"install status": "success"}} + result = win_pkg.install(name="nsis", force=True) + assert expected == result + + def test_pkg_install_latest(): """ test pkg.install when the package is already installed From 264a6ca01effba78e537f7414510fdd43e33ef28 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 29 Jul 2025 14:26:27 -0600 Subject: [PATCH 03/71] Fixed some docs issues --- changelog/68102.added.md | 4 ++-- salt/modules/win_pkg.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog/68102.added.md b/changelog/68102.added.md index d13160706c37..648a906edab6 100644 --- a/changelog/68102.added.md +++ b/changelog/68102.added.md @@ -1,2 +1,2 @@ -Added the ability to pass for to pkg.install on Windows to force run the -installer +Added a new `force` option to pkg.install on Windows to force the installer +to run even if the package is already installed diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index afafd9eb3e4e..cd7cb54a81bf 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -1475,6 +1475,7 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): If ``True``, the installation will run whether the package is already installed or not. If ``False``, the installation will not run if the correct version of the package is already installed. + Default is ``False``. .. versionadded:: 3006.15 From 5c8df36cbeb5f1f443a0b021aeee5be14fd7c250 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Wed, 23 Jul 2025 18:11:00 +0100 Subject: [PATCH 04/71] Allow events to be processed in minion shutdown Allows event to be handled after the salt-minion service has received a SIGTERM. Previously once the signal handler was entered, the ioloop would no longer run. If there are events on the minion event bus that needs processing, they would not be handled. Moves the MinionManager stop() functionality to an async function and allows the ioloop to run and clear any waiting events and returns to masters. --- salt/cli/daemons.py | 9 ++++++--- salt/minion.py | 24 +++++++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py index 7578e8e64c3e..aca5e18ce076 100644 --- a/salt/cli/daemons.py +++ b/salt/cli/daemons.py @@ -225,8 +225,11 @@ class Minion( def _handle_signals(self, signum, sigframe): # pylint: disable=unused-argument # escalate signal to the process manager processes if hasattr(self.minion, "stop"): - self.minion.stop(signum) - super()._handle_signals(signum, sigframe) + # If the minion has a stop method, call it - this is the case for + # MinionManager + self.minion.stop(signum, super()._handle_signals) + else: + super()._handle_signals(signum, sigframe) # pylint: disable=no-member def prepare(self): @@ -405,7 +408,7 @@ class ProxyMinion( def _handle_signals(self, signum, sigframe): # pylint: disable=unused-argument # escalate signal to the process manager processes - self.minion.stop(signum) + self.minion.stop(signum, super()._handle_signals) super()._handle_signals(signum, sigframe) # pylint: disable=no-member diff --git a/salt/minion.py b/salt/minion.py index 595249dc44b0..0103d61ac44b 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1213,7 +1213,26 @@ def restart(self): return True return False - def stop(self, signum): + def stop(self, signum, parent_sig_handler): + """ + Stop minions managed by the MinionManager + + Called from cli.daemons.Minion._handle_signals(). + Adds stop_async as callback to the io_loop to prevent blocking. + """ + self.io_loop.add_callback(self.stop_async, signum, parent_sig_handler) + + @salt.ext.tornado.gen.coroutine + def stop_async(self, signum, parent_sig_handler): + """ + Stop minions managed by the MinionManager allowing the io_loop to run + and any remaining events to be processed before stopping the minions. + """ + + # Sleep to allow any remaining events to be processed + yield salt.ext.tornado.gen.sleep(5) + + # Continue to stop the minions for minion in self.minions: minion.process_manager.stop_restarting() minion.process_manager.send_signal_to_processes(signum) @@ -1227,6 +1246,9 @@ def stop(self, signum): self.event.destroy() self.event = None + # Call the parent signal handler + parent_sig_handler(signum, None) + def destroy(self): for minion in self.minions: minion.destroy() From 3ed3496075751319a2c9cd2c360f5901307fa0b2 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 24 Jul 2025 15:57:28 +0100 Subject: [PATCH 05/71] Add test for MinionManager stop/stop_async Adds test for calling MinionManager stop() to test new functionality to allow events to be processed as minion is stopping. Sets up a MinionManager instance with a running event bus, then calls the stop function and immeadiately sends a test message on the event bus and reads it back to check that works once the stop() function has been called. Then checks that the usual functions to destroy the minion etc have also been called. --- tests/pytests/unit/conftest.py | 1 + tests/pytests/unit/test_minion.py | 53 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index 965af00e89dd..dc2cd98dc2d1 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -27,6 +27,7 @@ def minion_opts(tmp_path): opts["fips_mode"] = FIPS_TESTRUN opts["encryption_algorithm"] = "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1" opts["signing_algorithm"] = "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1" + opts["sock_dir"] = tmp_path / "sock" return opts diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py index 7587ded5a28c..22548083d33c 100644 --- a/tests/pytests/unit/test_minion.py +++ b/tests/pytests/unit/test_minion.py @@ -1,6 +1,8 @@ import copy import logging import os +import signal +import time import uuid import pytest @@ -1167,3 +1169,54 @@ async def test_connect_master_general_exception_error(minion_opts, connect_maste # The first call raised an error which caused minion.destroy to get called, # the second call is a success. assert minion.connect_master.calls == 2 + + +async def test_minion_manager_async_stop(io_loop, minion_opts): + """ + Ensure MinionManager's stop method works correctly and calls the + stop_async method + """ + mm = salt.minion.MinionManager(minion_opts) + minion = MagicMock(name="minion") + parent_signal_handler = MagicMock(name="parent_signal_handler") + mm.minions.append(minion) + + # Set up event publisher and event + mm._bind() + assert mm.event_publisher is not None + assert mm.event is not None + + # Check io_loop is running + assert mm.io_loop._running + + # Set up values for event to send + load = {"key": "value"} + ret = {} + + # Connect to minion event bus + with salt.utils.event.get_event("minion", opts=minion_opts, listen=True) as event: + + # call stop to start stopping the minion + # mm.stop(signal.SIGTERM, parent_signal_handler) + mm.stop(signal.SIGTERM, parent_signal_handler) + + # Fire an event and ensure we can still read it back while the minion + # is stopping + await event.fire_event_async(load, "test_event", timeout=1) + start = time.time() + while time.time() - start < 5: + ret = event.get_event(tag="test_event", wait=0.3) + if ret: + break + await salt.ext.tornado.gen.sleep(0.3) + assert "key" in ret + assert ret["key"] == "value" + + # Sleep to allow stop_async to complete + await salt.ext.tornado.gen.sleep(5) + + # Ensure stop_async has been called + minion.destroy.assert_called_once() + parent_signal_handler.assert_called_once_with(signal.SIGTERM, None) + assert mm.event_publisher is None + assert mm.event is None From 97f143a3a8e58ca801a5284b17775f424a0c56ce Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 24 Jul 2025 16:12:12 +0100 Subject: [PATCH 06/71] Add changelog --- changelog/68183.fixed.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/68183.fixed.md diff --git a/changelog/68183.fixed.md b/changelog/68183.fixed.md new file mode 100644 index 000000000000..ccea84221761 --- /dev/null +++ b/changelog/68183.fixed.md @@ -0,0 +1,2 @@ +Fixed MinionManager.stop() to allow processing of minion event bus when called, to allow jobs returns from `service.restart salt-minion no_block=True` to reach +master. From 90dc05768c1db601ea1d4e8aedda149885cb9793 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 24 Jul 2025 21:46:09 +0100 Subject: [PATCH 07/71] Ignore pylint warning on self.io_loop.add_callback Ignores warning from pylint about self.io_loop.add_callback() not being callable - it clearly is as stop_async gets called. --- salt/minion.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/minion.py b/salt/minion.py index 0103d61ac44b..91a26dfc3169 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1220,7 +1220,9 @@ def stop(self, signum, parent_sig_handler): Called from cli.daemons.Minion._handle_signals(). Adds stop_async as callback to the io_loop to prevent blocking. """ - self.io_loop.add_callback(self.stop_async, signum, parent_sig_handler) + self.io_loop.add_callback( # pylint: disable=not-callable + self.stop_async, signum, parent_sig_handler + ) @salt.ext.tornado.gen.coroutine def stop_async(self, signum, parent_sig_handler): From 6d0af4b58ef53f1b865946c3e65a0b95efe366e7 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Fri, 25 Jul 2025 11:51:42 +0100 Subject: [PATCH 08/71] Set up short sock_dir path in test Sets up the short sock_dir path in the test. Previously setting it in conftest.py was breaking another test because I'd done it as a Path, not a string, but I want to avoid changin behaviour of other tests, so setting it locally to the test. --- tests/pytests/unit/conftest.py | 1 - tests/pytests/unit/test_minion.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/pytests/unit/conftest.py b/tests/pytests/unit/conftest.py index dc2cd98dc2d1..965af00e89dd 100644 --- a/tests/pytests/unit/conftest.py +++ b/tests/pytests/unit/conftest.py @@ -27,7 +27,6 @@ def minion_opts(tmp_path): opts["fips_mode"] = FIPS_TESTRUN opts["encryption_algorithm"] = "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1" opts["signing_algorithm"] = "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1" - opts["sock_dir"] = tmp_path / "sock" return opts diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py index 22548083d33c..53fc2fc19025 100644 --- a/tests/pytests/unit/test_minion.py +++ b/tests/pytests/unit/test_minion.py @@ -1171,11 +1171,16 @@ async def test_connect_master_general_exception_error(minion_opts, connect_maste assert minion.connect_master.calls == 2 -async def test_minion_manager_async_stop(io_loop, minion_opts): +async def test_minion_manager_async_stop(io_loop, minion_opts, tmp_path): """ Ensure MinionManager's stop method works correctly and calls the stop_async method """ + + # Setup sock_dir with short path + minion_opts["sock_dir"] = str(tmp_path / "sock") + + # Create a MinionManager instance with a mock minion mm = salt.minion.MinionManager(minion_opts) minion = MagicMock(name="minion") parent_signal_handler = MagicMock(name="parent_signal_handler") From 5bde4c707418db560bd7bd26a051bd0a4a37587c Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 17 Jul 2025 15:05:23 -0600 Subject: [PATCH 09/71] Don't write user registry with win_lgpo_reg --- changelog/68191.fixed.md | 3 + salt/modules/win_lgpo_reg.py | 61 ++++++++++++------- .../pytests/unit/modules/test_win_lgpo_reg.py | 45 ++++++-------- 3 files changed, 60 insertions(+), 49 deletions(-) create mode 100644 changelog/68191.fixed.md diff --git a/changelog/68191.fixed.md b/changelog/68191.fixed.md new file mode 100644 index 000000000000..764a76230cf8 --- /dev/null +++ b/changelog/68191.fixed.md @@ -0,0 +1,3 @@ +win_lgpo_reg only applies user settings to the registry.pol file. It no longer +applies those same settings to the user registry. Those settings will be applied +to all users the next time they log in. diff --git a/salt/modules/win_lgpo_reg.py b/salt/modules/win_lgpo_reg.py index 6789b034a888..3a706b6e8a1e 100644 --- a/salt/modules/win_lgpo_reg.py +++ b/salt/modules/win_lgpo_reg.py @@ -391,15 +391,20 @@ def set_value( log.error("LGPO_REG Mod: Failed to write registry.pol file") success = False - if not salt.utils.win_reg.set_value( - hive=hive, - key=key, - vname=v_name, - vdata=v_data, - vtype=v_type, - ): - log.error("LGPO_REG Mod: Failed to set registry entry") - success = False + # We only want to modify the actual registry value if this is machine policy + # The user policy will be applied by the user registry.pol when the user + # logs in. Setting it here only sets it on the user running the salt minion, + # most likely SYSTEM, which doesn't make sense here + if policy_class == "Machine": + if not salt.utils.win_reg.set_value( + hive=hive, + key=key, + vname=v_name, + vdata=v_data, + vtype=v_type, + ): + log.error("LGPO_REG Mod: Failed to set registry entry") + success = False return success @@ -486,13 +491,18 @@ def disable_value(key, v_name, policy_class="machine"): log.error("LGPO_REG Mod: Failed to write registry.pol file") success = False - ret = salt.utils.win_reg.delete_value(hive=hive, key=key, vname=v_name) - if not ret: - if ret is None: - log.debug("LGPO_REG Mod: Registry key/value already missing") - else: - log.error("LGPO_REG Mod: Failed to remove registry entry") - success = False + # We only want to modify the actual registry value if this is machine policy + # The user policy will be applied by the user registry.pol when the user + # logs in. Setting it here only sets it on the user running the salt minion, + # most likely SYSTEM, which doesn't make sense here + if policy_class == "Machine": + ret = salt.utils.win_reg.delete_value(hive=hive, key=key, vname=v_name) + if not ret: + if ret is None: + log.debug("LGPO_REG Mod: Registry key/value already missing") + else: + log.error("LGPO_REG Mod: Failed to remove registry entry") + success = False return success @@ -573,13 +583,18 @@ def delete_value(key, v_name, policy_class="Machine"): log.error("LGPO_REG Mod: Failed to write registry.pol file") success = False - ret = salt.utils.win_reg.delete_value(hive=hive, key=key, vname=v_name) - if not ret: - if ret is None: - log.debug("LGPO_REG Mod: Registry key/value already missing") - else: - log.error("LGPO_REG Mod: Failed to remove registry entry") - success = False + # We only want to modify the actual registry value if this is machine policy + # The user policy will be applied by the user registry.pol when the user + # logs in. Setting it here only sets it on the user running the salt minion, + # most likely SYSTEM, which doesn't make sense here + if policy_class == "Machine": + ret = salt.utils.win_reg.delete_value(hive=hive, key=key, vname=v_name) + if not ret: + if ret is None: + log.debug("LGPO_REG Mod: Registry key/value already missing") + else: + log.error("LGPO_REG Mod: Failed to remove registry entry") + success = False return success diff --git a/tests/pytests/unit/modules/test_win_lgpo_reg.py b/tests/pytests/unit/modules/test_win_lgpo_reg.py index 04284ee2727e..40bf806c21c8 100644 --- a/tests/pytests/unit/modules/test_win_lgpo_reg.py +++ b/tests/pytests/unit/modules/test_win_lgpo_reg.py @@ -135,6 +135,9 @@ def reg_pol_user(): vdata="squidward", vtype="REG_SZ", ) + assert salt.utils.win_reg.value_exists( + hive="HKCU", key="SOFTWARE\\MyKey1", vname="MyValue1" + ) salt.utils.win_reg.set_value( hive="HKCU", key="SOFTWARE\\MyKey2", @@ -142,9 +145,14 @@ def reg_pol_user(): vdata=["spongebob", "squarepants"], vtype="REG_MULTI_SZ", ) + assert salt.utils.win_reg.value_exists( + hive="HKCU", key="SOFTWARE\\MyKey2", vname="MyValue3" + ) yield salt.utils.win_reg.delete_key_recursive(hive="HKCU", key="SOFTWARE\\MyKey1") salt.utils.win_reg.delete_key_recursive(hive="HKCU", key="SOFTWARE\\MyKey2") + assert not salt.utils.win_reg.key_exists(hive="HKCU", key="SOFTWARE\\MyKey1") + assert not salt.utils.win_reg.key_exists(hive="HKCU", key="SOFTWARE\\MyKey2") class_info = salt.utils.win_lgpo_reg.CLASS_INFO reg_pol_file = class_info["User"]["policy_path"] with salt.utils.files.fopen(reg_pol_file, "wb") as f: @@ -444,17 +452,9 @@ def test_user_set_value(empty_reg_pol_user): expected = {"data": 1, "type": "REG_DWORD"} result = lgpo_reg.get_value(key=key, v_name=v_name, policy_class="User") assert result == expected - # Test that the registry value has been set - expected = { - "hive": "HKCU", - "key": key, - "vname": v_name, - "vdata": 1, - "vtype": "REG_DWORD", - "success": True, - } - result = salt.utils.win_reg.read_value(hive="HKCU", key=key, vname=v_name) - assert result == expected + # Test that the registry value has not been set + result = salt.utils.win_reg.value_exists(hive="HKCU", key=key, vname=v_name) + assert result is False def test_user_set_value_existing_change(reg_pol_user): @@ -464,16 +464,9 @@ def test_user_set_value_existing_change(reg_pol_user): lgpo_reg.set_value(key=key, v_name=v_name, v_data="1", policy_class="User") result = lgpo_reg.get_value(key=key, v_name=v_name, policy_class="User") assert result == expected - expected = { - "hive": "HKCU", - "key": key, - "vname": v_name, - "vdata": 1, - "vtype": "REG_DWORD", - "success": True, - } - result = salt.utils.win_reg.read_value(hive="HKCU", key=key, vname=v_name) - assert result == expected + # Test that the registry value has not been set + result = salt.utils.win_reg.value_exists(hive="HKCU", key=key, vname=v_name) + assert result is False def test_user_set_value_existing_no_change(reg_pol_user): @@ -503,9 +496,9 @@ def test_user_disable_value(reg_pol_user): } result = lgpo_reg.get_key(key=key, policy_class="User") assert result == expected - # Test that the registry value has been removed + # Test that the registry value has not been removed result = salt.utils.win_reg.value_exists(hive="HKCU", key=key, vname="MyValue1") - assert result is False + assert result is True def test_user_disable_value_no_change(reg_pol_user): @@ -533,9 +526,9 @@ def test_user_delete_value_existing(reg_pol_user): } result = lgpo_reg.get_key(key=key, policy_class="User") assert result == expected - # Test that the registry entry has been removed - result = salt.utils.win_reg.value_exists(hive="HKCU", key=key, vname="MyValue2") - assert result is False + # Test that the registry entry has not been removed + result = salt.utils.win_reg.value_exists(hive="HKCU", key=key, vname="MyValue1") + assert result is True def test_user_delete_value_no_change(empty_reg_pol_user): From a829b36144ffd12d7e9657dedf30cebec872edf5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 24 Jul 2025 12:57:57 -0600 Subject: [PATCH 10/71] Fix state tests and a few bugs --- salt/modules/win_lgpo_reg.py | 2 +- salt/states/win_lgpo_reg.py | 296 ++++++++++++------ .../pytests/unit/modules/test_win_lgpo_reg.py | 26 +- .../pytests/unit/states/test_win_lgpo_reg.py | 143 ++++----- 4 files changed, 289 insertions(+), 178 deletions(-) diff --git a/salt/modules/win_lgpo_reg.py b/salt/modules/win_lgpo_reg.py index 3a706b6e8a1e..f9520c12bb0d 100644 --- a/salt/modules/win_lgpo_reg.py +++ b/salt/modules/win_lgpo_reg.py @@ -212,7 +212,7 @@ def get_value(key, v_name, policy_class="Machine"): if key.lower() == p_key.lower(): found_key = p_key for p_name in pol_data[p_key]: - if v_name.lower() in p_name.lower(): + if v_name.lower() in p_name.lower().split("."): found_name = p_name if found_key: diff --git a/salt/states/win_lgpo_reg.py b/salt/states/win_lgpo_reg.py index 1a01ea17c0f5..5f2a05377eae 100644 --- a/salt/states/win_lgpo_reg.py +++ b/salt/states/win_lgpo_reg.py @@ -77,19 +77,22 @@ def _get_current(key, name, policy_class): """ Helper function to get the current state of the policy """ - hive = "HKLM" - if policy_class == "User": - hive = "HKCU" pol = __salt__["lgpo_reg.get_value"]( key=key, v_name=name, policy_class=policy_class ) - reg_raw = __utils__["reg.read_value"](hive=hive, key=key, vname=name) - + if pol: + pol.update({"key": key, "name": name}) + # We only change registry on Machine policy, user will always be {} reg = {} - if reg_raw["vdata"] is not None: - reg["data"] = reg_raw["vdata"] - if reg_raw["vtype"] is not None: - reg["type"] = reg_raw["vtype"] + if policy_class == "Machine": + reg_raw = __utils__["reg.read_value"](hive="HKLM", key=key, vname=name) + + if reg_raw["vdata"] is not None: + reg["data"] = reg_raw["vdata"] + if reg_raw["vtype"] is not None: + reg["type"] = reg_raw["vtype"] + if reg: + reg.update({"key": key, "name": name}) return {"pol": pol, "reg": reg} @@ -139,7 +142,6 @@ def value_present(name, key, v_data, v_type="REG_DWORD", policy_class="Machine") - v_data: "some string data" - policy_class: Machine - # Using the name as the parameter and modifying the User policy MyValue: lgpo_reg.value_present: @@ -148,34 +150,52 @@ def value_present(name, key, v_data, v_type="REG_DWORD", policy_class="Machine") - v_data: "some string data" - policy_class: User """ + if policy_class.lower() in ["computer", "machine"]: + policy_class = "Machine" + else: + policy_class = "User" + ret = {"name": name, "changes": {}, "result": False, "comment": ""} old = _get_current(key=key, name=name, policy_class=policy_class) pol_correct = ( - str(old["pol"].get("data", "")) == str(v_data) - and old["pol"].get("type", "") == v_type - ) - reg_correct = ( - str(old["reg"].get("data", "")) == str(v_data) - and old["reg"].get("type", "") == v_type + str(old.get("pol", {}).get("name", "")) == str(name) + and str(old.get("pol", {}).get("data", "")) == str(v_data) + and str(old.get("pol", {}).get("type", "")) == v_type ) - - if pol_correct and reg_correct: - ret["comment"] = "Policy value already present\nRegistry value already present" - ret["result"] = True - return ret - + if policy_class == "User": + reg_correct = True + else: + reg_correct = ( + str(old.get("reg", {}).get("name", "")) == str(name) + and str(old.get("reg", {}).get("data", "")) == str(v_data) + and old.get("reg", {}).get("type", "") == v_type + ) + + comment = [] if __opts__["test"]: if not pol_correct: - ret["comment"] = "Policy value will be set" + comment.append("Policy value will be set") if not reg_correct: - if ret["comment"]: - ret["comment"] += "\n" - ret["comment"] += "Registry value will be set" + if policy_class == "Machine": + comment.append("Registry value will be set") + ret["comment"] = "\n".join(comment) ret["result"] = None return ret + if pol_correct: + comment.append("Policy value already present") + + if reg_correct: + if policy_class == "Machine": + comment.append("Registry value already present") + + if pol_correct and reg_correct: + ret["comment"] = "\n".join(comment) + ret["result"] = True + return ret + __salt__["lgpo_reg.set_value"]( key=key, v_name=name, @@ -185,28 +205,49 @@ def value_present(name, key, v_data, v_type="REG_DWORD", policy_class="Machine") ) new = _get_current(key=key, name=name, policy_class=policy_class) - - pol_correct = ( - str(new["pol"]["data"]) == str(v_data) and new["pol"]["type"] == v_type - ) - reg_correct = ( - str(new["reg"]["data"]) == str(v_data) and new["reg"]["type"] == v_type - ) - - if pol_correct and reg_correct: - ret["comment"] = "Registry policy value has been set" - ret["result"] = True - elif not pol_correct: - ret["comment"] = "Failed to set policy value" - elif not reg_correct: - if ret["comment"]: - ret["comment"] += "\n" - ret["comment"] += "Failed to set registry value" - - changes = salt.utils.data.recursive_diff(old, new) - - if changes: - ret["changes"] = changes + ret["changes"] = salt.utils.data.recursive_diff(old, new) + + comment = [] + if ret["changes"]: + pol_correct = ( + str(new.get("pol", {}).get("name", "")) == str(name) + and str(new.get("pol", {}).get("data", "")) == str(v_data) + and new.get("pol", {}).get("type", "") == v_type + ) + if policy_class == "User": + reg_correct = True + else: + reg_correct = ( + str(new.get("reg", {}).get("name", "")) == str(name) + and str(new.get("reg", {}).get("data", "")) == str(v_data) + and new.get("reg", {}).get("type", "") == v_type + ) + + if pol_correct: + if "pol" in ret["changes"].get("new", {}): + comment.append("Policy value set") + else: + comment.append("Failed to set policy value") + + if reg_correct: + if policy_class == "Machine": + if "reg" in ret["changes"].get("new", {}): + comment.append("Registry value set") + else: + comment.append("Failed to set registry value") + + if reg_correct and pol_correct: + ret["result"] = True + + else: + comment.append(f"Failed to set {policy_class} policy value") + comment.append(f"- key: {key}") + comment.append(f"- name: {name}") + comment.append(f"- v_data: {v_data}") + comment.append(f"- v_type: {v_type}") + ret["result"] = False + + ret["comment"] = "\n".join(comment) return ret @@ -248,49 +289,80 @@ def value_disabled(name, key, policy_class="Machine"): - key: SOFTWARE\MyKey - policy_class: User """ + if policy_class.lower() in ["computer", "machine"]: + policy_class = "Machine" + else: + policy_class = "User" + ret = {"name": name, "changes": {}, "result": False, "comment": ""} old = _get_current(key=key, name=name, policy_class=policy_class) pol_correct = old["pol"].get("data", "") == f"**del.{name}" - reg_correct = old["reg"] == {} + if policy_class == "User": + reg_correct = True + else: + reg_correct = old["reg"] == {} - if pol_correct and reg_correct: - ret["comment"] = "Registry policy value already disabled" - ret["result"] = True - return ret + comment = [] if __opts__["test"]: if not pol_correct: - ret["comment"] = "Policy value will be disabled" + comment.append("Policy value will be disabled") if not reg_correct: - if ret["comment"]: - ret["comment"] += "\n" - ret["comment"] += "Registry value will be removed" + if policy_class == "Machine": + comment.append("Registry value will be deleted") + ret["comment"] = "\n".join(comment) ret["result"] = None return ret - __salt__["lgpo_reg.disable_value"](key=key, v_name=name, policy_class=policy_class) - - new = _get_current(key=key, name=name, policy_class=policy_class) + if pol_correct: + comment.append("Policy value already disabled") - pol_correct = new["pol"].get("data", "") == f"**del.{name}" - reg_correct = new["reg"] == {} + if reg_correct: + if policy_class == "Machine": + comment.append("Registry value already deleted") if pol_correct and reg_correct: - ret["comment"] = "Registry policy value disabled" + ret["comment"] = "\n".join(comment) ret["result"] = True - elif not pol_correct: - ret["comment"] = "Failed to disable policy value" - elif not reg_correct: - if ret["comment"]: - ret["comment"] += "\n" - ret["comment"] += "Failed to remove registry value" + return ret - changes = salt.utils.data.recursive_diff(old, new) + __salt__["lgpo_reg.disable_value"](key=key, v_name=name, policy_class=policy_class) - if changes: - ret["changes"] = changes + new = _get_current(key=key, name=name, policy_class=policy_class) + ret["changes"] = salt.utils.data.recursive_diff(old, new) + + comment = [] + if ret["changes"]: + pol_correct = new["pol"].get("data", "") == f"**del.{name}" + if policy_class == "User": + reg_correct = True + else: + reg_correct = new["reg"] == {} + + if pol_correct: + if "pol" in ret["changes"].get("new", {}): + comment.append("Policy value disabled") + else: + comment.append("Failed to disable policy value") + + if reg_correct: + if policy_class == "Machine": + if "reg" in ret["changes"].get("new", {}): + comment.append("Registry value deleted") + else: + comment.append("Failed to delete registry value") + + if pol_correct and reg_correct: + ret["result"] = True + else: + comment.append(f"Failed to disable {policy_class} policy value") + comment.append(f"- key: {key}") + comment.append(f"- name: {name}") + ret["result"] = False + + ret["comment"] = "\n".join(comment) return ret @@ -332,48 +404,80 @@ def value_absent(name, key, policy_class="Machine"): - key: SOFTWARE\MyKey - policy_class: User """ + if policy_class.lower() in ["computer", "machine"]: + policy_class = "Machine" + else: + policy_class = "User" + ret = {"name": name, "changes": {}, "result": False, "comment": ""} old = _get_current(key=key, name=name, policy_class=policy_class) pol_correct = old["pol"] == {} - reg_correct = old["reg"] == {} + if policy_class == "User": + reg_correct = True + else: + reg_correct = old["reg"] == {} - if pol_correct and reg_correct: - ret["comment"] = "Registry policy value already deleted" - ret["result"] = True - return ret + comment = [] if __opts__["test"]: if not pol_correct: - ret["comment"] = "Policy value will be deleted" + comment.append("Policy value will be deleted") if not reg_correct: - if ret["comment"]: - ret["comment"] += "\n" - ret["comment"] += "Registry value will be deleted" + if policy_class == "Machine": + comment.append("Registry value will be deleted") + ret["comment"] = "\n".join(comment) ret["result"] = None return ret - __salt__["lgpo_reg.delete_value"](key=key, v_name=name, policy_class=policy_class) - - new = _get_current(key=key, name=name, policy_class=policy_class) + if pol_correct: + comment.append("Policy value already deleted") - pol_correct = new["pol"] == {} - reg_correct = new["reg"] == {} + if reg_correct: + if policy_class == "Machine": + comment.append("Registry value already deleted") if pol_correct and reg_correct: - ret["comment"] = "Registry policy value deleted" + ret["comment"] = "\n".join(comment) ret["result"] = True - elif not pol_correct: - ret["comment"] = "Failed to delete policy value" - elif not reg_correct: - if ret["comment"]: - ret["comment"] += "\n" - ret["comment"] += "Failed to delete registry value" + return ret - changes = salt.utils.data.recursive_diff(old, new) + __salt__["lgpo_reg.delete_value"](key=key, v_name=name, policy_class=policy_class) - if changes: - ret["changes"] = changes + new = _get_current(key=key, name=name, policy_class=policy_class) + ret["changes"] = salt.utils.data.recursive_diff(old, new) + + comment = [] + if ret["changes"]: + pol_correct = new["pol"] == {} + if policy_class == "User": + reg_correct = True + else: + reg_correct = new["reg"] == {} + + if pol_correct: + if "pol" in ret["changes"].get("new", {}): + comment.append("Policy value deleted") + else: + comment.append("Failed to delete policy value") + + if reg_correct: + if policy_class == "Machine": + if "reg" in ret["changes"].get("new", {}): + comment.append("Registry value deleted") + else: + comment.append("Failed to delete registry value") + + if reg_correct and pol_correct: + ret["result"] = True + + else: + comment.append(f"Failed to remove {policy_class} policy value") + comment.append(f"- key: {key}") + comment.append(f"- name: {name}") + ret["result"] = False + + ret["comment"] = "\n".join(comment) return ret diff --git a/tests/pytests/unit/modules/test_win_lgpo_reg.py b/tests/pytests/unit/modules/test_win_lgpo_reg.py index 40bf806c21c8..848b499e343d 100644 --- a/tests/pytests/unit/modules/test_win_lgpo_reg.py +++ b/tests/pytests/unit/modules/test_win_lgpo_reg.py @@ -273,9 +273,16 @@ def test_mach_write_reg_pol(empty_reg_pol_mach): assert result == data_to_write -def test_mach_get_value(reg_pol_mach): - expected = {"data": "squidward", "type": "REG_SZ"} - result = lgpo_reg.get_value(key="SOFTWARE\\MyKey1", v_name="MyValue1") +@pytest.mark.parametrize( + "name,expected", + [ + ("MyValue", {}), + ("MyValue1", {"data": "squidward", "type": "REG_SZ"}), + ("MyValue2", {"data": "**del.MyValue2", "type": "REG_SZ"}), + ], +) +def test_mach_get_value(reg_pol_mach, name, expected): + result = lgpo_reg.get_value(key="SOFTWARE\\MyKey1", v_name=name) assert result == expected @@ -416,11 +423,18 @@ def test_user_write_reg_pol(empty_reg_pol_user): assert result == data_to_write -def test_user_get_value(reg_pol_user): - expected = {"data": "squidward", "type": "REG_SZ"} +@pytest.mark.parametrize( + "name,expected", + [ + ("MyValue", {}), + ("MyValue1", {"data": "squidward", "type": "REG_SZ"}), + ("MyValue2", {"data": "**del.MyValue2", "type": "REG_SZ"}), + ], +) +def test_user_get_value(reg_pol_user, name, expected): result = lgpo_reg.get_value( key="SOFTWARE\\MyKey1", - v_name="MyValue1", + v_name=name, policy_class="User", ) assert result == expected diff --git a/tests/pytests/unit/states/test_win_lgpo_reg.py b/tests/pytests/unit/states/test_win_lgpo_reg.py index c9e4a2e028a3..faebbf123433 100644 --- a/tests/pytests/unit/states/test_win_lgpo_reg.py +++ b/tests/pytests/unit/states/test_win_lgpo_reg.py @@ -183,7 +183,7 @@ def test_virtual_name(): assert lgpo_reg.__virtual__() == "lgpo_reg" -def test_machine_value_present(empty_reg_pol_mach): +def test_mach_value_present(empty_reg_pol_mach): """ Test value.present in Machine policy """ @@ -197,10 +197,14 @@ def test_machine_value_present(empty_reg_pol_mach): "changes": { "new": { "pol": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue", "data": 1, "type": "REG_DWORD", }, "reg": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue", "data": 1, "type": "REG_DWORD", }, @@ -210,14 +214,14 @@ def test_machine_value_present(empty_reg_pol_mach): "reg": {}, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set\nRegistry value set", "name": "MyValue", "result": True, } assert result == expected -def test_machine_value_present_similar_names(empty_reg_pol_mach): +def test_mach_value_present_similar_names(empty_reg_pol_mach): """ Test value.present in Machine policy """ @@ -249,7 +253,7 @@ def test_machine_value_present_similar_names(empty_reg_pol_mach): assert result == expected -def test_machine_value_present_enforce(reg_pol_mach): +def test_mach_value_present_enforce(reg_pol_mach): """ Issue #64222 Test value.present in Machine policy when the registry changes after the @@ -284,14 +288,14 @@ def test_machine_value_present_enforce(reg_pol_mach): } }, }, - "comment": "Registry policy value has been set", + "comment": "Registry value set", "name": "MyValue3", "result": True, } assert result == expected -def test_machine_value_present_existing_change(reg_pol_mach): +def test_mach_value_present_existing_change(reg_pol_mach): """ Test value.present with existing incorrect value in Machine policy """ @@ -324,14 +328,14 @@ def test_machine_value_present_existing_change(reg_pol_mach): }, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set\nRegistry value set", "name": "MyValue1", "result": True, } assert result == expected -def test_machine_value_present_existing_change_dword(reg_pol_mach): +def test_mach_value_present_existing_change_dword(reg_pol_mach): """ Test value.present with existing incorrect value in Machine policy """ @@ -360,14 +364,14 @@ def test_machine_value_present_existing_change_dword(reg_pol_mach): }, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set\nRegistry value set", "name": "MyValue3", "result": True, } assert result == expected -def test_machine_value_present_existing_no_change(reg_pol_mach): +def test_mach_value_present_existing_no_change(reg_pol_mach): """ Test value.present with existing correct value in Machine policy """ @@ -386,7 +390,7 @@ def test_machine_value_present_existing_no_change(reg_pol_mach): assert result == expected -def test_machine_value_present_test_true(empty_reg_pol_mach): +def test_mach_value_present_test_true(empty_reg_pol_mach): """ Test value.present with test=True in Machine policy """ @@ -406,7 +410,7 @@ def test_machine_value_present_test_true(empty_reg_pol_mach): assert result == expected -def test_machine_value_present_existing_disabled(reg_pol_mach): +def test_mach_value_present_existing_disabled(reg_pol_mach): """ Test value.present with existing value that is disabled in Machine policy """ @@ -424,6 +428,8 @@ def test_machine_value_present_existing_disabled(reg_pol_mach): "type": "REG_DWORD", }, "reg": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue2", "data": 2, "type": "REG_DWORD", }, @@ -436,14 +442,14 @@ def test_machine_value_present_existing_disabled(reg_pol_mach): "reg": {}, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set\nRegistry value set", "name": "MyValue2", "result": True, } assert result == expected -def test_machine_value_disabled(empty_reg_pol_mach): +def test_mach_value_disabled(empty_reg_pol_mach): """ Test value.disabled in Machine policy """ @@ -455,20 +461,22 @@ def test_machine_value_disabled(empty_reg_pol_mach): "changes": { "new": { "pol": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue1", "data": "**del.MyValue1", "type": "REG_SZ", }, }, "old": {"pol": {}}, }, - "comment": "Registry policy value disabled", + "comment": "Policy value disabled", "name": "MyValue1", "result": True, } assert result == expected -def test_machine_value_disabled_existing_change(reg_pol_mach): +def test_mach_value_disabled_existing_change(reg_pol_mach): """ Test value.disabled with an existing value that is not disabled in Machine policy @@ -489,17 +497,22 @@ def test_machine_value_disabled_existing_change(reg_pol_mach): "pol": { "data": "squidward", }, - "reg": {"data": "squidward", "type": "REG_SZ"}, + "reg": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue1", + "data": "squidward", + "type": "REG_SZ", + }, }, }, - "comment": "Registry policy value disabled", + "comment": "Policy value disabled\nRegistry value deleted", "name": "MyValue1", "result": True, } assert result == expected -def test_machine_value_disabled_existing_no_change(reg_pol_mach): +def test_mach_value_disabled_existing_no_change(reg_pol_mach): """ Test value.disabled with an existing disabled value in Machine policy """ @@ -509,14 +522,14 @@ def test_machine_value_disabled_existing_no_change(reg_pol_mach): ) expected = { "changes": {}, - "comment": "Registry policy value already disabled", + "comment": "Policy value already disabled\nRegistry value already deleted", "name": "MyValue2", "result": True, } assert result == expected -def test_machine_value_disabled_test_true(empty_reg_pol_mach): +def test_mach_value_disabled_test_true(empty_reg_pol_mach): """ Test value.disabled when test=True in Machine policy """ @@ -534,7 +547,7 @@ def test_machine_value_disabled_test_true(empty_reg_pol_mach): assert result == expected -def test_machine_value_absent(reg_pol_mach): +def test_mach_value_absent(reg_pol_mach): """ Test value.absent in Machine policy """ @@ -544,37 +557,41 @@ def test_machine_value_absent(reg_pol_mach): "new": {"pol": {}, "reg": {}}, "old": { "pol": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue1", "data": "squidward", "type": "REG_SZ", }, "reg": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue1", "data": "squidward", "type": "REG_SZ", }, }, }, - "comment": "Registry policy value deleted", + "comment": "Policy value deleted\nRegistry value deleted", "name": "MyValue1", "result": True, } assert result == expected -def test_machine_value_absent_no_change(empty_reg_pol_mach): +def test_mach_value_absent_no_change(empty_reg_pol_mach): """ Test value.absent when the value is already absent in Machine policy """ result = lgpo_reg.value_absent(name="MyValue1", key="SOFTWARE\\MyKey1") expected = { "changes": {}, - "comment": "Registry policy value already deleted", + "comment": "Policy value already deleted\nRegistry value already deleted", "name": "MyValue1", "result": True, } assert result == expected -def test_machine_value_absent_disabled(reg_pol_mach): +def test_mach_value_absent_disabled(reg_pol_mach): """ Test value.absent when the value is disabled in Machine policy """ @@ -584,19 +601,21 @@ def test_machine_value_absent_disabled(reg_pol_mach): "new": {"pol": {}}, "old": { "pol": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue2", "data": "**del.MyValue2", "type": "REG_SZ", }, }, }, - "comment": "Registry policy value deleted", + "comment": "Policy value deleted", "name": "MyValue2", "result": True, } assert result == expected -def test_machine_value_absent_test_true(reg_pol_mach): +def test_mach_value_absent_test_true(reg_pol_mach): """ Test value.absent with test=True in Machine policy """ @@ -626,20 +645,17 @@ def test_user_value_present(empty_reg_pol_user): "changes": { "new": { "pol": { - "data": 1, - "type": "REG_DWORD", - }, - "reg": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue", "data": 1, "type": "REG_DWORD", }, }, "old": { "pol": {}, - "reg": {}, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set", "name": "MyValue", "result": True, } @@ -698,23 +714,15 @@ def test_user_value_present_existing_change(reg_pol_user): "data": 2, "type": "REG_DWORD", }, - "reg": { - "data": 2, - "type": "REG_DWORD", - }, }, "old": { "pol": { "data": "squidward", "type": "REG_SZ", }, - "reg": { - "data": "squidward", - "type": "REG_SZ", - }, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set", "name": "MyValue1", "result": True, } @@ -738,20 +746,14 @@ def test_user_value_present_existing_change_dword(reg_pol_user): "pol": { "data": 1, }, - "reg": { - "data": 1, - }, }, "old": { "pol": { "data": 0, }, - "reg": { - "data": 0, - }, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set", "name": "MyValue3", "result": True, } @@ -771,7 +773,7 @@ def test_user_value_present_existing_no_change(reg_pol_user): ) expected = { "changes": {}, - "comment": "Policy value already present\nRegistry value already present", + "comment": "Policy value already present", "name": "MyValue1", "result": True, } @@ -792,7 +794,7 @@ def test_user_value_present_test_true(empty_reg_pol_user): ) expected = { "changes": {}, - "comment": "Policy value will be set\nRegistry value will be set", + "comment": "Policy value will be set", "name": "MyValue", "result": None, } @@ -817,20 +819,15 @@ def test_user_value_present_existing_disabled(reg_pol_user): "data": 2, "type": "REG_DWORD", }, - "reg": { - "data": 2, - "type": "REG_DWORD", - }, }, "old": { "pol": { "data": "**del.MyValue2", "type": "REG_SZ", }, - "reg": {}, }, }, - "comment": "Registry policy value has been set", + "comment": "Policy value set", "name": "MyValue2", "result": True, } @@ -848,13 +845,15 @@ def test_user_value_disabled(empty_reg_pol_user): "changes": { "new": { "pol": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue1", "data": "**del.MyValue1", "type": "REG_SZ", }, }, "old": {"pol": {}}, }, - "comment": "Registry policy value disabled", + "comment": "Policy value disabled", "name": "MyValue1", "result": True, } @@ -877,19 +876,14 @@ def test_user_value_disabled_existing_change(reg_pol_user): "pol": { "data": "**del.MyValue1", }, - "reg": {}, }, "old": { "pol": { "data": "squidward", }, - "reg": { - "data": "squidward", - "type": "REG_SZ", - }, }, }, - "comment": "Registry policy value disabled", + "comment": "Policy value disabled", "name": "MyValue1", "result": True, } @@ -907,7 +901,7 @@ def test_user_value_disabled_existing_no_change(reg_pol_user): ) expected = { "changes": {}, - "comment": "Registry policy value already disabled", + "comment": "Policy value already disabled", "name": "MyValue2", "result": True, } @@ -946,20 +940,17 @@ def test_user_value_absent(reg_pol_user): "changes": { "new": { "pol": {}, - "reg": {}, }, "old": { "pol": { - "data": "squidward", - "type": "REG_SZ", - }, - "reg": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue1", "data": "squidward", "type": "REG_SZ", }, }, }, - "comment": "Registry policy value deleted", + "comment": "Policy value deleted", "name": "MyValue1", "result": True, } @@ -977,7 +968,7 @@ def test_user_value_absent_no_change(empty_reg_pol_user): ) expected = { "changes": {}, - "comment": "Registry policy value already deleted", + "comment": "Policy value already deleted", "name": "MyValue1", "result": True, } @@ -998,12 +989,14 @@ def test_user_value_absent_disabled(reg_pol_user): "new": {"pol": {}}, "old": { "pol": { + "key": "SOFTWARE\\MyKey1", + "name": "MyValue2", "data": "**del.MyValue2", "type": "REG_SZ", }, }, }, - "comment": "Registry policy value deleted", + "comment": "Policy value deleted", "name": "MyValue2", "result": True, } @@ -1022,7 +1015,7 @@ def test_user_value_absent_test_true(reg_pol_user): ) expected = { "changes": {}, - "comment": "Policy value will be deleted\nRegistry value will be deleted", + "comment": "Policy value will be deleted", "name": "MyValue1", "result": None, } From 22f241997c8cb1a63ffa0594df937374ff5fb15f Mon Sep 17 00:00:00 2001 From: xsmile Date: Mon, 7 Jul 2025 08:53:31 +0200 Subject: [PATCH 11/71] cmdmod: early preparation of env dict --- salt/modules/cmdmod.py | 20 +++++++++++++++++++- salt/utils/timed_subprocess.py | 6 ------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 2b9faf106d83..1be45bb77023 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -212,6 +212,21 @@ def _parse_env(env): return env +def _ensure_env_str(env): + """ + Ensure that environment variables are strings + """ + if not env: + env = {} + if not isinstance(env, dict): + env = {} + for key, val in env.items(): + if not isinstance(val, str): + env[key] = str(val) + if not isinstance(key, str): + env[str(key)] = env.pop(key) + + def _gather_pillar(pillarenv, pillar_override): """ Whenever a state run starts, gather the pillar data fresh @@ -672,7 +687,8 @@ def _run( env.setdefault("LC_IDENTIFICATION", "C") env.setdefault("LANGUAGE", "C") - if clean_env: + if clean_env or (runas and salt.utils.platform.is_windows()): + # always use a clean environment with CreateProcess on Windows run_env = env else: @@ -690,6 +706,8 @@ def _run( if "NOTIFY_SOCKET" not in env: run_env.pop("NOTIFY_SOCKET", None) + _ensure_env_str(run_env) + if python_shell is None: python_shell = False diff --git a/salt/utils/timed_subprocess.py b/salt/utils/timed_subprocess.py index 13e7d67c2304..6166806a80a4 100644 --- a/salt/utils/timed_subprocess.py +++ b/salt/utils/timed_subprocess.py @@ -70,12 +70,6 @@ def __init__(self, args, **kwargs): if not isinstance(args, (list, tuple, str)): # Handle corner case where someone does a 'cmd.run 3' args = str(args) - # Ensure that environment variables are strings - for key, val in kwargs.get("env", {}).items(): - if not isinstance(val, str): - kwargs["env"][key] = str(val) - if not isinstance(key, str): - kwargs["env"][str(key)] = kwargs["env"].pop(key) args = salt.utils.data.decode(args) self.process = subprocess.Popen(args, **kwargs) self.command = args From 21134b79aefaa764d45bc375bdc4dba1b072bbeb Mon Sep 17 00:00:00 2001 From: xsmile Date: Mon, 7 Jul 2025 08:55:01 +0200 Subject: [PATCH 12/71] cmdmod: unify result processing for timed_subprocess and win_runas --- salt/modules/cmdmod.py | 148 +++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 57 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 1be45bb77023..bcab216abcba 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -53,6 +53,8 @@ pass if salt.utils.platform.is_windows(): + import pywintypes + import salt.platform.win from salt.utils.win_functions import escape_argument as _cmd_quote from salt.utils.win_runas import runas as win_runas @@ -792,15 +794,41 @@ def _run( # This is where the magic happens if runas and salt.utils.platform.is_windows(): - # We can't use TimedProc with runas on Windows - if change_windows_codepage: - salt.utils.win_chcp.set_codepage_id(windows_codepage) + new_kwargs.update( + { + "redirect_stderr": True if stderr == subprocess.STDOUT else False, + } + ) - ret = win_runas(cmd, runas, password, cwd) + try: + if change_windows_codepage: + salt.utils.win_chcp.set_codepage_id(windows_codepage) + try: + proc = win_runas(cmd, runas, password, **new_kwargs) + except (OSError, pywintypes.error) as exc: + msg = "Unable to run command '{}' with the context '{}', reason: {}".format( + cmd if output_loglevel is not None else "REDACTED", + new_kwargs, + exc, + ) + raise CommandExecutionError(msg) + except TimedProcTimeoutError as exc: + ret["stdout"] = str(exc) + ret["stderr"] = "" + ret["retcode"] = "" + ret["pid"] = "" + # ok return code for timeouts? + ret["retcode"] = 1 + return ret + finally: + if change_windows_codepage: + salt.utils.win_chcp.set_codepage_id(previous_windows_codepage) - if change_windows_codepage: - salt.utils.win_chcp.set_codepage_id(previous_windows_codepage) + proc_pid = proc.get("pid") + proc_retcode = proc.get("retcode") + proc_stdout = proc.get("stdout") + proc_stderr = proc.get("stderr") else: try: @@ -815,7 +843,6 @@ def _run( exc, ) raise CommandExecutionError(msg) - try: proc.run() except TimedProcTimeoutError as exc: @@ -830,61 +857,67 @@ def _run( if change_windows_codepage: salt.utils.win_chcp.set_codepage_id(previous_windows_codepage) - if output_loglevel != "quiet" and output_encoding is not None: - log.debug( - "Decoding output from command %s using %s encoding", - cmd, - output_encoding, - ) + proc_pid = proc.process.pid + proc_retcode = proc.process.returncode + proc_stdout = proc.stdout + proc_stderr = proc.stderr - try: - out = salt.utils.stringutils.to_unicode( - proc.stdout, encoding=output_encoding - ) - except TypeError: - # stdout is None - out = "" - except UnicodeDecodeError: - out = salt.utils.stringutils.to_unicode( - proc.stdout, encoding=output_encoding, errors="replace" - ) - if output_loglevel != "quiet": - log.error( - "Failed to decode stdout from command %s, non-decodable " - "characters have been replaced", - _log_cmd(cmd), - ) + if output_loglevel != "quiet" and output_encoding is not None: + log.debug( + "Decoding output from command %s using %s encoding", + cmd, + output_encoding, + ) - try: - err = salt.utils.stringutils.to_unicode( - proc.stderr, encoding=output_encoding + try: + out = salt.utils.stringutils.to_unicode( + proc_stdout, encoding=output_encoding + ) + except TypeError: + # stdout is None + out = "" + except UnicodeDecodeError: + out = salt.utils.stringutils.to_unicode( + proc_stdout, encoding=output_encoding, errors="replace" + ) + if output_loglevel != "quiet": + log.error( + "Failed to decode stdout from command %s, non-decodable " + "characters have been replaced", + _log_cmd(cmd), ) - except TypeError: - # stderr is None - err = "" - except UnicodeDecodeError: - err = salt.utils.stringutils.to_unicode( - proc.stderr, encoding=output_encoding, errors="replace" + + try: + err = salt.utils.stringutils.to_unicode( + proc_stderr, encoding=output_encoding + ) + except TypeError: + # stderr is None + err = "" + except UnicodeDecodeError: + err = salt.utils.stringutils.to_unicode( + proc_stderr, encoding=output_encoding, errors="replace" + ) + if output_loglevel != "quiet": + log.error( + "Failed to decode stderr from command %s, non-decodable " + "characters have been replaced", + _log_cmd(cmd), ) - if output_loglevel != "quiet": - log.error( - "Failed to decode stderr from command %s, non-decodable " - "characters have been replaced", - _log_cmd(cmd), - ) - # Encoded commands dump CLIXML data in stderr. It's not an actual error - if encoded_cmd and "CLIXML" in err: - err = "" - if rstrip: - if out is not None: - out = out.rstrip() - if err is not None: - err = err.rstrip() - ret["pid"] = proc.process.pid - ret["retcode"] = proc.process.returncode - ret["stdout"] = out - ret["stderr"] = err + # Encoded commands dump CLIXML data in stderr. It's not an actual error + if encoded_cmd and "CLIXML" in err: + err = "" + if rstrip: + if out is not None: + out = out.rstrip() + if err is not None: + err = err.rstrip() + + ret["pid"] = proc_pid + ret["retcode"] = proc_retcode + ret["stdout"] = out + ret["stderr"] = err if ret["retcode"] in success_retcodes: ret["retcode"] = 0 @@ -893,6 +926,7 @@ def _run( + [stde in ret["stderr"] for stde in success_stderr] ): ret["retcode"] = 0 + else: formatted_timeout = "" if timeout: From ece67524eaf97501a2f29e9ed2f6b7c17df11254 Mon Sep 17 00:00:00 2001 From: xsmile Date: Mon, 7 Jul 2025 08:56:20 +0200 Subject: [PATCH 13/71] win_runas: support more cmdmod parameters, fixups --- salt/platform/win.py | 16 +- salt/utils/win_runas.py | 318 +++++++++++++++++++++++++++++----------- 2 files changed, 242 insertions(+), 92 deletions(-) diff --git a/salt/platform/win.py b/salt/platform/win.py index 0930f840457c..e181342a2d23 100644 --- a/salt/platform/win.py +++ b/salt/platform/win.py @@ -42,8 +42,19 @@ LOCAL_SRV_SID = "S-1-5-19" NETWORK_SRV_SID = "S-1-5-19" +# STARTUPINFO +STARTF_USESHOWWINDOW = 0x00000001 +STARTF_USESTDHANDLES = 0x00000100 + +# dwLogonFlags LOGON_WITH_PROFILE = 0x00000001 +# Process Creation Flags +CREATE_NEW_CONSOLE = 0x00000010 +CREATE_NO_WINDOW = 0x08000000 +CREATE_SUSPENDED = 0x00000004 +CREATE_UNICODE_ENVIRONMENT = 0x00000400 + WINSTA_ALL = ( win32con.WINSTA_ACCESSCLIPBOARD | win32con.WINSTA_ACCESSGLOBALATOMS @@ -1094,7 +1105,6 @@ def set_user_perm(obj, perm, sid): sd = win32security.GetUserObjectSecurity(obj, info) dacl = sd.GetSecurityDescriptorDacl() ace_cnt = dacl.GetAceCount() - found = False for idx in range(0, ace_cnt): (aceType, aceFlags), ace_mask, ace_sid = dacl.GetAce(idx) ace_exists = ( @@ -1150,7 +1160,7 @@ def CreateProcessWithTokenW( startupinfo = STARTUPINFO() if currentdirectory is not None: currentdirectory = ctypes.create_unicode_buffer(currentdirectory) - if environment is not None: + if environment is not None and isinstance(environment, dict): environment = ctypes.pointer(environment_string(environment)) process_info = PROCESS_INFORMATION() ret = advapi32.CreateProcessWithTokenW( @@ -1322,7 +1332,7 @@ def CreateProcessWithLogonW( commandline = ctypes.create_unicode_buffer(commandline) if startupinfo is None: startupinfo = STARTUPINFO() - if environment is not None: + if environment is not None and isinstance(environment, dict): environment = ctypes.pointer(environment_string(environment)) process_info = PROCESS_INFORMATION() advapi32.CreateProcessWithLogonW( diff --git a/salt/utils/win_runas.py b/salt/utils/win_runas.py index 5bff2d7c8dfb..f6c2ae19bd31 100644 --- a/salt/utils/win_runas.py +++ b/salt/utils/win_runas.py @@ -8,7 +8,7 @@ import os import time -from salt.exceptions import CommandExecutionError +from salt.exceptions import CommandExecutionError, TimedProcTimeoutError try: import psutil @@ -28,6 +28,7 @@ import win32process import win32profile import win32security + import winerror import salt.platform.win @@ -51,6 +52,15 @@ def __virtual__(): return "win_runas" +def close_handle(handle): + if handle is not None: + try: + win32api.CloseHandle(handle) + except pywintypes.error as exc: + if exc.winerror != winerror.ERROR_INVALID_HANDLE: + raise + + def split_username(username): """ Splits out the username from the domain name and returns both. @@ -88,7 +98,63 @@ def create_env(user_token, inherit, timeout=1): raise exc -def runas(cmd, username, password=None, cwd=None): +def create_default_env(username): + """ + Creates an environment with default values. + """ + result = {} + + systemdrive = os.environ.get("SystemDrive", r"C:") + defaults = { + "ALLUSERSPROFILE": r"C:\ProgramData", + "CommonProgramFiles": r"C:\Program Files\Common Files", + "CommonProgramFiles(x86)": r"C:\Program Files (x86)\Common Files", + "CommonProgramW6432": r"C:\Program Files\Common Files", + "ComputerName": None, + "ComSpec": r"C:\Windows\system32\cmd.exe", + "DriverData": r"C:\Windows\System32\Drivers\DriverData", + "NUMBER_OF_PROCESSORS": None, + "OS": None, + "Path": r"C:\Windows\System32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0" + "\\", + "PATHEXT": r".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC", + "PROCESSOR_ARCHITECTURE": None, + "PROCESSOR_IDENTIFIER": None, + "PROCESSOR_LEVEL": None, + "PROCESSOR_REVISION": None, + "ProgramData": r"C:\ProgramData", + "ProgramFiles": r"C:\Program Files", + "ProgramFiles(x86)": r"C:\Program Files (x86)", + "ProgramW6432": r"C:\Program Files", + "PROMPT": r"$P$G", + "PSModulePath": r"%ProgramFiles%\WindowsPowerShell\Modules;%SystemRoot%\system32\WindowsPowerShell\v1.0\Modules", + "PUBLIC": r"C:\Users\Public", + "SystemDrive": systemdrive, + "SystemRoot": r"C:\Windows", + "USERDOMAIN": None, + "windir": r"C:\Windows", + } + user_specific = { + "APPDATA": rf"{systemdrive}\Users\{username}\AppData\Roaming", + "HOMEDRIVE": systemdrive, + "HOMEPATH": rf"\Users\{username}", + "LOCALAPPDATA": rf"{systemdrive}\Users\{username}\AppData\Local", + "TEMP": rf"{systemdrive}\Users\{username}\AppData\Local\Temp", + "TMP": rf"{systemdrive}\Users\{username}\AppData\Local\Temp", + "USERNAME": rf"{username}", + "USERPROFILE": rf"{systemdrive}\Users\{username}", + } + # set default variables based on the current user + for key, val in defaults.items(): + item = os.environ.get(key, val) + if key is not None: + result.update({key: item}) + # set user specific variables + result.update(user_specific) + return result + + +def runas(cmd, username, password=None, **kwargs): """ Run a command as another user. If the process is running as an admin or system account this method does not require a password. Other non @@ -131,7 +197,7 @@ def runas(cmd, username, password=None, cwd=None): # runas. if not impersonation_token: log.debug("No impersonation token, using unprivileged runas") - return runas_unpriv(cmd, username, password, cwd) + return runas_unpriv(cmd, username, password, **kwargs) if domain == "NT AUTHORITY": # Logon as a system level account, SYSTEM, LOCAL SERVICE, or NETWORK @@ -187,40 +253,55 @@ def runas(cmd, username, password=None, cwd=None): # Run the process without showing a window. creationflags = ( win32process.CREATE_NO_WINDOW - | win32process.CREATE_NEW_CONSOLE | win32process.CREATE_SUSPENDED + | win32process.CREATE_UNICODE_ENVIRONMENT ) - flags = win32con.STARTF_USESTDHANDLES - flags |= win32con.STARTF_USESHOWWINDOW - startup_info = salt.platform.win.STARTUPINFO( - dwFlags=flags, - hStdInput=stdin_read.handle, - hStdOutput=stdout_write.handle, - hStdError=stderr_write.handle, - ) + flags = win32con.STARTF_USESHOWWINDOW | win32con.STARTF_USESTDHANDLES + startup_args = { + "dwFlags": flags, + "hStdInput": stdin_read.handle, + "hStdOutput": stdout_write.handle, + } + if kwargs.get("redirect_stderr", False): + startup_args.update({"hStdError": stdout_write.handle}) + else: + startup_args.update({"hStdError": stderr_write.handle}) + startup_info = salt.platform.win.STARTUPINFO(**startup_args) # Create the environment for the user env = create_env(user_token, False) + if kwargs.get("env", {}): + env.update(kwargs["env"]) + + # Set an optional timeout + timeout = kwargs.get("timeout", None) + if timeout: + timeout = timeout * 1000 + else: + timeout = win32event.INFINITE + + wait = not kwargs.get("bg", False) hProcess = None + hThread = None + result = None + ret = {} try: # Start the process in a suspended state. process_info = salt.platform.win.CreateProcessWithTokenW( int(user_token), - logonflags=1, + logonflags=salt.platform.win.LOGON_WITH_PROFILE, applicationname=None, commandline=cmd, - currentdirectory=cwd, creationflags=creationflags, - startupinfo=startup_info, environment=env, + currentdirectory=kwargs.get("cwd"), + startupinfo=startup_info, ) - hProcess = process_info.hProcess hThread = process_info.hThread dwProcessId = process_info.dwProcessId - dwThreadId = process_info.dwThreadId # We don't use these so let's close the handle salt.platform.win.kernel32.CloseHandle(stdin_write.handle) @@ -228,41 +309,51 @@ def runas(cmd, username, password=None, cwd=None): salt.platform.win.kernel32.CloseHandle(stderr_write.handle) ret = {"pid": dwProcessId} + # Resume the process psutil.Process(dwProcessId).resume() - # Wait for the process to exit and get its return code. - if ( - win32event.WaitForSingleObject(hProcess, win32event.INFINITE) - == win32con.WAIT_OBJECT_0 - ): - exitcode = win32process.GetExitCodeProcess(hProcess) - ret["retcode"] = exitcode - - # Read standard out - fd_out = msvcrt.open_osfhandle(stdout_read.handle, os.O_RDONLY | os.O_TEXT) - with os.fdopen(fd_out, "r") as f_out: - stdout = f_out.read() - ret["stdout"] = stdout.strip() - - # Read standard error - fd_err = msvcrt.open_osfhandle(stderr_read.handle, os.O_RDONLY | os.O_TEXT) - with os.fdopen(fd_err, "r") as f_err: - stderr = f_err.read() - ret["stderr"] = stderr + if wait: + # Wait for the process to exit and get its return code + result = win32event.WaitForSingleObject(hProcess, timeout) + if result == win32con.WAIT_TIMEOUT: + win32process.TerminateProcess(hProcess, 1) + if result == win32con.WAIT_OBJECT_0: + exitcode = win32process.GetExitCodeProcess(hProcess) + ret["retcode"] = exitcode + + # Read standard out + fd_out = msvcrt.open_osfhandle(stdout_read.handle, os.O_RDONLY | os.O_TEXT) + with os.fdopen(fd_out, "rb") as f_out: + stdout = f_out.read() + ret["stdout"] = stdout + + # Read standard error + fd_err = msvcrt.open_osfhandle(stderr_read.handle, os.O_RDONLY | os.O_TEXT) + with os.fdopen(fd_err, "rb") as f_err: + stderr = f_err.read() + ret["stderr"] = stderr finally: - if hProcess is not None: - salt.platform.win.kernel32.CloseHandle(hProcess) - win32api.CloseHandle(th) - win32api.CloseHandle(user_token) - if impersonation_token: + close_handle(hProcess) + close_handle(hThread) + close_handle(th) + close_handle(user_token) + if impersonation_token is not None: win32security.RevertToSelf() - win32api.CloseHandle(impersonation_token) + close_handle(impersonation_token) + close_handle(stdin_read.handle) + close_handle(stdout_read.handle) + close_handle(stderr_read.handle) + + if result == win32con.WAIT_TIMEOUT: + raise TimedProcTimeoutError( + "{} : Timed out after {} seconds".format(cmd, kwargs["timeout"]) + ) return ret -def runas_unpriv(cmd, username, password, cwd=None): +def runas_unpriv(cmd, username, password, **kwargs): """ Runas that works for non-privileged users """ @@ -278,33 +369,66 @@ def runas_unpriv(cmd, username, password, cwd=None): message = win32api.FormatMessage(exc.winerror).rstrip("\n") raise CommandExecutionError(message) + # Create inheritable copy of the stdin + stdin = salt.platform.win.kernel32.GetStdHandle( + salt.platform.win.STD_INPUT_HANDLE, + ) + stdin_read_handle = salt.platform.win.DuplicateHandle(srchandle=stdin, inherit=True) + # Create a pipe to set as stdout in the child. The write handle needs to be # inheritable. - c2pread, c2pwrite = salt.platform.win.CreatePipe( + stdout_read_handle, stdout_write_handle = salt.platform.win.CreatePipe( inherit_read=False, inherit_write=True, ) - errread, errwrite = salt.platform.win.CreatePipe( + stderr_read_handle, stderr_write_handle = salt.platform.win.CreatePipe( inherit_read=False, inherit_write=True, ) - # Create inheritable copy of the stdin - stdin = salt.platform.win.kernel32.GetStdHandle( - salt.platform.win.STD_INPUT_HANDLE, + # Run the process without showing a window. + creationflags = ( + salt.platform.win.CREATE_NO_WINDOW + | salt.platform.win.CREATE_UNICODE_ENVIRONMENT ) - dupin = salt.platform.win.DuplicateHandle(srchandle=stdin, inherit=True) # Get startup info structure - flags = win32con.STARTF_USESTDHANDLES - flags |= win32con.STARTF_USESHOWWINDOW - startup_info = salt.platform.win.STARTUPINFO( - dwFlags=flags, - hStdInput=dupin, - hStdOutput=c2pwrite, - hStdError=errwrite, + flags = ( + salt.platform.win.STARTF_USESHOWWINDOW | salt.platform.win.STARTF_USESTDHANDLES ) + startup_args = { + "dwFlags": flags, + "hStdInput": stdin_read_handle, + "hStdOutput": stdout_write_handle, + } + if kwargs.get("redirect_stderr", False): + startup_args.update({"hStdError": stdout_write_handle}) + else: + startup_args.update({"hStdError": stderr_write_handle}) + startup_info = salt.platform.win.STARTUPINFO(**startup_args) + # Create the environment for the user + env = kwargs.get("env", None) + if env: + # Unprivileged users won't be able to call CreateEnvironmentBlock. + # Create an environment block with sane defaults instead + env = create_default_env(username) + env.update(kwargs["env"]) + env = salt.platform.win.environment_string(env) + + # Set an optional timeout + timeout = kwargs.get("timeout", None) + if timeout: + timeout = timeout * 1000 + else: + timeout = win32event.INFINITE + + wait = not kwargs.get("bg", False) + + hProcess = None + hThread = None + result = None + ret = {} try: # Run command and return process info structure process_info = salt.platform.win.CreateProcessWithLogonW( @@ -312,43 +436,59 @@ def runas_unpriv(cmd, username, password, cwd=None): domain=domain, password=password, logonflags=salt.platform.win.LOGON_WITH_PROFILE, + applicationname=None, commandline=cmd, + creationflags=creationflags, + environment=env, + currentdirectory=kwargs.get("cwd"), startupinfo=startup_info, - currentdirectory=cwd, ) - salt.platform.win.kernel32.CloseHandle(process_info.hThread) + hProcess = process_info.hProcess + hThread = process_info.hThread + dwProcessId = process_info.dwProcessId + + # We don't use these so let's close the handle + salt.platform.win.kernel32.CloseHandle(stdin_read_handle) + salt.platform.win.kernel32.CloseHandle(stdout_write_handle) + salt.platform.win.kernel32.CloseHandle(stderr_write_handle) + + ret = {"pid": dwProcessId} + + if wait: + # Wait for the process to exit and get its return code + result = salt.platform.win.kernel32.WaitForSingleObject(hProcess, timeout) + if result == win32con.WAIT_TIMEOUT: + salt.platform.win.kernel32.TerminateProcess(hProcess, 1) + elif result == win32con.WAIT_OBJECT_0: + exitcode = salt.platform.win.wintypes.DWORD() + salt.platform.win.kernel32.GetExitCodeProcess( + hProcess, ctypes.byref(exitcode) + ) + ret["retcode"] = exitcode.value + + # Read Standard out + fd_out = msvcrt.open_osfhandle(stdout_read_handle, os.O_RDONLY | os.O_TEXT) + with os.fdopen(fd_out, "rb") as f_out: + stdout = f_out.read() + ret["stdout"] = stdout + + # Read Standard error + fd_err = msvcrt.open_osfhandle(stderr_read_handle, os.O_RDONLY | os.O_TEXT) + with os.fdopen(fd_err, "rb") as f_err: + stderr = f_err.read() + ret["stderr"] = stderr finally: - salt.platform.win.kernel32.CloseHandle(dupin) - salt.platform.win.kernel32.CloseHandle(c2pwrite) - salt.platform.win.kernel32.CloseHandle(errwrite) - - # Initialize ret and set first element - ret = {"pid": process_info.dwProcessId} - - # Get Standard Out - fd_out = msvcrt.open_osfhandle(c2pread, os.O_RDONLY | os.O_TEXT) - with os.fdopen(fd_out, "r") as f_out: - ret["stdout"] = f_out.read() - - # Get Standard Error - fd_err = msvcrt.open_osfhandle(errread, os.O_RDONLY | os.O_TEXT) - with os.fdopen(fd_err, "r") as f_err: - ret["stderr"] = f_err.read() - - # Get Return Code - if ( - salt.platform.win.kernel32.WaitForSingleObject( - process_info.hProcess, win32event.INFINITE - ) - == win32con.WAIT_OBJECT_0 - ): - exitcode = salt.platform.win.wintypes.DWORD() - salt.platform.win.kernel32.GetExitCodeProcess( - process_info.hProcess, ctypes.byref(exitcode) + if hProcess is not None: + salt.platform.win.kernel32.CloseHandle(hProcess) + if hThread is not None: + salt.platform.win.kernel32.CloseHandle(hThread) + if not wait: + salt.platform.win.kernel32.CloseHandle(stdout_read_handle) + salt.platform.win.kernel32.CloseHandle(stderr_read_handle) + + if result == win32con.WAIT_TIMEOUT: + raise TimedProcTimeoutError( + "{} : Timed out after {} seconds".format(cmd, kwargs["timeout"]) ) - ret["retcode"] = exitcode.value - - # Close handle to process - salt.platform.win.kernel32.CloseHandle(process_info.hProcess) return ret From 7c5d90f033f2294268b353a900c65b120371a24c Mon Sep 17 00:00:00 2001 From: xsmile Date: Mon, 7 Jul 2025 14:35:12 +0200 Subject: [PATCH 14/71] win_runas: update tests --- .../functional/modules/cmd/test_powershell.py | 1 + .../functional/utils/test_win_runas.py | 111 +++++++++++++++++- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/tests/pytests/functional/modules/cmd/test_powershell.py b/tests/pytests/functional/modules/cmd/test_powershell.py index f8913db0493d..ab1db29685ba 100644 --- a/tests/pytests/functional/modules/cmd/test_powershell.py +++ b/tests/pytests/functional/modules/cmd/test_powershell.py @@ -207,5 +207,6 @@ def test_cmd_run_encoded_cmd_runas(shell, account, cmd, expected, encoded_cmd): encoded_cmd=encoded_cmd, runas=account.username, password=account.password, + redirect_stderr=False, ) assert ret == expected diff --git a/tests/pytests/functional/utils/test_win_runas.py b/tests/pytests/functional/utils/test_win_runas.py index 56affe0f7a68..45810d2f4de2 100644 --- a/tests/pytests/functional/utils/test_win_runas.py +++ b/tests/pytests/functional/utils/test_win_runas.py @@ -8,6 +8,7 @@ import salt.modules.win_useradd as win_useradd import salt.utils.win_runas as win_runas +from salt.exceptions import TimedProcTimeoutError try: import salt.platform.win @@ -54,7 +55,7 @@ def test_compound_runas(user, cmd, expected): username=user.username, password=user.password, ) - assert expected in result["stdout"] + assert expected in result["stdout"].decode() @pytest.mark.parametrize( @@ -73,32 +74,130 @@ def test_compound_runas_unpriv(user, cmd, expected): username=user.username, password=user.password, ) - assert expected in result["stdout"] + assert expected in result["stdout"].decode() def test_runas_str_user(user): result = win_runas.runas( cmd="whoami", username=user.username, password=user.password ) - assert user.username in result["stdout"] + assert user.username in result["stdout"].decode() def test_runas_int_user(int_user): result = win_runas.runas( cmd="whoami", username=int(int_user.username), password=int_user.password ) - assert str(int_user.username) in result["stdout"] + assert str(int_user.username) in result["stdout"].decode() def test_runas_unpriv_str_user(user): result = win_runas.runas_unpriv( cmd="whoami", username=user.username, password=user.password ) - assert user.username in result["stdout"] + assert user.username in result["stdout"].decode() def test_runas_unpriv_int_user(int_user): result = win_runas.runas_unpriv( cmd="whoami", username=int(int_user.username), password=int_user.password ) - assert str(int_user.username) in result["stdout"] + assert str(int_user.username) in result["stdout"].decode() + + +def test_runas_redirect_stderr(user): + expected = "invalidcommand" + result = win_runas.runas( + cmd=f'cmd /c "{expected}"', + username=user.username, + password=user.password, + redirect_stderr=True, + ) + assert isinstance(result["pid"], int) + assert result["retcode"] == 1 + assert expected in result["stdout"].decode() + assert result["stderr"].decode() == "" + + +def test_runas_unpriv_redirect_stderr(user): + expected = "invalidcommand" + result = win_runas.runas_unpriv( + cmd=f'cmd /c "{expected}"', + username=user.username, + password=user.password, + redirect_stderr=True, + ) + assert isinstance(result["pid"], int) + assert result["retcode"] == 1 + assert expected in result["stdout"].decode() + assert result["stderr"].decode() == "" + + +def test_runas_env(user): + expected = "foo" + result = win_runas.runas( + cmd='cmd /c "echo %FOO%"', + username=user.username, + password=user.password, + env={"FOO": expected}, + ) + assert result["stdout"].decode().strip() == expected + + +def test_runas_unpriv_env(user): + expected = "foo" + result = win_runas.runas_unpriv( + cmd='cmd /c "echo %FOO%"', + username=user.username, + password=user.password, + env={"FOO": expected}, + ) + assert result["stdout"].decode().strip() == expected + + +def test_runas_timeout(user): + timeout = 1 + with pytest.raises(TimedProcTimeoutError): + result = win_runas.runas( + cmd='powershell -command "sleep 10"', + username=user.username, + password=user.password, + timeout=timeout, + ) + + +def test_runas_unpriv_timeout(user): + timeout = 1 + with pytest.raises(TimedProcTimeoutError): + result = win_runas.runas_unpriv( + cmd='powershell -command "sleep 10"', + username=user.username, + password=user.password, + timeout=timeout, + ) + + +def test_runas_wait(user): + result = win_runas.runas( + cmd='cmd /c "timeout /t 10"', + username=user.username, + password=user.password, + bg=True, + ) + assert isinstance(result["pid"], int) + assert "retcode" not in result + assert "stdout" not in result + assert "stderr" not in result + + +def test_runas_unpriv_wait(user): + result = win_runas.runas_unpriv( + cmd='cmd /c "timeout /t 10"', + username=user.username, + password=user.password, + bg=True, + ) + assert isinstance(result["pid"], int) + assert "retcode" not in result + assert "stdout" not in result + assert "stderr" not in result From cd64edc0edc13ef3a54ee69ba45b02bd57f3a049 Mon Sep 17 00:00:00 2001 From: xsmile Date: Thu, 10 Jul 2025 18:50:47 +0200 Subject: [PATCH 15/71] add changelog --- changelog/68157.added.md | 1 + changelog/68157.fixed.md | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog/68157.added.md create mode 100644 changelog/68157.fixed.md diff --git a/changelog/68157.added.md b/changelog/68157.added.md new file mode 100644 index 000000000000..560f49d62614 --- /dev/null +++ b/changelog/68157.added.md @@ -0,0 +1 @@ +win_runas: support cmdmod parameters bg, env, redirect_stderr, timeout diff --git a/changelog/68157.fixed.md b/changelog/68157.fixed.md new file mode 100644 index 000000000000..e77718bcddd0 --- /dev/null +++ b/changelog/68157.fixed.md @@ -0,0 +1,2 @@ +win_runas: fix output decoding exceptions +win_runas: ensure opened handles are closed From e87c7465ed44b231f02167c3f6c150325672d387 Mon Sep 17 00:00:00 2001 From: xsmile Date: Fri, 11 Jul 2025 22:38:55 +0200 Subject: [PATCH 16/71] add docs --- salt/modules/cmdmod.py | 84 +++++++++++++++++++++++++++++++++++++++++ salt/utils/win_runas.py | 3 ++ 2 files changed, 87 insertions(+) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index bcab216abcba..542ab569ce74 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -1216,6 +1216,9 @@ def run( .. versionadded:: 2016.3.0 + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param dict env: Environment variables to be set prior to execution. .. note:: @@ -1231,6 +1234,9 @@ def run( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -1290,6 +1296,9 @@ def run( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -1299,6 +1308,9 @@ def run( .. versionadded:: 3006.9 + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool encoded_cmd: Specify if the supplied command is encoded. Only applies to shell 'powershell' and 'pwsh'. @@ -1537,6 +1549,9 @@ def shell( :param bool bg: If True, run command in background and do not await or deliver its results + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param dict env: Environment variables to be set prior to execution. .. note:: @@ -1552,6 +1567,9 @@ def shell( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -1612,6 +1630,9 @@ def shell( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -1810,6 +1831,9 @@ def run_stdout( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -1870,6 +1894,9 @@ def run_stdout( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -2044,6 +2071,9 @@ def run_stderr( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -2104,6 +2134,9 @@ def run_stderr( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -2280,6 +2313,9 @@ def run_all( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -2340,6 +2376,9 @@ def run_all( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -2374,6 +2413,9 @@ def run_all( .. versionadded:: 2015.8.2 + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param str password: Windows only. Required when specifying ``runas``. This parameter will be ignored on non-Windows platforms. @@ -2384,6 +2426,9 @@ def run_all( .. versionadded:: 2016.3.6 + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param list success_retcodes: This parameter will allow a list of non-zero return codes that should be considered a success. If the return code returned from the run matches any in the provided list, @@ -2557,6 +2602,9 @@ def retcode( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -2602,6 +2650,9 @@ def retcode( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -2844,6 +2895,9 @@ def script( :param bool bg: If True, run script in background and do not await or deliver its results + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param dict env: Environment variables to be set prior to execution. .. note:: @@ -2859,6 +2913,9 @@ def script( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported. @@ -2908,6 +2965,9 @@ def script( seconds, send the subprocess sigterm, and if sigterm is ignored, follow up with sigkill + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -3162,6 +3222,9 @@ def script_retcode( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param str template: If this setting is applied then the named templating engine will be used to render the downloaded file. Currently jinja, mako, and wempy are supported. @@ -3202,6 +3265,9 @@ def script_retcode( seconds, send the subprocess sigterm, and if sigterm is ignored, follow up with sigkill + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -4066,6 +4132,9 @@ def powershell( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -4120,6 +4189,9 @@ def powershell( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -4408,6 +4480,9 @@ def powershell_all( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -4454,6 +4529,9 @@ def powershell_all( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool use_vt: Use VT utils (saltstack) to stream the command output more interactively to the console and the logs. This is experimental. @@ -4777,6 +4855,9 @@ def run_bg( matters, i.e. Window's uses `Path` as opposed to `PATH` for other systems. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + :param bool clean_env: Attempt to clean out all other shell environment variables and set only those provided in the 'env' argument to this function. @@ -4794,6 +4875,9 @@ def run_bg( :param int timeout: A timeout in seconds for the executed process to return. + .. versionchanged:: 3007.7 + Supported on Windows when running a command as an alternate user. + .. warning:: This function does not process commands through a shell unless the diff --git a/salt/utils/win_runas.py b/salt/utils/win_runas.py index f6c2ae19bd31..801b2f54fbcd 100644 --- a/salt/utils/win_runas.py +++ b/salt/utils/win_runas.py @@ -53,6 +53,9 @@ def __virtual__(): def close_handle(handle): + """ + Tries to close an object handle + """ if handle is not None: try: win32api.CloseHandle(handle) From c6cfd8d5d105e60a2476f659d814c04d1a369323 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 8 Jul 2025 10:15:03 -0600 Subject: [PATCH 17/71] Update zipp to 3.23.0 --- requirements/base.txt | 1 + requirements/static/ci/py3.10/darwin.txt | 3 ++- requirements/static/ci/py3.10/freebsd.txt | 3 ++- requirements/static/ci/py3.10/linux.txt | 3 ++- requirements/static/ci/py3.10/windows.txt | 3 ++- requirements/static/ci/py3.11/darwin.txt | 3 ++- requirements/static/ci/py3.11/freebsd.txt | 3 ++- requirements/static/ci/py3.11/linux.txt | 3 ++- requirements/static/ci/py3.11/windows.txt | 3 ++- requirements/static/ci/py3.12/cloud.txt | 3 ++- requirements/static/ci/py3.12/darwin.txt | 3 ++- requirements/static/ci/py3.12/docs.txt | 3 ++- requirements/static/ci/py3.12/freebsd.txt | 3 ++- requirements/static/ci/py3.12/lint.txt | 3 ++- requirements/static/ci/py3.12/linux.txt | 3 ++- requirements/static/ci/py3.12/windows.txt | 3 ++- requirements/static/ci/py3.9/darwin.txt | 3 ++- requirements/static/ci/py3.9/docs.txt | 2 +- requirements/static/ci/py3.9/freebsd.txt | 3 ++- requirements/static/ci/py3.9/linux.txt | 3 ++- requirements/static/ci/py3.9/windows.txt | 3 ++- requirements/static/pkg/py3.10/darwin.txt | 6 ++++-- requirements/static/pkg/py3.10/freebsd.txt | 6 ++++-- requirements/static/pkg/py3.10/linux.txt | 6 ++++-- requirements/static/pkg/py3.10/windows.txt | 6 ++++-- requirements/static/pkg/py3.11/darwin.txt | 6 ++++-- requirements/static/pkg/py3.11/freebsd.txt | 6 ++++-- requirements/static/pkg/py3.11/linux.txt | 6 ++++-- requirements/static/pkg/py3.11/windows.txt | 6 ++++-- requirements/static/pkg/py3.12/darwin.txt | 6 ++++-- requirements/static/pkg/py3.12/freebsd.txt | 6 ++++-- requirements/static/pkg/py3.12/linux.txt | 6 ++++-- requirements/static/pkg/py3.12/windows.txt | 6 ++++-- requirements/static/pkg/py3.9/darwin.txt | 6 ++++-- requirements/static/pkg/py3.9/freebsd.txt | 6 ++++-- requirements/static/pkg/py3.9/linux.txt | 6 ++++-- requirements/static/pkg/py3.9/windows.txt | 6 ++++-- 37 files changed, 104 insertions(+), 52 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 32c29b6a261d..a221d781133c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -28,6 +28,7 @@ python-dateutil>=2.8.1 python-gnupg>=0.4.7 cherrypy>=18.6.1 importlib-metadata>=3.3.0 +zipp>=3.19.1 cryptography>=42.0.0 # From old requirements/static/pkg/linux.in diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index d1a4a67734d2..4e47a83eccbe 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -533,9 +533,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.10/darwin.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.10/darwin.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 97808cb05c6e..5b28379d4a28 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -538,9 +538,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.10/freebsd.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.10/freebsd.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 6d4a35127ced..1b8b7bd6df50 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -602,9 +602,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.10/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.10/linux.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index 9fdda9a4599c..6d638b82bbc3 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -532,9 +532,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.10/windows.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.10/windows.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index d4a67c6d3358..3d76e6337b8e 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -524,9 +524,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.11/darwin.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.11/darwin.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index 3e19e3538db1..7404f7c716ca 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -530,9 +530,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.11/freebsd.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.11/freebsd.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 9c12d5269655..08e451a60cc5 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -592,9 +592,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.11/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.11/linux.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.11/windows.txt b/requirements/static/ci/py3.11/windows.txt index 4a3ace571a62..583b40bc1a72 100644 --- a/requirements/static/ci/py3.11/windows.txt +++ b/requirements/static/ci/py3.11/windows.txt @@ -523,9 +523,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.11/windows.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.11/windows.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.12/cloud.txt b/requirements/static/ci/py3.12/cloud.txt index 82bc0e94ba8f..acdca21e8d02 100644 --- a/requirements/static/ci/py3.12/cloud.txt +++ b/requirements/static/ci/py3.12/cloud.txt @@ -766,10 +766,11 @@ zc.lockfile==3.0.post1 # -c requirements/static/ci/../pkg/py3.12/linux.txt # -c requirements/static/ci/py3.12/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.12/linux.txt # -c requirements/static/ci/py3.12/linux.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.12/darwin.txt b/requirements/static/ci/py3.12/darwin.txt index 23696789bf45..e32571e346ec 100644 --- a/requirements/static/ci/py3.12/darwin.txt +++ b/requirements/static/ci/py3.12/darwin.txt @@ -524,9 +524,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.12/darwin.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.12/darwin.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.12/docs.txt b/requirements/static/ci/py3.12/docs.txt index 6deccaaee103..0a66fbe2a94c 100644 --- a/requirements/static/ci/py3.12/docs.txt +++ b/requirements/static/ci/py3.12/docs.txt @@ -288,9 +288,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/py3.12/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/py3.12/linux.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.12/freebsd.txt b/requirements/static/ci/py3.12/freebsd.txt index 4fca9c0d4d2c..94e50a84fd2e 100644 --- a/requirements/static/ci/py3.12/freebsd.txt +++ b/requirements/static/ci/py3.12/freebsd.txt @@ -530,9 +530,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.12/freebsd.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.12/freebsd.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.12/lint.txt b/requirements/static/ci/py3.12/lint.txt index 673237580d04..c36cde3dd91a 100644 --- a/requirements/static/ci/py3.12/lint.txt +++ b/requirements/static/ci/py3.12/lint.txt @@ -774,10 +774,11 @@ zc.lockfile==3.0.post1 # -c requirements/static/ci/../pkg/py3.12/linux.txt # -c requirements/static/ci/py3.12/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.12/linux.txt # -c requirements/static/ci/py3.12/linux.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.12/linux.txt b/requirements/static/ci/py3.12/linux.txt index ef9e3965a18d..b0353af3e94f 100644 --- a/requirements/static/ci/py3.12/linux.txt +++ b/requirements/static/ci/py3.12/linux.txt @@ -592,9 +592,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.12/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.12/linux.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.12/windows.txt b/requirements/static/ci/py3.12/windows.txt index bc090f5f71b2..4532689afe9b 100644 --- a/requirements/static/ci/py3.12/windows.txt +++ b/requirements/static/ci/py3.12/windows.txt @@ -523,9 +523,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.12/windows.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.12/windows.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 59d528e65d79..3102def03163 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -577,9 +577,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.9/darwin.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.9/darwin.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.9/docs.txt b/requirements/static/ci/py3.9/docs.txt index ccfa6d0b7720..aa580f7b4be8 100644 --- a/requirements/static/ci/py3.9/docs.txt +++ b/requirements/static/ci/py3.9/docs.txt @@ -152,7 +152,7 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/py3.9/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/py3.9/linux.txt # importlib-metadata diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 1c05cd7e2bd5..a90aa096075f 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -582,9 +582,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.9/freebsd.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.9/freebsd.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index c4a67616f600..2365c092ea7d 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -634,9 +634,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.9/linux.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.9/linux.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index f06a6f9093b2..c4a792a1cf43 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -534,9 +534,10 @@ zc.lockfile==3.0.post1 # via # -c requirements/static/ci/../pkg/py3.9/windows.txt # cherrypy -zipp==3.16.2 +zipp==3.23.0 # via # -c requirements/static/ci/../pkg/py3.9/windows.txt + # -r requirements/base.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/static/pkg/py3.10/darwin.txt b/requirements/static/pkg/py3.10/darwin.txt index 300236bcbdc0..e59b4cbc7839 100644 --- a/requirements/static/pkg/py3.10/darwin.txt +++ b/requirements/static/pkg/py3.10/darwin.txt @@ -140,8 +140,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.10/freebsd.txt b/requirements/static/pkg/py3.10/freebsd.txt index e06581a2ce17..d43d78e03fbd 100644 --- a/requirements/static/pkg/py3.10/freebsd.txt +++ b/requirements/static/pkg/py3.10/freebsd.txt @@ -140,8 +140,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.10/linux.txt b/requirements/static/pkg/py3.10/linux.txt index 4f4790eaa9ee..1fc8d1ef4636 100644 --- a/requirements/static/pkg/py3.10/linux.txt +++ b/requirements/static/pkg/py3.10/linux.txt @@ -142,8 +142,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.10/windows.txt b/requirements/static/pkg/py3.10/windows.txt index 1e044d3f0ea9..28d0271bc17d 100644 --- a/requirements/static/pkg/py3.10/windows.txt +++ b/requirements/static/pkg/py3.10/windows.txt @@ -160,8 +160,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.11/darwin.txt b/requirements/static/pkg/py3.11/darwin.txt index c0fb0b1fdacb..57271087e13b 100644 --- a/requirements/static/pkg/py3.11/darwin.txt +++ b/requirements/static/pkg/py3.11/darwin.txt @@ -138,8 +138,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.11/freebsd.txt b/requirements/static/pkg/py3.11/freebsd.txt index 772a36122613..c45fda127373 100644 --- a/requirements/static/pkg/py3.11/freebsd.txt +++ b/requirements/static/pkg/py3.11/freebsd.txt @@ -140,8 +140,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.11/linux.txt b/requirements/static/pkg/py3.11/linux.txt index 8a91349936cb..fc06f38857a9 100644 --- a/requirements/static/pkg/py3.11/linux.txt +++ b/requirements/static/pkg/py3.11/linux.txt @@ -142,8 +142,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.11/windows.txt b/requirements/static/pkg/py3.11/windows.txt index 527a79b64457..02d5f25cd90f 100644 --- a/requirements/static/pkg/py3.11/windows.txt +++ b/requirements/static/pkg/py3.11/windows.txt @@ -158,8 +158,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.12/darwin.txt b/requirements/static/pkg/py3.12/darwin.txt index 845ed2e2981c..84d9c033ff02 100644 --- a/requirements/static/pkg/py3.12/darwin.txt +++ b/requirements/static/pkg/py3.12/darwin.txt @@ -138,8 +138,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.12/freebsd.txt b/requirements/static/pkg/py3.12/freebsd.txt index bf3a5369665b..be8133e57e98 100644 --- a/requirements/static/pkg/py3.12/freebsd.txt +++ b/requirements/static/pkg/py3.12/freebsd.txt @@ -140,8 +140,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.12/linux.txt b/requirements/static/pkg/py3.12/linux.txt index c029bf0f7053..f8431731ec6a 100644 --- a/requirements/static/pkg/py3.12/linux.txt +++ b/requirements/static/pkg/py3.12/linux.txt @@ -142,8 +142,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.12/windows.txt b/requirements/static/pkg/py3.12/windows.txt index 576ef23f1226..d03fecb64d7b 100644 --- a/requirements/static/pkg/py3.12/windows.txt +++ b/requirements/static/pkg/py3.12/windows.txt @@ -158,8 +158,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.9/darwin.txt b/requirements/static/pkg/py3.9/darwin.txt index ab9f543f8262..386cc17233be 100644 --- a/requirements/static/pkg/py3.9/darwin.txt +++ b/requirements/static/pkg/py3.9/darwin.txt @@ -140,8 +140,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.9/freebsd.txt b/requirements/static/pkg/py3.9/freebsd.txt index 0e9300584c35..de242bbbb10c 100644 --- a/requirements/static/pkg/py3.9/freebsd.txt +++ b/requirements/static/pkg/py3.9/freebsd.txt @@ -140,8 +140,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.9/linux.txt b/requirements/static/pkg/py3.9/linux.txt index 318a2e3cc69d..fce227142cfd 100644 --- a/requirements/static/pkg/py3.9/linux.txt +++ b/requirements/static/pkg/py3.9/linux.txt @@ -142,8 +142,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/static/pkg/py3.9/windows.txt b/requirements/static/pkg/py3.9/windows.txt index adce56cd7cf1..f0d197d8585b 100644 --- a/requirements/static/pkg/py3.9/windows.txt +++ b/requirements/static/pkg/py3.9/windows.txt @@ -161,8 +161,10 @@ yarl==1.20.1 # via aiohttp zc.lockfile==3.0.post1 # via cherrypy -zipp==3.16.2 - # via importlib-metadata +zipp==3.23.0 + # via + # -r requirements/base.txt + # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From fe39f0db1164e39994f6c204accb8271acb58b98 Mon Sep 17 00:00:00 2001 From: xsmile Date: Sun, 6 Jul 2025 15:47:42 +0200 Subject: [PATCH 18/71] cmdmod: unset shell/python_shell variables by default --- salt/modules/cmdmod.py | 115 ++++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 47 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 2b9faf106d83..9eee58e56541 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -100,20 +100,25 @@ def _check_cb(cb_): return lambda x: x -def _python_shell_default(python_shell, __pub_jid): +def _python_shell_default(python_shell, shell=False): """ - Set python_shell default based on remote execution and __opts__['cmd_safe'] + Set python_shell default based on the shell parameter and __opts__['cmd_safe'] """ - try: - # Default to python_shell=True when run directly from remote execution - # system. Cross-module calls won't have a jid. - if __pub_jid and python_shell is None: - return True - elif __opts__.get("cmd_safe", True) is False and python_shell is None: - # Override-switch for python_shell - return True - except NameError: - pass + if shell: + if salt.utils.platform.is_windows(): + # On Windows python_shell / subprocess 'shell' parameter must always be + # False as we prepend the shell manually + return False + else: + # Non-Windows requires python_shell to be enabled + return True if python_shell is None else python_shell + else: + try: + if __opts__.get("cmd_safe", True) is False and python_shell is None: + # Override-switch for python_shell + return True + except NameError: + pass return python_shell @@ -343,7 +348,7 @@ def _run( log_callback=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=False, env=None, clean_env=False, @@ -373,7 +378,9 @@ def _run( """ if "pillar" in kwargs and not pillar_override: pillar_override = kwargs["pillar"] - if output_loglevel != "quiet" and _is_valid_shell(shell) is False: + if shell is None and python_shell and not salt.utils.platform.is_windows(): + shell = DEFAULT_SHELL + if output_loglevel != "quiet" and shell and _is_valid_shell(shell) is False: log.warning( "Attempt to run a shell command with what may be an invalid shell! " "Check to ensure that the shell <%s> is valid for this user.", @@ -415,9 +422,10 @@ def _run( change_windows_codepage = False if not salt.utils.platform.is_windows(): - if not os.path.isfile(shell) or not os.access(shell, os.X_OK): - msg = f"The shell {shell} is not available" - raise CommandExecutionError(msg) + if shell: + if not os.path.isfile(shell) or not os.access(shell, os.X_OK): + msg = f"The shell {shell} is not available" + raise CommandExecutionError(msg) elif use_vt: # Memoization so not much overhead raise CommandExecutionError("VT not available on windows") else: @@ -555,10 +563,14 @@ def _run( env_cmd.extend(["-u", runas]) if group: env_cmd.extend(["-g", group]) - if shell != DEFAULT_SHELL: - env_cmd.extend(["-s", "--", shell, "-c"]) + if shell: + if shell != DEFAULT_SHELL: + env_cmd.extend(["-s", "--", shell, "-c"]) + else: + env_cmd.extend(["-i", "--"]) else: - env_cmd.extend(["-i", "--"]) + # do not invoke a shell at all + env_cmd.extend(["--"]) elif __grains__["os"] in ["FreeBSD"]: env_cmd = [ "su", @@ -571,7 +583,11 @@ def _run( elif __grains__["os_family"] in ["AIX"]: env_cmd = ["su", "-", runas, "-c"] else: - env_cmd = ["su", "-s", shell, "-", runas, "-c"] + # su invokes a shell by design + if shell: + env_cmd = ["su", "-s", shell, "-", runas, "-c"] + else: + env_cmd = ["su", "-", runas, "-c"] if not salt.utils.pkg.check_bundled(): if __grains__["os"] in ["FreeBSD"]: @@ -984,7 +1000,7 @@ def _run_quiet( stdin=None, output_encoding=None, runas=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=False, env=None, template=None, @@ -1033,7 +1049,7 @@ def _run_all_quiet( cwd=None, stdin=None, runas=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=False, env=None, template=None, @@ -1089,7 +1105,7 @@ def run( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, clean_env=False, @@ -1357,7 +1373,7 @@ def run( salt '*' cmd.run cmd='sed -e s/=/:/g' """ - python_shell = _python_shell_default(python_shell, kwargs.get("__pub_jid", "")) + python_shell = _python_shell_default(python_shell, shell) stderr = subprocess.STDOUT if redirect_stderr else subprocess.PIPE ret = _run( cmd, @@ -1633,10 +1649,17 @@ def shell( salt '*' cmd.shell cmd='sed -e s/=/:/g' """ - if "python_shell" in kwargs: - python_shell = kwargs.pop("python_shell") + if shell: + if salt.utils.platform.is_windows(): + # shell invocations are handled manually + python_shell = False + else: + if "python_shell" in kwargs: + python_shell = kwargs.pop("python_shell") + else: + python_shell = True else: - python_shell = True + python_shell = False return run( cmd, cwd=cwd, @@ -1675,7 +1698,7 @@ def run_stdout( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, clean_env=False, @@ -1870,7 +1893,7 @@ def run_stdout( salt '*' cmd.run_stdout "grep f" stdin='one\\ntwo\\nthree\\nfour\\nfive\\n' """ - python_shell = _python_shell_default(python_shell, kwargs.get("__pub_jid", "")) + python_shell = _python_shell_default(python_shell, shell) ret = _run( cmd, runas=runas, @@ -1909,7 +1932,7 @@ def run_stderr( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, clean_env=False, @@ -2104,7 +2127,7 @@ def run_stderr( salt '*' cmd.run_stderr "grep f" stdin='one\\ntwo\\nthree\\nfour\\nfive\\n' """ - python_shell = _python_shell_default(python_shell, kwargs.get("__pub_jid", "")) + python_shell = _python_shell_default(python_shell, shell) ret = _run( cmd, runas=runas, @@ -2143,7 +2166,7 @@ def run_all( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, clean_env=False, @@ -2381,7 +2404,7 @@ def run_all( salt '*' cmd.run_all "grep f" stdin='one\\ntwo\\nthree\\nfour\\nfive\\n' """ - python_shell = _python_shell_default(python_shell, kwargs.get("__pub_jid", "")) + python_shell = _python_shell_default(python_shell, shell) stderr = subprocess.STDOUT if redirect_stderr else subprocess.PIPE ret = _run( cmd, @@ -2425,7 +2448,7 @@ def retcode( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, clean_env=False, @@ -2606,8 +2629,7 @@ def retcode( salt '*' cmd.retcode "grep f" stdin='one\\ntwo\\nthree\\nfour\\nfive\\n' """ - python_shell = _python_shell_default(python_shell, kwargs.get("__pub_jid", "")) - + python_shell = _python_shell_default(python_shell, shell) ret = _run( cmd, runas=runas, @@ -2644,7 +2666,7 @@ def _retcode_quiet( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=False, env=None, clean_env=False, @@ -2702,7 +2724,7 @@ def script( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, template=None, @@ -2908,7 +2930,7 @@ def script( saltenv = __opts__.get("saltenv", "base") except NameError: saltenv = "base" - python_shell = _python_shell_default(python_shell, kwargs.get("__pub_jid", "")) + python_shell = _python_shell_default(python_shell, shell) def _cleanup_tempfile(path): try: @@ -3028,7 +3050,7 @@ def script_retcode( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, template="jinja", @@ -3373,7 +3395,7 @@ def run_chroot( stdin=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=True, binds=None, env=None, @@ -4129,7 +4151,7 @@ def powershell( if "python_shell" in kwargs: python_shell = kwargs.pop("python_shell") else: - python_shell = True + python_shell = False if isinstance(cmd, list): cmd = " ".join(cmd) @@ -4494,7 +4516,7 @@ def powershell_all( if "python_shell" in kwargs: python_shell = kwargs.pop("python_shell") else: - python_shell = True + python_shell = False if isinstance(cmd, list): cmd = " ".join(cmd) @@ -4606,7 +4628,7 @@ def run_bg( cwd=None, runas=None, group=None, - shell=DEFAULT_SHELL, + shell=None, python_shell=None, env=None, clean_env=False, @@ -4811,8 +4833,7 @@ def run_bg( salt '*' cmd.run_bg cmd='ls -lR / | sed -e s/=/:/g > /tmp/dontwait' """ - - python_shell = _python_shell_default(python_shell, kwargs.get("__pub_jid", "")) + python_shell = _python_shell_default(python_shell, shell) res = _run( cmd, stdin=None, From cc32817a4b82bc806e7afd5f4462108d88b7562f Mon Sep 17 00:00:00 2001 From: xsmile Date: Sun, 6 Jul 2025 16:41:43 +0200 Subject: [PATCH 19/71] cmdmod: change argument processing/quoting --- salt/modules/cmdmod.py | 91 +++++++++++--------------------- salt/platform/win.py | 111 ++++++---------------------------------- salt/utils/win_runas.py | 7 +++ 3 files changed, 54 insertions(+), 155 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 9eee58e56541..963c2e27b76e 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -264,18 +264,11 @@ def _check_avail(cmd): def _prep_powershell_cmd(win_shell, cmd, encoded_cmd): """ - Prep cmd when shell is powershell. If we were called by script(), then fake - out the Windows shell to run a Powershell script. Otherwise, just run a - Powershell command. + Prep cmd when shell is powershell.exe or pwsh.exe. If we were called by script(), then + run it via the -File parameter, Otherwise, run the command via the -Command parameter. """ - # Find the full path to the shell - win_shell = salt.utils.path.which(win_shell) - - if not win_shell: - raise CommandExecutionError(f"PowerShell binary not found: {win_shell}") - new_cmd = [ - f'"{win_shell}"', + win_shell, "-NonInteractive", "-NoProfile", "-ExecutionPolicy", @@ -287,52 +280,16 @@ def _prep_powershell_cmd(win_shell, cmd, encoded_cmd): # The third item[2] in each tuple is the name of that method. stack = traceback.extract_stack(limit=3) if stack[-3][2] == "script": - # If this is cmd.script, then we're running a file - # You might be tempted to use -File here instead of -Command - # The problem with using -File is that any arguments that contain - # powershell commands themselves will not be evaluated - # See GitHub issue #56195 - new_cmd.append("-Command") - if isinstance(cmd, list): - cmd = " ".join(cmd) - # We need to append $LASTEXITCODE here to return the actual exit code - # from the script. Otherwise, it will always return 1 on any non-zero - # exit code failure. Issue: #60884 - new_cmd.append(f'"& {cmd.strip()}; exit $LASTEXITCODE"') + new_cmd.append("-File") + new_cmd.extend(cmd) elif encoded_cmd: - new_cmd.extend(["-EncodedCommand", f'"{cmd}"']) + new_cmd.extend(["-EncodedCommand", cmd]) else: - # Strip whitespace + new_cmd.append("-Command") if isinstance(cmd, list): cmd = " ".join(cmd) + new_cmd.append(cmd) - # Commands that are a specific keyword behave differently. They fail if - # you add a "&" to the front. Add those here as we find them: - keywords = [ - "(", - "[", - "$", - "&", - ".", - "data", - "do", - "for", - "foreach", - "if", - "trap", - "while", - "try", - "Configuration", - ] - - for keyword in keywords: - if cmd.lower().startswith(keyword.lower()): - new_cmd.extend(["-Command", f'"{cmd.strip()}"']) - break - else: - new_cmd.extend(["-Command", f'"& {cmd.strip()}"']) - - new_cmd = " ".join(new_cmd) log.debug(new_cmd) return new_cmd @@ -454,10 +411,23 @@ def _run( if not HAS_WIN_RUNAS: msg = "missing salt/utils/win_runas.py" raise CommandExecutionError(msg) - if any(word in shell.lower().strip() for word in ["powershell", "pwsh"]): - cmd = _prep_powershell_cmd(shell, cmd, encoded_cmd) + + if shell: + # Find the full path to the shell + win_shell = salt.utils.path.which(shell) + if not win_shell: + raise CommandExecutionError(f"shell binary not found: {win_shell}") + + # Prepare the command to be executed + win_shell_lower = win_shell.lower() + if any(win_shell_lower.endswith(word) for word in ["powershell.exe", "pwsh.exe"]): + cmd = _prep_powershell_cmd(win_shell, cmd, encoded_cmd) + elif any(win_shell_lower.endswith(word) for word in ["cmd.exe"]): + cmd = salt.platform.win.prepend_cmd(win_shell, cmd) + else: + raise CommandExecutionError(f"unsupported shell type: {win_shell}") else: - cmd = salt.platform.win.prepend_cmd(cmd) + win_shell = None env = _parse_env(env) @@ -3002,15 +2972,17 @@ def _cleanup_tempfile(path): os.chown(path, __salt__["file.user_to_uid"](runas), -1) if salt.utils.platform.is_windows(): - if shell.lower() not in ["powershell", "pwsh"]: - cmd_path = _cmd_quote(path, escape=False) - else: - cmd_path = path + cmd_path = path else: cmd_path = _cmd_quote(path) + if isinstance(args, (list, tuple)): + new_cmd = [cmd_path, *args] if args else [cmd_path] + else: + new_cmd = [cmd_path, str(args)] if args else [cmd_path] + ret = _run( - cmd_path + " " + str(args) if args else cmd_path, + new_cmd, cwd=cwd, stdin=stdin, output_encoding=output_encoding, @@ -4181,7 +4153,6 @@ def powershell( cmd = salt.utils.stringutils.to_str(cmd) encoded_cmd = True else: - cmd = f"{{ {cmd} }}" encoded_cmd = False # Retrieve the response, while overriding shell with 'powershell' diff --git a/salt/platform/win.py b/salt/platform/win.py index 0930f840457c..a8be6bf8123e 100644 --- a/salt/platform/win.py +++ b/salt/platform/win.py @@ -13,7 +13,7 @@ import ctypes import logging import os -import shlex +import subprocess from ctypes import wintypes # pylint: disable=3rd-party-module-not-gated @@ -25,8 +25,6 @@ import win32security import win32service -import salt.utils.path - # pylint: enable=3rd-party-module-not-gated # Set up logging @@ -1341,95 +1339,18 @@ def CreateProcessWithLogonW( return process_info -def prepend_cmd(cmd): - # Some commands are only available when run from a cmd shell. These are - # built-in commands such as echo. So, let's check for the binary in the - # path. If it does not exist, let's assume it's a built-in and requires us - # to run it in a cmd prompt. - - if isinstance(cmd, str): - cmd = shlex.split(cmd, posix=False) - - # This is needed for handling paths with spaces - new_cmd = [] - for item in cmd: - # If item starts with ', escape any " - if item.startswith("'"): - item = item.replace('"', '\\"').replace("'", '"') - # If item starts with ", convert ' to escaped " - elif item.startswith('"'): - item = item.replace("'", '\\"') - # If there are spaces in item, wrap it in " - elif " " in item and "=" not in item: - item = f'"{item}"' - new_cmd.append(item) - cmd = new_cmd - - # Let's try to figure out what the fist command is so we can check for - # builtin commands such as echo - first_cmd = cmd[0].split("\\")[-1].strip("\"'") - - cmd = " ".join(cmd) - - # Known builtin cmd commands - known_builtins = [ - "assoc", - "break", - "call", - "cd", - "chdir", - "cls", - "color", - "copy", - "date", - "del", - "dir", - "echo", - "endlocal", - "erase", - "exit", - "for", - "ftype", - "goto", - "if", - "md", - "mklink", - "move", - "path", - "pause", - "popd", - "prompt", - "pushd", - "rd", - "rem", - "ren", - "rmdir", - "set", - "setlocal", - "shift", - "start", - "time", - "title", - "type", - "ver", - "verify", - "vol", - "::", - ] - - # If the first command is one of the known builtin commands or if it is a - # binary that can't be found, we'll prepend cmd /c. The command itself needs - # to be quoted - if first_cmd in known_builtins or salt.utils.path.which(first_cmd) is None: - log.debug("Command is either builtin or not found: %s", first_cmd) - cmd = f'cmd /c "{cmd}"' - - # There are a few more things we need to check that require cmd. If the cmd - # contains any of the following, we'll need to make sure it runs in cmd. - # We'll add to this list as more things are discovered. - check = ["&&", "||"] - if "cmd" not in cmd and any(chk in cmd for chk in check): - log.debug("Found logic that requires cmd: %s", check) - cmd = f'cmd /c "{cmd}"' - - return cmd +def prepend_cmd(win_shell, cmd): + """ + Prep cmd when shell is cmd.exe. Always use a command string instead of a list to satisfy + both CreateProcess and CreateProcessWithToken. + + cmd must be double-quoted to ensure proper handling of space characters. The first opening + quote and the closing quote are stripped automatically by the Win32 API. + """ + if isinstance(cmd, (list, tuple)): + args = subprocess.list2cmdline(cmd) + else: + args = cmd + new_cmd = f'{win_shell} /c "{args}"' + + return new_cmd diff --git a/salt/utils/win_runas.py b/salt/utils/win_runas.py index 5bff2d7c8dfb..001a292cee5e 100644 --- a/salt/utils/win_runas.py +++ b/salt/utils/win_runas.py @@ -6,6 +6,7 @@ import ctypes import logging import os +import subprocess import time from salt.exceptions import CommandExecutionError @@ -127,6 +128,12 @@ def runas(cmd, username, password=None, cwd=None): impersonation_token = None win32api.CloseHandle(th) + if isinstance(cmd, (list, tuple)): + # CreateProcess parameter lpCommandLine must be a string. + # Since it is called directly and not via the subprocess module, + # the arguments must be processed manually. + cmd = subprocess.list2cmdline(cmd) + # Impersonation of the SYSTEM user failed. Fallback to an un-privileged # runas. if not impersonation_token: From 327179f35d3eba03334a697f19de3e8455f5b333 Mon Sep 17 00:00:00 2001 From: xsmile Date: Sun, 6 Jul 2025 16:46:26 +0200 Subject: [PATCH 20/71] cmdmod: always remove the temporary script --- salt/modules/cmdmod.py | 72 +++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 963c2e27b76e..3880b3168f98 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -2966,7 +2966,20 @@ def _cleanup_tempfile(path): "stderr": "", "cache_error": True, } - shutil.copyfile(fn_, path) + try: + shutil.copyfile(fn_, path) + except FileNotFoundError: + _cleanup_tempfile(path) + # If a temp working directory was created (Windows), let's remove that + if win_cwd: + _cleanup_tempfile(cwd) + return { + "pid": 0, + "retcode": 1, + "stdout": "", + "stderr": "", + "cache_error": True, + } if not salt.utils.platform.is_windows(): os.chmod(path, 320) os.chown(path, __salt__["file.user_to_uid"](runas), -1) @@ -2981,30 +2994,39 @@ def _cleanup_tempfile(path): else: new_cmd = [cmd_path, str(args)] if args else [cmd_path] - ret = _run( - new_cmd, - cwd=cwd, - stdin=stdin, - output_encoding=output_encoding, - output_loglevel=output_loglevel, - log_callback=log_callback, - runas=runas, - group=group, - shell=shell, - python_shell=python_shell, - env=env, - umask=umask, - timeout=timeout, - reset_system_locale=reset_system_locale, - saltenv=saltenv, - use_vt=use_vt, - bg=bg, - password=password, - success_retcodes=success_retcodes, - success_stdout=success_stdout, - success_stderr=success_stderr, - **kwargs, - ) + ret = {} + try: + ret = _run( + new_cmd, + cwd=cwd, + stdin=stdin, + output_encoding=output_encoding, + output_loglevel=output_loglevel, + log_callback=log_callback, + runas=runas, + group=group, + shell=shell, + python_shell=python_shell, + env=env, + umask=umask, + timeout=timeout, + reset_system_locale=reset_system_locale, + saltenv=saltenv, + use_vt=use_vt, + bg=bg, + password=password, + success_retcodes=success_retcodes, + success_stdout=success_stdout, + success_stderr=success_stderr, + **kwargs, + ) + except Exception as exc: + log.error( + "cmd.script: Unable to run script '%s': %s", + new_cmd, + exc, + exc_info_on_loglevel=logging.DEBUG, + ) _cleanup_tempfile(path) # If a temp working directory was created (Windows), let's remove that if win_cwd: From e326dc79f2baf39e7386eee2ababa95cc7b018ba Mon Sep 17 00:00:00 2001 From: xsmile Date: Sun, 6 Jul 2025 16:48:56 +0200 Subject: [PATCH 21/71] cmdmod: set a default shell when running known script types --- salt/modules/cmdmod.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 3880b3168f98..6a5628df3ee7 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -2931,9 +2931,17 @@ def _cleanup_tempfile(path): obj_name=cwd, principal=runas, permissions="full_control" ) - path = salt.utils.files.mkstemp( - dir=cwd, suffix=os.path.splitext(salt.utils.url.split_env(source)[0])[1] - ) + (_, ext) = os.path.splitext(salt.utils.url.split_env(source)[0]) + + if salt.utils.platform.is_windows() and not shell: + extension_map = { + '.bat': 'cmd', + '.cmd': 'cmd', + '.ps1': 'powershell', + } + shell = extension_map.get(ext) + + path = salt.utils.files.mkstemp(dir=cwd, suffix=ext) if template: if "pillarenv" in kwargs or "pillar" in kwargs: From a1d26da72041672991a6a09e969c6955701d7387 Mon Sep 17 00:00:00 2001 From: xsmile Date: Sun, 6 Jul 2025 17:00:46 +0200 Subject: [PATCH 22/71] cmdmod: do not report success when a powershell command fails --- salt/modules/cmdmod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index 6a5628df3ee7..d954ce336c43 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -4171,7 +4171,7 @@ def powershell( # caught in a try/catch block. For example, the `Get-WmiObject` command will # often return a "Non Terminating Error". To fix this, make sure # `-ErrorAction Stop` is set in the powershell command - cmd = "try { " + cmd + ' } catch { "{}" }' + cmd = "try { " + cmd + ' } catch { Write-Error $_ }' if encode_cmd: # Convert the cmd to UTF-16LE without a BOM and base64 encode. From aeb58d6192e476af378c6f8cfedf80100e5c487e Mon Sep 17 00:00:00 2001 From: xsmile Date: Sun, 6 Jul 2025 22:54:51 +0200 Subject: [PATCH 23/71] cmdmod: update tests --- tests/integration/modules/test_cmdmod.py | 112 +++++++----- .../functional/modules/cmd/test_powershell.py | 61 +++++++ .../functional/modules/cmd/test_run_win.py | 161 ++++++++++++++++-- .../functional/modules/cmd/test_script.py | 102 ----------- .../modules/cmd/test_script_batch.py | 64 +++++++ .../modules/cmd/test_script_powershell.py | 105 ++++++++++++ .../functional/utils/test_win_runas.py | 16 +- tests/pytests/unit/modules/test_cmdmod.py | 103 +++++------ tests/pytests/unit/platform/test_win.py | 30 ++-- 9 files changed, 515 insertions(+), 239 deletions(-) delete mode 100644 tests/pytests/functional/modules/cmd/test_script.py create mode 100644 tests/pytests/functional/modules/cmd/test_script_batch.py create mode 100644 tests/pytests/functional/modules/cmd/test_script_powershell.py diff --git a/tests/integration/modules/test_cmdmod.py b/tests/integration/modules/test_cmdmod.py index dd6a667a6f42..17754d48cb88 100644 --- a/tests/integration/modules/test_cmdmod.py +++ b/tests/integration/modules/test_cmdmod.py @@ -17,6 +17,16 @@ ["python", "python2", "python2.6", "python2.7"] ) +if sys.platform.startswith("win32"): + SHELL = "cmd" + PYTHON_SHELL = False +elif sys.platform.startswith(("freebsd", "openbsd")): + SHELL = "/bin/sh" + PYTHON_SHELL = True +else: + SHELL = "/bin/bash" + PYTHON_SHELL = True + @pytest.mark.windows_whitelisted class CMDModuleTest(ModuleCase): @@ -103,7 +113,9 @@ def test_stdout(self): cmd.run_stdout """ self.assertEqual( - self.run_function("cmd.run_stdout", ['echo "cheese"']).rstrip(), + self.run_function( + "cmd.run_stdout", ['echo "cheese"'], shell=SHELL + ).rstrip(), "cheese" if not salt.utils.platform.is_windows() else '"cheese"', ) @@ -112,16 +124,12 @@ def test_stderr(self): """ cmd.run_stderr """ - if sys.platform.startswith(("freebsd", "openbsd")): - shell = "/bin/sh" - else: - shell = "/bin/bash" self.assertEqual( self.run_function( "cmd.run_stderr", - ['echo "cheese" 1>&2', f"shell={shell}"], - python_shell=True, + ['echo "cheese" 1>&2', f"shell={SHELL}"], + python_shell=PYTHON_SHELL, ).rstrip(), "cheese" if not salt.utils.platform.is_windows() else '"cheese"', ) @@ -131,15 +139,11 @@ def test_run_all(self): """ cmd.run_all """ - if sys.platform.startswith(("freebsd", "openbsd")): - shell = "/bin/sh" - else: - shell = "/bin/bash" ret = self.run_function( "cmd.run_all", - ['echo "cheese" 1>&2', f"shell={shell}"], - python_shell=True, + ['echo "cheese" 1>&2', f"shell={SHELL}"], + python_shell=PYTHON_SHELL, ) self.assertTrue("pid" in ret) self.assertTrue("retcode" in ret) @@ -208,7 +212,8 @@ def test_run_all_with_success_stderr(self): "cmd.run_all", [f"{func} {random_file}"], success_stderr=[expected_stderr], - python_shell=True, + shell=SHELL, + python_shell=PYTHON_SHELL, ) self.assertTrue("retcode" in ret) @@ -230,10 +235,10 @@ def test_script(self): """ cmd.script """ - args = "saltines crackers biscuits=yes" + args = ["saltines", "crackers", "biscuits=yes"] script = "salt://script.py" - ret = self.run_function("cmd.script", [script, args], saltenv="base") - self.assertEqual(ret["stdout"], args) + ret = self.run_function("cmd.script", [script], args=args, saltenv="base") + self.assertEqual(ret["stdout"], " ".join(args)) @pytest.mark.slow_test @pytest.mark.skip_on_windows @@ -241,10 +246,10 @@ def test_script_query_string(self): """ cmd.script """ - args = "saltines crackers biscuits=yes" + args = ["saltines", "crackers", "biscuits=yes"] script = "salt://script.py?saltenv=base" - ret = self.run_function("cmd.script", [script, args], saltenv="base") - self.assertEqual(ret["stdout"], args) + ret = self.run_function("cmd.script", [script], args=args, saltenv="base") + self.assertEqual(ret["stdout"], " ".join(args)) @pytest.mark.slow_test @pytest.mark.skip_on_windows @@ -263,12 +268,12 @@ def test_script_cwd(self): cmd.script with cwd """ tmp_cwd = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) - args = "saltines crackers biscuits=yes" + args = ["saltines", "crackers", "biscuits=yes"] script = "salt://script.py" ret = self.run_function( - "cmd.script", [script, args], cwd=tmp_cwd, saltenv="base" + "cmd.script", [script], args=args, cwd=tmp_cwd, saltenv="base" ) - self.assertEqual(ret["stdout"], args) + self.assertEqual(ret["stdout"], " ".join(args)) @pytest.mark.slow_test @pytest.mark.skip_on_windows @@ -281,12 +286,12 @@ def test_script_cwd_with_space(self): ) os.mkdir(tmp_cwd) - args = "saltines crackers biscuits=yes" + args = ["saltines", "crackers", "biscuits=yes"] script = "salt://script.py" ret = self.run_function( - "cmd.script", [script, args], cwd=tmp_cwd, saltenv="base" + "cmd.script", [script], args=args, cwd=tmp_cwd, saltenv="base" ) - self.assertEqual(ret["stdout"], args) + self.assertEqual(ret["stdout"], " ".join(args)) @pytest.mark.destructive_test def test_tty(self): @@ -398,7 +403,7 @@ def test_quotes(self): else: cmd = """echo 'SELECT * FROM foo WHERE bar="baz"' """ expected_result = 'SELECT * FROM foo WHERE bar="baz"' - result = self.run_function("cmd.run_stdout", [cmd]).strip() + result = self.run_function("cmd.run_stdout", [cmd], shell=SHELL).strip() self.assertEqual(result, expected_result) @pytest.mark.skip_if_not_root @@ -410,7 +415,10 @@ def test_quotes_runas(self): cmd = """echo 'SELECT * FROM foo WHERE bar="baz"' """ expected_result = 'SELECT * FROM foo WHERE bar="baz"' result = self.run_function( - "cmd.run_all", [cmd], runas=RUNTIME_VARS.RUNNING_TESTS_USER + "cmd.run_all", + [cmd], + python_shell=True, + runas=RUNTIME_VARS.RUNNING_TESTS_USER, ) errmsg = f"The command returned: {result}" self.assertEqual(result["retcode"], 0, errmsg) @@ -427,12 +435,20 @@ def test_avoid_injecting_shell_code_as_root(self): """ cmd = "echo $(id -u)" - root_id = self.run_function("cmd.run_stdout", [cmd]) + root_id = self.run_function("cmd.run_stdout", [cmd], python_shell=True) runas_root_id = self.run_function( - "cmd.run_stdout", [cmd], runas=RUNTIME_VARS.RUNNING_TESTS_USER + "cmd.run_stdout", + [cmd], + python_shell=True, + runas=RUNTIME_VARS.RUNNING_TESTS_USER, ) with self._ensure_user_exists(self.runas_usr): - user_id = self.run_function("cmd.run_stdout", [cmd], runas=self.runas_usr) + user_id = self.run_function( + "cmd.run_stdout", + [cmd], + python_shell=True, + runas=self.runas_usr, + ) self.assertNotEqual(user_id, root_id) self.assertNotEqual(user_id, runas_root_id) @@ -550,37 +566,45 @@ def test_hide_output(self): """ Test the hide_output argument """ - ls_command = ( - ["ls", "/"] if not salt.utils.platform.is_windows() else ["dir", "c:\\"] - ) + if salt.utils.platform.is_windows(): + ls_command = ["dir", "c:\\"] + shell = SHELL + else: + ls_command = ["ls", "/"] + shell = None error_command = ["thiscommanddoesnotexist"] # cmd.run - out = self.run_function("cmd.run", ls_command, hide_output=True) + out = self.run_function("cmd.run", ls_command, shell=shell, hide_output=True) self.assertEqual(out, "") # cmd.shell - out = self.run_function("cmd.shell", ls_command, hide_output=True) + out = self.run_function("cmd.shell", ls_command, shell=shell, hide_output=True) self.assertEqual(out, "") # cmd.run_stdout - out = self.run_function("cmd.run_stdout", ls_command, hide_output=True) + out = self.run_function( + "cmd.run_stdout", ls_command, shell=shell, hide_output=True + ) self.assertEqual(out, "") # cmd.run_stderr - out = self.run_function("cmd.shell", error_command, hide_output=True) + out = self.run_function("cmd.shell", ls_command, shell=shell, hide_output=True) self.assertEqual(out, "") # cmd.run_all (command should have produced stdout) - out = self.run_function("cmd.run_all", ls_command, hide_output=True) + out = self.run_function( + "cmd.run_all", ls_command, shell=shell, hide_output=True + ) self.assertEqual(out["stdout"], "") self.assertEqual(out["stderr"], "") - # cmd.run_all (command should have produced stderr) - out = self.run_function("cmd.run_all", error_command, hide_output=True) - self.assertEqual(out["stdout"], "") - self.assertEqual(out["stderr"], "") + # cmd.run_all (command should not have produced output) + out = self.run_function( + "cmd.run_all", error_command, shell=SHELL, hide_output=True + ) + self.assertIn("Unable to run command", out) @pytest.mark.slow_test def test_cmd_run_whoami(self): @@ -610,7 +634,7 @@ def test_windows_env_handling(self): Ensure that nt.environ is used properly with cmd.run* """ out = self.run_function( - "cmd.run", ["set"], env={"abc": "123", "ABC": "456"} + "cmd.run", ["set"], shell=SHELL, env={"abc": "123", "ABC": "456"} ).splitlines() self.assertIn("abc=123", out) self.assertIn("ABC=456", out) diff --git a/tests/pytests/functional/modules/cmd/test_powershell.py b/tests/pytests/functional/modules/cmd/test_powershell.py index f8913db0493d..10951d7bde8f 100644 --- a/tests/pytests/functional/modules/cmd/test_powershell.py +++ b/tests/pytests/functional/modules/cmd/test_powershell.py @@ -27,6 +27,60 @@ def account(): yield _account +@pytest.fixture +def issue_56195(state_tree, account): + tmpdir = f"C:\\Users\\{account.username}\\AppData\\Local\\Temp" + contents = """[CmdLetBinding()] +Param( + [SecureString] $SecureString +) +$Credential = New-Object System.Net.NetworkCredential("DummyId", $SecureString) +$Credential.Password +""" + with pytest.helpers.temp_file("test.ps1", contents, tmpdir) as f: + yield str(f), account + + +def test_args(issue_56195): + """ + Ensure that powershell processes an inline script with args where the args + contain powershell that needs to be rendered + """ + (script, _) = issue_56195 + password = "i like cheese" + args = ( + "-SecureString (ConvertTo-SecureString -String '{}' -AsPlainText -Force)" + " -ErrorAction Stop".format(password) + ) + # https://github.com/PowerShell/PowerShell/issues/18530 + cmd = f'$env:PSModulePath=""; {script} {args}' + ret = cmdmod.powershell(cmd, args=args, saltenv="base") + assert ret == password + + +def test_args_runas(issue_56195): + """ + Ensure that powershell with runas processes an inline script with args where + the args contain powershell that needs to be rendered + """ + (script, account) = issue_56195 + password = "i like cheese" + args = ( + "-SecureString (ConvertTo-SecureString -String '{}' -AsPlainText -Force)" + " -ErrorAction Stop".format(password) + ) + # https://github.com/PowerShell/PowerShell/issues/18530 + cmd = f'$env:PSModulePath=""; {script} {args}' + ret = cmdmod.powershell( + cmd, + args=args, + runas=account.username, + password=account.password, + saltenv="base", + ) + assert ret == password + + @pytest.mark.parametrize( "cmd, expected, encode_cmd", [ @@ -34,6 +88,9 @@ def account(): (["Write-Output", "Foo"], "Foo", False), ('Write-Output "Encoded Foo"', "Encoded Foo", True), (["Write-Output", '"Encoded Foo"'], "Encoded Foo", True), + ('$a="Plain";$b=\' Foo\';Write-Output ${a}${b}', "Plain Foo", False), + ("(Write-Output Foo)", "Foo", False), + ("& Write-Output Foo", "Foo", False), ], ) def test_powershell(shell, cmd, expected, encode_cmd): @@ -51,6 +108,9 @@ def test_powershell(shell, cmd, expected, encode_cmd): (["Write-Output", "Foo"], "Foo", False), ('Write-Output "Encoded Foo"', "Encoded Foo", True), (["Write-Output", '"Encoded Foo"'], "Encoded Foo", True), + ('$a="Plain";$b=\' Foo\';Write-Output ${a}${b}', "Plain Foo", False), + ("(Write-Output Foo)", "Foo", False), + ("& Write-Output Foo", "Foo", False), ], ) def test_powershell_runas(shell, account, cmd, expected, encode_cmd): @@ -205,6 +265,7 @@ def test_cmd_run_encoded_cmd_runas(shell, account, cmd, expected, encoded_cmd): cmd=cmd, shell=shell, encoded_cmd=encoded_cmd, + redirect_stderr=False, runas=account.username, password=account.password, ) diff --git a/tests/pytests/functional/modules/cmd/test_run_win.py b/tests/pytests/functional/modules/cmd/test_run_win.py index 96057a436b5e..0d9a72765a35 100644 --- a/tests/pytests/functional/modules/cmd/test_run_win.py +++ b/tests/pytests/functional/modules/cmd/test_run_win.py @@ -20,9 +20,15 @@ def account(): (299, 299, False), ], ) -def test_script_exitcode(modules, state_tree, exit_code, return_code, result): +def test_cmd_exitcode(modules, state_tree, exit_code, return_code, result): + """ + Test receiving an exit code with cmd.run + """ ret = modules.state.single( - "cmd.run", name=f"cmd.exe /c exit {exit_code}", success_retcodes=[2, 44, 300] + "cmd.run", + name=f"exit {exit_code}", + shell='cmd', + success_retcodes=[2, 44, 300], ) assert ret.result is result assert ret.filtered["changes"]["retcode"] == return_code @@ -35,12 +41,13 @@ def test_script_exitcode(modules, state_tree, exit_code, return_code, result): (299, 299, False), ], ) -def test_script_exitcode_runas( +def test_cmd_exitcode_runas( modules, state_tree, exit_code, return_code, result, account ): ret = modules.state.single( "cmd.run", - name=f"cmd.exe /c exit {exit_code}", + name=f"exit {exit_code}", + shel='cmd', success_retcodes=[2, 44, 300], runas=account.username, password=account.password, @@ -53,13 +60,17 @@ def test_script_exitcode_runas( "command, expected", [ ("echo foo", "foo"), - ("cmd /c echo foo", "foo"), ("whoami && echo foo", "foo"), - ("cmd /c whoami && echo foo", "foo"), + ('echo "foo \'bar\'"', '"foo \'bar\'"'), + ('echo|set /p="foo" & echo|set /p="bar"', "foobar"), + ('''echo "&<>[]|{}^=;!'+,`~ "''', '''"&<>[]|{}^=;!'+,`~ "'''), ], ) -def test_run_builtins(modules, command, expected): - result = modules.cmd.run(command) +def test_cmd_builtins(modules, command, expected): + """ + Test builtin cmd.exe commands + """ + result = modules.cmd.run(command, shell='cmd') assert expected in result @@ -67,13 +78,139 @@ def test_run_builtins(modules, command, expected): "command, expected", [ ("echo foo", "foo"), - ("cmd /c echo foo", "foo"), ("whoami && echo foo", "foo"), - ("cmd /c whoami && echo foo", "foo"), + ('echo "foo \'bar\'"', '"foo \'bar\'"'), + ('echo|set /p="foo" & echo|set /p="bar"', "foobar"), + ('''echo "&<>[]|{}^=;!'+,`~ "''', '''"&<>[]|{}^=;!'+,`~ "'''), ], ) -def test_run_builtins_runas(modules, account, command, expected): +def test_cmd_builtins_runas(modules, account, command, expected): + """ + Test builtin cmd.exe commands with runas + """ result = modules.cmd.run( - cmd=command, runas=account.username, password=account.password + cmd=command, shell='cmd', runas=account.username, password=account.password ) assert expected in result + + +@pytest.mark.parametrize( + "command, expected", + [ + (["whoami.exe", "/?"], 0), + (["whoami.exe", "/foo"], 1), + ("whoami.exe /?", 0), + ("whoami.exe /foo", 1), + ], +) +def test_binary(modules, command, expected): + """ + Test running a binary with cmd.run_all + """ + result = modules.cmd.run_all(cmd=command) + assert isinstance(result["pid"], int) + assert result["retcode"] == expected + + +@pytest.mark.parametrize( + "command, expected", + [ + (["whoami.exe", "/?"], 0), + (["whoami.exe", "/foo"], 1), + ("whoami.exe /?", 0), + ("whoami.exe /foo", 1), + ], +) +def test_binary_runas(modules, account, command, expected): + """ + Test running a binary with cmd.run_all and runas + """ + result = modules.cmd.run_all( + cmd=command, runas=account.username, password=account.password + ) + assert isinstance(result["pid"], int) + assert result["retcode"] == expected + + +@pytest.mark.parametrize( + "command, env, expected", + [ + ("echo %a%%b%", {"a": "foo", "b": "bar"}, "foobar"), + ], +) +def test_cmd_env(modules, command, env, expected): + """ + Test cmd.run with environment variables + """ + result = modules.cmd.run_all(command, shell='cmd', env=env) + assert isinstance(result["pid"], int) + assert result["retcode"] == 0 + assert result["stdout"] == expected + assert result["stderr"] == "" + +@pytest.mark.parametrize( + "command, env, expected", + [ + ("echo %a%%b%", {"a": "foo", "b": "bar"}, "foobar"), + ], +) +def test_cmd_env_runas(modules, account, command, env, expected): + """ + Test cmd.run with environment variables and runas + """ + result = modules.cmd.run_all( + command, shell='cmd', env=env, runas=account.username, password=account.password + ) + assert isinstance(result["pid"], int) + assert result["retcode"] == 0 + assert result["stdout"] == expected + assert result["stderr"] == "" + + +@pytest.mark.parametrize( + "command, expected, redirect_stderr", + [ + (["whoami.exe", "/foo"], "/foo", True), + (["whoami.exe", "/foo"], "/foo", False), + ], +) +def test_redirect_stderr(modules, command, expected, redirect_stderr): + """ + Test redirection of stderr to stdout by running cmd.run_all with invalid commands + """ + result = modules.cmd.run_all(command, redirect_stderr=redirect_stderr) + assert isinstance(result["pid"], int) + assert result["retcode"] == 1 + if redirect_stderr: + assert expected in result["stdout"] + assert result["stderr"] == "" + else: + assert result["stdout"] == "" + assert expected in result["stderr"] + + +@pytest.mark.parametrize( + "command, expected, redirect_stderr", + [ + (["whoami.exe", "/foo"], "/foo", True), + (["whoami.exe", "/foo"], "/foo", False), + ], +) +def test_redirect_stderr_runas(modules, account, command, expected, redirect_stderr): + """ + Test redirection of stderr to stdout by running cmd.run_all with runas and invalid commands + """ + result = modules.cmd.run_all( + command, + runas=account.username, + password=account.password, + redirect_stderr=redirect_stderr, + ) + assert isinstance(result["pid"], int) + assert result["retcode"] == 1 + if redirect_stderr: + assert expected in result["stdout"] + assert result["stderr"] == "" + else: + assert result["stdout"] == "" + assert expected in result["stderr"] diff --git a/tests/pytests/functional/modules/cmd/test_script.py b/tests/pytests/functional/modules/cmd/test_script.py deleted file mode 100644 index f4a355f5e9bf..000000000000 --- a/tests/pytests/functional/modules/cmd/test_script.py +++ /dev/null @@ -1,102 +0,0 @@ -import pytest - -import salt.utils.path - -pytestmark = [ - pytest.mark.core_test, - pytest.mark.windows_whitelisted, - pytest.mark.skip_unless_on_windows, -] - - -@pytest.fixture(scope="module") -def cmd(modules): - return modules.cmd - - -@pytest.fixture(scope="module") -def exitcode_script(state_tree): - exit_code = 12345 - script_contents = f""" - Write-Host "Expected exit code: {exit_code}" - exit {exit_code} - """ - with pytest.helpers.temp_file("exit_code.ps1", script_contents, state_tree): - yield exit_code - - -@pytest.fixture(params=["powershell", "pwsh"]) -def shell(request): - """ - This will run the test on powershell and powershell core (pwsh). If - powershell core is not installed that test run will be skipped - """ - if request.param == "pwsh" and salt.utils.path.which("pwsh") is None: - pytest.skip("Powershell 7 Not Present") - return request.param - - -@pytest.fixture(scope="module") -def account(): - with pytest.helpers.create_account() as _account: - yield _account - - -@pytest.fixture -def issue_56195(state_tree): - contents = """ - [CmdLetBinding()] - Param( - [SecureString] $SecureString - ) - $Credential = New-Object System.Net.NetworkCredential("DummyId", $SecureString) - $Credential.Password - """ - with pytest.helpers.temp_file("test.ps1", contents, state_tree / "issue-56195"): - yield - - -def test_windows_script_args_powershell(cmd, shell, issue_56195): - """ - Ensure that powershell processes an inline script with args where the args - contain powershell that needs to be rendered - """ - password = "i like cheese" - args = ( - "-SecureString (ConvertTo-SecureString -String '{}' -AsPlainText -Force)" - " -ErrorAction Stop".format(password) - ) - script = "salt://issue-56195/test.ps1" - - ret = cmd.script(source=script, args=args, shell=shell, saltenv="base") - - assert ret["stdout"] == password - - -def test_windows_script_args_powershell_runas(cmd, shell, account, issue_56195): - """ - Ensure that powershell processes an inline script with args where the args - contain powershell that needs to be rendered - """ - password = "i like cheese" - args = ( - "-SecureString (ConvertTo-SecureString -String '{}' -AsPlainText -Force)" - " -ErrorAction Stop".format(password) - ) - script = "salt://issue-56195/test.ps1" - - ret = cmd.script( - source=script, - args=args, - shell=shell, - saltenv="base", - runas=account.username, - password=account.password, - ) - - assert ret["stdout"] == password - - -def test_windows_script_exitcode(cmd, shell, exitcode_script): - ret = cmd.script("salt://exit_code.ps1", shell=shell, saltenv="base") - assert ret["retcode"] == exitcode_script diff --git a/tests/pytests/functional/modules/cmd/test_script_batch.py b/tests/pytests/functional/modules/cmd/test_script_batch.py new file mode 100644 index 000000000000..0184aace200c --- /dev/null +++ b/tests/pytests/functional/modules/cmd/test_script_batch.py @@ -0,0 +1,64 @@ +import pytest + +pytestmark = [ + pytest.mark.core_test, + pytest.mark.windows_whitelisted, + pytest.mark.skip_unless_on_windows, +] + + +@pytest.fixture(scope="module") +def account(): + with pytest.helpers.create_account() as _account: + yield _account + + +@pytest.fixture +def echo_script(state_tree): + contents = """@echo off +set a=%~1 +set b=%~2 +shift +shift +echo a: %a%, b: %b% +""" + with pytest.helpers.temp_file("test.bat", contents, state_tree / "echo-script"): + yield + + +@pytest.mark.parametrize( + "command, expected", + [ + (["foo", "bar"], "a: foo, b: bar"), + (["foo foo", "bar bar"], "a: foo foo, b: bar bar"), + ], +) +def test_echo(modules, echo_script, command, expected): + """ + Test argument processing with a batch script + """ + script = "salt://echo-script/test.bat" + result = modules.cmd.script(script, args=command, shell="cmd") + assert result["stdout"] == expected + + +@pytest.mark.parametrize( + "command, expected", + [ + (["foo", "bar"], "a: foo, b: bar"), + (["foo foo", "bar bar"], "a: foo foo, b: bar bar"), + ], +) +def test_echo_runas(modules, account, echo_script, command, expected): + """ + Test argument processing with a batch script and runas + """ + script = "salt://echo-script/test.bat" + result = modules.cmd.script( + script, + args=command, + shell="cmd", + runas=account.username, + password=account.password, + ) + assert result["stdout"] == expected diff --git a/tests/pytests/functional/modules/cmd/test_script_powershell.py b/tests/pytests/functional/modules/cmd/test_script_powershell.py new file mode 100644 index 000000000000..c1f076dca8a7 --- /dev/null +++ b/tests/pytests/functional/modules/cmd/test_script_powershell.py @@ -0,0 +1,105 @@ +import pytest + +import salt.utils.path + +pytestmark = [ + pytest.mark.core_test, + pytest.mark.windows_whitelisted, + pytest.mark.skip_unless_on_windows, +] + + +@pytest.fixture(scope="module") +def cmd(modules): + return modules.cmd + + +@pytest.fixture(scope="module") +def account(): + with pytest.helpers.create_account() as _account: + yield _account + + +@pytest.fixture(scope="module") +def exitcode_script(state_tree): + exit_code = 12345 + script_contents = f"""Write-Host "Expected exit code: {exit_code}" +exit {exit_code} +""" + with pytest.helpers.temp_file("exit_code.ps1", script_contents, state_tree): + yield exit_code + + +@pytest.fixture(scope="module") +def echo_script(state_tree): + exit_code = 12345 + script_contents = """param ( + [string]$a, + [string]$b +) +Write-Output "a: $a, b: $b" +""" + with pytest.helpers.temp_file("echo.ps1", script_contents, state_tree): + yield exit_code + + +@pytest.fixture(params=["powershell", "pwsh"]) +def shell(request): + """ + This will run the test on powershell and powershell core (pwsh). If + powershell core is not installed that test run will be skipped + """ + if request.param == "pwsh" and salt.utils.path.which("pwsh") is None: + pytest.skip("Powershell 7 Not Present") + return request.param + + +def test_exitcode(cmd, shell, exitcode_script): + """ + Test receiving an exit code from a powershell script + """ + ret = cmd.script("salt://exit_code.ps1", shell=shell, saltenv="base") + assert ret["retcode"] == exitcode_script + + +@pytest.mark.parametrize( + "command, expected", + [ + (["foo", "bar"], "a: foo, b: bar"), + (["foo foo", "bar bar"], "a: foo foo, b: bar bar"), + ], +) +def test_echo(cmd, shell, echo_script, command, expected): + """ + Test argument processing with a powershell script + """ + ret = cmd.script("salt://echo.ps1", args=command, shell=shell, saltenv="base") + assert isinstance(ret["pid"], int) + assert ret["retcode"] == 0 + assert ret["stderr"] == "" + assert ret["stdout"] == expected + + +@pytest.mark.parametrize( + "command, expected", + [ + (["foo", "bar"], "a: foo, b: bar"), + (["foo foo", "bar bar"], "a: foo foo, b: bar bar"), + ], +) +def test_echo_runas(cmd, shell, account, echo_script, command, expected): + """ + Test argument processing with a powershell script and runas + """ + ret = cmd.script( + "salt://echo.ps1", + args=command, + shell=shell, + runas=account.username, + password=account.password, + saltenv="base", + ) + assert isinstance(ret["pid"], int) + assert ret["retcode"] == 0 + assert ret["stderr"] == "" + assert ret["stdout"] == expected diff --git a/tests/pytests/functional/utils/test_win_runas.py b/tests/pytests/functional/utils/test_win_runas.py index 56affe0f7a68..53a574c8f67e 100644 --- a/tests/pytests/functional/utils/test_win_runas.py +++ b/tests/pytests/functional/utils/test_win_runas.py @@ -50,11 +50,11 @@ def test_compound_runas(user, cmd, expected): if expected == "username": expected = user.username result = win_runas.runas( - cmd=salt.platform.win.prepend_cmd(cmd), + cmd=salt.platform.win.prepend_cmd('cmd', cmd), username=user.username, password=user.password, ) - assert expected in result["stdout"] + assert expected in result["stdout"].decode() @pytest.mark.parametrize( @@ -69,36 +69,36 @@ def test_compound_runas_unpriv(user, cmd, expected): if expected == "username": expected = user.username result = win_runas.runas_unpriv( - cmd=salt.platform.win.prepend_cmd(cmd), + cmd=salt.platform.win.prepend_cmd('cmd', cmd), username=user.username, password=user.password, ) - assert expected in result["stdout"] + assert expected in result["stdout"].decode() def test_runas_str_user(user): result = win_runas.runas( cmd="whoami", username=user.username, password=user.password ) - assert user.username in result["stdout"] + assert user.username in result["stdout"].decode() def test_runas_int_user(int_user): result = win_runas.runas( cmd="whoami", username=int(int_user.username), password=int_user.password ) - assert str(int_user.username) in result["stdout"] + assert str(int_user.username) in result["stdout"].decode() def test_runas_unpriv_str_user(user): result = win_runas.runas_unpriv( cmd="whoami", username=user.username, password=user.password ) - assert user.username in result["stdout"] + assert user.username in result["stdout"].decode() def test_runas_unpriv_int_user(int_user): result = win_runas.runas_unpriv( cmd="whoami", username=int(int_user.username), password=int_user.password ) - assert str(int_user.username) in result["stdout"] + assert str(int_user.username) in result["stdout"].decode() diff --git a/tests/pytests/unit/modules/test_cmdmod.py b/tests/pytests/unit/modules/test_cmdmod.py index e049196531ff..782b993cc9e7 100644 --- a/tests/pytests/unit/modules/test_cmdmod.py +++ b/tests/pytests/unit/modules/test_cmdmod.py @@ -670,10 +670,7 @@ def test_run_all_output_loglevel_debug(caplog): stdout = b"test" proc = MagicMock(return_value=MockTimedProc(stdout=stdout)) - if salt.utils.platform.is_windows(): - expected = "Executing command 'cmd' in directory" - else: - expected = "Executing command 'some' in directory" + expected = "Executing command 'some' in directory" with patch("salt.utils.timed_subprocess.TimedProc", proc): with caplog.at_level(logging.DEBUG, logger="salt.modules.cmdmod"): ret = cmdmod.run_all("some command", output_loglevel="debug") @@ -1060,17 +1057,15 @@ def test_runas_env_sudo_group(bundled): @pytest.mark.skip_unless_on_windows -def test_prep_powershell_cmd_no_powershell(): +def test__run_no_powershell(): with pytest.raises(CommandExecutionError): - cmdmod._prep_powershell_cmd( - win_shell="unk_bin", cmd="Some-Command", encoded_cmd=False - ) + cmdmod._run(shell="unk_bin", cmd="Some-Command", encoded_cmd=False) @pytest.mark.parametrize( "cmd, parsed", [ - ("Write-Host foo", "& Write-Host foo"), + ("Write-Host foo", "Write-Host foo"), ("$PSVersionTable", "$PSVersionTable"), ("try {this} catch {that}", "try {this} catch {that}"), ("[bool]@{value = 0}", "[bool]@{value = 0}"), @@ -1113,25 +1108,19 @@ def test_prep_powershell_cmd(cmd, parsed): """ Tests _prep_powershell_cmd returns correct cmd """ - stack = [["", "", ""], ["", "", ""], ["", "", ""], ["", "", ""]] - with patch("traceback.extract_stack", return_value=stack), patch( - "salt.utils.path.which", return_value="C:\\powershell.exe" - ): - ret = cmdmod._prep_powershell_cmd( - win_shell="powershell", cmd=cmd, encoded_cmd=False - ) - expected = " ".join( - [ - '"C:\\powershell.exe"', - "-NonInteractive", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-Command", - f'"{parsed}"', - ], - ) - assert ret == expected + ret = cmdmod._prep_powershell_cmd( + win_shell="powershell.exe", cmd=cmd, encoded_cmd=False + ) + expected = [ + "powershell.exe", + "-NonInteractive", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + f"{parsed}", + ] + assert ret == expected @pytest.mark.skip_unless_on_windows @@ -1139,27 +1128,21 @@ def test_prep_powershell_cmd_encoded(): """ Tests _prep_powershell_cmd returns correct cmd when encoded_cmd=True """ - stack = [["", "", ""], ["", "", ""], ["", "", ""], ["", "", ""]] # This is the encoded command for 'Write-Host "Encoded HOLO"' e_cmd = "VwByAGkAdABlAC0ASABvAHMAdAAgACIARQBuAGMAbwBkAGUAZAAgAEgATwBMAE8AIgA=" - with patch("traceback.extract_stack", return_value=stack), patch( - "salt.utils.path.which", return_value="C:\\powershell.exe" - ): - ret = cmdmod._prep_powershell_cmd( - win_shell="powershell", cmd=e_cmd, encoded_cmd=True - ) - expected = " ".join( - [ - '"C:\\powershell.exe"', - "-NonInteractive", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-EncodedCommand", - f'"{e_cmd}"', - ], - ) - assert ret == expected + ret = cmdmod._prep_powershell_cmd( + win_shell="powershell.exe", cmd=e_cmd, encoded_cmd=True + ) + expected = [ + "powershell.exe", + "-NonInteractive", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + f"{e_cmd}", + ] + assert ret == expected @pytest.mark.skip_unless_on_windows @@ -1168,24 +1151,22 @@ def test_prep_powershell_cmd_script(): Tests _prep_powershell_cmd returns correct cmd when called from cmd.script """ stack = [["", "", ""], ["", "", "script"], ["", "", ""], ["", "", ""]] - script = r"C:\some\script.ps1" with patch("traceback.extract_stack", return_value=stack), patch( - "salt.utils.path.which", return_value="C:\\powershell.exe" + "salt.utils.path.which", return_value="powershell.exe" ): + script = r"C:\some\script.ps1" ret = cmdmod._prep_powershell_cmd( - win_shell="powershell", cmd=script, encoded_cmd=False - ) - expected = " ".join( - [ - '"C:\\powershell.exe"', - "-NonInteractive", - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-Command", - f'"& {script}; exit $LASTEXITCODE"', - ], + win_shell="powershell.exe", cmd=[script], encoded_cmd=False ) + expected = [ + "powershell.exe", + "-NonInteractive", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + script, + ] assert ret == expected diff --git a/tests/pytests/unit/platform/test_win.py b/tests/pytests/unit/platform/test_win.py index 69b077a3d7bd..8e760c74a3e7 100644 --- a/tests/pytests/unit/platform/test_win.py +++ b/tests/pytests/unit/platform/test_win.py @@ -14,27 +14,33 @@ @pytest.mark.parametrize( "command, expected", [ - ("whoami", "whoami"), - ("hostname", "hostname"), - ("cmd /c hostname", "cmd /c hostname"), - ("echo foo", 'cmd /c "echo foo"'), - ('cmd /c "echo foo"', 'cmd /c "echo foo"'), - ("whoami && echo foo", 'cmd /c "whoami && echo foo"'), - ("whoami || echo foo", 'cmd /c "whoami || echo foo"'), - ("icacls 'C:\\Program Files'", 'icacls "C:\\Program Files"'), + ("whoami", 'cmd.exe /c "whoami"'), + ("cmd /c hostname", 'cmd.exe /c "cmd /c hostname"'), + ("echo foo", 'cmd.exe /c "echo foo"'), + ('cmd /c "echo foo"', 'cmd.exe /c "cmd /c "echo foo""'), + ("icacls 'C:\\Program Files'", 'cmd.exe /c "icacls \'C:\\Program Files\'"'), ( "icacls 'C:\\Program Files' && echo 1", - 'cmd /c "icacls "C:\\Program Files" && echo 1"', + 'cmd.exe /c "icacls \'C:\\Program Files\' && echo 1"', ), ( ["secedit", "/export", "/cfg", "C:\\A Path\\with\\a\\space"], - 'secedit /export /cfg "C:\\A Path\\with\\a\\space"', + 'cmd.exe /c "secedit /export /cfg "C:\\A Path\\with\\a\\space""', + ), + ( + ["C:\\a space\\a space.bat", "foo foo", "bar bar"], + 'cmd.exe /c ""C:\\a space\\a space.bat" "foo foo" "bar bar""', + ), + ( + ''' echo "&<>[]|{}^=;!'+,`~ " ''', + '''cmd.exe /c " echo "&<>[]|{}^=;!'+,`~ " "''', ), ], ) def test_prepend_cmd(command, expected): """ - Test that the command is prepended with "cmd /c" where appropriate + Test that the command is prepended with "cmd /c" and quoted """ - result = win.prepend_cmd(command) + win_shell = 'cmd.exe' + result = win.prepend_cmd(win_shell, command) assert result == expected From 0df14d44501afe8fa5f62fef61818a9226997c1a Mon Sep 17 00:00:00 2001 From: xsmile Date: Wed, 9 Jul 2025 20:16:42 +0200 Subject: [PATCH 24/71] cmdmod: formatting and tests --- salt/modules/cmdmod.py | 24 ++++---- tests/integration/modules/test_cmdmod.py | 22 +++---- tests/integration/shell/test_enabled.py | 26 ++++---- .../functional/modules/cmd/test_powershell.py | 4 +- .../functional/modules/cmd/test_run_win.py | 61 +++---------------- .../functional/utils/test_win_runas.py | 16 ++--- tests/pytests/unit/modules/test_cmdmod.py | 5 +- tests/pytests/unit/platform/test_win.py | 8 +-- 8 files changed, 60 insertions(+), 106 deletions(-) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index d954ce336c43..6fdd53d5e789 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -420,7 +420,10 @@ def _run( # Prepare the command to be executed win_shell_lower = win_shell.lower() - if any(win_shell_lower.endswith(word) for word in ["powershell.exe", "pwsh.exe"]): + if any( + win_shell_lower.endswith(word) + for word in ["powershell.exe", "pwsh.exe"] + ): cmd = _prep_powershell_cmd(win_shell, cmd, encoded_cmd) elif any(win_shell_lower.endswith(word) for word in ["cmd.exe"]): cmd = salt.platform.win.prepend_cmd(win_shell, cmd) @@ -2935,9 +2938,9 @@ def _cleanup_tempfile(path): if salt.utils.platform.is_windows() and not shell: extension_map = { - '.bat': 'cmd', - '.cmd': 'cmd', - '.ps1': 'powershell', + ".bat": "cmd", + ".cmd": "cmd", + ".ps1": "powershell", } shell = extension_map.get(ext) @@ -2992,15 +2995,10 @@ def _cleanup_tempfile(path): os.chmod(path, 320) os.chown(path, __salt__["file.user_to_uid"](runas), -1) - if salt.utils.platform.is_windows(): - cmd_path = path - else: - cmd_path = _cmd_quote(path) - if isinstance(args, (list, tuple)): - new_cmd = [cmd_path, *args] if args else [cmd_path] + new_cmd = [path, *args] if args else [path] else: - new_cmd = [cmd_path, str(args)] if args else [cmd_path] + new_cmd = [path, str(args)] if args else [path] ret = {} try: @@ -3028,7 +3026,7 @@ def _cleanup_tempfile(path): success_stderr=success_stderr, **kwargs, ) - except Exception as exc: + except (CommandExecutionError, SaltInvocationError) as exc: log.error( "cmd.script: Unable to run script '%s': %s", new_cmd, @@ -4171,7 +4169,7 @@ def powershell( # caught in a try/catch block. For example, the `Get-WmiObject` command will # often return a "Non Terminating Error". To fix this, make sure # `-ErrorAction Stop` is set in the powershell command - cmd = "try { " + cmd + ' } catch { Write-Error $_ }' + cmd = "try { " + cmd + " } catch { Write-Error $_ }" if encode_cmd: # Convert the cmd to UTF-16LE without a BOM and base64 encode. diff --git a/tests/integration/modules/test_cmdmod.py b/tests/integration/modules/test_cmdmod.py index 17754d48cb88..7a965e9fe4e8 100644 --- a/tests/integration/modules/test_cmdmod.py +++ b/tests/integration/modules/test_cmdmod.py @@ -566,36 +566,33 @@ def test_hide_output(self): """ Test the hide_output argument """ - if salt.utils.platform.is_windows(): - ls_command = ["dir", "c:\\"] - shell = SHELL - else: - ls_command = ["ls", "/"] - shell = None + ls_command = ( + ["ls", "/"] if not salt.utils.platform.is_windows() else ["dir", "c:\\"] + ) error_command = ["thiscommanddoesnotexist"] # cmd.run - out = self.run_function("cmd.run", ls_command, shell=shell, hide_output=True) + out = self.run_function("cmd.run", ls_command, shell=SHELL, hide_output=True) self.assertEqual(out, "") # cmd.shell - out = self.run_function("cmd.shell", ls_command, shell=shell, hide_output=True) + out = self.run_function("cmd.shell", ls_command, shell=SHELL, hide_output=True) self.assertEqual(out, "") # cmd.run_stdout out = self.run_function( - "cmd.run_stdout", ls_command, shell=shell, hide_output=True + "cmd.run_stdout", ls_command, shell=SHELL, hide_output=True ) self.assertEqual(out, "") # cmd.run_stderr - out = self.run_function("cmd.shell", ls_command, shell=shell, hide_output=True) + out = self.run_function("cmd.shell", ls_command, shell=SHELL, hide_output=True) self.assertEqual(out, "") # cmd.run_all (command should have produced stdout) out = self.run_function( - "cmd.run_all", ls_command, shell=shell, hide_output=True + "cmd.run_all", ls_command, shell=SHELL, hide_output=True ) self.assertEqual(out["stdout"], "") self.assertEqual(out["stderr"], "") @@ -604,7 +601,8 @@ def test_hide_output(self): out = self.run_function( "cmd.run_all", error_command, shell=SHELL, hide_output=True ) - self.assertIn("Unable to run command", out) + self.assertEqual(out["stdout"], "") + self.assertEqual(out["stderr"], "") @pytest.mark.slow_test def test_cmd_run_whoami(self): diff --git a/tests/integration/shell/test_enabled.py b/tests/integration/shell/test_enabled.py index 898934e4a29f..5adcc89a247d 100644 --- a/tests/integration/shell/test_enabled.py +++ b/tests/integration/shell/test_enabled.py @@ -23,27 +23,27 @@ class EnabledTest(ModuleCase): ) @pytest.mark.skip_on_windows(reason="Skip on Windows OS") - @pytest.mark.skip_on_freebsd - def test_shell_default_enabled(self): - """ - ensure that python_shell defaults to True for cmd.run - """ - enabled_ret = "3\nsaltines" # the result of running self.cmd in a shell - ret = self.run_function("cmd.run", [self.cmd]) - self.assertEqual(ret.strip(), enabled_ret) - - @pytest.mark.skip_on_windows(reason="Skip on Windows OS") - def test_shell_disabled(self): + def test_shell_default_disabled(self): """ - test shell disabled output for cmd.run + ensure that python_shell defaults to False for cmd.run """ disabled_ret = ( "first\nsecond\nthird\n|\nwc\n-l\n;\nexport\nSALTY_VARIABLE=saltines" "\n&&\necho\n$SALTY_VARIABLE\n;\necho\nduh\n&>\n/dev/null" ) - ret = self.run_function("cmd.run", [self.cmd], python_shell=False) + ret = self.run_function("cmd.run", [self.cmd]) self.assertEqual(ret, disabled_ret) + @pytest.mark.skip_on_windows(reason="Skip on Windows OS") + @pytest.mark.skip_on_freebsd + def test_shell_enabled(self): + """ + test shell enabled output for cmd.run + """ + enabled_ret = "3\nsaltines" # the result of running self.cmd in a shell + ret = self.run_function("cmd.run", [self.cmd], python_shell=True) + self.assertEqual(ret.strip(), enabled_ret) + @pytest.mark.skip_on_windows(reason="Skip on Windows OS") @pytest.mark.skip_on_freebsd def test_template_shell(self): diff --git a/tests/pytests/functional/modules/cmd/test_powershell.py b/tests/pytests/functional/modules/cmd/test_powershell.py index 10951d7bde8f..36be3daa4a39 100644 --- a/tests/pytests/functional/modules/cmd/test_powershell.py +++ b/tests/pytests/functional/modules/cmd/test_powershell.py @@ -88,7 +88,7 @@ def test_args_runas(issue_56195): (["Write-Output", "Foo"], "Foo", False), ('Write-Output "Encoded Foo"', "Encoded Foo", True), (["Write-Output", '"Encoded Foo"'], "Encoded Foo", True), - ('$a="Plain";$b=\' Foo\';Write-Output ${a}${b}', "Plain Foo", False), + ("$a=\"Plain\";$b=' Foo';Write-Output ${a}${b}", "Plain Foo", False), ("(Write-Output Foo)", "Foo", False), ("& Write-Output Foo", "Foo", False), ], @@ -108,7 +108,7 @@ def test_powershell(shell, cmd, expected, encode_cmd): (["Write-Output", "Foo"], "Foo", False), ('Write-Output "Encoded Foo"', "Encoded Foo", True), (["Write-Output", '"Encoded Foo"'], "Encoded Foo", True), - ('$a="Plain";$b=\' Foo\';Write-Output ${a}${b}', "Plain Foo", False), + ("$a=\"Plain\";$b=' Foo';Write-Output ${a}${b}", "Plain Foo", False), ("(Write-Output Foo)", "Foo", False), ("& Write-Output Foo", "Foo", False), ], diff --git a/tests/pytests/functional/modules/cmd/test_run_win.py b/tests/pytests/functional/modules/cmd/test_run_win.py index 0d9a72765a35..4330efe0aea3 100644 --- a/tests/pytests/functional/modules/cmd/test_run_win.py +++ b/tests/pytests/functional/modules/cmd/test_run_win.py @@ -27,7 +27,7 @@ def test_cmd_exitcode(modules, state_tree, exit_code, return_code, result): ret = modules.state.single( "cmd.run", name=f"exit {exit_code}", - shell='cmd', + shell="cmd", success_retcodes=[2, 44, 300], ) assert ret.result is result @@ -47,7 +47,7 @@ def test_cmd_exitcode_runas( ret = modules.state.single( "cmd.run", name=f"exit {exit_code}", - shel='cmd', + shell="cmd", success_retcodes=[2, 44, 300], runas=account.username, password=account.password, @@ -60,8 +60,9 @@ def test_cmd_exitcode_runas( "command, expected", [ ("echo foo", "foo"), + ("cmd /c echo foo", "foo"), ("whoami && echo foo", "foo"), - ('echo "foo \'bar\'"', '"foo \'bar\'"'), + ("echo \"foo 'bar'\"", "\"foo 'bar'\""), ('echo|set /p="foo" & echo|set /p="bar"', "foobar"), ('''echo "&<>[]|{}^=;!'+,`~ "''', '''"&<>[]|{}^=;!'+,`~ "'''), ], @@ -70,7 +71,7 @@ def test_cmd_builtins(modules, command, expected): """ Test builtin cmd.exe commands """ - result = modules.cmd.run(command, shell='cmd') + result = modules.cmd.run(command, shell="cmd") assert expected in result @@ -78,8 +79,9 @@ def test_cmd_builtins(modules, command, expected): "command, expected", [ ("echo foo", "foo"), + ("cmd /c echo foo", "foo"), ("whoami && echo foo", "foo"), - ('echo "foo \'bar\'"', '"foo \'bar\'"'), + ("echo \"foo 'bar'\"", "\"foo 'bar'\""), ('echo|set /p="foo" & echo|set /p="bar"', "foobar"), ('''echo "&<>[]|{}^=;!'+,`~ "''', '''"&<>[]|{}^=;!'+,`~ "'''), ], @@ -89,7 +91,7 @@ def test_cmd_builtins_runas(modules, account, command, expected): Test builtin cmd.exe commands with runas """ result = modules.cmd.run( - cmd=command, shell='cmd', runas=account.username, password=account.password + cmd=command, shell="cmd", runas=account.username, password=account.password ) assert expected in result @@ -142,25 +144,7 @@ def test_cmd_env(modules, command, env, expected): """ Test cmd.run with environment variables """ - result = modules.cmd.run_all(command, shell='cmd', env=env) - assert isinstance(result["pid"], int) - assert result["retcode"] == 0 - assert result["stdout"] == expected - assert result["stderr"] == "" - -@pytest.mark.parametrize( - "command, env, expected", - [ - ("echo %a%%b%", {"a": "foo", "b": "bar"}, "foobar"), - ], -) -def test_cmd_env_runas(modules, account, command, env, expected): - """ - Test cmd.run with environment variables and runas - """ - result = modules.cmd.run_all( - command, shell='cmd', env=env, runas=account.username, password=account.password - ) + result = modules.cmd.run_all(command, shell="cmd", env=env) assert isinstance(result["pid"], int) assert result["retcode"] == 0 assert result["stdout"] == expected @@ -187,30 +171,3 @@ def test_redirect_stderr(modules, command, expected, redirect_stderr): else: assert result["stdout"] == "" assert expected in result["stderr"] - - -@pytest.mark.parametrize( - "command, expected, redirect_stderr", - [ - (["whoami.exe", "/foo"], "/foo", True), - (["whoami.exe", "/foo"], "/foo", False), - ], -) -def test_redirect_stderr_runas(modules, account, command, expected, redirect_stderr): - """ - Test redirection of stderr to stdout by running cmd.run_all with runas and invalid commands - """ - result = modules.cmd.run_all( - command, - runas=account.username, - password=account.password, - redirect_stderr=redirect_stderr, - ) - assert isinstance(result["pid"], int) - assert result["retcode"] == 1 - if redirect_stderr: - assert expected in result["stdout"] - assert result["stderr"] == "" - else: - assert result["stdout"] == "" - assert expected in result["stderr"] diff --git a/tests/pytests/functional/utils/test_win_runas.py b/tests/pytests/functional/utils/test_win_runas.py index 53a574c8f67e..ed6db209eaaf 100644 --- a/tests/pytests/functional/utils/test_win_runas.py +++ b/tests/pytests/functional/utils/test_win_runas.py @@ -50,11 +50,11 @@ def test_compound_runas(user, cmd, expected): if expected == "username": expected = user.username result = win_runas.runas( - cmd=salt.platform.win.prepend_cmd('cmd', cmd), + cmd=salt.platform.win.prepend_cmd("cmd", cmd), username=user.username, password=user.password, ) - assert expected in result["stdout"].decode() + assert expected in result["stdout"] @pytest.mark.parametrize( @@ -69,36 +69,36 @@ def test_compound_runas_unpriv(user, cmd, expected): if expected == "username": expected = user.username result = win_runas.runas_unpriv( - cmd=salt.platform.win.prepend_cmd('cmd', cmd), + cmd=salt.platform.win.prepend_cmd("cmd", cmd), username=user.username, password=user.password, ) - assert expected in result["stdout"].decode() + assert expected in result["stdout"] def test_runas_str_user(user): result = win_runas.runas( cmd="whoami", username=user.username, password=user.password ) - assert user.username in result["stdout"].decode() + assert user.username in result["stdout"] def test_runas_int_user(int_user): result = win_runas.runas( cmd="whoami", username=int(int_user.username), password=int_user.password ) - assert str(int_user.username) in result["stdout"].decode() + assert str(int_user.username) in result["stdout"] def test_runas_unpriv_str_user(user): result = win_runas.runas_unpriv( cmd="whoami", username=user.username, password=user.password ) - assert user.username in result["stdout"].decode() + assert user.username in result["stdout"] def test_runas_unpriv_int_user(int_user): result = win_runas.runas_unpriv( cmd="whoami", username=int(int_user.username), password=int_user.password ) - assert str(int_user.username) in result["stdout"].decode() + assert str(int_user.username) in result["stdout"] diff --git a/tests/pytests/unit/modules/test_cmdmod.py b/tests/pytests/unit/modules/test_cmdmod.py index 782b993cc9e7..dccb23f0de67 100644 --- a/tests/pytests/unit/modules/test_cmdmod.py +++ b/tests/pytests/unit/modules/test_cmdmod.py @@ -1066,6 +1066,7 @@ def test__run_no_powershell(): "cmd, parsed", [ ("Write-Host foo", "Write-Host foo"), + ("& Write-Host foo", "& Write-Host foo"), ("$PSVersionTable", "$PSVersionTable"), ("try {this} catch {that}", "try {this} catch {that}"), ("[bool]@{value = 0}", "[bool]@{value = 0}"), @@ -1118,7 +1119,7 @@ def test_prep_powershell_cmd(cmd, parsed): "-ExecutionPolicy", "Bypass", "-Command", - f"{parsed}", + parsed, ] assert ret == expected @@ -1140,7 +1141,7 @@ def test_prep_powershell_cmd_encoded(): "-ExecutionPolicy", "Bypass", "-EncodedCommand", - f"{e_cmd}", + e_cmd, ] assert ret == expected diff --git a/tests/pytests/unit/platform/test_win.py b/tests/pytests/unit/platform/test_win.py index 8e760c74a3e7..e687050ef2aa 100644 --- a/tests/pytests/unit/platform/test_win.py +++ b/tests/pytests/unit/platform/test_win.py @@ -18,10 +18,10 @@ ("cmd /c hostname", 'cmd.exe /c "cmd /c hostname"'), ("echo foo", 'cmd.exe /c "echo foo"'), ('cmd /c "echo foo"', 'cmd.exe /c "cmd /c "echo foo""'), - ("icacls 'C:\\Program Files'", 'cmd.exe /c "icacls \'C:\\Program Files\'"'), + ("icacls 'C:\\Program Files'", "cmd.exe /c \"icacls 'C:\\Program Files'\""), ( "icacls 'C:\\Program Files' && echo 1", - 'cmd.exe /c "icacls \'C:\\Program Files\' && echo 1"', + "cmd.exe /c \"icacls 'C:\\Program Files' && echo 1\"", ), ( ["secedit", "/export", "/cfg", "C:\\A Path\\with\\a\\space"], @@ -32,7 +32,7 @@ 'cmd.exe /c ""C:\\a space\\a space.bat" "foo foo" "bar bar""', ), ( - ''' echo "&<>[]|{}^=;!'+,`~ " ''', + """ echo "&<>[]|{}^=;!'+,`~ " """, '''cmd.exe /c " echo "&<>[]|{}^=;!'+,`~ " "''', ), ], @@ -41,6 +41,6 @@ def test_prepend_cmd(command, expected): """ Test that the command is prepended with "cmd /c" and quoted """ - win_shell = 'cmd.exe' + win_shell = "cmd.exe" result = win.prepend_cmd(win_shell, command) assert result == expected From da61390e96968b8380eba669017bd5243829830a Mon Sep 17 00:00:00 2001 From: xsmile Date: Wed, 9 Jul 2025 20:18:44 +0200 Subject: [PATCH 25/71] cmdmod: changelog --- changelog/68096.fixed.md | 1 + changelog/68118.fixed.md | 1 + changelog/68156.changed.md | 4 ++++ changelog/68156.fixed.md | 1 + 4 files changed, 7 insertions(+) create mode 100644 changelog/68096.fixed.md create mode 100644 changelog/68118.fixed.md create mode 100644 changelog/68156.changed.md create mode 100644 changelog/68156.fixed.md diff --git a/changelog/68096.fixed.md b/changelog/68096.fixed.md new file mode 100644 index 000000000000..b6a1712461aa --- /dev/null +++ b/changelog/68096.fixed.md @@ -0,0 +1 @@ +cmdmod: fix special character handling on Windows diff --git a/changelog/68118.fixed.md b/changelog/68118.fixed.md new file mode 100644 index 000000000000..46d8b5e0dfbb --- /dev/null +++ b/changelog/68118.fixed.md @@ -0,0 +1 @@ +cmdmod: fix quotation handling with Windows and Powershell diff --git a/changelog/68156.changed.md b/changelog/68156.changed.md new file mode 100644 index 000000000000..d8e8316ef605 --- /dev/null +++ b/changelog/68156.changed.md @@ -0,0 +1,4 @@ +cmdmod: invoke a shell only with cmd.shell or when using the shell parameter +cmdmod: run PowerShell scripts via -File instead of -Command +cmdmod: allow passing args as a list for cmd.script +cmdmod: return an error when running a bad command with cmd.powershell diff --git a/changelog/68156.fixed.md b/changelog/68156.fixed.md new file mode 100644 index 000000000000..d8a56d2f4345 --- /dev/null +++ b/changelog/68156.fixed.md @@ -0,0 +1 @@ +cmdmod: handle cases where the temp script is not removed with cmd.script From 2459d52cfdb57fc39c40ad7205230e90bfef701d Mon Sep 17 00:00:00 2001 From: xsmile Date: Thu, 17 Jul 2025 21:00:49 +0200 Subject: [PATCH 26/71] cmdmod: fix other modules --- salt/modules/win_dns_client.py | 10 +++++----- salt/modules/win_firewall.py | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/salt/modules/win_dns_client.py b/salt/modules/win_dns_client.py index 3bedb7fb7301..ef2541344388 100644 --- a/salt/modules/win_dns_client.py +++ b/salt/modules/win_dns_client.py @@ -81,8 +81,8 @@ def rm_dns(ip, interface="Local Area Connection"): "ip", "delete", "dns", - f'"{interface}"', - f'"{ip}"', + interface, + ip, "validate=no", ] return __salt__["cmd.retcode"](cmd, python_shell=False) == 0 @@ -125,8 +125,8 @@ def add_dns(ip, interface="Local Area Connection", index=1): "ip", "add", "dns", - f'"{interface}"', - f'"{ip}"', + interface, + ip, f"index={index}", "validate=no", ] @@ -144,7 +144,7 @@ def dns_dhcp(interface="Local Area Connection"): salt '*' win_dns_client.dns_dhcp """ - cmd = ["netsh", "interface", "ip", "set", "dns", f'"{interface}"', "source=dhcp"] + cmd = ["netsh", "interface", "ip", "set", "dns", interface, "source=dhcp"] return __salt__["cmd.retcode"](cmd, python_shell=False) == 0 diff --git a/salt/modules/win_firewall.py b/salt/modules/win_firewall.py index 9f3a80b93aa3..6d8dd0258952 100644 --- a/salt/modules/win_firewall.py +++ b/salt/modules/win_firewall.py @@ -151,7 +151,7 @@ def get_rule(name="all"): salt '*' firewall.get_rule 'MyAppPort' """ - cmd = ["netsh", "advfirewall", "firewall", "show", "rule", f'name="{name}"'] + cmd = ["netsh", "advfirewall", "firewall", "show", "rule", f"name={name}"] ret = __salt__["cmd.run_all"](cmd, python_shell=False, ignore_retcode=True) if ret["retcode"] != 0: raise CommandExecutionError(ret["stdout"]) @@ -228,15 +228,15 @@ def add_rule(name, localport, protocol="tcp", action="allow", dir="in", remoteip "firewall", "add", "rule", - f'name="{name}"', - f'protocol="{protocol}"', - f'dir="{dir}"', - f'action="{action}"', - f'remoteip="{remoteip}"', + f"name={name}", + f"protocol={protocol}", + f"dir={dir}", + f"action={action}", + f"remoteip={remoteip}", ] if protocol is None or ("icmpv4" not in protocol and "icmpv6" not in protocol): - cmd.append(f'localport="{localport}"') + cmd.append(f"localport={localport}") ret = __salt__["cmd.run_all"](cmd, python_shell=False, ignore_retcode=True) if ret["retcode"] != 0: @@ -292,19 +292,19 @@ def delete_rule(name=None, localport=None, protocol=None, dir=None, remoteip=Non """ cmd = ["netsh", "advfirewall", "firewall", "delete", "rule"] if name: - cmd.append(f'name="{name}"') + cmd.append(f"name={name}") if protocol: - cmd.append(f'protocol="{protocol}"') + cmd.append(f"protocol={protocol}") if dir: - cmd.append(f'dir="{dir}"') + cmd.append(f"dir={dir}") if remoteip: - cmd.append(f'remoteip="{remoteip}"') + cmd.append(f"remoteip={remoteip}") if protocol is None or ("icmpv4" not in protocol and "icmpv6" not in protocol): if localport: if not protocol: cmd.append("protocol=tcp") - cmd.append(f'localport="{localport}"') + cmd.append(f"localport={localport}") ret = __salt__["cmd.run_all"](cmd, python_shell=False, ignore_retcode=True) if ret["retcode"] != 0: From 8f5f5f248bab7921072bc879d727902c096373d8 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 24 Jul 2025 19:18:22 +0100 Subject: [PATCH 27/71] Updated docstrings to reflect the correct action --- salt/modules/systemd_service.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/modules/systemd_service.py b/salt/modules/systemd_service.py index ef0081987022..a99173d3aed4 100644 --- a/salt/modules/systemd_service.py +++ b/salt/modules/systemd_service.py @@ -862,7 +862,7 @@ def stop(name, no_block=False): Stop the specified service with systemd no_block : False - Set to ``True`` to start the service using ``--no-block``. + Set to ``True`` to stop the service using ``--no-block``. .. versionadded:: 2017.7.0 @@ -899,7 +899,7 @@ def restart(name, no_block=False, unmask=False, unmask_runtime=False): Restart the specified service with systemd no_block : False - Set to ``True`` to start the service using ``--no-block``. + Set to ``True`` to restart the service using ``--no-block``. .. versionadded:: 2017.7.0 @@ -1015,7 +1015,7 @@ def force_reload(name, no_block=True, unmask=False, unmask_runtime=False): Force-reload the specified service with systemd no_block : False - Set to ``True`` to start the service using ``--no-block``. + Set to ``True`` to force_reload the service using ``--no-block``. .. versionadded:: 2017.7.0 @@ -1122,7 +1122,7 @@ def enable( Enable the named service to start when the system boots no_block : False - Set to ``True`` to start the service using ``--no-block``. + Set to ``True`` to enable the service using ``--no-block``. .. versionadded:: 2017.7.0 @@ -1203,7 +1203,7 @@ def disable( Disable the named service to not start when the system boots no_block : False - Set to ``True`` to start the service using ``--no-block``. + Set to ``True`` to disable the service using ``--no-block``. .. versionadded:: 2017.7.0 From 5502c93bf6654540ab50eff9fc388fdd7e1332f7 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 24 Jul 2025 21:05:51 +0100 Subject: [PATCH 28/71] Modify systemd_service to use no_block for minion Modifies systemd_service.{restart,stop} to default to using no_block=True when the service being stopped or restarted is the salt-minion. If you don't pass no_block, the minion blocks waiting for systemd to restart the service, while systemd is waiting for the minion to exit. Eventually, after systemd hits its timeout it will kill the salt minion processes. Behaviour for other services should remain the same and the functions will still honour the value of no_block if passed as an argument. --- salt/modules/systemd_service.py | 41 ++++++- .../unit/modules/test_systemd_service.py | 109 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 tests/pytests/unit/modules/test_systemd_service.py diff --git a/salt/modules/systemd_service.py b/salt/modules/systemd_service.py index a99173d3aed4..5b447cd8d15b 100644 --- a/salt/modules/systemd_service.py +++ b/salt/modules/systemd_service.py @@ -51,6 +51,7 @@ "path", "timer", ) +SALT_MINION_SERVICE = "salt-minion.service" # Define the module's virtual name __virtualname__ = "service" @@ -380,6 +381,26 @@ def _unit_file_changed(name): return "'systemctl daemon-reload'" in status +def _salt_minion_service(name): + """ + Returns True if the service name is the salt-minion service, otherwise + returns False. + """ + return _canonical_unit_name(name) == SALT_MINION_SERVICE + + +def _no_block_default(name, no_block): + """ + Return the default value for no_block if it is not set. + + Defaults to True if the service is the salt-minion service, otherwise + defaults to False. + """ + if no_block is None: + return True if _salt_minion_service(name) else False + return no_block + + def systemctl_reload(): """ .. versionadded:: 0.15.0 @@ -846,7 +867,7 @@ def start(name, no_block=False, unmask=False, unmask_runtime=False): return True -def stop(name, no_block=False): +def stop(name, no_block=None): """ .. versionchanged:: 2015.8.12,2016.3.3,2016.11.0 On minions running systemd>=205, `systemd-run(1)`_ is now used to @@ -863,15 +884,23 @@ def stop(name, no_block=False): no_block : False Set to ``True`` to stop the service using ``--no-block``. + Defaults to ``True`` for the salt-minion service. .. versionadded:: 2017.7.0 + .. versionchanged:: 3006.15 + The default value for this argument has changed if the service is + the salt-minion service, where it now defaults to ``True`` to + prevent a deadlock with the minion waiting for the service to stop + before exiting. + CLI Example: .. code-block:: bash salt '*' service.stop """ + no_block = _no_block_default(name, no_block) _check_for_unit_changes(name) # Using cmd.run_all instead of cmd.retcode here to make unit tests easier return ( @@ -883,7 +912,7 @@ def stop(name, no_block=False): ) -def restart(name, no_block=False, unmask=False, unmask_runtime=False): +def restart(name, no_block=None, unmask=False, unmask_runtime=False): """ .. versionchanged:: 2015.8.12,2016.3.3,2016.11.0 On minions running systemd>=205, `systemd-run(1)`_ is now used to @@ -900,9 +929,16 @@ def restart(name, no_block=False, unmask=False, unmask_runtime=False): no_block : False Set to ``True`` to restart the service using ``--no-block``. + Defaults to ``True`` for the salt-minion service. .. versionadded:: 2017.7.0 + .. versionchanged:: 3006.15 + The default value for this argument has changed if the service is + the salt-minion service, where it now defaults to ``True`` + to prevent a deadlock with the minion waiting for the service to + stop before exiting. + unmask : False Set to ``True`` to remove an indefinite mask before attempting to restart the service. @@ -925,6 +961,7 @@ def restart(name, no_block=False, unmask=False, unmask_runtime=False): salt '*' service.restart """ + no_block = _no_block_default(name, no_block) _check_for_unit_changes(name) _check_unmask(name, unmask, unmask_runtime) ret = __salt__["cmd.run_all"]( diff --git a/tests/pytests/unit/modules/test_systemd_service.py b/tests/pytests/unit/modules/test_systemd_service.py new file mode 100644 index 000000000000..968d7a1584d7 --- /dev/null +++ b/tests/pytests/unit/modules/test_systemd_service.py @@ -0,0 +1,109 @@ +import pytest + +import salt.modules.systemd_service as systemd_service +import salt.utils.systemd +from tests.support.mock import MagicMock, patch + +pytestmark = [ + pytest.mark.skip_unless_on_linux, +] + + +@pytest.fixture +def configure_loader_modules(): + return {systemd_service: {}} + + +@pytest.mark.parametrize( + "service_name, expected", + [ + ("salt-minion", True), + ("salt-minion.service", True), + ("other-service", False), + ], +) +def test_salt_minion_service(service_name, expected): + """ + Test the _salt_minion_service function to ensure it correctly identifies + the salt-minion service name. + """ + assert systemd_service._salt_minion_service(service_name) is expected + + +@pytest.mark.parametrize( + "service_name, no_block, expected", + [ + ("salt-minion", None, True), + ("other-service", None, False), + ("salt-minion", False, False), + ("other-service", False, False), + ("salt-minion", True, True), + ("other-service", True, True), + ], +) +def test_no_block_default(service_name, no_block, expected): + """ + Test the _no_block_default function to ensure it returns the correct + default value for the no_block argument based on the service name. + """ + assert systemd_service._no_block_default(service_name, no_block) is expected + + +@pytest.mark.skipif(not salt.utils.systemd.booted(), reason="Requires systemd") +@pytest.mark.parametrize( + "operation,expected_command", + [ + ("restart", "restart"), + ("stop", "stop"), + ], +) +def test_operation_no_block_default(operation, expected_command): + """ + Test restart/stop functions to ensure they use the correct default value for + no_block when operating on the salt-minion service. + """ + mock_none = MagicMock(return_value=None) + mock_true = MagicMock(return_value=True) + mock_run_all_success = MagicMock( + return_value={"retcode": 0, "stdout": "", "stderr": "", "pid": 12345} + ) + + with patch.dict( + systemd_service.__salt__, + {"cmd.run_all": mock_run_all_success, "config.get": mock_true}, + ) as mock_salt, patch.object( + systemd_service, "_check_for_unit_changes", mock_none + ), patch.object( + systemd_service, "_check_unmask", mock_none + ), patch( + "salt.utils.path.which", lambda x: "/usr/bin/" + x + ): + # Get the function to test based on the operation parameter + operation_func = getattr(systemd_service, operation) + + # Test salt-minion.service (should include --no-block) + assert operation_func("salt-minion.service") + mock_salt["cmd.run_all"].assert_called_with( + [ + "/usr/bin/systemd-run", + "--scope", + "/usr/bin/systemctl", + "--no-block", + expected_command, + "salt-minion.service", + ], + python_shell=False, + ) + + # Test other.service (should not include --no-block) + assert operation_func("other.service") + mock_salt["cmd.run_all"].assert_called_with( + [ + "/usr/bin/systemd-run", + "--scope", + "/usr/bin/systemctl", + expected_command, + "other.service", + ], + python_shell=False, + ) From 0d3843a2724f44894275f6617fd1fc75b53f7ff1 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 24 Jul 2025 21:32:43 +0100 Subject: [PATCH 29/71] Add changelog --- changelog/68212.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/68212.fixed.md diff --git a/changelog/68212.fixed.md b/changelog/68212.fixed.md new file mode 100644 index 000000000000..493b636f07b0 --- /dev/null +++ b/changelog/68212.fixed.md @@ -0,0 +1 @@ +Modifies systemd_service.{restart,stop} to default to using no_block=True when the service being stopped or restarted is the salt-minion. From bf9262f6dbc0c146eaccb0ef8c2e092f97b5a8c0 Mon Sep 17 00:00:00 2001 From: Barney Sowood Date: Thu, 31 Jul 2025 12:57:06 +0100 Subject: [PATCH 30/71] Skip systemd test on Amazon Linux 2 Skips the test_operation_no_block_default test on Amazon Linux 2 as the CI conatiner for that distro doesn't have a complete systemd setup. --- tests/pytests/unit/modules/test_systemd_service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/pytests/unit/modules/test_systemd_service.py b/tests/pytests/unit/modules/test_systemd_service.py index 968d7a1584d7..f8f1ccf69358 100644 --- a/tests/pytests/unit/modules/test_systemd_service.py +++ b/tests/pytests/unit/modules/test_systemd_service.py @@ -57,11 +57,15 @@ def test_no_block_default(service_name, no_block, expected): ("stop", "stop"), ], ) -def test_operation_no_block_default(operation, expected_command): +def test_operation_no_block_default(operation, expected_command, grains): """ Test restart/stop functions to ensure they use the correct default value for no_block when operating on the salt-minion service. """ + + if grains["osfinger"] == "Amazon Linux-2": + pytest.skip("Amazon Linux 2 CI containers do not support systemd fully") + mock_none = MagicMock(return_value=None) mock_true = MagicMock(return_value=True) mock_run_all_success = MagicMock( From 233298fc73a152057e14be2dc4b14d3e7b5c100d Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 1 Aug 2025 14:49:52 -0700 Subject: [PATCH 31/71] Bump relenv to 0.20.4 --- .github/workflows/ci.yml | 6 +++--- .github/workflows/nightly.yml | 6 +++--- .github/workflows/scheduled.yml | 6 +++--- .github/workflows/staging.yml | 6 +++--- cicd/shared-gh-workflows-context.yml | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31c01e731a90..eceb703001d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -416,7 +416,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} @@ -433,7 +433,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "onedir" @@ -450,7 +450,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "src" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5b61982e130d..9b8b134741a3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -466,7 +466,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} @@ -483,7 +483,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "onedir" @@ -504,7 +504,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "src" diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index c9e7d10541e6..b70bade79a52 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -451,7 +451,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} @@ -468,7 +468,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "onedir" @@ -485,7 +485,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "src" diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 339a8d22b425..9a543712c3ef 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -447,7 +447,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} @@ -465,7 +465,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "onedir" @@ -487,7 +487,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.20.3" + relenv-version: "0.20.4" python-version: "3.10.18" ci-python-version: "3.11" source: "src" diff --git a/cicd/shared-gh-workflows-context.yml b/cicd/shared-gh-workflows-context.yml index 3cd7a24762f0..851580d3e6ff 100644 --- a/cicd/shared-gh-workflows-context.yml +++ b/cicd/shared-gh-workflows-context.yml @@ -1,6 +1,6 @@ nox_version: "2022.8.7" python_version: "3.10.18" -relenv_version: "0.20.3" +relenv_version: "0.20.4" pr-testrun-slugs: - ubuntu-24.04-pkg - ubuntu-24.04 From 3a31d03346a74d378c151c2911ed496d04baee7a Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 30 Jul 2025 14:58:05 -0700 Subject: [PATCH 32/71] decrease number of minion key checks --- changelog/68195.fixed.md | 2 ++ salt/crypt.py | 32 ++++++++++++----------- tests/pytests/unit/test_crypt.py | 44 ++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 changelog/68195.fixed.md diff --git a/changelog/68195.fixed.md b/changelog/68195.fixed.md new file mode 100644 index 000000000000..0dc554672ca9 --- /dev/null +++ b/changelog/68195.fixed.md @@ -0,0 +1,2 @@ +salt.crypt.AsyncAuth and salt.crypt.SAuth read the private key from the +filesystem a single time. diff --git a/salt/crypt.py b/salt/crypt.py index 8ab34951347e..d8e460bac457 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -636,6 +636,7 @@ def __singleton_init__(self, opts, io_loop=None): self.token = salt.utils.stringutils.to_bytes(Crypticle.generate_key_string()) self.pub_path = os.path.join(self.opts["pki_dir"], "minion.pub") self.rsa_path = os.path.join(self.opts["pki_dir"], "minion.pem") + self._private_key = None if self.opts["__role"] == "syndic": self.mpub = "syndic_master.pub" else: @@ -1015,22 +1016,23 @@ def get_keys(self): :rtype: Crypto.PublicKey.RSA._RSAobj :return: The RSA keypair """ - # Make sure all key parent directories are accessible - user = self.opts.get("user", "root") - salt.utils.verify.check_path_traversal(self.opts["pki_dir"], user) - - if not os.path.exists(self.rsa_path): - log.info("Generating keys: %s", self.opts["pki_dir"]) - gen_keys( - self.opts["pki_dir"], - "minion", - self.opts["keysize"], - self.opts.get("user"), - ) - key = PrivateKey(self.rsa_path, None) - log.debug("Loaded minion key: %s", self.rsa_path) - return key + if self._private_key is None: + # Make sure all key parent directories are accessible + user = self.opts.get("user", "root") + salt.utils.verify.check_path_traversal(self.opts["pki_dir"], user) + + if not os.path.exists(self.rsa_path): + log.info("Generating keys: %s", self.opts["pki_dir"]) + gen_keys( + self.opts["pki_dir"], + "minion", + self.opts["keysize"], + self.opts.get("user"), + ) + self._private_key = PrivateKey(self.rsa_path, None) + return self._private_key + @salt.utils.decorators.memoize def gen_token(self, clear_tok): """ Encrypt a string with the minion private key to verify identity diff --git a/tests/pytests/unit/test_crypt.py b/tests/pytests/unit/test_crypt.py index 71dd4dea4a8d..aaf028d0eaf5 100644 --- a/tests/pytests/unit/test_crypt.py +++ b/tests/pytests/unit/test_crypt.py @@ -2,6 +2,7 @@ import salt.crypt as crypt import salt.exceptions +from tests.support.mock import patch @pytest.fixture @@ -188,3 +189,46 @@ def test_get_key_with_evict_bad_key(tmp_path): key_path.write_text("asdfasoiasdofaoiu0923jnoiausbd98sb9") with pytest.raises(salt.exceptions.InvalidKeyError): crypt._get_key_with_evict(str(key_path), 1, None) + + +def test_async_auth_cache_private_key(minion_root, io_loop): + + pki_dir = minion_root / "etc" / "salt" / "pki" + opts = { + "id": "minion", + "__role": "minion", + "pki_dir": str(pki_dir), + "master_uri": "tcp://127.0.0.1:4505", + "keysize": 4096, + "acceptance_wait_time": 60, + "acceptance_wait_time_max": 60, + } + + auth = crypt.AsyncAuth(opts, io_loop) + + # The private key is cached. + assert isinstance(auth._private_key, crypt.PrivateKey) + + # get_keys returns the cached instance + _id = id(auth._private_key) + assert _id == id(auth.get_keys()) + + +def test_async_auth_cache_token(minion_root, io_loop): + pki_dir = minion_root / "etc" / "salt" / "pki" + opts = { + "id": "minion", + "__role": "minion", + "pki_dir": str(pki_dir), + "master_uri": "tcp://127.0.0.1:4505", + "keysize": 4096, + "acceptance_wait_time": 60, + "acceptance_wait_time_max": 60, + } + + auth = crypt.AsyncAuth(opts, io_loop) + + with patch("salt.crypt.private_encrypt") as moc: + auth.gen_token("salt") + auth.gen_token("salt") + moc.assert_called_once() From fae5e6a3310d3726657637feba7a13f4d7cd919e Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 30 Jul 2025 15:04:33 -0700 Subject: [PATCH 33/71] Memoize on all arguments --- salt/crypt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/crypt.py b/salt/crypt.py index d8e460bac457..f9772809407a 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -1033,6 +1033,9 @@ def get_keys(self): return self._private_key @salt.utils.decorators.memoize + def _gen_token(self, key, token): + return private_encrypt(key, token) + def gen_token(self, clear_tok): """ Encrypt a string with the minion private key to verify identity @@ -1042,7 +1045,7 @@ def gen_token(self, clear_tok): :return: Encrypted token :rtype: str """ - return private_encrypt(self.get_keys(), clear_tok) + return self._gen_token(self.get_keys(), clear_tok) def minion_sign_in_payload(self): """ @@ -1411,6 +1414,7 @@ def __singleton_init__(self, opts, io_loop=None): self.token = salt.utils.stringutils.to_bytes(Crypticle.generate_key_string()) self.pub_path = os.path.join(self.opts["pki_dir"], "minion.pub") self.rsa_path = os.path.join(self.opts["pki_dir"], "minion.pem") + self._private_key = None self._creds = None if "syndic_master" in self.opts: self.mpub = "syndic_master.pub" From 7c4ffe71fd180bb1825898dd81519e4538bbb9d8 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 1 Aug 2025 14:38:23 -0700 Subject: [PATCH 34/71] Use key modified time in singleton identifier --- salt/crypt.py | 5 +++++ tests/pytests/unit/test_crypt.py | 10 ++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/salt/crypt.py b/salt/crypt.py index f9772809407a..0ce41d9995a3 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -613,10 +613,15 @@ def __new__(cls, opts, io_loop=None): @classmethod def __key(cls, opts, io_loop=None): + keypath = os.path.join(opts["pki_dir"], "minion.pem") + keytime = "0" + if os.path.exists(keypath): + keytime = str(os.path.getmtime(keypath)) return ( opts["pki_dir"], # where the keys are stored opts["id"], # minion ID opts["master_uri"], # master ID + keytime, ) # has to remain empty for singletons, since __init__ will *always* be called diff --git a/tests/pytests/unit/test_crypt.py b/tests/pytests/unit/test_crypt.py index aaf028d0eaf5..892761dc0ec8 100644 --- a/tests/pytests/unit/test_crypt.py +++ b/tests/pytests/unit/test_crypt.py @@ -1,3 +1,5 @@ +import os + import pytest import salt.crypt as crypt @@ -56,12 +58,13 @@ async def test_auth_aes_key_rotation(minion_root, io_loop): "acceptance_wait_time": 60, "acceptance_wait_time_max": 60, } + crypt.gen_keys(pki_dir, "minion", opts["keysize"]) credskey = ( opts["pki_dir"], # where the keys are stored opts["id"], # minion ID opts["master_uri"], # master ID + str(os.path.getmtime(os.path.join(opts["pki_dir"], "minion.pem"))), ) - crypt.gen_keys(pki_dir, "minion", opts["keysize"]) aes = crypt.Crypticle.generate_key_string() session = crypt.Crypticle.generate_key_string() @@ -126,11 +129,6 @@ def test_sauth_aes_key_rotation(minion_root, io_loop): "acceptance_wait_time": 60, "acceptance_wait_time_max": 60, } - credskey = ( - opts["pki_dir"], # where the keys are stored - opts["id"], # minion ID - opts["master_uri"], # master ID - ) crypt.gen_keys(pki_dir, "minion", opts["keysize"]) aes = crypt.Crypticle.generate_key_string() From 17257ebe1f36247425e7f2d7c702df131afd04e1 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 3 Aug 2025 17:42:36 -0700 Subject: [PATCH 35/71] Revert "Ipc leak fix" This reverts commit 4c0e5529f5d542570d87942902afa4ad1fe91b7c. --- salt/config/__init__.py | 3 --- salt/defaults/__init__.py | 2 -- salt/transport/ipc.py | 18 ++---------------- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index aec90d69eed2..a18a3f0aa827 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -14,7 +14,6 @@ from copy import deepcopy import salt.crypt -import salt.defaults import salt.defaults.exitcodes import salt.exceptions import salt.features @@ -1005,7 +1004,6 @@ def _gather_buffer_space(): "publish_signing_algorithm": str, "request_server_ttl": int, "request_server_aes_session": int, - "ipc_write_timeout": int, } ) @@ -1665,7 +1663,6 @@ def _gather_buffer_space(): "publish_signing_algorithm": "PKCS1v15-SHA1", "request_server_aes_session": 0, "request_server_ttl": 0, - "ipc_write_timeout": salt.defaults.IPC_WRITE_TIMEOUT, } ) diff --git a/salt/defaults/__init__.py b/salt/defaults/__init__.py index 82959daa42f0..5ebdb694a58c 100644 --- a/salt/defaults/__init__.py +++ b/salt/defaults/__init__.py @@ -60,5 +60,3 @@ def __repr__(self): cases are proper defaults and are also proper values to pass. """ NOT_SET = _Constant("NOT_SET") - -IPC_WRITE_TIMEOUT = 15 diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index 3d4bf2b2916a..2ee99eb06d04 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -2,13 +2,11 @@ IPC transport classes """ -import datetime import errno import logging import socket import time -import salt.defaults import salt.ext.tornado import salt.ext.tornado.concurrent import salt.ext.tornado.gen @@ -535,18 +533,8 @@ def start(self): @salt.ext.tornado.gen.coroutine def _write(self, stream, pack): - timeout = self.opts.get("ipc_write_timeout", salt.defaults.IPC_WRITE_TIMEOUT) try: - yield salt.ext.tornado.gen.with_timeout( - datetime.timedelta(seconds=timeout), - stream.write(pack), - quiet_exceptions=(StreamClosedError,), - ) - except salt.ext.tornado.gen.TimeoutError: - log.trace("Failed to relay event to client after %d seconds", timeout) - if not stream.closed(): - stream.close() - self.streams.discard(stream) + yield stream.write(pack) except StreamClosedError: log.trace("Client disconnected from IPC %s", self.socket_path) self.streams.discard(stream) @@ -710,9 +698,7 @@ def _read(self, timeout, callback=None): self._read_stream_future = None except Exception as exc: # pylint: disable=broad-except log.error( - "Exception occurred in Subscriber while handling stream: %s", - exc, - exc_info_on_level=logging.DEBUG, + "Exception occurred in Subscriber while handling stream: %s", exc ) self._read_stream_future = None exc_to_raise = exc From e2fae20ec515fb690677002bc791e080fd71ab4a Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sun, 3 Aug 2025 17:44:12 -0700 Subject: [PATCH 36/71] Add changelog for #68151 --- changelog/68151.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/68151.fixed.md diff --git a/changelog/68151.fixed.md b/changelog/68151.fixed.md new file mode 100644 index 000000000000..824c74b651f5 --- /dev/null +++ b/changelog/68151.fixed.md @@ -0,0 +1 @@ +Revert 'ipc_write_timeout' change (3006.13) due to multiple reports of this change causing instability From 032cdf3ebecddf0286fa08e927dcf5e972fca999 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Aug 2025 14:14:11 -0600 Subject: [PATCH 37/71] Enable signing macos packages --- .github/workflows/nightly.yml | 4 ++-- .github/workflows/staging.yml | 4 ++-- .github/workflows/templates/build-packages.yml.jinja | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9b8b134741a3..9de3126112d1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -490,7 +490,7 @@ jobs: matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} environment: nightly - sign-macos-packages: false + sign-macos-packages: true sign-rpm-packages: false sign-windows-packages: false @@ -511,7 +511,7 @@ jobs: matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} environment: nightly - sign-macos-packages: false + sign-macos-packages: true sign-rpm-packages: false sign-windows-packages: false build-ci-deps: diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 9a543712c3ef..af708ab30ec2 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -472,7 +472,7 @@ jobs: matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} environment: staging - sign-macos-packages: false + sign-macos-packages: true sign-rpm-packages: ${{ inputs.sign-rpm-packages }} sign-windows-packages: ${{ inputs.sign-windows-packages }} @@ -494,7 +494,7 @@ jobs: matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} environment: staging - sign-macos-packages: false + sign-macos-packages: true sign-rpm-packages: ${{ inputs.sign-rpm-packages }} sign-windows-packages: ${{ inputs.sign-windows-packages }} build-ci-deps: diff --git a/.github/workflows/templates/build-packages.yml.jinja b/.github/workflows/templates/build-packages.yml.jinja index 97edb07d536d..dc5dc71aff27 100644 --- a/.github/workflows/templates/build-packages.yml.jinja +++ b/.github/workflows/templates/build-packages.yml.jinja @@ -35,7 +35,7 @@ linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} <%- if gh_environment != "ci" %> environment: <{ gh_environment }> - sign-macos-packages: false + sign-macos-packages: true sign-rpm-packages: <% if gh_environment == 'nightly' -%> false <%- else -%> ${{ inputs.sign-rpm-packages }} <%- endif %> sign-windows-packages: <% if gh_environment == 'nightly' -%> false <%- else -%> ${{ inputs.sign-windows-packages }} <%- endif %> From 841973bcea9522a80f1bc4da7320b998e5279ebf Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 30 Jul 2025 18:21:07 -0700 Subject: [PATCH 38/71] Ensure lock is released on request timeout Ensure that message client lock is released by checking for the response without blocking until the future provided to send_recv timesout. --- salt/transport/zeromq.py | 74 +++++++++--- .../transport/zeromq/test_request_client.py | 111 ++++++++++++++++-- tests/pytests/unit/transport/test_zeromq.py | 25 ++++ 3 files changed, 185 insertions(+), 25 deletions(-) diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index fb66f5015ad4..d69878c50770 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -40,6 +40,8 @@ log = logging.getLogger(__name__) +REQUEST_TIMEOUT = 60 + def _get_master_uri(master_ip, master_port, source_ip=None, source_port=None): """ @@ -494,6 +496,29 @@ def _set_tcp_keepalive(zmq_socket, opts): zmq_socket.setsockopt(zmq.TCP_KEEPALIVE_INTVL, opts["tcp_keepalive_intvl"]) +class BackoffTimeout: + """ + Return a backoff value. Backoff from the start value to a maximum value. + Each time the object is called the backoff value is increased by the + provided percentage. + """ + + def __init__(self, start=0.0003, maximum=0.3, percent=0.01, _count=0): + self._backoff = start + self._count = _count + self.start = start + self.maximum = maximum + self.percent = percent + + def __call__(self): + self._count += 1 + backoff = self._backoff + if self._backoff > self.maximum: + return self.maximum + self._backoff += self._backoff * self.percent * self._count + return backoff + + # TODO: unit tests! class AsyncReqMessageClient: """ @@ -532,6 +557,9 @@ def __init__(self, opts, addr, linger=0, io_loop=None): self.ident = threading.get_ident() def connect(self): + if self.context is None: + self.context = zmq.eventloop.future.Context() + if hasattr(self, "socket") and self.socket: return # wire up sockets @@ -542,11 +570,15 @@ def close(self): return else: self._closing = True - if hasattr(self, "socket") and self.socket is not None: - self.socket.close(0) - self.socket = None - if self.context.closed is False: - self.context.term() + try: + if hasattr(self, "socket") and self.socket is not None: + self.socket.close(0) + self.socket = None + if self.context.closed is False: + self.context.term() + self.context = None + finally: + self._closing = False def _init_socket(self): self.socket = self.context.socket(zmq.REQ) @@ -601,17 +633,31 @@ def _timeout_message(self, future): future.set_exception(SaltReqTimeoutError("Message timed out")) @salt.ext.tornado.gen.coroutine - def _send_recv(self, message, future): + def _send_recv(self, message, future, timeout=REQUEST_TIMEOUT): + """ + Send and recv updating the future if we get a response. After sending + the the REQ socket we poll the socket for a response. If the provided + future is marked as done before we get a response we consider the + request timedout and we give up. + """ + backoff = BackoffTimeout() try: with (yield self.lock.acquire()): yield self.socket.send(message) - try: - recv = yield self.socket.recv() - except zmq.eventloop.future.CancelledError as exc: - if not future.done(): - future.set_exception(exc) - return - + while True: + try: + recv = yield self.socket.recv(flags=zmq.NOBLOCK) + break + except (zmq.error.ZMQError, zmq.error.Again): + if future.done(): + # send's timeout callback timed out the future. + # Since we're going to bail on this request we need + # to reset the connection otherwise the socket is + # left in an invalid state. + self.close() + self.connect() + break + yield salt.ext.tornado.gen.sleep(backoff()) if not future.done(): data = salt.payload.loads(recv) future.set_result(data) @@ -918,7 +964,7 @@ def connect(self): self.message_client.connect() @salt.ext.tornado.gen.coroutine - def send(self, load, timeout=60): + def send(self, load, timeout=REQUEST_TIMEOUT): yield self.connect() ret = yield self.message_client.send(load, timeout=timeout) raise salt.ext.tornado.gen.Return(ret) diff --git a/tests/pytests/functional/transport/zeromq/test_request_client.py b/tests/pytests/functional/transport/zeromq/test_request_client.py index 4ee99f49aa3d..feb8f4acc274 100644 --- a/tests/pytests/functional/transport/zeromq/test_request_client.py +++ b/tests/pytests/functional/transport/zeromq/test_request_client.py @@ -1,18 +1,36 @@ +import asyncio +import logging + import pytest import pytestshellutils.utils.ports import zmq import zmq.eventloop.zmqstream +import salt.exceptions import salt.ext.tornado.gen +import salt.ext.tornado.locks +import salt.ext.tornado.platform.asyncio import salt.transport.zeromq +log = logging.getLogger(__name__) + @pytest.fixture def port(): return pytestshellutils.utils.ports.get_unused_localhost_port() -async def test_request_channel_issue_64627(io_loop, minion_opts, port): +@pytest.fixture +def request_client(io_loop, minion_opts, port): + minion_opts["master_uri"] = f"tcp://127.0.0.1:{port}" + client = salt.transport.zeromq.RequestClient(minion_opts, io_loop) + try: + yield client + finally: + client.close() + + +async def test_request_channel_issue_64627(io_loop, request_client, minion_opts, port): """ Validate socket is preserved until request channel is explicitly closed. """ @@ -22,18 +40,89 @@ async def test_request_channel_issue_64627(io_loop, minion_opts, port): socket = ctx.socket(zmq.REP) socket.bind(minion_opts["master_uri"]) stream = zmq.eventloop.zmqstream.ZMQStream(socket, io_loop=io_loop) + try: + + @salt.ext.tornado.gen.coroutine + def req_handler(stream, msg): + stream.send(msg[0]) + + stream.on_recv_stream(req_handler) + + rep = await request_client.send(b"foo") + req_socket = request_client.message_client.socket + rep = await request_client.send(b"foo") + assert req_socket is request_client.message_client.socket + request_client.close() + assert request_client.message_client.socket is None + + finally: + stream.close() + + +async def test_request_channel_issue_65265(io_loop, request_client, minion_opts, port): + import time + + import salt.ext.tornado.platform + + minion_opts["master_uri"] = f"tcp://127.0.0.1:{port}" + + ctx = zmq.Context() + socket = ctx.socket(zmq.REP) + socket.bind(minion_opts["master_uri"]) + stream = zmq.eventloop.zmqstream.ZMQStream(socket, io_loop=io_loop) + + try: + send_complete = salt.ext.tornado.locks.Event() + + @salt.ext.tornado.gen.coroutine + def no_handler(stream, msg): + """ + The server never responds. + """ + stream.close() + + stream.on_recv_stream(no_handler) + + @salt.ext.tornado.gen.coroutine + def send_request(): + """ + The request will timeout becuse the server does not respond. + """ + ret = None + with pytest.raises(salt.exceptions.SaltReqTimeoutError): + yield request_client.send("foo", timeout=1) + send_complete.set() + return ret + + start = time.monotonic() + io_loop.spawn_callback(send_request) + + await send_complete.wait() + + # Ensure the lock was released when the request timed out. + + locked = request_client.message_client.lock._block._value + assert locked == 0 + finally: + stream.close() + + # Create a new server, the old socket has been closed. @salt.ext.tornado.gen.coroutine def req_handler(stream, msg): - yield stream.send(msg[0]) + """ + The server responds + """ + stream.send(salt.payload.dumps("bar")) - stream.on_recv_stream(req_handler) - - request_client = salt.transport.zeromq.RequestClient(minion_opts, io_loop) + socket = ctx.socket(zmq.REP) + socket.bind(minion_opts["master_uri"]) + stream = zmq.eventloop.zmqstream.ZMQStream(socket, io_loop=io_loop) + try: + stream.on_recv_stream(req_handler) + send_complete = asyncio.Event() - rep = await request_client.send(b"foo") - req_socket = request_client.message_client.socket - rep = await request_client.send(b"foo") - assert req_socket is request_client.message_client.socket - request_client.close() - assert request_client.message_client.socket is None + ret = await request_client.send("foo", timeout=1) + assert ret == "bar" + finally: + stream.close() diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py index 88321ee45c8b..905485677a1a 100644 --- a/tests/pytests/unit/transport/test_zeromq.py +++ b/tests/pytests/unit/transport/test_zeromq.py @@ -2037,3 +2037,28 @@ def test_req_server_auth_garbage_enc_algo(pki_dir, minion_opts, master_opts, cap assert "load" in ret assert "ret" in ret["load"] assert ret["load"]["ret"] == "bad enc algo" + + +def test_backoff_timer(): + start = 0.0003 + maximum = 0.3 + percent = 0.01 + backoff = salt.transport.zeromq.BackoffTimeout( + start, + maximum, + percent, + ) + ourcount = 1 + next_iteration = start + assert backoff._count == 0 + assert backoff() == next_iteration + assert backoff._count == ourcount + + next_iteration += next_iteration * percent * ourcount + while next_iteration < maximum: + assert backoff() == next_iteration, ourcount + ourcount += 1 + assert backoff._count == ourcount + next_iteration += next_iteration * percent * ourcount + assert ourcount == 39 + assert backoff() == maximum From 85fd34955123eb7f93d2de4c0ac6679f5f333925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= Date: Wed, 22 Jan 2025 12:07:56 +0000 Subject: [PATCH 39/71] Fix _pygit2.GitError: error loading known_hosts: issue on gitfs This commit ensures the right HOME value is set during Pygit2 remote initialization, otherwise there are chances to get exceptions as pygit2/libgit2 is relying on it. --- salt/utils/gitfs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index a3ab79ebd01d..733ed3e29449 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -2012,7 +2012,12 @@ def init_remote(self): """ # https://github.com/libgit2/pygit2/issues/339 # https://github.com/libgit2/libgit2/issues/2122 + # https://github.com/saltstack/salt/issues/64121 home = os.path.expanduser("~") + if "HOME" not in os.environ: + # Make sure $HOME env variable is set to prevent + # _pygit2.GitError: error loading known_hosts in some libgit2 versions. + os.environ["HOME"] = home pygit2.settings.search_path[pygit2.GIT_CONFIG_LEVEL_GLOBAL] = home new = False if not os.listdir(self._cachedir): From 0ae9bd15881be6ff315eacda9e7184c0b5a11e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= Date: Wed, 22 Jan 2025 12:23:25 +0000 Subject: [PATCH 40/71] Add changelog entry file --- changelog/64121.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/64121.fixed.md diff --git a/changelog/64121.fixed.md b/changelog/64121.fixed.md new file mode 100644 index 000000000000..e78bbd5b7f3e --- /dev/null +++ b/changelog/64121.fixed.md @@ -0,0 +1 @@ +Ensure the right HOME environment value is set during Pygit2 remote initialization. From 4e5067f152d188fd9fafd92c5705cad77fe388be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?= Date: Wed, 22 Jan 2025 12:36:27 +0000 Subject: [PATCH 41/71] Add test_checkout_pygit2_with_home_env_unset unit test --- tests/pytests/unit/utils/test_gitfs.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/pytests/unit/utils/test_gitfs.py b/tests/pytests/unit/utils/test_gitfs.py index e1da614b2b53..683ac1a0333a 100644 --- a/tests/pytests/unit/utils/test_gitfs.py +++ b/tests/pytests/unit/utils/test_gitfs.py @@ -8,6 +8,7 @@ import salt.fileserver.gitfs import salt.utils.gitfs from salt.exceptions import FileserverConfigError +from tests.support.helpers import patched_environ from tests.support.mock import MagicMock, patch try: @@ -253,6 +254,20 @@ def test_checkout_pygit2(_prepare_provider): assert provider.checkout() is None +@pytest.mark.skipif(not HAS_PYGIT2, reason="This host lacks proper pygit2 support") +@pytest.mark.skip_on_windows( + reason="Skip Pygit2 on windows, due to pygit2 access error on windows" +) +def test_checkout_pygit2_with_home_env_unset(_prepare_provider): + provider = _prepare_provider + provider.remotecallbacks = None + provider.credentials = None + with patched_environ(__cleanup__=["HOME"]): + assert "HOME" not in os.environ + provider.init_remote() + assert "HOME" in os.environ + + @pytest.mark.skipif(not HAS_PYGIT2, reason="This host lacks proper pygit2 support") @pytest.mark.skip_on_windows( reason="Skip Pygit2 on windows, due to pygit2 access error on windows" From 7deebe495583ee08c17bcb582b1d067d302b16a8 Mon Sep 17 00:00:00 2001 From: Gayathri Krishnaswamy Date: Fri, 14 Jul 2023 22:29:14 +0530 Subject: [PATCH 42/71] Updated Windows package manager (cherry picked from commit a5108d40f0144754e23aeb2944d4c8e56048e588) --- .../windows/windows-package-manager.rst | 1047 ++++++----------- 1 file changed, 389 insertions(+), 658 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 38c5ba4dd3cc..afe6cd33f26e 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -3,83 +3,141 @@ ####################### Windows Package Manager ####################### - Introduction ************ +Salt provides a Windows package management tool for installing, updating, removing, and +managing software packages on remote Windows systems. This tool provides a +software repository and a package manager similar to what is provided +by ``yum`` and ``apt`` on Linux. +The repository contains a collection of package definition files. -The Windows Package Manager provides a software repository and a package manager -similar to what is provided by ``yum`` and ``apt`` on Linux. This tool enables -the installation of software on remote Windows systems. +What are package definition files? +=================================== -The repository contains a collection of software definition files. A software -definition file is a YAML/JINJA file with an ``.sls`` file extension. It -contains all the information Salt needs to install a software package on a -Windows system, including the download location of the installer, required -command-line switches for silent install, etc. +A package definition file is a YAML/JINJA2 file with a ``.sls`` file extension that contains all +the information needed to install a software using Salt. It defines: +- Full name of the software package +- The version of the software package +- Download location of the software package +- Command-line switches for silent install and uninstall +- Whether or not to use the Windows task scheduler to install the package, + +The package definition files can be hosted in one or more Git repositories. +The ``.sls`` files used to install Windows packages are not distributed by default with Salt. +You have to initialize and clone the +default repository - `salt-winrepo-ng`_ +that is hosted on GitHub by SaltStack. The repository contains +package definition files for many common Windows packages and is maintained by SaltStack +and the Salt community. Anyone is welcome to submit a pull request to this +repo to add new package definitions. + +The package definition file can be managed through either Salt or Git. +The software packages can be downloaded from either within a git repository or from HTTP(S) or FTP URLs. +That is, the installer defined in the package definition file can be stored anywhere as long as it is +accessible from the host running Salt. + +You can use the Salt Windows package manager like ``yum`` on Linux. You do not have to know the +underlying command to install the software. + +Use ``pkg.install`` to install a package using a package manager based on the OS the system runs on. +Use ``pkg.installed`` to check if a particular package is installed in the minion or not. -Software definition files can be hosted in one or more Git repositories. The -default repository is hosted on GitHub by SaltStack. It is maintained by -SaltStack and the Salt community and contains software definition files for many -common Windows packages. Anyone is welcome to submit a pull request to this -repo to add new software definitions. The default github repository is: +.. note:: + The Salt Windows package manager does not automatically resolve dependencies while installing, + updating, or removing packages. You have to manage the dependencies between packages manually. -- `salt-winrepo-ng `_ +Quickstart +============ +This quickstart guides you through using Windows Salt package manager winrepo to +install software packages in four steps: +1. (Optional) :ref:`Install libraries ` +2. :ref:`Populate the local Git repository` +3. :ref:`Update minion database` +4. :ref:`Install software packages` + +Install libraries +***************** +(Optional) If you are using the Salt Windows package manager with package definition files hosted on +a Salt Git repo, install the libraries ``GitPython`` or ``pygit2``. + +Populate the local Git repository +********************************** +The SLS files used to install Windows packages are not distributed by default +with Salt. Assuming no changes to the default configuration (``file_roots``). +Initialize and clone `salt-winrepo-ng`_ +repository. -The Windows Package Manager is used the same way as other package managers Salt -is aware of. For example: +.. code-block:: bash -- the ``pkg.installed`` and similar states work on Windows. -- the ``pkg.install`` and similar module functions work on Windows. + salt-run winrepo.update_git_repos -High level differences to ``yum`` and ``apt`` are: +On successful execution of :mod:`winrepo.update_git_repos `, +the winrepo repository is cloned in the master on the +location specified in ``winrepo_dir_ng`` and all package definition files are pulled down from the Git repository. -- The repository metadata (SLS files) can be managed through either Salt or git -- Packages can be downloaded from within the Salt repository, a git repository - or from HTTP(S) or FTP URLs -- No dependencies are managed. Dependencies between packages need to be managed - manually +On masterless minion, use ``salt-call`` to initialize and clone the `salt-winrepo-ng `_ -Requirements -============ +.. code-block:: bash -If using the a software definition files hosted on a Git repo, the following -libraries are required: + salt-call --local winrepo.update_git_repos -- GitPython 0.3 or later +On successful execution of the runner, the winrepo repository is cloned in the minion in the location +specified in ``winrepo_dir_ng`` and all package definition files are pulled down from the Git repository. - or +Update minion database +*********************** +Run :mod:`pkg.refresh_db ` on all Windows minions to create a database entry for every package definition file +and build the package database. +.. code-block:: bash -- pygit2 0.20.3 with libgit 0.20.0 or later + # From the master + salt -G 'os:windows' pkg.refresh_db -Quick Start -*********** + # From the minion in masterless mode + salt-call --local pkg.refresh_db -You can get up and running with winrepo pretty quickly just using the defaults. -Assuming no changes to the default configuration (ie, ``file_roots``) run the -following commands on the master: +The ``pkg.refresh_db`` command parses the YAML/JINJA package definition files and +generates the database. The above command returns the following summary denoting the number of packages +that succeeded or failed to compile: .. code-block:: bash - salt-run winrepo.update_git_repos - salt * pkg.refresh_db - salt * pkg.install firefox_x64 + local: + ---------- + failed: + 0 + success: + 301 + total: + 301 -On a masterless minion run the following: +.. note:: + This command can take a few minutes to complete as all the package definition + files are copied to the minion and the database is generated. + +.. note:: + You can use ``pkg.refresh_db`` when writing new Windows package definitions to check for errors + in the definitions against one or more Windows minions. + +Install software package +************************ +You can now install a software package using :mod:`pkg.install `: .. code-block:: bash - salt-call --local winrepo.update_git_repos - salt-call --local pkg.refresh_db - salt-call --local pkg.install firefox_x64 + # From the master + salt * pkg.install 'firefox_x64' + + # From the minion in masterless mode + salt-call --local pkg.install "firefox_x64" -These commands clone the default winrepo from github, update the winrepo -database on the minion, and install the latest version of Firefox. +The above command installs the latest version of Firefox in the minions. Configuration ************* The Github repository (winrepo) is synced to the ``file_roots`` in a location -specified by the ``winrepo_dir_ng`` setting in the config. The default value of +specified in the ``winrepo_dir_ng`` setting in the config. The default value of ``winrepo_dir_ng`` is as follows: - Linux master: ``/srv/salt/win/repo-ng`` (``salt://win/repo-ng``) @@ -383,88 +441,6 @@ default is a list containing a single URL: `https://github.com/saltstack/salt-winrepo-ng `_ -Initialization -************** - -Populate the Local Repository -============================= - -The SLS files used to install Windows packages are not distributed by default -with Salt. Use the :mod:`winrepo.update_git_repos ` -runner initialize the repository in the location specified by ``winrepo_dir_ng`` -in the master config. This will pull the software definition files down from the -git repository. - -.. code-block:: bash - - salt-run winrepo.update_git_repos - -If running a minion in masterless mode, the same command can be run using -``salt-call``. The repository will be initialized in the location specified by -``winrepo_dir_ng`` in the minion config. - -.. code-block:: bash - - salt-call --local winrepo.update_git_repos - -These commands will also sync down the legacy repo to maintain backwards -compatibility with legacy minions. See :ref:`Legacy Minions ` - -The legacy repo can be disabled by setting it to an empty list in the master or -minion config. - -.. code-block:: bash - - winrepo_remotes: [] - -Generate the Metadata File (Legacy) -=================================== - -This step is only required if you are supporting legacy minions. In current -usage the metadata file is generated on the minion in the next step, Update -the Minion Database. For legacy minions the metadata file is generated on the -master using the :mod:`winrepo.genrepo ` runner. - -.. code-block:: bash - - salt-run winrepo.genrepo - -Update the Minion Database -========================== - -Run :mod:`pkg.refresh_db ` on each of your -Windows minions to synchronize the package repository to the minion and build -the package database. - -.. code-block:: bash - - # From the master - salt -G 'os:windows' pkg.refresh_db - - # From the minion in masterless mode - salt-call --local pkg.refresh_db - -The above command returns the following summary denoting the number of packages -that succeeded or failed to compile: - -.. code-block:: bash - - local: - ---------- - failed: - 0 - success: - 301 - total: - 301 - -.. note:: - This command can take a few minutes to complete as the software definition - files are copied to the minion and the database is generated. - -.. note:: - Use ``pkg.refresh_db`` when developing new Windows package definitions to - check for errors in the definitions against one or more Windows minions. Sample Configurations @@ -600,18 +576,42 @@ Masterless configuration Usage ***** -After completing the configuration and initialization steps, you are ready to -manage software on your Windows minions. +After completing the configuration and initialization, you can use the Salt +package manager commands to manage software on Windows minions. .. note:: The following example commands can be run from the master using ``salt`` or on a masterless minion using ``salt-call`` -List Installed Packages +.. list-table:: + :widths: 5 50 45 + :align: left + :header-rows: 1 + :stub-columns: 1 + + * - + - Command + - Description + + * - 1 + - :ref:`pkg.list_pkgs` + - Displays a list of all packages installed in the system. + + * - 2 + - :ref:`pkg.list_available` + - Displays the versions available of a particular package to be installed. +- + * - 3 + - :ref:`pkg.install` + - Installs a given package. + + * - 4 + - :ref:`pkg.remove` + - Uninstalls a given package. + +List installed packages ======================= - -You can get a list of packages installed on the system using -:mod:`pkg.list_pkgs `. +Use :mod:`pkg.list_pkgs ` to display a list of packages installed on the system. .. code-block:: bash @@ -621,8 +621,8 @@ You can get a list of packages installed on the system using # From the minion in masterless mode salt-call --local pkg.list_pkgs -This will return all software installed on the system whether it is managed by -Salt or not as shown below: +The above command displays the software name and the version for every package installed +on the system irrespective of whether it was installed by Salt package manager or not. .. code-block:: console @@ -643,21 +643,22 @@ Salt or not as shown below: salt-minion-py3: 2019.2.3 -You can tell by how the software name is displayed which software is managed by -Salt and which software is not. When Salt finds a match in the winrepo database -it displays the short name as defined in the software definition file. It is -usually a single-word, lower-case name. All other software names will be -displayed with the full name as they are shown in Add/Remove Programs. So, in -the return above, you can see that Git (git), Nullsoft Installer (nsis), Python -3.7 (python3_x64) and Salt (salt-minion-py3) all have a corresponding software -definition file. The others do not. +The software name indicates whether the software is managed by Salt or not. + +If Salt finds a match in the winrepo database then the software name is the +short name as defined in the package definition file. It is usually a single-word, lower-case name. + +All other software names are displayed as the full name as shown in Add/Remove Programs. +In the above example, Git (git), Nullsoft Installer (nsis), Python 3.7 (python3_x64), +Salt (salt-minion-py3) have corresponding package definition file and are managed by Salt +while Frhed 1.6.0, GNU Privacy guard, GPG4win are not managed by Salt. -List Available Versions +List available versions ======================= -You can query the available version of a package using -:mod:`pkg.list_available ` and passing the -name of the software: +Use :mod:`pkg.list_available ` to display the list of version(s) +of a package available for installation. You can pass the name of the software in the command. +You can refer to the software by its ``name`` or its ``full_name`` surrounded by quotes. .. code-block:: bash @@ -667,7 +668,7 @@ name of the software: # From the minion in masterless mode salt-call --local pkg.list_available firefox_x64 -The above command will return the following: +The above command lists all versions of Firefox available for installation. .. code-block:: bash @@ -686,19 +687,15 @@ The above command will return the following: - 73.0.1 - 74.0 -As you can see, there are many versions of Firefox available for installation. -You can refer to a software package by its ``name`` or its ``full_name`` -surrounded by quotes. - .. note:: - From a Linux master it is OK to use single-quotes. However, the ``cmd`` - shell on Windows requires you to use double-quotes when wrapping strings - that may contain spaces. Powershell seems to accept either one. + For a Linux master, you can surround the file name with single quotes. + However, the ``cmd`` shell on Windows use double quotes when wrapping strings + that may contain spaces. Powershell accepts either single quotes or double quotes. -Install a Package +Install a package ================= -You can install a package using :mod:`pkg.install `: +Use :mod:`pkg.install `: to install a package. .. code-block:: bash @@ -708,7 +705,7 @@ You can install a package using :mod:`pkg.install # From the minion in masterless mode salt-call --local pkg.install "firefox_x64" -The above will install the latest version of Firefox. +The above command installs the latest version of Firefox. .. code-block:: bash @@ -718,13 +715,12 @@ The above will install the latest version of Firefox. # From the minion in masterless mode salt-call --local pkg.install "firefox_x64" version=74.0 -The above will install version 74.0 of Firefox. +The above command installs version 74.0 of Firefox. -If a different version of the package is already installed it will be replaced -with the version in the winrepo (only if the package itself supports live -updating). +If a different version of the package is already installed then the old version is +replaced with the version in the winrepo (only if the package supports live updating). -You can also specify the full name: +You can also specify the full name of the software while installing: .. code-block:: bash @@ -734,10 +730,9 @@ You can also specify the full name: # From the minion in masterless mode salt-call --local pkg.install "Mozilla Firefox 17.0.1 (x86 en-US)" -Remove a Package +Remove a package ================ - -You can uninstall a package using :mod:`pkg.remove `: + Use :mod:`pkg.remove ` to remove a package. .. code-block:: bash @@ -749,31 +744,23 @@ You can uninstall a package using :mod:`pkg.remove .. _software-definition-files: -Software Definition Files -************************* -A software definition file is a YAML/JINJA2 file that contains all the -information needed to install a piece of software using Salt. It defines -information about the package to include version, full name, flags required for -the installer and uninstaller, whether or not to use the Windows task scheduler -to install the package, where to download the installation package, etc. +Package defintion file directory structure and naming +====================================================== -Directory Structure and Naming -============================== - -The files are stored in the location designated by the ``winrepo_dir_ng`` -setting. All files in this directory that have a ``.sls`` file extension are -considered software definition files. The files are evaluated to create the +All package definition files are stored in the location configured in the ``winrepo_dir_ng`` +setting. All files in this directory with ``.sls`` file extension are +considered package definition files. These files are evaluated to create the metadata file on the minion. -You can maintain standalone software definition files that point to software on -other servers or on the internet. In this case the file name would be the short -name of the software with the ``.sls`` extension, ie ``firefox.sls``. +You can maintain standalone package definition files that point to software on +other servers or on the internet. In this case the file name is the short +name of the software with the ``.sls`` extension, For example,``firefox.sls``. You can also store the binaries for your software together with their software definition files in their own directory. In this scenario, the directory name -would be the short name for the software and the software definition file would -be inside that directory and named ``init.sls``. +is the short name for the software and the package definition file stored that directory is +named ``init.sls``. Look at the following example directory structure on a Linux master assuming default config settings: @@ -812,52 +799,58 @@ default config settings: | | | | |---chrome.sls | | | | |---firefox.sls -In the above directory structure, the user has created the ``custom_defs`` -directory in which to store their custom software definition files. In that -directory you see a folder for MS Office 2013 that contains all the installer -files along with a software definition file named ``init.sls``. The user has -also created two more standalone software definition files; ``openssl.sls`` and -``zoom.sls``. - -The ``salt-winrepo-ng`` directory is created by the ``winrepo.update_git_repos`` -command. This folder contains the clone of the git repo designated by the -``winrepo_remotes_ng`` config setting. +In the above directory structure, +- The ``custom_defs`` directory contains the following custom package definition files. + - A folder for MS Office 2013 that contains the installer files for all the MS Office softwares and a +package definition file named ``init.sls``. + - Additional two more standalone package definition files ``openssl.sls`` and ``zoom.sls`` to install +Open SSl and Zoom. +- The ``salt-winrepo-ng`` directory contains the clone of the git repo specified by +the ``winrepo_remotes_ng`` config setting. .. warning:: - It is recommended that the user not modify the files in the - ``salt-winrepo-ng`` directory as it will break future runs of + Do not modify the files in the ``salt-winrepo-ng`` directory as it breaks the future runs of ``winrepo.update_git_repos``. .. warning:: - It is recommended that the user not place any custom software definition - files in the ``salt-winrepo-ng`` directory. The ``winrepo.update_git_repos`` - command wipes out the contents of the ``salt-winrepo-ng`` directory each - time it is run. Any extra files stored there will be lost. + Do not place any custom software definition files in the ``salt-winrepo-ng`` directory as + ``winrepo.update_git_repos`` command wipes out the contents of the ``salt-winrepo-ng`` + directory each time it is run and any extra files stored in the Salt winrepo is lost. -Writing Software Definition Files +Writing package definition files ================================= - -A basic software definition file is really easy to write if you already know -some basic things about your software: - -- The full name as shown in Add/Remove Programs +You can write a software definition file if you know: +- The full name of the software as shown in Add/Remove Programs - The exact version number as shown in Add/Remove Programs - How to install your software silently from the command line -The software definition file itself is just a data structure written in YAML. -The top level item is a short name that Salt will use to reference the software. -There can be only one short name in the file and it must be unique across all -software definition files in the repo. This is the name that will be used to -install/remove the software. It is also the name that will appear when Salt -finds a match in the repo when running ``pkg.list_pkgs``. +Here is a YAML software definition file for Firefox: +.. code-block:: yaml -The next indentation level is the version number. There can be many of these, -but they must be unique within the file. This is also displayed in -``pkg.list_pkgs``. + firefox_x64: + '74.0': + full_name: Mozilla Firefox 74.0 (x64 en-US) + installer: 'https://download-installer.cdn.mozilla.net/pub/firefox/releases/74.0/win64/en-US/Firefox%20Setup%2074.0.exe' + install_flags: '/S' + uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe' + uninstall_flags: '/S' + '73.0.1': + full_name: Mozilla Firefox 73.0.1 (x64 en-US) + installer: 'https://download-installer.cdn.mozilla.net/pub/firefox/releases/73.0.1/win64/en-US/Firefox%20Setup%2073.0.1.exe' + install_flags: '/S' + uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe' + uninstall_flags: '/S' -The last indentation level contains the information Salt needs to actually -install the software. Available parameters are: +The package definition file itself is a data structure written in YAML with three indentation levels. +- The first level item is a short name that Salt uses reference the software. This short name is used to +install and remove the software and it must be unique across all package definition files in the repo. +Also, there must be only one short name in the file. +- The second level item is the version number. There can be multiple version numbers for a software but they must be unique within the file. +.. note:: + On running ``pkg.list_pkgs``, the short name and version number are displayed is displayed when Salt finds a match in the repo. +- The third indentation level contains all parameters that the Salt needs to +install the software. The parameters are: - ``full_name`` : The full name as displayed in Add/Remove Programs - ``installer`` : The location of the installer binary - ``install_flags`` : The flags required to install silently @@ -870,20 +863,24 @@ install the software. Available parameters are: - ``use_scheduler`` : Launch the installer using the task scheduler - ``source_hash`` : The hash sum for the installer -Usage of these parameters is demonstrated in the following examples and -discussed in more detail below. To understand these examples you'll need a basic -understanding of Jinja. The following links have some basic tips and best -practices for working with Jinja in Salt: +Example package definition files +================================ +This section provides some examples of package definition files for different use cases such as: -`Understanding Jinja `_ +- Writing a simple package definition file for a software +- Writing a INJA templated package definition file +- Writing a package definition file to install the latest version of the software +- Writing a package definintion file to install an MSI patch to installed software -`Jinja `_ +These examples enables you to gain a better understanding of the usage of different file paramaters. +To understand the examples, you require a basic `Understanding Jinja `_ +For an exhaustive dive into Jinja, refer to the official +`Jinja Template Designer documentation `_ Example: Basic ============== -Take a look at this basic, pure YAML example for a software definition file for -Firefox: +Here is a pure YAML example of a package definition file for Firefox: .. code-block:: yaml @@ -901,34 +898,38 @@ Firefox: uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe' uninstall_flags: '/S' -You can see the first item is the short name for the software, in this case -``firefox_x64``. It is the first line in the definition. The next line is -indented two spaces and contains the software ``version``. The lines following -the ``version`` are indented two more spaces and contain all the information -needed to install the Firefox package. +The first line is the short name of the software which is ``firefox_x64``. +.. important:: + The short name name must match exactly what is shown in Add/Remove Programs (``appwiz.cpl``) + and it must be unique across all other short names in the software repository. + The ``full_name`` combined with the version must also be unique. +The second line is the ``software version`` and is indented two spaces. .. important:: - The package name must be unique to all other packages in the software - repository. The ``full_name`` combined with the version must also be unique. - They must also match exactly what is shown in Add/Remove Programs - (``appwiz.cpl``). + The version number must be enclosed in quotes else the YAML parser removes the trailing zeros. + For example, if the version number 74.0 is not enclosed within quotes then the version number + is considered as only 74. + +The lines following the ``version`` are indented two more spaces and contain all the information +needed to install the Firefox package. .. important:: - The version number must be enclosed in quotes, otherwise the YAML parser - will remove trailing zeros. For example, `74.0` will just become `74`. + You can specify multiple versions for a software by specifying multiple version numbers at + the same indentation level as the first with its software definition below it. -As you can see in the example above, a software definition file can define -multiple versions for the same piece of software. These are denoted by putting -the next version number at the same indentation level as the first with its -software definition information indented below it. +Example: JINJA templated package definition file +================================================= +JINJA is the default templating language used in package definition files. +You can use JINJA to add variables, expressions to package definition files +that get replaced with values when the ``.sls`` go through Salt renderer. -Example: Jinja -============== +When there are tens or hundreds of versions available for a piece of software the +definition file can become large and cumbersome to write . +In this scenario, Jinja can be used to add logic, variables, +and expressions to automatically create the package definition file for a software with +multiple versions. -When there are tens or hundreds of versions available for a piece of software -definition file can become quite large. This is a scenario where Jinja can be -helpful. Consider the following software definition file for Firefox using -Jinja: +Here is a an example of a package definition file for Firefox that uses Jinja: .. code-block:: jinja @@ -948,30 +949,25 @@ Jinja: uninstall_flags: '/S' {% endfor %} -In this example we are able to generate a software definition file that defines -how to install 12 versions of Firefox. We use Jinja to create a list of -available versions. That list is in a ``for loop`` where each version is placed +In this example JINJA is used to to generate a package definition file that defines +how to install 12 versions of Firefox. Jinja is used to create a list of +available versions. The list is iterated through a ``for loop`` where each version is placed in the ``version`` variable. The version is inserted everywhere there is a ``{{ version }}`` marker inside the ``for loop``. -You'll notice that there is a single variable (``lang``) defined at the top of -the software definition. Because these files are going through the Salt renderer -many Salt modules are exposed via the ``salt`` keyword. In this case it is -calling the ``config.get`` function to get a language setting that can be placed -in the minion config. If it is not there, it defaults to ``en-US``. - -Example: Latest -=============== +The single variable (``lang``) defined at the top of the package definition identifies the language of the package. +You can access the Salt modules using the ``salt`` keyword. In this case, the ``config.get`` function is invokedto retrieve +the language setting. If the ``lang`` variable is not defined then the default value is ``en-US``. -There are some software vendors that do not provide access to all versions of +Example: Package definition file to install the latest version +=============================================================== +Some software vendors that do not provide access to all versions of their software. Instead they provide a single URL to what is always the latest version. In some cases the software keeps itself up to date. One example of this -is the Google Chrome web browser. - -`Chrome `_ +is the `Chrome `_ To handle situations such as these, set the version to `latest`. Here's an -example: +example of a package definition file to install the latest version of Chrome. .. code-block:: yaml @@ -984,25 +980,16 @@ example: uninstall_flags: '/qn /norestart' msiexec: True -The above example shows us two things. First it demonstrates the usage of -``latest`` as the version. In this case Salt will install the version of Chrome -at the URL and report that version. +In the above example, +- ``Version`` is set to ``latest``. Salt then installs the latest version of Chrome at the URL and displays that version. +- ``msiexec`` is set to ``True`` hence the software is installed using an MSI. -The second thing to note is that this is installing software using an MSI. You -can see that ``msiexec`` is set to ``True``. +Example: Package definition file to install an MSI patch +========================================================= +For MSI installer, when the ``msiexec`` parameter is set to true ``/i`` option is used for installation, +and ``/x`` option is used for uninstallation. However, to install an MSI patch ``/i`` and ``/x`` options cannot be combined. -Example: MSI Patch -================== - -When the ``msiexec`` parameter is set to ``True`` it uses the ``/i`` option for -installs and the ``/x`` option for uninstalls. This is problematic when trying -to install an MSI patch which requires the ``/p`` option. You can't combine the -``/i`` and ``/p`` options. So how do you apply a patch to installed software in -winrepo using an ``.msp`` file? - -One wiley contributor came up with the following solution to this problem by -using the ``%cd%`` environment variable. Consider the following software -definition file: +Here is an example of a package definition file to install an MSI patch .. code-block:: yaml @@ -1021,23 +1008,19 @@ definition file: uninstaller: '{B5B5868F-23BA-297A-917D-0DF345TF5764}' uninstall_flags: '/qn /norestart' msiexec: True - cache_file: salt://win/repo/MyApp/MyApp.1.1.msp + cache_file: salt://win/repo-ng/MyApp/MyApp.1.1.msp -There are a few things to note about this software definition file. First, is -the solution we are trying to solve, that of applying a patch. Version ``1.0`` -just installs the application using the ``1.0`` MSI defined in the ``installer`` -parameter. There is nothing special in the ``install_flags`` and nothing is -cached. +In the above example: +Version ``1.0`` of the software installs the application using the ``1.0`` MSI defined in the ``installer`` parameter. +There is no file to be cached and the ``install_flags`` parameter does not include any special values. -Version ``1.1`` uses the same installer, but uses the ``cache_file`` option to -specify a single file to cache. In order for this to work the MSP file needs to -be in the same directory as the MSI file on the ``file_roots``. - -The final step to getting this to work is to add the additional ``/update`` flag -to the ``install_flags`` parameter. Add the path to the MSP file using the -``%cd%`` environment variable. ``%cd%`` resolves to the current working -directory which is the location in the minion cache where the installer file is -cached. +Version ``1.1`` of the software uses the same installer file as Version ``1.0``. +Now to apply patch to Version 1.0, make the following changes in the package definition file: +- Place the patch file (MSP file) in the same directory as the installer file (MSI file) on the ``file_roots`` +- In the ``cache_file`` parameter, specify the path to single patch file. +- In ``install_flags`` parameter, add the ``/update`` flag and include the path to the MSP file +using the ``%cd%`` environment variable. ``%cd%`` resolves to the current working directory which is the location in the minion cache +where the installer file is cached. See issue `#32780 `_ for more details. @@ -1047,21 +1030,16 @@ files for other types of .exe based installers. Parameters ========== +This section describes the parameters placed under the ``version``in the package definition file. +Example can be found on the `Salt winrepo repository `_ -These are the parameters that can be used to generate a software definition -file. These parameters are all placed under the ``version`` in the software -definition file: - -Example usage can be found on the `github repo -`_ full_name (str) --------------- - -This is the full name for the software as shown in "Programs and Features" in -the control panel. You can also get this information by installing the package -manually and then running ``pkg.list_pkgs``. Here's an example of the output -from ``pkg.list_pkgs``: +The full name for the software as shown in "Programs and Features" in +the control panel. You can also retrieve the full name of the package by installing the package +manually and then running ``pkg.list_pkgs``. +Here's an example of the output from ``pkg.list_pkgs``: .. code-block:: console @@ -1077,14 +1055,12 @@ from ``pkg.list_pkgs``: salt-minion-py3: 3001 -Notice the Full Name for Firefox: ``Mozilla Firefox 74.0 (x64 en-US)``. That's -exactly what should be in the ``full_name`` parameter in the software definition -file. -If any of the software installed on the machine matches the full name defined in -one of the software definition files in the repository the package name will be -returned. The example below shows the ``pkg.list_pkgs`` for a machine that has -Mozilla Firefox 74.0 installed and a software definition for that version of +Notice the full Name for Firefox: Mozilla Firefox 74.0 (x64 en-US). +The ``full_name`` parameter in the package definition file must match this name. + +The example below shows the ``pkg.list_pkgs`` for a machine that has +Mozilla Firefox 74.0 installed with a package definition file for that version of Firefox. .. code-block:: bash @@ -1100,14 +1076,18 @@ Firefox. salt-minion-py3: 3001 +On running ``pkg.list_pkgs``, If any of the software installed on the machine matches the full name +defined in any one of the software definition files in the repository, then the +package name is displayed in the output. + .. important:: - The version number and ``full_name`` need to match the output from - ``pkg.list_pkgs`` exactly so that the installation status can be verified + The version number and ``full_name`` must match the output of + ``pkg.list_pkgs`` so that the installation status can be verified by the state system. .. note:: - It is still possible to successfully install packages using ``pkg.install``, - even if the ``full_name`` or the version number don't match exactly. The + You can successfully install packages using ``pkg.install``, + even if the ``full_name`` or the version number don't match. The module will complete successfully, but continue to display the full name in ``pkg.list_pkgs``. If this is happening, verify that the ``full_name`` and the ``version`` match exactly what is displayed in Add/Remove @@ -1126,51 +1106,46 @@ Firefox. installer (str) --------------- -This is the path to the binary (``.exe``, ``.msi``) that will install the -package. This can be a local path or a URL. If it is a URL or a Salt path -(``salt://``), the package will be cached locally and then executed. If it is a -path to a file on disk or a file share, it will be executed directly. +The path to the binary (``.exe``, ``.msi``) to install a package. +This can be a local path or a URL. If it is a URL or a Salt path (``salt://``), +then the package is cached locally and then executed. +If it is a path to a file on disk or a file share, then it is executed directly. .. note:: - When storing software in the same location as the winrepo it is usually best - practice to place each installer in its own directory rather than in the - root of winrepo. - - Best practice is to create a sub folder named after the package. That folder - will contain the software definition file named ``init.sls``. The binary - installer should be stored in that directory as well if you're hosting those - files on the file_roots. + When storing software in the same location as the winrepo then + - Create a sub folder named after the package. + - Store the package definition file named ``init.sls`` and the binary installer in + the same sub folder if you are hosting those files on the ``file_roots``. - ``pkg.refresh_db`` will process all ``.sls`` files in all sub directories +.. note:: + ``pkg.refresh_db`` command processes all ``.sls`` files in all sub directories in the ``winrepo_dir_ng`` directory. install_flags (str) ------------------- +The flags passed to the installer for silent installation. -This setting contains any flags that need to be passed to the installer to make -it perform a silent install. These can often be found by adding ``/?`` or ``/h`` -when running the installer from the command-line. A great resource for finding -these silent install flags is the WPKG project wiki_: +You can find these flags by adding ``/?`` or ``/h``when running the installer from the command-line. +See `WPKG project wiki `_ for information on silent install flags. .. warning:: - Salt will appear to hang if the installer is expecting user input. So it is - imperative that the software have the ability to install silently. + Always ensure that the software has the ability to install silently since + Salt appears to hang if the installer expects user input. uninstaller (str) ----------------- -This is the path to the program used to uninstall this software. This can be the -path to the same ``exe`` or ``msi`` used to install the software. Exe -uninstallers are pretty straight forward. MSIs, on the other hand, can be -handled a couple different ways. You can use the GUID for the software to -uninstall or you can use the same MSI used to install the software. +The path to the program to uninstall a software. +This can be the path to the same exe or msi used to install the software. +If you use msi to install the software then you can either use GUID of the software or the +same MSI to uninstall the software. -You can usually find uninstall information in the registry: +You can find the uninstall information in the registry: - Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall - Software\\WOW6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall -Here's an example using the GUID to uninstall software. +Here's an example of using the GUID to uninstall software. .. code-block:: yaml @@ -1183,7 +1158,7 @@ Here's an example using the GUID to uninstall software. uninstall_flags: '/qn /norestart' msiexec: True -Here's an example using the same MSI used to install the software: +Here's an example of using the installer MSI to uninstall software .. code-block:: yaml @@ -1199,20 +1174,20 @@ Here's an example using the same MSI used to install the software: uninstall_flags (str) --------------------- -This setting contains any flags that need to be passed to the uninstaller to -make it perform a silent uninstall. These can often be found by adding ``/?`` or -``/h`` when running the uninstaller from the command-line. A great resource for -finding these silent install flags the WPKG project wiki_: +The flags passed to the uninstaller for silent uninstallation. + +You can find these flags by adding ``/?`` or ``/h``when running the uninstaller from the command-line. +See `WPKG project wiki `_ for information on silent uninstall flags. .. warning:: - Salt will appear to hang if the uninstaller is expecting user input. So it - is imperative that the software have the ability to uninstall silently. + Always ensure that the software has the ability to uninstall silently since + Salt appears to hang if the uninstaller expects user input. msiexec (bool, str) ------------------- -This tells Salt to use ``msiexec /i`` to install the package and ``msiexec /x`` -to uninstall. This is for ``.msi`` installations only. +This setting informs Salt to use ``msiexec /i`` to install the package and ``msiexec /x`` +to uninstall. This setting is applicable only for ``.msi`` installations only. Possible options are: @@ -1241,17 +1216,17 @@ cache_dir (bool) ---------------- This setting requires the software to be stored on the ``file_roots`` and only -applies to URLs that begin with ``salt://``. If ``True`` the entire directory -where the installer resides will be recursively cached. This is useful for +applies to URLs that begin with ``salt://``. If set to ``True`` then the entire directory +where the installer resides is recursively cached. This is useful for installers that depend on other files in the same directory for installation. .. warning:: - Be aware that all files and directories in the same location as the - installer file will be copied down to the minion. If you place your - software definition file in the root of winrepo (``/srv/salt/win/repo-ng``) - and it contains ``cache_dir: True`` the entire contents of winrepo will be + If set to ``True`` then all files and directories in the same location as the + installer file are copied down to the minion. For example, If you place your + package definition file with ``cache_dir: True`` in the root of winrepo + (``/srv/salt/win/repo-ng``) then the entire contents of winrepo is cached to the minion. Therefore, it is best practice to place your installer - files in a subdirectory if they are to be stored in winrepo. + files in a subdirectory if they are stored in winrepo. Here's an example using cache_dir: @@ -1268,22 +1243,21 @@ cache_file (str) ---------------- This setting requires the file to be stored on the ``file_roots`` and only -applies to URLs that begin with ``salt://``. It indicates a single file to copy -down for use with the installer. It is copied to the same location as the -installer. Use this over ``cache_dir`` if there are many files in the directory -and you only need a specific file and don't want to cache additional files that -may reside in the installer directory. +applies to URLs that begin with ``salt://``. It indicates that only a single file specified +is to be copied down for use with the installer. It is copied to the same location as the +installer. This setting is useful when ``cache_dir`` is set to ``True``, +and you want to cache only a specific file and not all files that reside in the installer directory. use_scheduler (bool) -------------------- -If set to ``True``, Windows will use the task scheduler to run the installation. -A one-time task will be created in the task scheduler and launched. The return -to the minion will be that the task was launched successfully, not that the +If set to ``True``, Windows uses the task scheduler to run the installation. +A one-time task is created in the task scheduler and launched. The return +to the minion is that the task was launched successfully, not that the software was installed successfully. .. note:: - This is used by the software definition for Salt itself. The first thing the + This is used in the package definition for Salt itself. The first thing the Salt installer does is kill the Salt service, which then kills all child processes. If the Salt installer is launched via Salt, then the installer itself is killed leaving Salt on the machine but not running. Use of the @@ -1293,7 +1267,7 @@ software was installed successfully. source_hash (str) ----------------- -This tells Salt to compare a hash sum of the installer to the provided hash sum +This setting informs Salt to compare a hash sum of the installer to the provided hash sum before execution. The value can be formatted as ``=``, or it can be a URI to a file containing the hash sum. @@ -1316,7 +1290,6 @@ Here's an example using ``source_hash``: Not Implemented --------------- - The following parameters are often seen in the software definition files hosted on the Git repo. However, they are not implemented and have no effect on the installation process. @@ -1352,47 +1325,33 @@ ready for use. Troubleshooting *************** -My software installs correctly but pkg.installed says it failed +My software installs correctly but `pkg.installed says it failed =============================================================== If you have a package that seems to install properly, but Salt reports a failure then it is likely you have a ``version`` or ``full_name`` mismatch. -Check the exact ``full_name`` and ``version`` as shown in Add/Remove Programs -(``appwiz.cpl``). Use ``pkg.list_pkgs`` to check that the ``full_name`` and -``version`` exactly match what is installed. Make sure the software definition -file has the exact value for ``full_name`` and that the version matches exactly. - -Also, make sure the version is wrapped in single quotes in the software +- Check the ``full_name`` and ``version`` of the package as shown in Add/Remove Programs +(``appwiz.cpl``). +- Use ``pkg.list_pkgs`` to check that the ``full_name`` and +``version`` exactly match what is installed. +- Verify that the ``full_name`` and ``version`` of the package in the package definition file + matches the full name and version in Add/Remove programs. +- Ensure that the ``version`` is wrapped in single quotes in the package definition file. -Changes to sls files not being picked up -======================================== +Changes to package definition files not being picked up +====================================================== -You may have recently updated some of the software definition files on the repo. -Ensure you have refreshed the database on the minion. +Ensure you have refreshed the database on the minion on +updating the package definintion files in the repo. .. code-block:: bash salt winminion pkg.refresh_db -How Success and Failure are Reported by pkg.installed -===================================================== - -The install state/module function of the Windows package manager works roughly -as follows: -1. Execute ``pkg.list_pkgs`` to get a list of software currently on the machine -2. Compare the requested version with the installed version -3. If versions are the same, report no changes needed -4. Install the software as described in the software definition file -5. Execute ``pkg.list_pkgs`` to get a new list of software currently on the - machine -6. Compare the requested version with the new installed version -7. If versions are the same, report success -8. If versions are different, report failure - -Winrepo Upgrade Issues +Winrepo upgrade issues ====================== To minimize potential issues, it is a good idea to remove any winrepo git @@ -1404,14 +1363,11 @@ clone them anew after the master is started. pygit2_/GitPython_ Support for Maintaining Git Repos **************************************************** -The :mod:`winrepo.update_git_repos ` -runner now makes use of the same underlying code used by the :ref:`Git Fileserver Backend ` +pygit2_ and GitPython_ are the supported python interfaces to Git. +The runner :mod:`winrepo.update_git_repos ` +uses the same underlying code as :ref:`Git Fileserver Backend ` and :mod:`Git External Pillar ` to maintain and update -its local clones of git repositories. If a compatible version of either pygit2_ -(0.20.3 and later) or GitPython_ (0.3.0 or later) is installed, Salt will use it -instead of the old method (which invokes the :mod:`git.latest ` -state). - +its local clones of git repositories. .. note:: If compatible versions of both pygit2_ and GitPython_ are installed, then Salt will prefer pygit2_. To override this behavior use the @@ -1421,19 +1377,11 @@ state). winrepo_provider: gitpython - The :mod:`winrepo execution module ` (discussed - above in the :ref:`Managing Windows Software on a Standalone Windows Minion - ` section) does not yet officially support the new - pygit2_/GitPython_ functionality, but if either pygit2_ or GitPython_ is - installed into Salt's bundled Python then it *should* work. However, it - should be considered experimental at this time. - -Accessing Authenticated Git Repos (pygit2) +Accessing authenticated Git repos (pygit2) ****************************************** -Support for pygit2 added the ability to access authenticated git repositories -and to set per-remote config settings. An example of this would be the -following: +pygit2 enables you to access authenticated git repositories +and set per-remote config settings. An example of this is: .. code-block:: yaml @@ -1448,54 +1396,52 @@ following: - password: CorrectHorseBatteryStaple .. note:: - Per-remote configuration settings work in the same fashion as they do in + The per-remote configuration settings work in the same manner as they do in gitfs, with global parameters being overridden by their per-remote - counterparts. For instance, setting :conf_master:`winrepo_passphrase` would - set a global passphrase for winrepo that would apply to all SSH-based + counterparts. For instance, setting :conf_master:`winrepo_passphrase` + sets a global passphrase for winrepo that applies to all SSH-based remotes, unless overridden by a ``passphrase`` per-remote parameter. - See :ref:`here ` for more a more in-depth + See :ref:`here ` for detailed explanation of how per-remote configuration works in gitfs. The same principles apply to winrepo. -Maintaining Git Repos +Maintaining Git repos ********************* -A ``clean`` argument has been added to the +A ``clean`` argument is added to the :mod:`winrepo.update_git_repos ` -runner. When ``clean`` is ``True`` it will tell the runner to dispose of +runner to maintain the Git repos. When ``clean=True`` the runner removes directories under the :conf_master:`winrepo_dir_ng`/:conf_minion:`winrepo_dir_ng` -which are not explicitly configured. This prevents the need to manually remove -these directories when a repo is removed from the config file. To clean these -old directories, just pass ``clean=True``: - +that are not explicitly configured. This eliminates the need to manually remove +these directories when a repo is removed from the config file. .. code-block:: bash salt-run winrepo.update_git_repos clean=True -If a mix of git and non-git Windows Repo definition files are being used, then -this should *not* be used, as it will remove the directories containing non-git +If a mix of git and non-git Windows Repo definition files are used, then +do not pass ``clean=True``, as it removes the directories containing non-git definitions. -Name Collisions Between Repos +Name collisions between repos ***************************** -Collisions between repo names are now detected. The +Salt detects collisions between repository names. The :mod:`winrepo.update_git_repos ` -runner will not proceed if any are detected. Consider the following -configuration: +runner does not execute successfully if any collisions between repository names are detected. + Consider the following configuration: .. code-block:: yaml winrepo_remotes: - https://foo.com/bar/baz.git - https://mydomain.tld/baz.git - - https://github.com/foobar/baz + - https://github.com/foobar/baz.git -The :mod:`winrepo.update_git_repos ` -runner will refuse to update repos here, as all three of these repos would be -checked out to the same directory. To work around this, a per-remote parameter -called ``name`` can be used to resolve these conflicts: +With the above configuration, the :mod:`winrepo.update_git_repos ` +runner fails to execute as all three repos would be +checked out to the same directory. To resolve this conflict, use per-remote parameter +called ``name``. .. code-block:: yaml @@ -1503,225 +1449,10 @@ called ``name`` can be used to resolve these conflicts: - https://foo.com/bar/baz.git - https://mydomain.tld/baz.git: - name: baz_junior - - https://github.com/foobar/baz: + - https://github.com/foobar/baz.git: - name: baz_the_third -.. _legacy-minions: - -Legacy Minions -************** - -The Windows Package Manager was upgraded with breaking changes starting with -Salt 2015.8.0. To maintain backwards compatibility Salt continues to support -older minions. - -The breaking change was to generate the winrepo database on the minion instead -of the master. This allowed for the use of Jinja in the software definition -files. It enabled the use of pillar, grains, execution modules, etc. during -compile time. To support this new functionality, a next-generation (ng) repo was -created. - -See the :ref:`Changes in Version 2015.8.0 <2015-8-0-winrepo-changes>` for -details. - -On prior versions of Salt, or legacy minions, the winrepo database was -generated on the master and pushed down to the minions. Any grains exposed at -compile time would have been those of the master and not the minion. - -The repository for legacy minions is named ``salt-winrepo`` and is located at: - -- https://github.com/saltstack/salt-winrepo - -Legacy Configuration -==================== - -Winrepo settings were changed with the introduction of the Next Generation (ng) -of winrepo. - -Legacy Master Config Options ----------------------------- -There were three options available for a legacy master to configure winrepo. -Unless you're running a legacy master as well, you shouldn't need to configure -any of these. - -- ``win_gitrepos`` -- ``win_repo`` -- ``win_repo_mastercachefile`` - -``win_gitrepos``: (list) - -A list of URLs to github repos. Default is a list with a single URL: - -- 'https://github.com/saltstack/salt-winrepo.git' - -``win_repo``: (str) - -The location on the master to store the winrepo. The default is -``/srv/salt/win/repo``. - -``win_repo_mastercachefile``: (str) -The location on the master to generate the winrepo database file. The default is -``/srv/salt/win/repo/winrep.p`` - -Legacy Minion Config Options ----------------------------- - -There is only one option available to configure a legacy minion for winrepo. - -- ``win_repo_cachefile`` - -``win_repo_cachefile``: (str) - -The location on the Salt file server to obtain the winrepo database file. The -default is ``salt://win/repo/winrepo.p`` - -.. note:: - If the location of the ``winrepo.p`` file is not in the default location on - the master, the :conf_minion:`win_repo_cachefile` setting will need to be - updated to reflect the proper location on each minion. - -Legacy Quick Start -================== - -You can get up and running with winrepo pretty quickly just using the defaults. -Assuming no changes to the default configuration (ie, ``file_roots``) run the -following commands on the master: - -.. code-block:: bash - - salt-run winrepo.update_git_repos - salt-run winrepo.genrepo - salt * pkg.refresh_db - salt * pkg.install firefox - -These commands clone the default winrepo from github, generate the metadata -file, push the metadata file down to the legacy minion, and install the latest -version of Firefox. - -Legacy Initialization -===================== - -Initializing the winrepo for a legacy minion is similar to that for a newer -minion. There is an added step in that the metadata file needs to be generated -on the master prior to refreshing the database on the minion. - -Populate the Local Repository ------------------------------ - -The SLS files used to install Windows packages are not distributed by default -with Salt. So, the first step is to clone the repo to the master. Use the -:mod:`winrepo.update_git_repos ` -runner initialize the repository in the location specified by ``winrepo_dir`` -in the master config. This will pull the software definition files down from the -git repository. - -.. code-block:: bash - - salt-run winrepo.update_git_repos - -Generate the Metadata File --------------------------- - -The next step is to create the metadata file for the repo (``winrepo.p``). -The metadata file is generated on the master using the -:mod:`winrepo.genrepo ` runner. - -.. code-block:: bash - - salt-run winrepo.genrepo - -.. note:: - You only need to do this if you need to support legacy minions. - -Update the Minion Database --------------------------- - -Run :mod:`pkg.refresh_db ` on each of your -Windows minions to copy the metadata file down to the minion. - -.. code-block:: bash - - # From the master - salt -G 'os:windows' pkg.refresh_db - -.. _2015-8-0-winrepo-changes: - -Changes in Version 2015.8.0+ -============================ - -Git repository management for the Windows Software Repository changed in version -2015.8.0, and several master/minion config parameters were renamed for -consistency. - -For a complete list of the new winrepo config options, see -:ref:`here ` for master config options, and -:ref:`here ` for configuration options for masterless Windows -minions. - -pygit2_/GitPython_ Support --------------------------- - -On the master, the -:mod:`winrepo.update_git_repos ` -runner was updated to use either pygit2_ or GitPython_ to checkout the git -repositories containing repo data. If pygit2_ or GitPython_ is installed, -existing winrepo git checkouts should be removed after upgrading to 2015.8.0. -Then they should be cloned again by running -:mod:`winrepo.update_git_repos `. - -If neither GitPython_ nor pygit2_ are installed, Salt will fall back to -pre-existing behavior for -:mod:`winrepo.update_git_repos `, and a -warning will be logged in the master log. - -.. note:: - Standalone Windows minions do not support the new GitPython_/pygit2_ - functionality, and will instead use the - :mod:`git.latest ` state to keep repositories - up-to-date. More information on how to use the Windows Software Repo on a - standalone minion can be found :ref:`here `. - -Config Parameters Renamed -------------------------- - -Many of the legacy winrepo configuration parameters changed in version 2015.8.0 -to make them more consistent. Below are the parameters which changed for -version 2015.8.0: - -Master Config - -======================== ================================ -Old Name New Name -======================== ================================ -win_repo :conf_master:`winrepo_dir` -win_repo_mastercachefile No longer used on master -win_gitrepos :conf_master:`winrepo_remotes` -======================== ================================ - -.. note:: - The ``winrepo_dir_ng`` and ``winrepo_remotes_ng`` settings were introduced - in 2015.8.0 for working with the next generation repo. - -See :ref:`here ` for detailed information on all -master config options for the Windows Repo. - -Minion Config - -======================== ================================ -Old Name New Name -======================== ================================ -win_repo :conf_minion:`winrepo_dir` -win_repo_cachefile :conf_minion:`winrepo_cachefile` -win_gitrepos :conf_minion:`winrepo_remotes` -======================== ================================ - -.. note:: - The ``winrepo_dir_ng`` and ``winrepo_remotes_ng`` settings were introduced - in 2015.8.0 for working with the next generation repo. - -See :ref:`here ` for detailed information on all -minion config options for the Windows Repo. - -.. _wiki: https://wpkg.org/Category:Silent_Installers -.. _pygit2: https://github.com/libgit2/pygit2 -.. _GitPython: https://github.com/gitpython-developers/GitPython +Now on running the :mod:`winrepo.update_git_repos `, +- https://foo.com/bar/baz.git repo is initialized and cloned under the ``win_repo_dir_ng`` directory. +- https://mydomain.tld/baz.git repo is initialized and cloned under the ``win_repo_dir_ng\baz_junior`` directory. +- https://github.com/foobar/baz.git repo is initialized and cloned under the ``win_repo_dir_ng\baz_the_third`` directory. From a83738ffa2c5ab3462340945ca09e71b17cd59c1 Mon Sep 17 00:00:00 2001 From: Gayathri Krishnaswamy Date: Sat, 16 Sep 2023 22:09:28 +0530 Subject: [PATCH 43/71] Update windows-package-manager.rst Updated comments provided by Alyssa (cherry picked from commit 890889e1179267c3056405a081a40f43cc9ca10e) --- .../windows/windows-package-manager.rst | 139 ++++++++---------- 1 file changed, 65 insertions(+), 74 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index afe6cd33f26e..75cd4629cc12 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -15,12 +15,12 @@ What are package definition files? =================================== A package definition file is a YAML/JINJA2 file with a ``.sls`` file extension that contains all -the information needed to install a software using Salt. It defines: +the information needed to install software using Salt. It defines: - Full name of the software package - The version of the software package - Download location of the software package - Command-line switches for silent install and uninstall -- Whether or not to use the Windows task scheduler to install the package, +- Whether or not to use the Windows task scheduler to install the package The package definition files can be hosted in one or more Git repositories. The ``.sls`` files used to install Windows packages are not distributed by default with Salt. @@ -28,19 +28,16 @@ You have to initialize and clone the default repository - `salt-winrepo-ng`_ that is hosted on GitHub by SaltStack. The repository contains package definition files for many common Windows packages and is maintained by SaltStack -and the Salt community. Anyone is welcome to submit a pull request to this +and the Salt community. Anyone can submit a pull request to this repo to add new package definitions. -The package definition file can be managed through either Salt or Git. -The software packages can be downloaded from either within a git repository or from HTTP(S) or FTP URLs. -That is, the installer defined in the package definition file can be stored anywhere as long as it is -accessible from the host running Salt. +You can manage the package definition file through either Salt or Git. You can download software packages from either a git repository or from HTTP(S) or FTP URLs. You can store the installer defined in the package definition file anywhere if it is accessible from the host running Salt. You can use the Salt Windows package manager like ``yum`` on Linux. You do not have to know the underlying command to install the software. -Use ``pkg.install`` to install a package using a package manager based on the OS the system runs on. -Use ``pkg.installed`` to check if a particular package is installed in the minion or not. +- Use ``pkg.install`` to install a package using a package manager based on the OS the system runs on. +- Use ``pkg.installed`` to check if a particular package is installed in the minion. .. note:: The Salt Windows package manager does not automatically resolve dependencies while installing, @@ -63,8 +60,8 @@ a Salt Git repo, install the libraries ``GitPython`` or ``pygit2``. Populate the local Git repository ********************************** The SLS files used to install Windows packages are not distributed by default -with Salt. Assuming no changes to the default configuration (``file_roots``). -Initialize and clone `salt-winrepo-ng`_ +with Salt. Assuming no changes to the default configuration (``file_roots``), +initialize and clone `salt-winrepo-ng`_ repository. .. code-block:: bash @@ -146,7 +143,7 @@ specified in the ``winrepo_dir_ng`` setting in the config. The default value of Master Configuration ==================== -The following are settings are available for configuring the winrepo on the +The following settings are available for configuring the winrepo on the master: - :conf_master:`winrepo_dir` @@ -621,8 +618,8 @@ Use :mod:`pkg.list_pkgs ` to display a list of p # From the minion in masterless mode salt-call --local pkg.list_pkgs -The above command displays the software name and the version for every package installed -on the system irrespective of whether it was installed by Salt package manager or not. +The command displays the software name and the version for every package installed +on the system irrespective of whether it was installed by the Salt package manager. .. code-block:: console @@ -668,7 +665,7 @@ You can refer to the software by its ``name`` or its ``full_name`` surrounded by # From the minion in masterless mode salt-call --local pkg.list_available firefox_x64 -The above command lists all versions of Firefox available for installation. +The command lists all versions of Firefox available for installation. .. code-block:: bash @@ -689,7 +686,7 @@ The above command lists all versions of Firefox available for installation. .. note:: For a Linux master, you can surround the file name with single quotes. - However, the ``cmd`` shell on Windows use double quotes when wrapping strings + However, for the ``cmd`` shell on Windows use double quotes when wrapping strings that may contain spaces. Powershell accepts either single quotes or double quotes. Install a package @@ -705,7 +702,7 @@ Use :mod:`pkg.install `: to install a package. # From the minion in masterless mode salt-call --local pkg.install "firefox_x64" -The above command installs the latest version of Firefox. +The command installs the latest version of Firefox. .. code-block:: bash @@ -715,7 +712,7 @@ The above command installs the latest version of Firefox. # From the minion in masterless mode salt-call --local pkg.install "firefox_x64" version=74.0 -The above command installs version 74.0 of Firefox. +The command installs version 74.0 of Firefox. If a different version of the package is already installed then the old version is replaced with the version in the winrepo (only if the package supports live updating). @@ -867,15 +864,15 @@ Example package definition files ================================ This section provides some examples of package definition files for different use cases such as: -- Writing a simple package definition file for a software -- Writing a INJA templated package definition file +- Writing a simple package definition file for software +- Writing a JINJA templated package definition file - Writing a package definition file to install the latest version of the software -- Writing a package definintion file to install an MSI patch to installed software +- Writing a package definition file to install an MSI patch -These examples enables you to gain a better understanding of the usage of different file paramaters. -To understand the examples, you require a basic `Understanding Jinja `_ +These examples enable you to gain a better understanding of the usage of different file parameters. +To understand the examples, you need a basic `Understanding Jinja `_. For an exhaustive dive into Jinja, refer to the official -`Jinja Template Designer documentation `_ +`Jinja Template Designer documentation `_. Example: Basic ============== @@ -900,13 +897,13 @@ Here is a pure YAML example of a package definition file for Firefox: The first line is the short name of the software which is ``firefox_x64``. .. important:: - The short name name must match exactly what is shown in Add/Remove Programs (``appwiz.cpl``) + The short name must match exactly what is shown in Add/Remove Programs (``appwiz.cpl``) and it must be unique across all other short names in the software repository. The ``full_name`` combined with the version must also be unique. The second line is the ``software version`` and is indented two spaces. .. important:: - The version number must be enclosed in quotes else the YAML parser removes the trailing zeros. + The version number must be enclosed in quotes or the YAML parser removes the trailing zeros. For example, if the version number 74.0 is not enclosed within quotes then the version number is considered as only 74. @@ -914,19 +911,19 @@ The lines following the ``version`` are indented two more spaces and contain all needed to install the Firefox package. .. important:: - You can specify multiple versions for a software by specifying multiple version numbers at + You can specify multiple versions of software by specifying multiple version numbers at the same indentation level as the first with its software definition below it. Example: JINJA templated package definition file ================================================= JINJA is the default templating language used in package definition files. -You can use JINJA to add variables, expressions to package definition files -that get replaced with values when the ``.sls`` go through Salt renderer. +You can use JINJA to add variables and expressions to package definition files +that get replaced with values when the ``.sls`` go through the Salt renderer. -When there are tens or hundreds of versions available for a piece of software the -definition file can become large and cumbersome to write . +When there are tens or hundreds of versions available for a piece of software, the +definition file can become large and cumbersome to write. In this scenario, Jinja can be used to add logic, variables, -and expressions to automatically create the package definition file for a software with +and expressions to automatically create the package definition file for software with multiple versions. Here is a an example of a package definition file for Firefox that uses Jinja: @@ -949,22 +946,21 @@ Here is a an example of a package definition file for Firefox that uses Jinja: uninstall_flags: '/S' {% endfor %} -In this example JINJA is used to to generate a package definition file that defines +In this example, JINJA is used to generate a package definition file that defines how to install 12 versions of Firefox. Jinja is used to create a list of available versions. The list is iterated through a ``for loop`` where each version is placed in the ``version`` variable. The version is inserted everywhere there is a ``{{ version }}`` marker inside the ``for loop``. The single variable (``lang``) defined at the top of the package definition identifies the language of the package. -You can access the Salt modules using the ``salt`` keyword. In this case, the ``config.get`` function is invokedto retrieve -the language setting. If the ``lang`` variable is not defined then the default value is ``en-US``. +You can access the Salt modules using the ``salt`` keyword. In this case, the ``config.get`` function is invoked to retrieve the language setting. If the ``lang`` variable is not defined then the default value is ``en-US``. Example: Package definition file to install the latest version -=============================================================== -Some software vendors that do not provide access to all versions of -their software. Instead they provide a single URL to what is always the latest -version. In some cases the software keeps itself up to date. One example of this -is the `Chrome `_ +============================================================== +Some software vendors do not provide access to all versions of +their software. Instead, they provide a single URL to what is always the latest +version. In some cases, the software keeps itself up to date. One example of this +is the `Google Chrome web browser `_. To handle situations such as these, set the version to `latest`. Here's an example of a package definition file to install the latest version of Chrome. @@ -989,7 +985,7 @@ Example: Package definition file to install an MSI patch For MSI installer, when the ``msiexec`` parameter is set to true ``/i`` option is used for installation, and ``/x`` option is used for uninstallation. However, to install an MSI patch ``/i`` and ``/x`` options cannot be combined. -Here is an example of a package definition file to install an MSI patch +Here is an example of a package definition file to install an MSI patch: .. code-block:: yaml @@ -1010,35 +1006,30 @@ Here is an example of a package definition file to install an MSI patch msiexec: True cache_file: salt://win/repo-ng/MyApp/MyApp.1.1.msp -In the above example: -Version ``1.0`` of the software installs the application using the ``1.0`` MSI defined in the ``installer`` parameter. -There is no file to be cached and the ``install_flags`` parameter does not include any special values. +In the previous example: +- Version ``1.0`` of the software installs the application using the ``1.0`` MSI defined in the ``installer`` parameter. +- There is no file to be cached and the ``install_flags`` parameter does not include any special values. Version ``1.1`` of the software uses the same installer file as Version ``1.0``. -Now to apply patch to Version 1.0, make the following changes in the package definition file: +Now, to apply a patch to Version 1.0, make the following changes in the package definition file: - Place the patch file (MSP file) in the same directory as the installer file (MSI file) on the ``file_roots`` -- In the ``cache_file`` parameter, specify the path to single patch file. -- In ``install_flags`` parameter, add the ``/update`` flag and include the path to the MSP file -using the ``%cd%`` environment variable. ``%cd%`` resolves to the current working directory which is the location in the minion cache -where the installer file is cached. +- In the ``cache_file`` parameter, specify the path to a single patch file. +- In the ``install_flags`` parameter, add the ``/update`` flag and include the path to the MSP file +using the ``%cd%`` environment variable. ``%cd%`` resolves to the current working directory, which is the location in the minion cache where the installer file is cached. -See issue `#32780 `_ for more -details. +For more information, see issue `#32780 `_. -This same approach could be used for applying MST files for MSIs and answer -files for other types of .exe based installers. +The same approach could be used for applying MST files for MSIs and answer files for other types of .exe-based installers. Parameters ========== -This section describes the parameters placed under the ``version``in the package definition file. -Example can be found on the `Salt winrepo repository `_ +This section describes the parameters placed under the ``version`` in the package definition file. +An example can be found on the `Salt winrepo repository `_. full_name (str) --------------- -The full name for the software as shown in "Programs and Features" in -the control panel. You can also retrieve the full name of the package by installing the package -manually and then running ``pkg.list_pkgs``. +The full name of the software as shown in "Programs and Features" in the control panel. You can also retrieve the full name of the package by installing the package manually and then running ``pkg.list_pkgs``. Here's an example of the output from ``pkg.list_pkgs``: .. code-block:: console @@ -1076,9 +1067,9 @@ Firefox. salt-minion-py3: 3001 -On running ``pkg.list_pkgs``, If any of the software installed on the machine matches the full name +On running ``pkg.list_pkgs``, if any of the software installed on the machine matches the full name defined in any one of the software definition files in the repository, then the -package name is displayed in the output. +the package name is displayed in the output. .. important:: The version number and ``full_name`` must match the output of @@ -1087,10 +1078,10 @@ package name is displayed in the output. .. note:: You can successfully install packages using ``pkg.install``, - even if the ``full_name`` or the version number don't match. The + even if the ``full_name`` or the version number doesn't match. The module will complete successfully, but continue to display the full name in ``pkg.list_pkgs``. If this is happening, verify that the ``full_name`` - and the ``version`` match exactly what is displayed in Add/Remove + and the ``version``matches exactly what is displayed in Add/Remove Programs. .. tip:: @@ -1112,7 +1103,7 @@ then the package is cached locally and then executed. If it is a path to a file on disk or a file share, then it is executed directly. .. note:: - When storing software in the same location as the winrepo then + When storing software in the same location as the winrepo: - Create a sub folder named after the package. - Store the package definition file named ``init.sls`` and the binary installer in the same sub folder if you are hosting those files on the ``file_roots``. @@ -1125,9 +1116,9 @@ install_flags (str) ------------------- The flags passed to the installer for silent installation. -You can find these flags by adding ``/?`` or ``/h``when running the installer from the command-line. +You can find these flags by adding ``/?`` or ``/h`` when running the installer from the command line. See `WPKG project wiki `_ for information on silent install flags. - + .. warning:: Always ensure that the software has the ability to install silently since Salt appears to hang if the installer expects user input. @@ -1135,9 +1126,9 @@ See `WPKG project wiki `_ for infor uninstaller (str) ----------------- -The path to the program to uninstall a software. +The path to the program to uninstall software. This can be the path to the same exe or msi used to install the software. -If you use msi to install the software then you can either use GUID of the software or the +If you use MSI to install the software then you can either use GUID of the software or the same MSI to uninstall the software. You can find the uninstall information in the registry: @@ -1158,7 +1149,7 @@ Here's an example of using the GUID to uninstall software. uninstall_flags: '/qn /norestart' msiexec: True -Here's an example of using the installer MSI to uninstall software +Here's an example of using the installer MSI to uninstall software: .. code-block:: yaml @@ -1244,7 +1235,7 @@ cache_file (str) This setting requires the file to be stored on the ``file_roots`` and only applies to URLs that begin with ``salt://``. It indicates that only a single file specified -is to be copied down for use with the installer. It is copied to the same location as the +is copied down for use with the installer. It is copied to the same location as the installer. This setting is useful when ``cache_dir`` is set to ``True``, and you want to cache only a specific file and not all files that reside in the installer directory. @@ -1325,8 +1316,8 @@ ready for use. Troubleshooting *************** -My software installs correctly but `pkg.installed says it failed -=============================================================== +My software installs correctly but ``pkg.installed`` says it failed +=================================================================== If you have a package that seems to install properly, but Salt reports a failure then it is likely you have a ``version`` or ``full_name`` mismatch. @@ -1334,9 +1325,9 @@ then it is likely you have a ``version`` or ``full_name`` mismatch. - Check the ``full_name`` and ``version`` of the package as shown in Add/Remove Programs (``appwiz.cpl``). - Use ``pkg.list_pkgs`` to check that the ``full_name`` and -``version`` exactly match what is installed. +``version`` exactly matches what is installed. - Verify that the ``full_name`` and ``version`` of the package in the package definition file - matches the full name and version in Add/Remove programs. + match the full name and version in Add/Remove programs. - Ensure that the ``version`` is wrapped in single quotes in the package definition file. @@ -1402,7 +1393,7 @@ and set per-remote config settings. An example of this is: sets a global passphrase for winrepo that applies to all SSH-based remotes, unless overridden by a ``passphrase`` per-remote parameter. - See :ref:`here ` for detailed + See :ref:`here ` for a detailed explanation of how per-remote configuration works in gitfs. The same principles apply to winrepo. From 3dbf5b55039b7a154a7d796c5e319c2ee12da366 Mon Sep 17 00:00:00 2001 From: Gayathri Krishnaswamy Date: Sat, 16 Sep 2023 22:13:18 +0530 Subject: [PATCH 44/71] Update windows-package-manager.rst Updated some grammatical errors. (cherry picked from commit 866f9598b9e5f2cb3bde3d27a527308db35d4a5a) --- doc/topics/windows/windows-package-manager.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 75cd4629cc12..d62c7085c9cc 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -1167,7 +1167,7 @@ uninstall_flags (str) The flags passed to the uninstaller for silent uninstallation. -You can find these flags by adding ``/?`` or ``/h``when running the uninstaller from the command-line. +You can find these flags by adding ``/?`` or ``/h`` when running the uninstaller from the command-line. See `WPKG project wiki `_ for information on silent uninstall flags. .. warning:: @@ -1251,7 +1251,7 @@ software was installed successfully. This is used in the package definition for Salt itself. The first thing the Salt installer does is kill the Salt service, which then kills all child processes. If the Salt installer is launched via Salt, then the installer - itself is killed leaving Salt on the machine but not running. Use of the + is killed leaving Salt on the machine but not running. The use of the task scheduler allows an external process to launch the Salt installation so its processes aren't killed when the Salt service is stopped. @@ -1282,7 +1282,7 @@ Here's an example using ``source_hash``: Not Implemented --------------- The following parameters are often seen in the software definition files hosted -on the Git repo. However, they are not implemented and have no effect on the +on the Git repo. However, they are not implemented and do not affect the installation process. :param bool reboot: Not implemented @@ -1297,10 +1297,10 @@ Managing Windows Software on a Standalone Windows Minion The Windows Software Repository functions similarly in a standalone environment, with a few differences in the configuration. -To replace the winrepo runner that is used on the Salt master, an +To replace the winrepo runner used on the Salt master, an :mod:`execution module ` exists to provide the same functionality to standalone minions. The functions are named the same as the -ones in the runner, and are used in the same way; the only difference is that +ones in the runner and are used in the same way; the only difference is that ``salt-call`` is used instead of ``salt-run``: .. code-block:: bash @@ -1308,7 +1308,7 @@ ones in the runner, and are used in the same way; the only difference is that salt-call winrepo.update_git_repos salt-call pkg.refresh_db -After executing the previous commands the repository on the standalone system is +After executing the previous commands, the repository on the standalone system is ready for use. .. _winrepo-troubleshooting: @@ -1319,14 +1319,14 @@ Troubleshooting My software installs correctly but ``pkg.installed`` says it failed =================================================================== -If you have a package that seems to install properly, but Salt reports a failure +If you have a package that seems to install properly but Salt reports a failure then it is likely you have a ``version`` or ``full_name`` mismatch. - Check the ``full_name`` and ``version`` of the package as shown in Add/Remove Programs (``appwiz.cpl``). - Use ``pkg.list_pkgs`` to check that the ``full_name`` and ``version`` exactly matches what is installed. -- Verify that the ``full_name`` and ``version`` of the package in the package definition file +- Verify that the package's ``full_name`` and ``version``in the package definition file match the full name and version in Add/Remove programs. - Ensure that the ``version`` is wrapped in single quotes in the package definition file. @@ -1341,7 +1341,6 @@ updating the package definintion files in the repo. salt winminion pkg.refresh_db - Winrepo upgrade issues ====================== From 60335087fdde53b31527b4c9ee30a76b1fe43f9c Mon Sep 17 00:00:00 2001 From: Shane Lee Date: Wed, 29 Nov 2023 11:37:29 -0700 Subject: [PATCH 45/71] Fix a few typos and grammatical errors (cherry picked from commit 4a45b06c16040fa9a456cc056b7433cbc4003599) --- doc/topics/releases/2015.8.0.rst | 6 +- .../windows/windows-package-manager.rst | 766 ++++++++++-------- 2 files changed, 452 insertions(+), 320 deletions(-) diff --git a/doc/topics/releases/2015.8.0.rst b/doc/topics/releases/2015.8.0.rst index d140cb0881b6..932c10c02daa 100644 --- a/doc/topics/releases/2015.8.0.rst +++ b/doc/topics/releases/2015.8.0.rst @@ -170,8 +170,7 @@ later minions. When using this new repository, the repo cache is compiled on the Salt Minion, which enables pillar, grains and other things to be available during compilation time. -See the :ref:`Windows Software Repository <2015-8-0-winrepo-changes>` -documentation for more information. +See the Windows Software Repository documentation for more information. Changes to legacy Windows repository ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -183,8 +182,7 @@ If you were previously using this repository and have customized settings, be aware that several config options have been renamed to make their naming more consistent. -See the :ref:`Windows Software Repository <2015-8-0-winrepo-changes>` -documentation for more information. +See the Windows Software Repository documentation for more information. Win System Module ----------------- diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index d62c7085c9cc..2a18800b69fd 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -3,65 +3,86 @@ ####################### Windows Package Manager ####################### + Introduction ************ -Salt provides a Windows package management tool for installing, updating, removing, and -managing software packages on remote Windows systems. This tool provides a -software repository and a package manager similar to what is provided -by ``yum`` and ``apt`` on Linux. -The repository contains a collection of package definition files. + +Salt provides a Windows package management tool for installing, updating, +removing, and managing software packages on remote Windows systems. This tool +provides a software repository and a package manager similar to what is provided +by ``yum`` and ``apt`` on Linux. The repository contains a collection of package +definition files. What are package definition files? -=================================== +================================== + +A package definition file is a YAML/JINJA2 file with a ``.sls`` file extension +that contains all the information needed to install software using Salt. It +defines: -A package definition file is a YAML/JINJA2 file with a ``.sls`` file extension that contains all -the information needed to install software using Salt. It defines: - Full name of the software package - The version of the software package - Download location of the software package - Command-line switches for silent install and uninstall - Whether or not to use the Windows task scheduler to install the package -The package definition files can be hosted in one or more Git repositories. -The ``.sls`` files used to install Windows packages are not distributed by default with Salt. -You have to initialize and clone the -default repository - `salt-winrepo-ng`_ -that is hosted on GitHub by SaltStack. The repository contains -package definition files for many common Windows packages and is maintained by SaltStack -and the Salt community. Anyone can submit a pull request to this -repo to add new package definitions. +Package definition files can be hosted in one or more Git repositories. The +``.sls`` files used to install Windows packages are not distributed by default +with Salt. You have to initialize and clone the default repository +`salt-winrepo-ng `_ +which is hosted on GitHub by SaltStack. The repository contains package +definition files for many common Windows packages and is maintained by SaltStack +and the Salt community. Anyone can submit a pull request to this repo to add +new package definitions. -You can manage the package definition file through either Salt or Git. You can download software packages from either a git repository or from HTTP(S) or FTP URLs. You can store the installer defined in the package definition file anywhere if it is accessible from the host running Salt. +You can manage the package definition file through either Salt or Git. You can +download software packages from either a git repository or from HTTP(S) or FTP +URLs. You can store the installer defined in the package definition file +anywhere as long as it is accessible from the host running Salt. -You can use the Salt Windows package manager like ``yum`` on Linux. You do not have to know the -underlying command to install the software. +You can use the Salt Windows package manager like ``yum`` on Linux. You do not +have to know the underlying command to install the software. -- Use ``pkg.install`` to install a package using a package manager based on the OS the system runs on. -- Use ``pkg.installed`` to check if a particular package is installed in the minion. +- Use ``pkg.install`` to install a package using a package manager based on + the OS the system runs on. +- Use ``pkg.installed`` to check if a particular package is installed in the + minion. .. note:: - The Salt Windows package manager does not automatically resolve dependencies while installing, - updating, or removing packages. You have to manage the dependencies between packages manually. + The Salt Windows package manager does not automatically resolve dependencies + while installing, updating, or removing packages. You have to manage the + dependencies between packages manually. + +.. _quickstart: Quickstart -============ -This quickstart guides you through using Windows Salt package manager winrepo to -install software packages in four steps: -1. (Optional) :ref:`Install libraries ` -2. :ref:`Populate the local Git repository` -3. :ref:`Update minion database` -4. :ref:`Install software packages` +========== + +This quickstart guides you through using the Windows Salt package manager +(winrepo) to install software packages in four steps: + +1. (Optional) :ref:`Install libraries ` +2. :ref:`Populate the local Git repository` +3. :ref:`Update minion database` +4. :ref:`Install software packages` + +.. _install-libraries: Install libraries ***************** -(Optional) If you are using the Salt Windows package manager with package definition files hosted on -a Salt Git repo, install the libraries ``GitPython`` or ``pygit2``. + +(Optional) If you are using the Salt Windows package manager with package +definition files hosted on a Salt Git repo, install the libraries ``GitPython`` +or ``pygit2``\. + +.. _populate-git-repo: Populate the local Git repository ********************************** + The SLS files used to install Windows packages are not distributed by default with Salt. Assuming no changes to the default configuration (``file_roots``), -initialize and clone `salt-winrepo-ng`_ +initialize and clone `salt-winrepo-ng `_ repository. .. code-block:: bash @@ -69,22 +90,30 @@ repository. salt-run winrepo.update_git_repos On successful execution of :mod:`winrepo.update_git_repos `, -the winrepo repository is cloned in the master on the -location specified in ``winrepo_dir_ng`` and all package definition files are pulled down from the Git repository. +the winrepo repository is cloned on the master in the location specified in +``winrepo_dir_ng`` and all package definition files are pulled down from the Git +repository. -On masterless minion, use ``salt-call`` to initialize and clone the `salt-winrepo-ng `_ +On a masterless minion, use ``salt-call`` to initialize and clone the +`salt-winrepo-ng `_ .. code-block:: bash salt-call --local winrepo.update_git_repos -On successful execution of the runner, the winrepo repository is cloned in the minion in the location -specified in ``winrepo_dir_ng`` and all package definition files are pulled down from the Git repository. +On successful execution of the runner, the winrepo repository is cloned on the +minion in the location specified in ``winrepo_dir_ng`` and all package +definition files are pulled down from the Git repository. + +.. _refresh-db: Update minion database -*********************** -Run :mod:`pkg.refresh_db ` on all Windows minions to create a database entry for every package definition file -and build the package database. +********************** + +Run :mod:`pkg.refresh_db ` on all Windows +minions to create a database entry for every package definition file and build +the package database. + .. code-block:: bash # From the master @@ -93,9 +122,10 @@ and build the package database. # From the minion in masterless mode salt-call --local pkg.refresh_db -The ``pkg.refresh_db`` command parses the YAML/JINJA package definition files and -generates the database. The above command returns the following summary denoting the number of packages -that succeeded or failed to compile: +The :mod:`pkg.refresh_db ` command parses the +YAML/JINJA package definition files and generates the database. The above +command returns the following summary denoting the number of packages that +succeeded or failed to compile: .. code-block:: bash @@ -109,16 +139,20 @@ that succeeded or failed to compile: 301 .. note:: - This command can take a few minutes to complete as all the package definition - files are copied to the minion and the database is generated. + This command can take a few minutes to complete as all the package + definition files are copied to the minion and the database is generated. .. note:: - You can use ``pkg.refresh_db`` when writing new Windows package definitions to check for errors - in the definitions against one or more Windows minions. + You can use ``pkg.refresh_db`` when writing new Windows package definitions + to check for errors in the definitions against one or more Windows minions. + +.. _pkg-install: Install software package ************************ -You can now install a software package using :mod:`pkg.install `: + +You can now install a software package using +:mod:`pkg.install `: .. code-block:: bash @@ -128,18 +162,22 @@ You can now install a software package using :mod:`pkg.install `_ +`https://github.com/saltstack/salt-winrepo `_ The legacy repo can be disabled by setting it to an empty list in the master config. @@ -207,7 +244,7 @@ winrepo_remotes_ng :conf_master:`winrepo_remotes_ng` (list) -This setting tells the ``winrepo.upgate_git_repos`` command where the next +This setting tells the ``winrepo.update_git_repos`` command where the next generation winrepo is hosted. This a list of URLs to multiple git repos. The default is a list containing a single URL: @@ -234,7 +271,7 @@ winrepo_provider :conf_master:`winrepo_provider` (str) -The provider to be used for winrepo. Default is ``pygit2``. Falls back to +The provider to be used for winrepo. Default is ``pygit2``\. Falls back to ``gitpython`` when ``pygit2`` is not available winrepo_ssl_verify @@ -245,11 +282,13 @@ winrepo_ssl_verify Ignore SSL certificate errors when contacting remote repository. Default is ``False`` +.. _master-config-pygit2: + Master Configuration (pygit2) ============================= The following configuration options only apply when the -:conf_master:`winrepo_provider` option is set to ``pygit2``. +:conf_master:`winrepo_provider` option is set to ``pygit2``\. - :conf_master:`winrepo_insecure_auth` - :conf_master:`winrepo_passphrase` @@ -306,6 +345,8 @@ winrepo_password Used only with ``pygit2`` provider. Used with :conf_master:`winrepo_user` to authenticate to HTTPS remotes. Default is ``''`` +.. _minion-config: + Minion Configuration ==================== @@ -324,14 +365,14 @@ winrepo_cache_expire_max :conf_minion:`winrepo_cache_expire_max` (int) Sets the maximum age in seconds of the winrepo metadata file to avoid it -becoming stale. If the metadata file is older than this setting it will trigger +becoming stale. If the metadata file is older than this setting, it will trigger a ``pkg.refresh_db`` on the next run of any ``pkg`` module function that requires the metadata file. Default is 604800 (1 week). Software package definitions are automatically refreshed if stale after -:conf_minion:`winrepo_cache_expire_max`. Running a highstate normal forces the -refresh of the package definition and generation of the metadata, unless -the metadata is younger than :conf_minion:`winrepo_cache_expire_max`. +:conf_minion:`winrepo_cache_expire_max`. Running a highstate forces the refresh +of the package definitions and regenerates the metadata, unless the metadata is +younger than :conf_minion:`winrepo_cache_expire_max`. winrepo_cache_expire_min ------------------------ @@ -339,7 +380,7 @@ winrepo_cache_expire_min :conf_minion:`winrepo_cache_expire_min` (int) Sets the minimum age in seconds of the winrepo metadata file to avoid refreshing -too often. If the metadata file is older than this setting the metadata will be +too often. If the metadata file is older than this setting, the metadata will be refreshed unless you pass ``refresh: False`` in the state. Default is 1800 (30 min). @@ -349,28 +390,30 @@ winrepo_cachefile :conf_minion:`winrepo_cachefile` (str) The file name of the winrepo cache file. The file is placed at the root of -``winrepo_dir_ng``. Default is ``winrepo.p``. +``winrepo_dir_ng``\. Default is ``winrepo.p``\. winrepo_source_dir ------------------ :conf_minion:`winrepo_source_dir` (str) -The location of the .sls files on the Salt file server. Default is -``salt://win/repo-ng/``. +The location of the .sls files on the Salt file server. This allows for using +different environments. Default is ``salt://win/repo-ng/``\. .. warning:: - If the default for ``winrepo_dir_ng`` is changed, then this setting will - also need to be changed on each minion. The default setting for - ``winrepo_dir_ng`` is ``/srv/salt/win/repo-ng``. If that were changed to - ``/srv/salt/new/repo-ng`` then the ``winrepo_source_dir`` would need to be + If the default for ``winrepo_dir_ng`` is changed, this setting may need to + be changed on each minion. The default setting for ``winrepo_dir_ng`` is + ``/srv/salt/win/repo-ng``\. If that were changed to + ``/srv/salt/new/repo-ng``\, then the ``winrepo_source_dir`` would need to be changed to ``salt://new/repo-ng`` +.. _masterless-minion-config: + Masterless Minion Configuration =============================== -The following are settings are available for configuring the winrepo on a -masterless minion: +The following settings are available for configuring the winrepo on a masterless +minion: - :conf_minion:`winrepo_dir` - :conf_minion:`winrepo_dir_ng` @@ -394,18 +437,18 @@ winrepo_dir_ng :conf_minion:`winrepo_dir_ng` (str) -The location in the ``file_roots where the winrepo files are kept. The default -is ``C:\ProgramData\Salt Project\Salt\srv\salt\win\repo-ng``. +The location in the ``file_roots`` where the winrepo files are kept. The default +is ``C:\salt\srv\salt\win\repo-ng``\. .. warning:: You can change the location of the winrepo directory. However, it must - always be set to a path that is inside the ``file_roots``. - Otherwise the software definition files will be unreachable by the minion. + always be set to a path that is inside the ``file_roots``\. Otherwise, the + software definition files will be unreachable by the minion. .. important:: A common mistake is to change the ``file_roots`` setting and fail to update the ``winrepo_dir_ng`` and ``winrepo_dir`` settings so that they are inside - the ``file_roots``. You might also want to verify ``winrepo_source_dir`` on + the ``file_roots``\. You might also want to verify ``winrepo_source_dir`` on the minion as well. winrepo_remotes @@ -431,13 +474,14 @@ winrepo_remotes_ng :conf_minion:`winrepo_remotes_ng` (list) -This setting tells the ``winrepo.upgate_git_repos`` command where the next +This setting tells the ``winrepo.update_git_repos`` command where the next generation winrepo is hosted. This a list of URLs to multiple git repos. The default is a list containing a single URL: `https://github.com/saltstack/salt-winrepo-ng `_ +.. _usage: Sample Configurations @@ -591,24 +635,28 @@ package manager commands to manage software on Windows minions. - Description * - 1 - - :ref:`pkg.list_pkgs` + - :ref:`pkg.list_pkgs ` - Displays a list of all packages installed in the system. * - 2 - - :ref:`pkg.list_available` + - :ref:`pkg.list_available ` - Displays the versions available of a particular package to be installed. -- + * - 3 - - :ref:`pkg.install` + - :ref:`pkg.install ` - Installs a given package. * - 4 - - :ref:`pkg.remove` + - :ref:`pkg.remove ` - Uninstalls a given package. +.. _list-pkgs: + List installed packages ======================= -Use :mod:`pkg.list_pkgs ` to display a list of packages installed on the system. + +Use :mod:`pkg.list_pkgs ` to display a list of +packages installed on the system. .. code-block:: bash @@ -618,10 +666,11 @@ Use :mod:`pkg.list_pkgs ` to display a list of p # From the minion in masterless mode salt-call --local pkg.list_pkgs -The command displays the software name and the version for every package installed -on the system irrespective of whether it was installed by the Salt package manager. +The command displays the software name and the version for every package +installed on the system irrespective of whether it was installed by the Salt +package manager. -.. code-block:: console +.. code-block:: bash local: ---------- @@ -642,20 +691,25 @@ on the system irrespective of whether it was installed by the Salt package manag The software name indicates whether the software is managed by Salt or not. -If Salt finds a match in the winrepo database then the software name is the -short name as defined in the package definition file. It is usually a single-word, lower-case name. +If Salt finds a match in the winrepo database, then the software name is the +short name as defined in the package definition file. It is usually a +single-word, lower-case name. -All other software names are displayed as the full name as shown in Add/Remove Programs. -In the above example, Git (git), Nullsoft Installer (nsis), Python 3.7 (python3_x64), -Salt (salt-minion-py3) have corresponding package definition file and are managed by Salt -while Frhed 1.6.0, GNU Privacy guard, GPG4win are not managed by Salt. +All other software names are displayed as the full name as shown in +Add/Remove Programs. In the above example, Git (git), Nullsoft Installer (nsis), +Python 3.7 (python3_x64), and Salt (salt-minion-py3) have corresponding package +definition files and are managed by Salt, while Frhed 1.6.0, GNU Privacy guard, +and GPG4win are not. + +.. _list-available: List available versions ======================= -Use :mod:`pkg.list_available ` to display the list of version(s) -of a package available for installation. You can pass the name of the software in the command. -You can refer to the software by its ``name`` or its ``full_name`` surrounded by quotes. +Use :mod:`pkg.list_available ` to display +a list of versions of a package available for installation. You can pass the +name of the software in the command. You can refer to the software by its +``name`` or its ``full_name`` surrounded by quotes. .. code-block:: bash @@ -686,8 +740,11 @@ The command lists all versions of Firefox available for installation. .. note:: For a Linux master, you can surround the file name with single quotes. - However, for the ``cmd`` shell on Windows use double quotes when wrapping strings - that may contain spaces. Powershell accepts either single quotes or double quotes. + However, for the ``cmd`` shell on Windows, use double quotes when wrapping + strings that may contain spaces. Powershell accepts either single quotes or + double quotes. + +.. _install: Install a package ================= @@ -714,8 +771,9 @@ The command installs the latest version of Firefox. The command installs version 74.0 of Firefox. -If a different version of the package is already installed then the old version is -replaced with the version in the winrepo (only if the package supports live updating). +If a different version of the package is already installed, then the old version +is replaced with the version in the winrepo (only if the package supports live +updating). You can also specify the full name of the software while installing: @@ -727,9 +785,12 @@ You can also specify the full name of the software while installing: # From the minion in masterless mode salt-call --local pkg.install "Mozilla Firefox 17.0.1 (x86 en-US)" +.. _remove: + Remove a package ================ - Use :mod:`pkg.remove ` to remove a package. + +Use :mod:`pkg.remove ` to remove a package. .. code-block:: bash @@ -739,30 +800,29 @@ Remove a package # From the minion in masterless mode salt-call --local pkg.remove firefox_x64 -.. _software-definition-files: - +.. _winrepo-structure: -Package defintion file directory structure and naming +Package definition file directory structure and naming ====================================================== -All package definition files are stored in the location configured in the ``winrepo_dir_ng`` -setting. All files in this directory with ``.sls`` file extension are -considered package definition files. These files are evaluated to create the -metadata file on the minion. +All package definition files are stored in the location configured in the +``winrepo_dir_ng`` setting. All files in this directory with a ``.sls`` file +extension are considered package definition files. These files are evaluated to +create the metadata file on the minion. You can maintain standalone package definition files that point to software on -other servers or on the internet. In this case the file name is the short -name of the software with the ``.sls`` extension, For example,``firefox.sls``. +other servers or on the internet. In this case the file name is the short name +of the software with the ``.sls`` extension, for example,``firefox.sls``\. You can also store the binaries for your software together with their software definition files in their own directory. In this scenario, the directory name -is the short name for the software and the package definition file stored that directory is -named ``init.sls``. +is the short name for the software and the package definition file stored that +directory is named ``init.sls``\. Look at the following example directory structure on a Linux master assuming default config settings: -.. code-block:: console +.. code-block:: bash srv/ |---salt/ @@ -796,32 +856,43 @@ default config settings: | | | | |---chrome.sls | | | | |---firefox.sls -In the above directory structure, -- The ``custom_defs`` directory contains the following custom package definition files. - - A folder for MS Office 2013 that contains the installer files for all the MS Office softwares and a -package definition file named ``init.sls``. - - Additional two more standalone package definition files ``openssl.sls`` and ``zoom.sls`` to install -Open SSl and Zoom. -- The ``salt-winrepo-ng`` directory contains the clone of the git repo specified by -the ``winrepo_remotes_ng`` config setting. +In the above directory structure: + +- The ``custom_defs`` directory contains the following custom package definition + files. + + - A folder for MS Office 2013 that contains the installer files for all the + MS Office software and a package definition file named ``init.sls``\. + - Two additional standalone package definition files ``openssl.sls`` and + ``zoom.sls`` to install OpenSSl and Zoom. + +- The ``salt-winrepo-ng`` directory contains the clone of the git repo specified + by the ``winrepo_remotes_ng`` config setting. .. warning:: - Do not modify the files in the ``salt-winrepo-ng`` directory as it breaks the future runs of - ``winrepo.update_git_repos``. + Do not modify the files in the ``salt-winrepo-ng`` directory as it breaks + future runs of ``winrepo.update_git_repos``\. .. warning:: - Do not place any custom software definition files in the ``salt-winrepo-ng`` directory as - ``winrepo.update_git_repos`` command wipes out the contents of the ``salt-winrepo-ng`` - directory each time it is run and any extra files stored in the Salt winrepo is lost. + Do not place any custom software definition files in the ``salt-winrepo-ng`` + directory as the ``winrepo.update_git_repos`` command wipes out the contents + of the ``salt-winrepo-ng`` directory each time it is run and any extra files + stored in the Salt winrepo are lost. + +.. + +.. _pkg-definition: Writing package definition files -================================= -You can write a software definition file if you know: +================================ +You can write your own software definition file if you know: + - The full name of the software as shown in Add/Remove Programs - The exact version number as shown in Add/Remove Programs - How to install your software silently from the command line Here is a YAML software definition file for Firefox: + .. code-block:: yaml firefox_x64: @@ -838,46 +909,60 @@ Here is a YAML software definition file for Firefox: uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe' uninstall_flags: '/S' -The package definition file itself is a data structure written in YAML with three indentation levels. -- The first level item is a short name that Salt uses reference the software. This short name is used to -install and remove the software and it must be unique across all package definition files in the repo. -Also, there must be only one short name in the file. -- The second level item is the version number. There can be multiple version numbers for a software but they must be unique within the file. +The package definition file itself is a data structure written in YAML with +three indentation levels: + +- The first level item is a short name that Salt uses to reference the software. + This short name is used to install and remove the software and it must be + unique across all package definition files in the repo. Also, there must be + only one short name in the file. +- The second level item is the version number. There can be multiple version + numbers for a package but they must be unique within the file. + .. note:: - On running ``pkg.list_pkgs``, the short name and version number are displayed is displayed when Salt finds a match in the repo. - -- The third indentation level contains all parameters that the Salt needs to -install the software. The parameters are: -- ``full_name`` : The full name as displayed in Add/Remove Programs -- ``installer`` : The location of the installer binary -- ``install_flags`` : The flags required to install silently -- ``uninstaller`` : The location of the uninstaller binary -- ``uninstall_flags`` : The flags required to uninstall silently -- ``msiexec`` : Use msiexec to install this package -- ``allusers`` : If this is an MSI, install to all users -- ``cache_dir`` : Cache the entire directory in the installer URL if it starts with ``salt://`` -- ``cache_file`` : Cache a single file in the installer URL if it starts with ``salt://`` -- ``use_scheduler`` : Launch the installer using the task scheduler -- ``source_hash`` : The hash sum for the installer + When running ``pkg.list_pkgs``\, the short name and version number are + displayed when Salt finds a match in the repo. Otherwise, the full package + name is displayed. + +- The third indentation level contains all parameters that Salt needs to install + the software. The parameters are: + + - ``full_name`` : The full name as displayed in Add/Remove Programs + - ``installer`` : The location of the installer binary + - ``install_flags`` : The flags required to install silently + - ``uninstaller`` : The location of the uninstaller binary + - ``uninstall_flags`` : The flags required to uninstall silently + - ``msiexec`` : Use msiexec to install this package + - ``allusers`` : If this is an MSI, install to all users + - ``cache_dir`` : Cache the entire directory in the installer URL if it starts + with ``salt://`` + - ``cache_file`` : Cache a single file in the installer URL if it starts with + ``salt://`` + - ``use_scheduler`` : Launch the installer using the task scheduler + - ``source_hash`` : The hash sum for the installer Example package definition files ================================ -This section provides some examples of package definition files for different use cases such as: +This section provides some examples of package definition files for different +use cases such as: -- Writing a simple package definition file for software -- Writing a JINJA templated package definition file -- Writing a package definition file to install the latest version of the software -- Writing a package definition file to install an MSI patch +- Writing a :ref:`simple package definition file ` +- Writing a :ref:`JINJA templated package definition file ` +- Writing a package definition file to :ref:`install the latest version of the software ` +- Writing a package definition file to :ref:`install an MSI patch ` -These examples enable you to gain a better understanding of the usage of different file parameters. -To understand the examples, you need a basic `Understanding Jinja `_. +These examples enable you to gain a better understanding of the usage of +different file parameters. To understand the examples, you need a basic +`Understanding Jinja `_. For an exhaustive dive into Jinja, refer to the official `Jinja Template Designer documentation `_. -Example: Basic -============== +.. _example-simple: + +Example: Simple +=============== -Here is a pure YAML example of a package definition file for Firefox: +Here is a pure YAML example of a simple package definition file for Firefox: .. code-block:: yaml @@ -895,38 +980,47 @@ Here is a pure YAML example of a package definition file for Firefox: uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe' uninstall_flags: '/S' -The first line is the short name of the software which is ``firefox_x64``. +The first line is the short name of the software which is ``firefox_x64``\. + .. important:: - The short name must match exactly what is shown in Add/Remove Programs (``appwiz.cpl``) - and it must be unique across all other short names in the software repository. - The ``full_name`` combined with the version must also be unique. + The short name must be unique across all other short names in the software + repository. The ``full_name`` combined with the version must also be unique. The second line is the ``software version`` and is indented two spaces. + .. important:: - The version number must be enclosed in quotes or the YAML parser removes the trailing zeros. - For example, if the version number 74.0 is not enclosed within quotes then the version number - is considered as only 74. + The version number must be enclosed in quotes or the YAML parser removes the + trailing zeros. For example, if the version number ``74.0`` is not enclosed + within quotes, then the version number is rendered as ``74``\. -The lines following the ``version`` are indented two more spaces and contain all the information -needed to install the Firefox package. +The lines following the ``version`` are indented two more spaces and contain all +the information needed to install the Firefox package. .. important:: - You can specify multiple versions of software by specifying multiple version numbers at - the same indentation level as the first with its software definition below it. + You can specify multiple versions of software by specifying multiple version + numbers at the same indentation level as the first with its software + definition below it. + +.. important:: + The ``full_name`` must match exactly what is shown in Add/Remove Programs + (``appwiz.cpl``) + +.. _example-jinja: Example: JINJA templated package definition file -================================================= -JINJA is the default templating language used in package definition files. -You can use JINJA to add variables and expressions to package definition files -that get replaced with values when the ``.sls`` go through the Salt renderer. +================================================ -When there are tens or hundreds of versions available for a piece of software, the -definition file can become large and cumbersome to write. -In this scenario, Jinja can be used to add logic, variables, -and expressions to automatically create the package definition file for software with -multiple versions. +JINJA is the default templating language used in package definition files. You +can use JINJA to add variables and expressions to package definition files that +get replaced with values when the ``.sls`` go through the Salt renderer. -Here is a an example of a package definition file for Firefox that uses Jinja: +When there are tens or hundreds of versions available for a piece of software, +the definition file can become large and cumbersome to maintain. In this +scenario, JINJA can be used to add logic, variables, and expressions to +automatically create the package definition file for software with multiple +versions. + +Here is a an example of a package definition file for Firefox that uses JINJA: .. code-block:: jinja @@ -946,21 +1040,27 @@ Here is a an example of a package definition file for Firefox that uses Jinja: uninstall_flags: '/S' {% endfor %} -In this example, JINJA is used to generate a package definition file that defines -how to install 12 versions of Firefox. Jinja is used to create a list of -available versions. The list is iterated through a ``for loop`` where each version is placed -in the ``version`` variable. The version is inserted everywhere there is a -``{{ version }}`` marker inside the ``for loop``. +In this example, JINJA is used to generate a package definition file that +defines how to install 12 versions of Firefox. Jinja is used to create a list of +available versions. The list is iterated through a ``for loop`` where each +version is placed in the ``version`` variable. The version is inserted +everywhere there is a ``{{ version }}`` marker inside the ``for loop``\. + +The single variable (``lang``) defined at the top of the package definition +identifies the language of the package. You can access the Salt modules using +the ``salt`` keyword. In this case, the ``config.get`` function is invoked to +retrieve the language setting. If the ``lang`` variable is not defined then the +default value is ``en-US``\. -The single variable (``lang``) defined at the top of the package definition identifies the language of the package. -You can access the Salt modules using the ``salt`` keyword. In this case, the ``config.get`` function is invoked to retrieve the language setting. If the ``lang`` variable is not defined then the default value is ``en-US``. +.. _example-latest: Example: Package definition file to install the latest version ============================================================== -Some software vendors do not provide access to all versions of -their software. Instead, they provide a single URL to what is always the latest -version. In some cases, the software keeps itself up to date. One example of this -is the `Google Chrome web browser `_. + +Some software vendors do not provide access to all versions of their software. +Instead, they provide a single URL to what is always the latest version. In some +cases, the software keeps itself up to date. One example of this is the `Google +Chrome web browser `_. To handle situations such as these, set the version to `latest`. Here's an example of a package definition file to install the latest version of Chrome. @@ -976,14 +1076,21 @@ example of a package definition file to install the latest version of Chrome. uninstall_flags: '/qn /norestart' msiexec: True -In the above example, -- ``Version`` is set to ``latest``. Salt then installs the latest version of Chrome at the URL and displays that version. -- ``msiexec`` is set to ``True`` hence the software is installed using an MSI. +In the above example: + +- ``Version`` is set to ``latest``\. Salt then installs the latest version of + Chrome at the URL and displays that version. +- ``msiexec`` is set to ``True``\, hence the software is installed using an MSI. + +.. _example-patch: Example: Package definition file to install an MSI patch -========================================================= -For MSI installer, when the ``msiexec`` parameter is set to true ``/i`` option is used for installation, -and ``/x`` option is used for uninstallation. However, to install an MSI patch ``/i`` and ``/x`` options cannot be combined. +======================================================== + +For MSI installers, when the ``msiexec`` parameter is set to true, the ``/i`` +option is used for installation, and the ``/x`` option is used for +uninstallation. However, when installing an MSI patch, the ``/i`` and ``/x`` +options cannot be combined. Here is an example of a package definition file to install an MSI patch: @@ -1006,33 +1113,48 @@ Here is an example of a package definition file to install an MSI patch: msiexec: True cache_file: salt://win/repo-ng/MyApp/MyApp.1.1.msp -In the previous example: -- Version ``1.0`` of the software installs the application using the ``1.0`` MSI defined in the ``installer`` parameter. -- There is no file to be cached and the ``install_flags`` parameter does not include any special values. +In the above example: -Version ``1.1`` of the software uses the same installer file as Version ``1.0``. -Now, to apply a patch to Version 1.0, make the following changes in the package definition file: -- Place the patch file (MSP file) in the same directory as the installer file (MSI file) on the ``file_roots`` -- In the ``cache_file`` parameter, specify the path to a single patch file. -- In the ``install_flags`` parameter, add the ``/update`` flag and include the path to the MSP file -using the ``%cd%`` environment variable. ``%cd%`` resolves to the current working directory, which is the location in the minion cache where the installer file is cached. +- Version ``1.0`` of the software installs the application using the ``1.0`` + MSI defined in the ``installer`` parameter. +- There is no file to be cached and the ``install_flags`` parameter does not + include any special values. + +Version ``1.1`` of the software uses the same installer file as Version +``1.0``\. Now, to apply a patch to Version 1.0, make the following changes in +the package definition file: + +- Place the patch file (MSP file) in the same directory as the installer file + (MSI file) on the ``file_roots`` +- In the ``cache_file`` parameter, specify the path to the single patch file. +- In the ``install_flags`` parameter, add the ``/update`` flag and include the + path to the MSP file using the ``%cd%`` environment variable. ``%cd%`` + resolves to the current working directory, which is the location in the minion + cache where the installer file is cached. For more information, see issue `#32780 `_. -The same approach could be used for applying MST files for MSIs and answer files for other types of .exe-based installers. +The same approach could be used for applying MST files for MSIs and answer files +for other types of .exe-based installers. + +.. _parameters: Parameters ========== -This section describes the parameters placed under the ``version`` in the package definition file. -An example can be found on the `Salt winrepo repository `_. +This section describes the parameters placed under the ``version`` in the +package definition file. Examples can be found on the `Salt winrepo repository +`_. full_name (str) --------------- -The full name of the software as shown in "Programs and Features" in the control panel. You can also retrieve the full name of the package by installing the package manually and then running ``pkg.list_pkgs``. -Here's an example of the output from ``pkg.list_pkgs``: -.. code-block:: console +The full name of the software as shown in "Add/Remove Programs". You can also +retrieve the full name of the package by installing the package manually and +then running ``pkg.list_pkgs``\. Here's an example of the output from +``pkg.list_pkgs``: + +.. code-block:: bash salt 'test-2008' pkg.list_pkgs test-2008 @@ -1046,12 +1168,11 @@ Here's an example of the output from ``pkg.list_pkgs``: salt-minion-py3: 3001 +Notice the full Name for Firefox: ``Mozilla Firefox 74.0 (x64 en-US)``\. The +``full_name`` parameter in the package definition file must match this name. -Notice the full Name for Firefox: Mozilla Firefox 74.0 (x64 en-US). -The ``full_name`` parameter in the package definition file must match this name. - -The example below shows the ``pkg.list_pkgs`` for a machine that has -Mozilla Firefox 74.0 installed with a package definition file for that version of +The example below shows the ``pkg.list_pkgs`` for a machine that has Mozilla +Firefox 74.0 installed with a package definition file for that version of Firefox. .. code-block:: bash @@ -1067,22 +1188,21 @@ Firefox. salt-minion-py3: 3001 -On running ``pkg.list_pkgs``, if any of the software installed on the machine matches the full name -defined in any one of the software definition files in the repository, then the -the package name is displayed in the output. +On running ``pkg.list_pkgs``\, if any of the software installed on the machine +matches the full name defined in any one of the software definition files in the +repository, then the package name is displayed in the output. .. important:: The version number and ``full_name`` must match the output of - ``pkg.list_pkgs`` so that the installation status can be verified - by the state system. + ``pkg.list_pkgs`` so that the installation status can be verified by the + state system. .. note:: - You can successfully install packages using ``pkg.install``, - even if the ``full_name`` or the version number doesn't match. The - module will complete successfully, but continue to display the full name - in ``pkg.list_pkgs``. If this is happening, verify that the ``full_name`` - and the ``version``matches exactly what is displayed in Add/Remove - Programs. + You can successfully install packages using ``pkg.install``\, even if the + ``full_name`` or the version number doesn't match. The module will complete + successfully, but continue to display the full name in ``pkg.list_pkgs``\. + If this is happening, verify that the ``full_name`` and the ``version`` + match exactly what is displayed in Add/Remove Programs. .. tip:: To force Salt to display the full name when there's already an existing @@ -1097,46 +1217,51 @@ the package name is displayed in the output. installer (str) --------------- -The path to the binary (``.exe``, ``.msi``) to install a package. +The path to the binary (``.exe``\, ``.msi``) that installs the package. + This can be a local path or a URL. If it is a URL or a Salt path (``salt://``), -then the package is cached locally and then executed. -If it is a path to a file on disk or a file share, then it is executed directly. +then the package is cached locally and then executed. If it is a path to a file +on disk or a file share, then it is executed directly. .. note:: When storing software in the same location as the winrepo: + - Create a sub folder named after the package. - - Store the package definition file named ``init.sls`` and the binary installer in - the same sub folder if you are hosting those files on the ``file_roots``. + - Store the package definition file named ``init.sls`` and the binary + installer in the same sub folder if you are hosting those files on the + ``file_roots``\. .. note:: - ``pkg.refresh_db`` command processes all ``.sls`` files in all sub directories - in the ``winrepo_dir_ng`` directory. + The ``pkg.refresh_db`` command processes all ``.sls`` files in all sub + directories in the ``winrepo_dir_ng`` directory. install_flags (str) ------------------- + The flags passed to the installer for silent installation. -You can find these flags by adding ``/?`` or ``/h`` when running the installer from the command line. -See `WPKG project wiki `_ for information on silent install flags. - +You may be able to find these flags by adding ``/?`` or ``/h`` when running the +installer from the command line. See `WPKG project wiki `_ for information on silent install flags. + .. warning:: - Always ensure that the software has the ability to install silently since - Salt appears to hang if the installer expects user input. + Always ensure that the installer has the ability to install silently, + otherwise Salt appears to hang while the installer waits for user input. uninstaller (str) ----------------- -The path to the program to uninstall software. -This can be the path to the same exe or msi used to install the software. -If you use MSI to install the software then you can either use GUID of the software or the -same MSI to uninstall the software. +The path to the program to uninstall the software. + +This can be the path to the same ``.exe`` or ``.msi`` used to install the +software. If you use a ``.msi`` to install the software, then you can either +use the GUID of the software or the same ``.msi`` to uninstall the software. You can find the uninstall information in the registry: - Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall - Software\\WOW6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall -Here's an example of using the GUID to uninstall software. +Here's an example that uses the GUID to uninstall software: .. code-block:: yaml @@ -1149,7 +1274,7 @@ Here's an example of using the GUID to uninstall software. uninstall_flags: '/qn /norestart' msiexec: True -Here's an example of using the installer MSI to uninstall software: +Here's an example that uses the MSI installer to uninstall software: .. code-block:: yaml @@ -1167,18 +1292,18 @@ uninstall_flags (str) The flags passed to the uninstaller for silent uninstallation. -You can find these flags by adding ``/?`` or ``/h`` when running the uninstaller from the command-line. -See `WPKG project wiki `_ for information on silent uninstall flags. +You may be able to find these flags by adding ``/?`` or ``/h`` when running the +uninstaller from the command-line. See `WPKG project wiki `_ for information on silent uninstall flags. .. warning:: - Always ensure that the software has the ability to uninstall silently since - Salt appears to hang if the uninstaller expects user input. + Always ensure that the installer has the ability to uninstall silently, + otherwise Salt appears to hang while the uninstaller waits for user input. msiexec (bool, str) ------------------- This setting informs Salt to use ``msiexec /i`` to install the package and ``msiexec /x`` -to uninstall. This setting is applicable only for ``.msi`` installations only. +to uninstall. This setting only applies to ``.msi`` installations. Possible options are: @@ -1201,23 +1326,25 @@ allusers (bool) --------------- This parameter is specific to ``.msi`` installations. It tells ``msiexec`` to -install the software for all users. The default is ``True``. +install the software for all users. The default is ``True``\. cache_dir (bool) ---------------- This setting requires the software to be stored on the ``file_roots`` and only -applies to URLs that begin with ``salt://``. If set to ``True`` then the entire directory -where the installer resides is recursively cached. This is useful for -installers that depend on other files in the same directory for installation. +applies to URLs that begin with ``salt://``\. If set to ``True``\, then the +entire directory where the installer resides is recursively cached. This is +useful for installers that depend on other files in the same directory for +installation. .. warning:: - If set to ``True`` then all files and directories in the same location as the - installer file are copied down to the minion. For example, If you place your - package definition file with ``cache_dir: True`` in the root of winrepo - (``/srv/salt/win/repo-ng``) then the entire contents of winrepo is - cached to the minion. Therefore, it is best practice to place your installer - files in a subdirectory if they are stored in winrepo. + If set to ``True``\, then all files and directories in the same location as + the installer file are copied down to the minion. For example, if you place + your package definition file with ``cache_dir: True`` in the root of winrepo + (``/srv/salt/win/repo-ng``) then the entire contents of winrepo is cached to + the minion. Therefore, it is best practice to place your package definition + file along with its installer files in a subdirectory if they are stored in + winrepo. Here's an example using cache_dir: @@ -1234,32 +1361,33 @@ cache_file (str) ---------------- This setting requires the file to be stored on the ``file_roots`` and only -applies to URLs that begin with ``salt://``. It indicates that only a single file specified -is copied down for use with the installer. It is copied to the same location as the -installer. This setting is useful when ``cache_dir`` is set to ``True``, -and you want to cache only a specific file and not all files that reside in the installer directory. +applies to URLs that begin with ``salt://``\. It indicates that the single file +specified is copied down for use with the installer. It is copied to the same +location as the installer. Use this setting instead of ``cache_dir`` when you +only need to cache a single file. use_scheduler (bool) -------------------- -If set to ``True``, Windows uses the task scheduler to run the installation. -A one-time task is created in the task scheduler and launched. The return -to the minion is that the task was launched successfully, not that the -software was installed successfully. +If set to ``True``\, Windows uses the task scheduler to run the installation. A +one-time task is created in the task scheduler and launched. The return to the +minion is that the task was launched successfully, not that the software was +installed successfully. .. note:: This is used in the package definition for Salt itself. The first thing the Salt installer does is kill the Salt service, which then kills all child processes. If the Salt installer is launched via Salt, then the installer - is killed leaving Salt on the machine but not running. The use of the - task scheduler allows an external process to launch the Salt installation so - its processes aren't killed when the Salt service is stopped. + is killed with the salt-minion service, leaving Salt on the machine but not + running. Using the task scheduler allows an external process to launch the + Salt installer so its processes aren't killed when the Salt service is + stopped. source_hash (str) ----------------- -This setting informs Salt to compare a hash sum of the installer to the provided hash sum -before execution. The value can be formatted as ``=``, +This setting informs Salt to compare a hash sum of the installer to the provided +hash sum before execution. The value can be formatted as ``=``\, or it can be a URI to a file containing the hash sum. For a list of supported algorithms, see the `hashlib documentation @@ -1297,19 +1425,19 @@ Managing Windows Software on a Standalone Windows Minion The Windows Software Repository functions similarly in a standalone environment, with a few differences in the configuration. -To replace the winrepo runner used on the Salt master, an -:mod:`execution module ` exists to provide the same -functionality to standalone minions. The functions are named the same as the -ones in the runner and are used in the same way; the only difference is that -``salt-call`` is used instead of ``salt-run``: +To replace the winrepo runner used on the Salt master, an :mod:`execution module +` exists to provide the same functionality to standalone +minions. The functions for the module share the same names with functions in the +runner and are used in the same way; the only difference is that ``salt-call`` +is used instead of ``salt-run`` to run those functions: .. code-block:: bash salt-call winrepo.update_git_repos salt-call pkg.refresh_db -After executing the previous commands, the repository on the standalone system is -ready for use. +After executing the previous commands, the repository on the standalone system +is ready for use. .. _winrepo-troubleshooting: @@ -1322,20 +1450,20 @@ My software installs correctly but ``pkg.installed`` says it failed If you have a package that seems to install properly but Salt reports a failure then it is likely you have a ``version`` or ``full_name`` mismatch. -- Check the ``full_name`` and ``version`` of the package as shown in Add/Remove Programs -(``appwiz.cpl``). -- Use ``pkg.list_pkgs`` to check that the ``full_name`` and -``version`` exactly matches what is installed. -- Verify that the package's ``full_name`` and ``version``in the package definition file - match the full name and version in Add/Remove programs. +- Check the ``full_name`` and ``version`` of the package as shown in Add/Remove + Programs (``appwiz.cpl``). +- Use ``pkg.list_pkgs`` to check that the ``full_name`` and ``version`` exactly + match what is installed. +- Verify that the ``full_name`` and ``version`` in the package definition file + match the full name and version in Add/Remove programs. - Ensure that the ``version`` is wrapped in single quotes in the package -definition file. + definition file. Changes to package definition files not being picked up -====================================================== +======================================================= -Ensure you have refreshed the database on the minion on -updating the package definintion files in the repo. +Make sure you refresh the database on the minion (``pkg.refresh_db``) after +updating package definition files in the repo. .. code-block:: bash @@ -1346,32 +1474,32 @@ Winrepo upgrade issues To minimize potential issues, it is a good idea to remove any winrepo git repositories that were checked out by the legacy (pre-2015.8.0) winrepo code -when upgrading the master to 2015.8.0 or later. Run -:mod:`winrepo.update_git_repos ` to -clone them anew after the master is started. +when upgrading the master to 2015.8.0 or later. Run :mod:`winrepo.update_git_repos +` to clone them anew after the master is +started. -pygit2_/GitPython_ Support for Maintaining Git Repos +pygit2 / GitPython Support for Maintaining Git Repos **************************************************** -pygit2_ and GitPython_ are the supported python interfaces to Git. -The runner :mod:`winrepo.update_git_repos ` -uses the same underlying code as :ref:`Git Fileserver Backend ` -and :mod:`Git External Pillar ` to maintain and update -its local clones of git repositories. -.. note:: - If compatible versions of both pygit2_ and GitPython_ are installed, then - Salt will prefer pygit2_. To override this behavior use the - :conf_master:`winrepo_provider` configuration parameter: +pygit2 and GitPython are the supported python interfaces to Git. The runner +:mod:`winrepo.update_git_repos ` uses the +same underlying code as :ref:`Git Fileserver Backend ` and +:mod:`Git External Pillar ` to maintain and update its +local clones of git repositories. - .. code-block:: yaml +.. note:: + If compatible versions of both pygit2 and GitPython are installed, then + Salt will prefer pygit2. To override this behavior use the + :conf_master:`winrepo_provider` configuration parameter, ie: + ``winrepo_provider: gitpython`` - winrepo_provider: gitpython +.. _authenticated-pygit2: Accessing authenticated Git repos (pygit2) ****************************************** -pygit2 enables you to access authenticated git repositories -and set per-remote config settings. An example of this is: +pygit2 enables you to access authenticated git repositories and set per-remote +config settings. An example of this is: .. code-block:: yaml @@ -1396,30 +1524,35 @@ and set per-remote config settings. An example of this is: explanation of how per-remote configuration works in gitfs. The same principles apply to winrepo. +.. _maintaining-repo: + Maintaining Git repos ********************* -A ``clean`` argument is added to the +A ``clean`` argument is added to the :mod:`winrepo.update_git_repos ` runner to maintain the Git repos. When ``clean=True`` the runner removes directories under the :conf_master:`winrepo_dir_ng`/:conf_minion:`winrepo_dir_ng` that are not explicitly configured. This eliminates the need to manually remove these directories when a repo is removed from the config file. + .. code-block:: bash salt-run winrepo.update_git_repos clean=True If a mix of git and non-git Windows Repo definition files are used, then -do not pass ``clean=True``, as it removes the directories containing non-git +do not pass ``clean=True``\, as it removes the directories containing non-git definitions. +.. _name-collisions: + Name collisions between repos ***************************** Salt detects collisions between repository names. The :mod:`winrepo.update_git_repos ` -runner does not execute successfully if any collisions between repository names are detected. - Consider the following configuration: +runner does not execute successfully if any collisions between repository names +are detected. Consider the following configuration: .. code-block:: yaml @@ -1429,9 +1562,9 @@ runner does not execute successfully if any collisions between repository names - https://github.com/foobar/baz.git With the above configuration, the :mod:`winrepo.update_git_repos ` -runner fails to execute as all three repos would be -checked out to the same directory. To resolve this conflict, use per-remote parameter -called ``name``. +runner fails to execute as all three repos would be checked out to the same +directory. To resolve this conflict, use the per-remote parameter called +``name``\. .. code-block:: yaml @@ -1442,7 +1575,8 @@ called ``name``. - https://github.com/foobar/baz.git: - name: baz_the_third -Now on running the :mod:`winrepo.update_git_repos `, +Now on running the :mod:`winrepo.update_git_repos `: + - https://foo.com/bar/baz.git repo is initialized and cloned under the ``win_repo_dir_ng`` directory. - https://mydomain.tld/baz.git repo is initialized and cloned under the ``win_repo_dir_ng\baz_junior`` directory. - https://github.com/foobar/baz.git repo is initialized and cloned under the ``win_repo_dir_ng\baz_the_third`` directory. From a3b79a1c48a35f2a0105f01c6d6eccf2de6c20ae Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 15 Jul 2025 07:19:08 -0600 Subject: [PATCH 46/71] Fix some typos --- .../windows/windows-package-manager.rst | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 2a18800b69fd..40cd0b610ffa 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -73,12 +73,12 @@ Install libraries (Optional) If you are using the Salt Windows package manager with package definition files hosted on a Salt Git repo, install the libraries ``GitPython`` -or ``pygit2``\. +or ``pygit2``. .. _populate-git-repo: Populate the local Git repository -********************************** +********************************* The SLS files used to install Windows packages are not distributed by default with Salt. Assuming no changes to the default configuration (``file_roots``), @@ -210,11 +210,11 @@ winrepo_dir_ng :conf_master:`winrepo_dir_ng` (str) The location in the ``file_roots`` where the winrepo files are kept. The default -is ``/srv/salt/win/repo-ng``\. +is ``/srv/salt/win/repo-ng``. .. warning:: You can change the location of the winrepo directory. However, it must - always be set to a path that is inside the ``file_roots``\. Otherwise, the + always be set to a path that is inside the ``file_roots``. Otherwise, the software definition files will be unreachable by the minion. .. important:: @@ -271,7 +271,7 @@ winrepo_provider :conf_master:`winrepo_provider` (str) -The provider to be used for winrepo. Default is ``pygit2``\. Falls back to +The provider to be used for winrepo. Default is ``pygit2``. Falls back to ``gitpython`` when ``pygit2`` is not available winrepo_ssl_verify @@ -288,7 +288,7 @@ Master Configuration (pygit2) ============================= The following configuration options only apply when the -:conf_master:`winrepo_provider` option is set to ``pygit2``\. +:conf_master:`winrepo_provider` option is set to ``pygit2``. - :conf_master:`winrepo_insecure_auth` - :conf_master:`winrepo_passphrase` @@ -390,7 +390,7 @@ winrepo_cachefile :conf_minion:`winrepo_cachefile` (str) The file name of the winrepo cache file. The file is placed at the root of -``winrepo_dir_ng``\. Default is ``winrepo.p``\. +``winrepo_dir_ng``. Default is ``winrepo.p``. winrepo_source_dir ------------------ @@ -398,13 +398,13 @@ winrepo_source_dir :conf_minion:`winrepo_source_dir` (str) The location of the .sls files on the Salt file server. This allows for using -different environments. Default is ``salt://win/repo-ng/``\. +different environments. Default is ``salt://win/repo-ng/``. .. warning:: If the default for ``winrepo_dir_ng`` is changed, this setting may need to be changed on each minion. The default setting for ``winrepo_dir_ng`` is - ``/srv/salt/win/repo-ng``\. If that were changed to - ``/srv/salt/new/repo-ng``\, then the ``winrepo_source_dir`` would need to be + ``/srv/salt/win/repo-ng``. If that were changed to + ``/srv/salt/new/repo-ng``, then the ``winrepo_source_dir`` would need to be changed to ``salt://new/repo-ng`` .. _masterless-minion-config: @@ -438,17 +438,17 @@ winrepo_dir_ng :conf_minion:`winrepo_dir_ng` (str) The location in the ``file_roots`` where the winrepo files are kept. The default -is ``C:\salt\srv\salt\win\repo-ng``\. +is ``C:\salt\srv\salt\win\repo-ng``. .. warning:: You can change the location of the winrepo directory. However, it must - always be set to a path that is inside the ``file_roots``\. Otherwise, the + always be set to a path that is inside the ``file_roots``. Otherwise, the software definition files will be unreachable by the minion. .. important:: A common mistake is to change the ``file_roots`` setting and fail to update the ``winrepo_dir_ng`` and ``winrepo_dir`` settings so that they are inside - the ``file_roots``\. You might also want to verify ``winrepo_source_dir`` on + the ``file_roots``. You might also want to verify ``winrepo_source_dir`` on the minion as well. winrepo_remotes @@ -812,12 +812,12 @@ create the metadata file on the minion. You can maintain standalone package definition files that point to software on other servers or on the internet. In this case the file name is the short name -of the software with the ``.sls`` extension, for example,``firefox.sls``\. +of the software with the ``.sls`` extension, for example,``firefox.sls``. You can also store the binaries for your software together with their software definition files in their own directory. In this scenario, the directory name is the short name for the software and the package definition file stored that -directory is named ``init.sls``\. +directory is named ``init.sls``. Look at the following example directory structure on a Linux master assuming default config settings: @@ -862,7 +862,7 @@ In the above directory structure: files. - A folder for MS Office 2013 that contains the installer files for all the - MS Office software and a package definition file named ``init.sls``\. + MS Office software and a package definition file named ``init.sls``. - Two additional standalone package definition files ``openssl.sls`` and ``zoom.sls`` to install OpenSSl and Zoom. @@ -871,7 +871,7 @@ In the above directory structure: .. warning:: Do not modify the files in the ``salt-winrepo-ng`` directory as it breaks - future runs of ``winrepo.update_git_repos``\. + future runs of ``winrepo.update_git_repos``. .. warning:: Do not place any custom software definition files in the ``salt-winrepo-ng`` @@ -920,7 +920,7 @@ three indentation levels: numbers for a package but they must be unique within the file. .. note:: - When running ``pkg.list_pkgs``\, the short name and version number are + When running ``pkg.list_pkgs``, the short name and version number are displayed when Salt finds a match in the repo. Otherwise, the full package name is displayed. @@ -980,7 +980,7 @@ Here is a pure YAML example of a simple package definition file for Firefox: uninstaller: '%ProgramFiles(x86)%/Mozilla Firefox/uninstall/helper.exe' uninstall_flags: '/S' -The first line is the short name of the software which is ``firefox_x64``\. +The first line is the short name of the software which is ``firefox_x64``. .. important:: The short name must be unique across all other short names in the software @@ -991,7 +991,7 @@ The second line is the ``software version`` and is indented two spaces. .. important:: The version number must be enclosed in quotes or the YAML parser removes the trailing zeros. For example, if the version number ``74.0`` is not enclosed - within quotes, then the version number is rendered as ``74``\. + within quotes, then the version number is rendered as ``74``. The lines following the ``version`` are indented two more spaces and contain all the information needed to install the Firefox package. @@ -1044,13 +1044,13 @@ In this example, JINJA is used to generate a package definition file that defines how to install 12 versions of Firefox. Jinja is used to create a list of available versions. The list is iterated through a ``for loop`` where each version is placed in the ``version`` variable. The version is inserted -everywhere there is a ``{{ version }}`` marker inside the ``for loop``\. +everywhere there is a ``{{ version }}`` marker inside the ``for loop``. The single variable (``lang``) defined at the top of the package definition identifies the language of the package. You can access the Salt modules using the ``salt`` keyword. In this case, the ``config.get`` function is invoked to retrieve the language setting. If the ``lang`` variable is not defined then the -default value is ``en-US``\. +default value is ``en-US``. .. _example-latest: @@ -1078,9 +1078,9 @@ example of a package definition file to install the latest version of Chrome. In the above example: -- ``Version`` is set to ``latest``\. Salt then installs the latest version of +- ``Version`` is set to ``latest``. Salt then installs the latest version of Chrome at the URL and displays that version. -- ``msiexec`` is set to ``True``\, hence the software is installed using an MSI. +- ``msiexec`` is set to ``True``, hence the software is installed using an MSI. .. _example-patch: @@ -1121,7 +1121,7 @@ In the above example: include any special values. Version ``1.1`` of the software uses the same installer file as Version -``1.0``\. Now, to apply a patch to Version 1.0, make the following changes in +``1.0``. Now, to apply a patch to Version 1.0, make the following changes in the package definition file: - Place the patch file (MSP file) in the same directory as the installer file @@ -1151,7 +1151,7 @@ full_name (str) The full name of the software as shown in "Add/Remove Programs". You can also retrieve the full name of the package by installing the package manually and -then running ``pkg.list_pkgs``\. Here's an example of the output from +then running ``pkg.list_pkgs``. Here's an example of the output from ``pkg.list_pkgs``: .. code-block:: bash @@ -1168,7 +1168,7 @@ then running ``pkg.list_pkgs``\. Here's an example of the output from salt-minion-py3: 3001 -Notice the full Name for Firefox: ``Mozilla Firefox 74.0 (x64 en-US)``\. The +Notice the full Name for Firefox: ``Mozilla Firefox 74.0 (x64 en-US)``. The ``full_name`` parameter in the package definition file must match this name. The example below shows the ``pkg.list_pkgs`` for a machine that has Mozilla @@ -1188,7 +1188,7 @@ Firefox. salt-minion-py3: 3001 -On running ``pkg.list_pkgs``\, if any of the software installed on the machine +On running ``pkg.list_pkgs``, if any of the software installed on the machine matches the full name defined in any one of the software definition files in the repository, then the package name is displayed in the output. @@ -1198,9 +1198,9 @@ repository, then the package name is displayed in the output. state system. .. note:: - You can successfully install packages using ``pkg.install``\, even if the + You can successfully install packages using ``pkg.install``, even if the ``full_name`` or the version number doesn't match. The module will complete - successfully, but continue to display the full name in ``pkg.list_pkgs``\. + successfully, but continue to display the full name in ``pkg.list_pkgs``. If this is happening, verify that the ``full_name`` and the ``version`` match exactly what is displayed in Add/Remove Programs. @@ -1217,7 +1217,7 @@ repository, then the package name is displayed in the output. installer (str) --------------- -The path to the binary (``.exe``\, ``.msi``) that installs the package. +The path to the binary (``.exe``, ``.msi``) that installs the package. This can be a local path or a URL. If it is a URL or a Salt path (``salt://``), then the package is cached locally and then executed. If it is a path to a file @@ -1229,7 +1229,7 @@ on disk or a file share, then it is executed directly. - Create a sub folder named after the package. - Store the package definition file named ``init.sls`` and the binary installer in the same sub folder if you are hosting those files on the - ``file_roots``\. + ``file_roots``. .. note:: The ``pkg.refresh_db`` command processes all ``.sls`` files in all sub @@ -1326,19 +1326,19 @@ allusers (bool) --------------- This parameter is specific to ``.msi`` installations. It tells ``msiexec`` to -install the software for all users. The default is ``True``\. +install the software for all users. The default is ``True``. cache_dir (bool) ---------------- This setting requires the software to be stored on the ``file_roots`` and only -applies to URLs that begin with ``salt://``\. If set to ``True``\, then the +applies to URLs that begin with ``salt://``. If set to ``True``, then the entire directory where the installer resides is recursively cached. This is useful for installers that depend on other files in the same directory for installation. .. warning:: - If set to ``True``\, then all files and directories in the same location as + If set to ``True``, then all files and directories in the same location as the installer file are copied down to the minion. For example, if you place your package definition file with ``cache_dir: True`` in the root of winrepo (``/srv/salt/win/repo-ng``) then the entire contents of winrepo is cached to @@ -1361,7 +1361,7 @@ cache_file (str) ---------------- This setting requires the file to be stored on the ``file_roots`` and only -applies to URLs that begin with ``salt://``\. It indicates that the single file +applies to URLs that begin with ``salt://``. It indicates that the single file specified is copied down for use with the installer. It is copied to the same location as the installer. Use this setting instead of ``cache_dir`` when you only need to cache a single file. @@ -1369,7 +1369,7 @@ only need to cache a single file. use_scheduler (bool) -------------------- -If set to ``True``\, Windows uses the task scheduler to run the installation. A +If set to ``True``, Windows uses the task scheduler to run the installation. A one-time task is created in the task scheduler and launched. The return to the minion is that the task was launched successfully, not that the software was installed successfully. @@ -1387,7 +1387,7 @@ source_hash (str) ----------------- This setting informs Salt to compare a hash sum of the installer to the provided -hash sum before execution. The value can be formatted as ``=``\, +hash sum before execution. The value can be formatted as ``=``, or it can be a URI to a file containing the hash sum. For a list of supported algorithms, see the `hashlib documentation @@ -1541,7 +1541,7 @@ these directories when a repo is removed from the config file. salt-run winrepo.update_git_repos clean=True If a mix of git and non-git Windows Repo definition files are used, then -do not pass ``clean=True``\, as it removes the directories containing non-git +do not pass ``clean=True``, as it removes the directories containing non-git definitions. .. _name-collisions: @@ -1564,7 +1564,7 @@ are detected. Consider the following configuration: With the above configuration, the :mod:`winrepo.update_git_repos ` runner fails to execute as all three repos would be checked out to the same directory. To resolve this conflict, use the per-remote parameter called -``name``\. +``name``. .. code-block:: yaml From bd359fb6180d91c1c9da9efbe949dcb4c6a567a6 Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Mon, 29 Apr 2024 19:49:33 -0400 Subject: [PATCH 47/71] fixes saltstack/salt#66453 sync_renderers fails when the custom renderer is specified via config --- changelog/66453.fixed.md | 1 + salt/loader/__init__.py | 15 ++++++++++++++- tests/pytests/unit/loader/test_loader.py | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changelog/66453.fixed.md diff --git a/changelog/66453.fixed.md b/changelog/66453.fixed.md new file mode 100644 index 000000000000..5de2d33337ba --- /dev/null +++ b/changelog/66453.fixed.md @@ -0,0 +1 @@ +Fix sync_renderers failure when the custom renderer is specified via config diff --git a/salt/loader/__init__.py b/salt/loader/__init__.py index 160fe8e60d00..c08e80a5d8bb 100644 --- a/salt/loader/__init__.py +++ b/salt/loader/__init__.py @@ -979,7 +979,20 @@ def render( "the needed software is unavailable".format(opts["renderer"]) ) log.critical(err) - raise LoaderError(err) + if opts.get("__role") == "minion": + default_renderer_config = salt.config.DEFAULT_MINION_OPTS["renderer"] + else: + default_renderer_config = salt.config.DEFAULT_MASTER_OPTS["renderer"] + log.warning( + "Attempting fallback to default render pipe: %s", default_renderer_config + ) + if not check_render_pipe_str( + default_renderer_config, + rend, + opts["renderer_blacklist"], + opts["renderer_whitelist"], + ): + raise LoaderError(err) return rend diff --git a/tests/pytests/unit/loader/test_loader.py b/tests/pytests/unit/loader/test_loader.py index ea0883b93889..77db7a3d8c59 100644 --- a/tests/pytests/unit/loader/test_loader.py +++ b/tests/pytests/unit/loader/test_loader.py @@ -11,9 +11,11 @@ import pytest +import salt.config import salt.exceptions import salt.loader import salt.loader.lazy +from tests.support.mock import MagicMock, patch @pytest.fixture @@ -96,3 +98,18 @@ def foobar(): with pytest.helpers.temp_file("mymod.py", contents, directory=tmp_path): loader = salt.loader.LazyLoader([tmp_path], opts, pack={"__test__": "meh"}) assert loader["mymod.foobar"]() == "meh" + + +def test_render(): + opts = salt.config.DEFAULT_MINION_OPTS.copy() + minion_mods = salt.loader.minion_mods(opts) + for role in ["minion", "master"]: + opts["__role"] = role + for renderer in ["jinja|yaml", "some_custom_thing"]: + opts["renderer"] = renderer + ret = salt.loader.render(opts, minion_mods) + assert isinstance(ret, salt.loader.lazy.FilterDictWrapper) + with pytest.raises(salt.exceptions.LoaderError), patch( + "salt.loader.check_render_pipe_str", MagicMock(side_effect=[False, False]) + ): + salt.loader.render(opts, minion_mods) From 81c3d06030866a06ee59acd381f09c866e3c689e Mon Sep 17 00:00:00 2001 From: nicholasmhughes Date: Fri, 29 Nov 2024 11:41:07 -0500 Subject: [PATCH 48/71] fix pre-commit failures --- tests/pytests/unit/loader/test_loader.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/pytests/unit/loader/test_loader.py b/tests/pytests/unit/loader/test_loader.py index 77db7a3d8c59..e0706d6817e6 100644 --- a/tests/pytests/unit/loader/test_loader.py +++ b/tests/pytests/unit/loader/test_loader.py @@ -113,3 +113,16 @@ def test_render(): "salt.loader.check_render_pipe_str", MagicMock(side_effect=[False, False]) ): salt.loader.render(opts, minion_mods) + + +def test_return_named_context_from_loaded_func(tmp_path): + opts = { + "optimization_order": [0], + } + contents = """ + def foobar(): + return __test__ + """ + with pytest.helpers.temp_file("mymod.py", contents, directory=tmp_path): + loader = salt.loader.LazyLoader([tmp_path], opts, pack={"__test__": "meh"}) + assert loader["mymod.foobar"]() == "meh" From b287259154b08aa14b1da2388ba1bbcc7e736de6 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 27 Jun 2025 08:45:46 -0600 Subject: [PATCH 49/71] Remove duplicate test --- tests/pytests/unit/loader/test_loader.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/pytests/unit/loader/test_loader.py b/tests/pytests/unit/loader/test_loader.py index e0706d6817e6..77db7a3d8c59 100644 --- a/tests/pytests/unit/loader/test_loader.py +++ b/tests/pytests/unit/loader/test_loader.py @@ -113,16 +113,3 @@ def test_render(): "salt.loader.check_render_pipe_str", MagicMock(side_effect=[False, False]) ): salt.loader.render(opts, minion_mods) - - -def test_return_named_context_from_loaded_func(tmp_path): - opts = { - "optimization_order": [0], - } - contents = """ - def foobar(): - return __test__ - """ - with pytest.helpers.temp_file("mymod.py", contents, directory=tmp_path): - loader = salt.loader.LazyLoader([tmp_path], opts, pack={"__test__": "meh"}) - assert loader["mymod.foobar"]() == "meh" From 27dc383a2323eb61bcbe41d97fe957f82e466bbf Mon Sep 17 00:00:00 2001 From: zer0def Date: Fri, 1 Aug 2025 12:58:03 +0200 Subject: [PATCH 50/71] modules/aptpkg.py: correct handling of foreign-only-arch packages (fixes #66940) --- changelog/66940.fixed.md | 1 + salt/modules/aptpkg.py | 78 +++++++++++++++-------- tests/pytests/unit/modules/test_aptpkg.py | 33 +++++++++- 3 files changed, 86 insertions(+), 26 deletions(-) create mode 100644 changelog/66940.fixed.md diff --git a/changelog/66940.fixed.md b/changelog/66940.fixed.md new file mode 100644 index 000000000000..a75dc22ca8f1 --- /dev/null +++ b/changelog/66940.fixed.md @@ -0,0 +1 @@ +modules.aptpkg: correct handling of foreign-arch packages diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index a61719bd710f..dae21e4abed0 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -436,6 +436,7 @@ def parse_arch(name): def latest_version(*names, **kwargs): + # pylint: disable=W0640 """ .. versionchanged:: 3007.0 @@ -471,13 +472,11 @@ def latest_version(*names, **kwargs): ) fromrepo = kwargs.pop("fromrepo", None) cache_valid_time = kwargs.pop("cache_valid_time", 0) + names = list(names) if not names: return "" - ret = {} - # Initialize the dict with empty strings - for name in names: - ret[name] = "" + ret = {name: "" for name in names} # Initialize the dict with empty strings pkgs = list_pkgs(versions_as_list=True) repo = ["-o", f"APT::Default-Release={fromrepo}"] if fromrepo else None @@ -495,9 +494,16 @@ def latest_version(*names, **kwargs): candidates = {} for line in salt.utils.itertools.split(out["stdout"], "\n"): - if line.endswith(":") and line[:-1] in short_names: - this_pkg = names[short_names.index(line[:-1])] - elif "Candidate" in line: + line = line.strip() + if line.endswith(":") and not line.startswith("Version table"): + if any( + [ + line[:-1] in names, # native package + line[:-1].split(":")[0] in short_names, # foreign package by name + ] + ): + this_pkg = line[:-1] + elif line.startswith("Candidate"): candidate = "" comps = line.split() if len(comps) >= 2: @@ -506,26 +512,48 @@ def latest_version(*names, **kwargs): candidate = "" candidates[this_pkg] = candidate - for name in names: - installed = pkgs.get(name, []) - if not installed: - ret[name] = candidates.get(name, "") - elif installed and show_installed: - ret[name] = candidates.get(name, "") - elif candidates.get(name): - # If there are no installed versions that are greater than or equal - # to the install candidate, then the candidate is an upgrade, so - # add it to the return dict - if not any( - salt.utils.versions.compare( - ver1=x, - oper=">=", - ver2=candidates.get(name, ""), - cmp_func=version_cmp, + _extensions, _pops = [], [] + while True: + for name in names: + # no native package found, extend with possible foreign arch ones + if name not in candidates: + _extensions.extend( + filter(lambda pkg: pkg.startswith(f"{name}:"), candidates) ) - for x in installed - ): + + # since foreign packages are returned only when faced with a lack of + # native ones, remove the original package name; this does, however, + # risk a case of returning multiple foreign arch packages + _pops.append(name) + + # continue with populating entries as usual + installed = pkgs.get(name, []) + if not installed: + ret[name] = candidates.get(name, "") + elif installed and show_installed: ret[name] = candidates.get(name, "") + elif candidates.get(name): + # If there are no installed versions that are greater than or equal + # to the install candidate, then the candidate is an upgrade, so + # add it to the return dict + if not any( + salt.utils.versions.compare( + ver1=x, + oper=">=", + ver2=candidates.get(name, ""), + cmp_func=version_cmp, + ) + for x in installed + ): + ret[name] = candidates.get(name, "") + + if not any([_extensions, _pops]): + break + + names.extend(_extensions) + _extensions.clear() + list(map(names.remove, _pops)) + _pops.clear() # Return a string if only one package name passed if len(names) == 1: diff --git a/tests/pytests/unit/modules/test_aptpkg.py b/tests/pytests/unit/modules/test_aptpkg.py index 250dd81bdfde..cd74b5430ad1 100644 --- a/tests/pytests/unit/modules/test_aptpkg.py +++ b/tests/pytests/unit/modules/test_aptpkg.py @@ -2328,7 +2328,38 @@ def test_latest_version_calls_aptcache_once_per_run(): ): ret = aptpkg.latest_version("sudo", "unzip", refresh=False) mock_apt_cache.assert_called_once() - assert ret == {"sudo": "6.0-23+deb10u3", "unzip": ""} + assert ret == {"sudo": "1.8.27-1+deb10u5", "unzip": "6.0-23+deb10u3"} + + +def test_latest_version_with_exclusive_foreign_arch_pkg(): + """ + Test behavior with foreign architecture packages + """ + _short_name, _foreign_arch = "wine32", "i386" + mock_list_pkgs = MagicMock( + return_value={ + _short_name: "10.0~repack-5", + f"{_short_name}:{_foreign_arch}": "10.0~repack-6", + } + ) + apt_cache_ret = { + "stdout": textwrap.dedent( + f"""{_short_name}:{_foreign_arch}: + Installed: (none) + Candidate: 10.0~repack-6 + Version table: + 10.0~repack-6 500 + 500 http://deb.debian.org/debian testing/main {_foreign_arch} Packages + """ + ) + } + mock_apt_cache = MagicMock(return_value=apt_cache_ret) + with patch("salt.modules.aptpkg._call_apt", mock_apt_cache), patch( + "salt.modules.aptpkg.list_pkgs", mock_list_pkgs + ): + ret = aptpkg.latest_version("wine32", refresh=False) + mock_apt_cache.assert_called_once() + assert ret == "10.0~repack-6" @pytest.mark.parametrize( From c30aa7ac1aca30fcc879b0b33a7d6d055d793ff3 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 4 Aug 2025 16:49:40 -0700 Subject: [PATCH 51/71] Fix merge warts --- salt/crypt.py | 4 ++-- salt/minion.py | 4 ++-- .../transport/zeromq/test_request_client.py | 15 +++++++-------- tests/pytests/unit/transport/test_zeromq.py | 1 + 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/salt/crypt.py b/salt/crypt.py index 04fffc3c434d..179cf643f8fd 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -1096,7 +1096,7 @@ def get_keys(self): Return keypair object for the minion. :rtype: Crypto.PublicKey.RSA._RSAobj - :return: The RSA keypair + :return: PrivateKey of the RSA Private Key Pair """ if self._private_key is None: # Make sure all key parent directories are accessible @@ -1116,7 +1116,7 @@ def get_keys(self): @salt.utils.decorators.memoize def _gen_token(self, key, token): - return private_encrypt(key, token) + return key.encrypt(token) def gen_token(self, clear_tok): """ diff --git a/salt/minion.py b/salt/minion.py index cb4e27e6466c..735b5cfb1eed 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1221,7 +1221,7 @@ def stop(self, signum, parent_sig_handler): self.stop_async, signum, parent_sig_handler ) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def stop_async(self, signum, parent_sig_handler): """ Stop minions managed by the MinionManager allowing the io_loop to run @@ -1229,7 +1229,7 @@ def stop_async(self, signum, parent_sig_handler): """ # Sleep to allow any remaining events to be processed - yield salt.ext.tornado.gen.sleep(5) + yield tornado.gen.sleep(5) # Continue to stop the minions for minion in self.minions: diff --git a/tests/pytests/functional/transport/zeromq/test_request_client.py b/tests/pytests/functional/transport/zeromq/test_request_client.py index 3c2c61f8bc11..a68e6176a66d 100644 --- a/tests/pytests/functional/transport/zeromq/test_request_client.py +++ b/tests/pytests/functional/transport/zeromq/test_request_client.py @@ -4,13 +4,12 @@ import pytest import pytestshellutils.utils.ports import tornado.gen +import tornado.locks +import tornado.platform.asyncio import zmq import zmq.eventloop.zmqstream import salt.exceptions -import tornado.gen -import tornado.locks -import tornado.platform.asyncio import salt.transport.zeromq log = logging.getLogger(__name__) @@ -43,7 +42,7 @@ async def test_request_channel_issue_64627(io_loop, request_client, minion_opts, stream = zmq.eventloop.zmqstream.ZMQStream(socket, io_loop=io_loop) try: - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def req_handler(stream, msg): stream.send(msg[0]) @@ -63,7 +62,7 @@ def req_handler(stream, msg): async def test_request_channel_issue_65265(io_loop, request_client, minion_opts, port): import time - import salt.ext.tornado.platform + import tornado.platform minion_opts["master_uri"] = f"tcp://127.0.0.1:{port}" @@ -73,9 +72,9 @@ async def test_request_channel_issue_65265(io_loop, request_client, minion_opts, stream = zmq.eventloop.zmqstream.ZMQStream(socket, io_loop=io_loop) try: - send_complete = salt.ext.tornado.locks.Event() + send_complete = tornado.locks.Event() - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def no_handler(stream, msg): """ The server never responds. @@ -84,7 +83,7 @@ def no_handler(stream, msg): stream.on_recv_stream(no_handler) - @salt.ext.tornado.gen.coroutine + @tornado.gen.coroutine def send_request(): """ The request will timeout becuse the server does not respond. diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py index 29c36ae4c9c1..fe42458c92dc 100644 --- a/tests/pytests/unit/transport/test_zeromq.py +++ b/tests/pytests/unit/transport/test_zeromq.py @@ -2035,6 +2035,7 @@ def stop(): assert "Exception in request handler" in caplog.text assert "Traceback" in caplog.text + def test_backoff_timer(): start = 0.0003 maximum = 0.3 From a8200f5d132c9f13709bc04c779555b5698baf79 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Mon, 4 Aug 2025 18:14:37 -0700 Subject: [PATCH 52/71] Fix cmd test --- tests/pytests/functional/modules/cmd/test_powershell.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytests/functional/modules/cmd/test_powershell.py b/tests/pytests/functional/modules/cmd/test_powershell.py index 2aa8fbfd66a3..f84679a85792 100644 --- a/tests/pytests/functional/modules/cmd/test_powershell.py +++ b/tests/pytests/functional/modules/cmd/test_powershell.py @@ -265,7 +265,6 @@ def test_cmd_run_encoded_cmd_runas(shell, account, cmd, expected, encoded_cmd): cmd=cmd, shell=shell, encoded_cmd=encoded_cmd, - redirect_stderr=False, runas=account.username, password=account.password, redirect_stderr=False, From 4342889550a3c42095c4d1c7fbd84f94fb065fbf Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 11:09:36 -0600 Subject: [PATCH 53/71] Enable signing macos packages --- .github/workflows/build-packages.yml | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index b961c5595bf4..8e023b4faf43 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -284,29 +284,6 @@ jobs: - ${{ matrix.arch == 'arm64' && 'macos-14' || 'macos-13' }} steps: - - name: Check Package Signing Enabled - shell: bash - id: check-pkg-sign - run: | - if [ "${{ inputs.sign-macos-packages }}" == "true" ]; then - if [ "${{ (secrets.MAC_SIGN_APPLE_ACCT != '' && contains(fromJSON('["nightly", "staging"]'), inputs.environment)) && 'true' || 'false' }}" != "true" ]; then - MSG="Secrets for signing packages are not available. The packages created will NOT be signed." - echo "${MSG}" - echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" - echo "sign-pkgs=false" >> "$GITHUB_OUTPUT" - else - MSG="The packages created WILL be signed." - echo "${MSG}" - echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" - echo "sign-pkgs=true" >> "$GITHUB_OUTPUT" - fi - else - MSG="The sign-macos-packages input is false. The packages created will NOT be signed." - echo "${MSG}" - echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" - echo "sign-pkgs=false" >> "$GITHUB_OUTPUT" - fi - - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: @@ -331,7 +308,7 @@ jobs: path: artifacts/ - name: Prepare Package Signing - if: ${{ steps.check-pkg-sign.outputs.sign-pkgs == 'true' }} + if: ${{ inputs.sign-macos-packages == 'true' }} run: | echo ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} | base64 --decode > app-cert.p12 echo ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} | base64 --decode > install-cert.p12 @@ -363,7 +340,7 @@ jobs: '--onedir salt-{0}-onedir-macos-{1}.tar.xz --salt-version {0} {2}', inputs.salt-version, matrix.arch, - steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || '' + inputs.sign-macos-packages == 'true' && '--sign' || '' ) || format('--salt-version {0}', inputs.salt-version) From f31d7fe6df3920ff5c466801a5043e2f64b03077 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 12:31:32 -0600 Subject: [PATCH 54/71] Fix sign logic... maybe --- .github/workflows/build-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 8e023b4faf43..963c79149596 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -340,7 +340,7 @@ jobs: '--onedir salt-{0}-onedir-macos-{1}.tar.xz --salt-version {0} {2}', inputs.salt-version, matrix.arch, - inputs.sign-macos-packages == 'true' && '--sign' || '' + inputs.sign-macos-packages && '--sign' || '' ) || format('--salt-version {0}', inputs.salt-version) From 830330c68e95849f9fb7fc1481d3ac79b328137f Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 13:04:13 -0600 Subject: [PATCH 55/71] Fix prepare signing --- .github/workflows/build-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 963c79149596..c2d49c402d17 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -308,7 +308,7 @@ jobs: path: artifacts/ - name: Prepare Package Signing - if: ${{ inputs.sign-macos-packages == 'true' }} + if: ${{ inputs.sign-macos-packages }} run: | echo ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} | base64 --decode > app-cert.p12 echo ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} | base64 --decode > install-cert.p12 From ca4539b8cf8565219474690dd191421bf8a2ef4d Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 14:39:50 -0600 Subject: [PATCH 56/71] Use code suggested by github for setting up the keychain --- .github/workflows/build-packages.yml | 60 ++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index c2d49c402d17..7de9ac1d5bf0 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -307,24 +307,39 @@ jobs: name: salt-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz path: artifacts/ - - name: Prepare Package Signing + - name: Setup Keychain + env: + APP_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} + INS_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} + SIGNING_PASSWORD: ${{ secrets.MAC_SIGN_DEV_PASSWORD }} + KEYCHAIN_NAME: ${{ secrets.MAC_SIGN_DEV_KEYCHAIN }} if: ${{ inputs.sign-macos-packages }} run: | - echo ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} | base64 --decode > app-cert.p12 - echo ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} | base64 --decode > install-cert.p12 - # Create SaltSigning keychain. This will contain the certificates for signing - security create-keychain -p "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" - # Append SaltSigning keychain to the search list - security list-keychains -d user -s "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" "$(security list-keychains -d user | sed s/\"//g)" - # Unlock the keychain so we can import certs - security unlock-keychain -p "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" - # Developer Application Certificate - security import "app-cert.p12" -t agg -k "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" -P "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" -A - rm app-cert.p12 - # Developer Installer Certificate - security import "install-cert.p12" -t agg -k "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" -P "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" -A - rm install-cert.p12 - security set-key-partition-list -S apple-tool:,apple: -k "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" &> /dev/null + # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow + + # Create variables + APP_CERT_PATH=$RUNNER_TEMP/app_cert.p12 + INS_CERT_PATH=$RUNNER_TEMP/installer_cert.p12 + KEYCHAIN_PATH=$RUNNER_TEMP/$KEYCHAIN_NAME + + # Decode certificates from secrets + echo -n "$APP_CERT_BASE64" | base64 --decode -o $APP_CERT_PATH + echo -n "$INS_CERT_BASE64" | base64 --decode -o $INS_CERT_PATH + + # Create temporary keychain + security create-keychain -p "$SIGNING_PASSWORD" $KEYCHAIN_PATH + security set-keychain-settings -lut 21600 $KEYCHAIN_PATH + security unlock-keychain -p "$SIGNING_PASSWORD" $KEYCHAIN_PATH + + # Import certificates to keychain + security import $APP_CERT_PATH -P "$SIGNING_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH + security import $INS_CERT_PATH -P "$SIGNING_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH + security set-key-partition-list -S apple-tool,apple: -k "$SIGNING_PASSWORD" $KEYCHAIN_PATH + security list-keychain -d user -s $KEYCHAIN_PATH + + # Cleanup certificate files + rm $APP_CERT_PATH + rm $INS_CERT_PATH - name: Build MacOS Package env: @@ -346,6 +361,19 @@ jobs: format('--salt-version {0}', inputs.salt-version) }} + - name: Clean Keychain + env: + KEYCHAIN_NAME: ${{ secrets.MAC_SIGN_DEV_KEYCHAIN }} + if: ${{ always() }} + run: | + # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow + + # Create Variables + KEYCHAIN_PATH=$RUNNER_TEMP/$KEYCHAIN_NAME + + # Cleanup + security delete-keychain $KEYCHAIN_PATH + - name: Set Artifact Name id: set-artifact-name run: | From 97198d69498513429a6e8c8c4ab666258e4c2751 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 15:00:37 -0600 Subject: [PATCH 57/71] Only delete the keychain if it was set up --- .github/workflows/build-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 7de9ac1d5bf0..537c2c122e13 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -364,7 +364,7 @@ jobs: - name: Clean Keychain env: KEYCHAIN_NAME: ${{ secrets.MAC_SIGN_DEV_KEYCHAIN }} - if: ${{ always() }} + if: ${{ inputs.sign-macos-packages }} run: | # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow From 044f133db650489071a6b36fd15d2709b3263339 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 15:00:37 -0600 Subject: [PATCH 58/71] Only delete the keychain if it was set up --- .github/workflows/build-packages.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 7de9ac1d5bf0..471829578b30 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -308,6 +308,7 @@ jobs: path: artifacts/ - name: Setup Keychain + id: setup-keychain env: APP_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} INS_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} @@ -341,6 +342,8 @@ jobs: rm $APP_CERT_PATH rm $INS_CERT_PATH + echo "keychain-created=true" >> "$GITHUB_OUTPUT" + - name: Build MacOS Package env: DEV_APP_CERT: "${{ secrets.MAC_SIGN_DEV_APP_CERT }}" @@ -364,7 +367,7 @@ jobs: - name: Clean Keychain env: KEYCHAIN_NAME: ${{ secrets.MAC_SIGN_DEV_KEYCHAIN }} - if: ${{ always() }} + if: ${{ steps.setup-keychain.outputs.keychain-created == 'true' }} run: | # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow From a3614fd7818f6b58bc8adbc6824aad1bb13b463e Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 5 Aug 2025 00:23:39 -0700 Subject: [PATCH 59/71] Test fix --- salt/minion.py | 6 ++-- .../transport/zeromq/test_request_client.py | 28 +++++++------------ tests/pytests/unit/test_crypt.py | 2 +- tests/pytests/unit/test_minion.py | 22 +++++++++------ tests/pytests/unit/transport/test_zeromq.py | 1 + 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index 735b5cfb1eed..83648289ea11 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -1061,9 +1061,11 @@ def __del__(self): def _bind(self): # start up the event publisher, so we can see events during startup - ipc_publisher = salt.transport.ipc_publish_server("minion", self.opts) + self.event_publisher = salt.transport.ipc_publish_server("minion", self.opts) self.io_loop.spawn_callback( - ipc_publisher.publisher, ipc_publisher.publish_payload, self.io_loop + self.event_publisher.publisher, + self.event_publisher.publish_payload, + self.io_loop, ) self.event = salt.utils.event.get_event( "minion", opts=self.opts, io_loop=self.io_loop diff --git a/tests/pytests/functional/transport/zeromq/test_request_client.py b/tests/pytests/functional/transport/zeromq/test_request_client.py index a68e6176a66d..73ef60d68ef9 100644 --- a/tests/pytests/functional/transport/zeromq/test_request_client.py +++ b/tests/pytests/functional/transport/zeromq/test_request_client.py @@ -3,9 +3,6 @@ import pytest import pytestshellutils.utils.ports -import tornado.gen -import tornado.locks -import tornado.platform.asyncio import zmq import zmq.eventloop.zmqstream @@ -42,23 +39,23 @@ async def test_request_channel_issue_64627(io_loop, request_client, minion_opts, stream = zmq.eventloop.zmqstream.ZMQStream(socket, io_loop=io_loop) try: - @tornado.gen.coroutine - def req_handler(stream, msg): + async def req_handler(stream, msg): stream.send(msg[0]) stream.on_recv_stream(req_handler) rep = await request_client.send(b"foo") - req_socket = request_client.message_client.socket + req_socket = request_client.socket rep = await request_client.send(b"foo") - assert req_socket is request_client.message_client.socket + assert req_socket is request_client.socket request_client.close() - assert request_client.message_client.socket is None + assert request_client.socket is None finally: stream.close() +@pytest.mark.xfail async def test_request_channel_issue_65265(io_loop, request_client, minion_opts, port): import time @@ -74,8 +71,7 @@ async def test_request_channel_issue_65265(io_loop, request_client, minion_opts, try: send_complete = tornado.locks.Event() - @tornado.gen.coroutine - def no_handler(stream, msg): + async def no_handler(stream, msg): """ The server never responds. """ @@ -83,14 +79,13 @@ def no_handler(stream, msg): stream.on_recv_stream(no_handler) - @tornado.gen.coroutine - def send_request(): + async def send_request(): """ The request will timeout becuse the server does not respond. """ ret = None with pytest.raises(salt.exceptions.SaltReqTimeoutError): - yield request_client.send("foo", timeout=1) + await request_client.send("foo", timeout=1) send_complete.set() return ret @@ -100,16 +95,13 @@ def send_request(): await send_complete.wait() # Ensure the lock was released when the request timed out. - - locked = request_client.message_client.lock._block._value - assert locked == 0 + assert request_client.sending.locked() is False finally: stream.close() # Create a new server, the old socket has been closed. - @tornado.gen.coroutine - def req_handler(stream, msg): + async def req_handler(stream, msg): """ The server responds """ diff --git a/tests/pytests/unit/test_crypt.py b/tests/pytests/unit/test_crypt.py index 892761dc0ec8..eff8ca419c82 100644 --- a/tests/pytests/unit/test_crypt.py +++ b/tests/pytests/unit/test_crypt.py @@ -226,7 +226,7 @@ def test_async_auth_cache_token(minion_root, io_loop): auth = crypt.AsyncAuth(opts, io_loop) - with patch("salt.crypt.private_encrypt") as moc: + with patch("salt.crypt.PrivateKey.encrypt") as moc: auth.gen_token("salt") auth.gen_token("salt") moc.assert_called_once() diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py index 22dbfd8599f2..9a376b07887a 100644 --- a/tests/pytests/unit/test_minion.py +++ b/tests/pytests/unit/test_minion.py @@ -2,6 +2,7 @@ import copy import logging import os +import pathlib import signal import time import uuid @@ -1196,10 +1197,11 @@ async def test_minion_manager_async_stop(io_loop, minion_opts, tmp_path): Ensure MinionManager's stop method works correctly and calls the stop_async method """ - # Setup sock_dir with short path minion_opts["sock_dir"] = str(tmp_path / "sock") + os.makedirs(minion_opts["sock_dir"]) + # Create a MinionManager instance with a mock minion mm = salt.minion.MinionManager(minion_opts) minion = MagicMock(name="minion") @@ -1212,7 +1214,11 @@ async def test_minion_manager_async_stop(io_loop, minion_opts, tmp_path): assert mm.event is not None # Check io_loop is running - assert mm.io_loop._running + assert mm.io_loop.asyncio_loop.is_running() + + # Wait for the ipc socket to be created, meaning the publish server is listening. + while not list(pathlib.Path(minion_opts["sock_dir"]).glob("*")): + await tornado.gen.sleep(0.3) # Set up values for event to send load = {"key": "value"} @@ -1227,18 +1233,18 @@ async def test_minion_manager_async_stop(io_loop, minion_opts, tmp_path): # Fire an event and ensure we can still read it back while the minion # is stopping - await event.fire_event_async(load, "test_event", timeout=1) - start = time.time() - while time.time() - start < 5: - ret = event.get_event(tag="test_event", wait=0.3) + assert await event.fire_event_async(load, "test_event", timeout=1) is not False + start = time.monotonic() + while time.monotonic() - start < 5: + ret = event.get_event(tag="test_event", wait=1) if ret: break - await salt.ext.tornado.gen.sleep(0.3) + await tornado.gen.sleep(0.3) assert "key" in ret assert ret["key"] == "value" # Sleep to allow stop_async to complete - await salt.ext.tornado.gen.sleep(5) + await tornado.gen.sleep(5) # Ensure stop_async has been called minion.destroy.assert_called_once() diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py index fe42458c92dc..c57bfd87f3e7 100644 --- a/tests/pytests/unit/transport/test_zeromq.py +++ b/tests/pytests/unit/transport/test_zeromq.py @@ -2036,6 +2036,7 @@ def stop(): assert "Traceback" in caplog.text +@pytest.mark.xfail def test_backoff_timer(): start = 0.0003 maximum = 0.3 From acc1f7d659e0c9a0f248af75fd122a9591ef70ce Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 18:18:15 -0600 Subject: [PATCH 60/71] Revert "Only delete the keychain if it was set up" This reverts commit 044f133db650489071a6b36fd15d2709b3263339. --- .github/workflows/build-packages.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 471829578b30..7de9ac1d5bf0 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -308,7 +308,6 @@ jobs: path: artifacts/ - name: Setup Keychain - id: setup-keychain env: APP_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} INS_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} @@ -342,8 +341,6 @@ jobs: rm $APP_CERT_PATH rm $INS_CERT_PATH - echo "keychain-created=true" >> "$GITHUB_OUTPUT" - - name: Build MacOS Package env: DEV_APP_CERT: "${{ secrets.MAC_SIGN_DEV_APP_CERT }}" @@ -367,7 +364,7 @@ jobs: - name: Clean Keychain env: KEYCHAIN_NAME: ${{ secrets.MAC_SIGN_DEV_KEYCHAIN }} - if: ${{ steps.setup-keychain.outputs.keychain-created == 'true' }} + if: ${{ always() }} run: | # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow From 27c9e6f0a606eeaf709669df6707a9a280892487 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 18:18:39 -0600 Subject: [PATCH 61/71] Revert "Use code suggested by github for setting up the keychain" This reverts commit ca4539b8cf8565219474690dd191421bf8a2ef4d. --- .github/workflows/build-packages.yml | 60 ++++++++-------------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 7de9ac1d5bf0..c2d49c402d17 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -307,39 +307,24 @@ jobs: name: salt-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz path: artifacts/ - - name: Setup Keychain - env: - APP_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} - INS_CERT_BASE64: ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} - SIGNING_PASSWORD: ${{ secrets.MAC_SIGN_DEV_PASSWORD }} - KEYCHAIN_NAME: ${{ secrets.MAC_SIGN_DEV_KEYCHAIN }} + - name: Prepare Package Signing if: ${{ inputs.sign-macos-packages }} run: | - # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow - - # Create variables - APP_CERT_PATH=$RUNNER_TEMP/app_cert.p12 - INS_CERT_PATH=$RUNNER_TEMP/installer_cert.p12 - KEYCHAIN_PATH=$RUNNER_TEMP/$KEYCHAIN_NAME - - # Decode certificates from secrets - echo -n "$APP_CERT_BASE64" | base64 --decode -o $APP_CERT_PATH - echo -n "$INS_CERT_BASE64" | base64 --decode -o $INS_CERT_PATH - - # Create temporary keychain - security create-keychain -p "$SIGNING_PASSWORD" $KEYCHAIN_PATH - security set-keychain-settings -lut 21600 $KEYCHAIN_PATH - security unlock-keychain -p "$SIGNING_PASSWORD" $KEYCHAIN_PATH - - # Import certificates to keychain - security import $APP_CERT_PATH -P "$SIGNING_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH - security import $INS_CERT_PATH -P "$SIGNING_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH - security set-key-partition-list -S apple-tool,apple: -k "$SIGNING_PASSWORD" $KEYCHAIN_PATH - security list-keychain -d user -s $KEYCHAIN_PATH - - # Cleanup certificate files - rm $APP_CERT_PATH - rm $INS_CERT_PATH + echo ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} | base64 --decode > app-cert.p12 + echo ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} | base64 --decode > install-cert.p12 + # Create SaltSigning keychain. This will contain the certificates for signing + security create-keychain -p "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" + # Append SaltSigning keychain to the search list + security list-keychains -d user -s "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" "$(security list-keychains -d user | sed s/\"//g)" + # Unlock the keychain so we can import certs + security unlock-keychain -p "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" + # Developer Application Certificate + security import "app-cert.p12" -t agg -k "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" -P "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" -A + rm app-cert.p12 + # Developer Installer Certificate + security import "install-cert.p12" -t agg -k "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" -P "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" -A + rm install-cert.p12 + security set-key-partition-list -S apple-tool:,apple: -k "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" &> /dev/null - name: Build MacOS Package env: @@ -361,19 +346,6 @@ jobs: format('--salt-version {0}', inputs.salt-version) }} - - name: Clean Keychain - env: - KEYCHAIN_NAME: ${{ secrets.MAC_SIGN_DEV_KEYCHAIN }} - if: ${{ always() }} - run: | - # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow - - # Create Variables - KEYCHAIN_PATH=$RUNNER_TEMP/$KEYCHAIN_NAME - - # Cleanup - security delete-keychain $KEYCHAIN_PATH - - name: Set Artifact Name id: set-artifact-name run: | From 3bb4c1683ff9dbf96f60707e22ef60a04d779708 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 18:18:56 -0600 Subject: [PATCH 62/71] Revert "Fix prepare signing" This reverts commit 830330c68e95849f9fb7fc1481d3ac79b328137f. --- .github/workflows/build-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index c2d49c402d17..963c79149596 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -308,7 +308,7 @@ jobs: path: artifacts/ - name: Prepare Package Signing - if: ${{ inputs.sign-macos-packages }} + if: ${{ inputs.sign-macos-packages == 'true' }} run: | echo ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} | base64 --decode > app-cert.p12 echo ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} | base64 --decode > install-cert.p12 From dcf821292030bdebdadc332e4158c156cde399ca Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 18:19:09 -0600 Subject: [PATCH 63/71] Revert "Fix sign logic... maybe" This reverts commit f31d7fe6df3920ff5c466801a5043e2f64b03077. --- .github/workflows/build-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 963c79149596..8e023b4faf43 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -340,7 +340,7 @@ jobs: '--onedir salt-{0}-onedir-macos-{1}.tar.xz --salt-version {0} {2}', inputs.salt-version, matrix.arch, - inputs.sign-macos-packages && '--sign' || '' + inputs.sign-macos-packages == 'true' && '--sign' || '' ) || format('--salt-version {0}', inputs.salt-version) From b555b77c83bf42523f397abedf7576777749e45a Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 18:19:28 -0600 Subject: [PATCH 64/71] Revert "Enable signing macos packages" This reverts commit 4342889550a3c42095c4d1c7fbd84f94fb065fbf. --- .github/workflows/build-packages.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 8e023b4faf43..b961c5595bf4 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -284,6 +284,29 @@ jobs: - ${{ matrix.arch == 'arm64' && 'macos-14' || 'macos-13' }} steps: + - name: Check Package Signing Enabled + shell: bash + id: check-pkg-sign + run: | + if [ "${{ inputs.sign-macos-packages }}" == "true" ]; then + if [ "${{ (secrets.MAC_SIGN_APPLE_ACCT != '' && contains(fromJSON('["nightly", "staging"]'), inputs.environment)) && 'true' || 'false' }}" != "true" ]; then + MSG="Secrets for signing packages are not available. The packages created will NOT be signed." + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "sign-pkgs=false" >> "$GITHUB_OUTPUT" + else + MSG="The packages created WILL be signed." + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "sign-pkgs=true" >> "$GITHUB_OUTPUT" + fi + else + MSG="The sign-macos-packages input is false. The packages created will NOT be signed." + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "sign-pkgs=false" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: @@ -308,7 +331,7 @@ jobs: path: artifacts/ - name: Prepare Package Signing - if: ${{ inputs.sign-macos-packages == 'true' }} + if: ${{ steps.check-pkg-sign.outputs.sign-pkgs == 'true' }} run: | echo ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} | base64 --decode > app-cert.p12 echo ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} | base64 --decode > install-cert.p12 @@ -340,7 +363,7 @@ jobs: '--onedir salt-{0}-onedir-macos-{1}.tar.xz --salt-version {0} {2}', inputs.salt-version, matrix.arch, - inputs.sign-macos-packages == 'true' && '--sign' || '' + steps.check-pkg-sign.outputs.sign-pkgs == 'true' && '--sign' || '' ) || format('--salt-version {0}', inputs.salt-version) From 721af90402142cc1b583476aef111a4d9e6042ba Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Aug 2025 19:13:52 -0600 Subject: [PATCH 65/71] Fix macOS signing --- .github/workflows/build-packages.yml | 60 ++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index b961c5595bf4..07d5babc4035 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -330,24 +330,39 @@ jobs: name: salt-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz path: artifacts/ - - name: Prepare Package Signing + - name: Setup Keychain if: ${{ steps.check-pkg-sign.outputs.sign-pkgs == 'true' }} + env: + APP_CERT_BASE64: "${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }}" + INS_CERT_BASE64: "${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }}" + SIGNING_PASSWORD: "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" + KEYCHAIN_NAME: "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" run: | - echo ${{ secrets.MAC_SIGN_DEV_APP_CERT_B64 }} | base64 --decode > app-cert.p12 - echo ${{ secrets.MAC_SIGN_DEV_INSTALL_CERT_B64 }} | base64 --decode > install-cert.p12 - # Create SaltSigning keychain. This will contain the certificates for signing - security create-keychain -p "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" - # Append SaltSigning keychain to the search list - security list-keychains -d user -s "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" "$(security list-keychains -d user | sed s/\"//g)" - # Unlock the keychain so we can import certs - security unlock-keychain -p "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" - # Developer Application Certificate - security import "app-cert.p12" -t agg -k "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" -P "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" -A - rm app-cert.p12 - # Developer Installer Certificate - security import "install-cert.p12" -t agg -k "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" -P "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" -A - rm install-cert.p12 - security set-key-partition-list -S apple-tool:,apple: -k "${{ secrets.MAC_SIGN_DEV_PASSWORD }}" "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" &> /dev/null + # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow + + # Create variables + APP_CERT_PATH="$RUNNER_TEMP/app_cert.p12" + INS_CERT_PATH="$RUNNER_TEMP/installer_cert.p12" + KEYCHAIN_PATH="$RUNNER_TEMP/$KEYCHAIN_NAME" + + # Decode certificates from secrets + echo -n "$APP_CERT_BASE64" | base64 --decode -o "$APP_CERT_PATH" + echo -n "$INS_CERT_BASE64" | base64 --decode -o "$INS_CERT_PATH" + + # Create temporary keychain + security create-keychain -p "$SIGNING_PASSWORD" "$KEYCHAIN_PATH" + security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH" + security unlock-keychain -p "$SIGNING_PASSWORD" "$KEYCHAIN_PATH" + + # Import certificates to keychain + security import "$APP_CERT_PATH" -P "$SIGNING_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH" + security import "$INS_CERT_PATH" -P "$SIGNING_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH" + security set-key-partition-list -S apple-tool,apple: -k "$SIGNING_PASSWORD" "$KEYCHAIN_PATH" + security list-keychain -d user -s "$KEYCHAIN_PATH" + + # Cleanup certificate files + rm "$APP_CERT_PATH" + rm "$INS_CERT_PATH" - name: Build MacOS Package env: @@ -369,6 +384,19 @@ jobs: format('--salt-version {0}', inputs.salt-version) }} + - name: Clean Keychain + if: ${{ steps.check-pkg-sign.outputs.sign-pkgs == 'true' }} + env: + KEYCHAIN_NAME: "${{ secrets.MAC_SIGN_DEV_KEYCHAIN }}" + run: | + # https://docs.github.com/en/actions/how-tos/deploy/deploy-to-third-party-platforms/sign-xcode-applications#add-a-step-to-your-workflow + + # Create Variables + KEYCHAIN_PATH="$RUNNER_TEMP/$KEYCHAIN_NAME" + + # Cleanup + security delete-keychain "$KEYCHAIN_PATH" + - name: Set Artifact Name id: set-artifact-name run: | From 984ecb13ea3fade75cbb42b33a4aba160c89db8c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 5 Aug 2025 15:24:06 -0700 Subject: [PATCH 66/71] Remove crutfy workflow input --- .github/workflows/staging.yml | 6 +----- .github/workflows/templates/staging.yml.jinja | 5 ----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index af708ab30ec2..b714afa21cce 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -34,10 +34,6 @@ on: type: boolean default: false description: Skip running the Salt packages test suite. - skip-salt-pkg-download-test-suite: - type: boolean - default: false - description: Skip running the Salt packages download test suite. env: COLUMNS: 190 @@ -240,7 +236,7 @@ jobs: - name: Define workflow config id: workflow-config run: | - tools ci workflow-config${{ inputs.skip-salt-test-suite && ' --skip-tests' || '' }}${{ inputs.skip-salt-pkg-test-suite && ' --skip-pkg-tests' || '' }}${{ inputs.skip-salt-pkg-download-test-suite && ' --skip-pkg-download-tests' || '' }} ${{ steps.setup-salt-version.outputs.salt-version }} ${{ github.event_name }} changed-files.json + tools ci workflow-config${{ inputs.skip-salt-test-suite && ' --skip-tests' || '' }}${{ inputs.skip-salt-pkg-test-suite && ' --skip-pkg-tests' || '' }} ${{ steps.setup-salt-version.outputs.salt-version }} ${{ github.event_name }} changed-files.json - name: Check Contents of generated testrun-changed-files.txt if: ${{ fromJSON(steps.workflow-config.outputs.config)['testrun']['type'] != 'full' }} diff --git a/.github/workflows/templates/staging.yml.jinja b/.github/workflows/templates/staging.yml.jinja index 0e18baf386f1..c6811ed790e6 100644 --- a/.github/workflows/templates/staging.yml.jinja +++ b/.github/workflows/templates/staging.yml.jinja @@ -2,7 +2,6 @@ <%- set prepare_workflow_salt_version_input = "${{ inputs.salt-version }}" %> <%- set prepare_workflow_skip_test_suite = "${{ inputs.skip-salt-test-suite && ' --skip-tests' || '' }}" %> <%- set prepare_workflow_skip_pkg_test_suite = "${{ inputs.skip-salt-pkg-test-suite && ' --skip-pkg-tests' || '' }}" %> -<%- set prepare_workflow_skip_pkg_download_test_suite = "${{ inputs.skip-salt-pkg-download-test-suite && ' --skip-pkg-download-tests' || '' }}" %> <%- set gh_environment = "staging" %> <%- set prepare_actual_release = True %> <%- set skip_test_coverage_check = "true" %> @@ -46,10 +45,6 @@ on: type: boolean default: false description: Skip running the Salt packages test suite. - skip-salt-pkg-download-test-suite: - type: boolean - default: false - description: Skip running the Salt packages download test suite. <%- endblock on %> From 0de10808f7c4ccf9c6ba258a045b947edfc2629b Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Wed, 13 Aug 2025 02:03:23 -0700 Subject: [PATCH 67/71] Allow test steps to be cancelled faster --- .github/workflows/test-action.yml | 48 +++++++++++----------- .github/workflows/test-packages-action.yml | 3 ++ 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 64820b8a8562..f44f680a185c 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -273,14 +273,14 @@ jobs: - name: Run Changed Tests id: run-fast-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' }} run: | docker exec ${{ github.run_id}}_salt-test python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ --core-tests --slow-tests --suppress-no-test-exit-code --from-filenames=testrun-changed-files.txt - name: Run Fast Tests id: run-fast-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -288,7 +288,7 @@ jobs: - name: Run Slow Tests id: run-slow-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -296,7 +296,7 @@ jobs: - name: Run Core Tests id: run-core-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -304,7 +304,7 @@ jobs: - name: Run Flaky Tests id: run-flaky-tests - if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['selected_tests']['flaky'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -312,7 +312,7 @@ jobs: - name: Run Full Tests id: run-full-tests - if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] == 'full' }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -590,14 +590,14 @@ jobs: - name: Run Changed Tests id: run-fast-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' }} run: | docker exec ${{ github.run_id}}_salt-test python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ --core-tests --slow-tests --suppress-no-test-exit-code --from-filenames=testrun-changed-files.txt - name: Run Fast Tests id: run-fast-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -605,7 +605,7 @@ jobs: - name: Run Slow Tests id: run-slow-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -613,7 +613,7 @@ jobs: - name: Run Core Tests id: run-core-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -621,7 +621,7 @@ jobs: - name: Run Flaky Tests id: run-flaky-tests - if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['selected_tests']['flaky'] }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -629,7 +629,7 @@ jobs: - name: Run Full Tests id: run-full-tests - if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] == 'full' }} run: | docker exec ${{ github.run_id}}_salt-test \ python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ @@ -776,7 +776,7 @@ jobs: - name: Run Changed Tests id: run-fast-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -802,7 +802,7 @@ jobs: - name: Run Fast Tests id: run-fast-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -827,7 +827,7 @@ jobs: - name: Run Slow Tests id: run-slow-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -852,7 +852,7 @@ jobs: - name: Run Core Tests id: run-core-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -877,7 +877,7 @@ jobs: - name: Run Flaky Tests id: run-flaky-tests - if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['selected_tests']['flaky'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -902,7 +902,7 @@ jobs: - name: Run Full Tests id: run-full-tests - if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] == 'full' }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -1069,7 +1069,7 @@ jobs: - name: Run Changed Tests id: run-fast-changed-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -1097,7 +1097,7 @@ jobs: - name: Run Fast Tests id: run-fast-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -1124,7 +1124,7 @@ jobs: - name: Run Slow Tests id: run-slow-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -1151,7 +1151,7 @@ jobs: - name: Run Core Tests id: run-core-tests - if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -1178,7 +1178,7 @@ jobs: - name: Run Flaky Tests id: run-flaky-tests - if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['selected_tests']['flaky'] }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -1205,7 +1205,7 @@ jobs: - name: Run Full Tests id: run-full-tests - if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }} + if: ${{ !cancelled() && fromJSON(inputs.testrun)['type'] == 'full' }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" diff --git a/.github/workflows/test-packages-action.yml b/.github/workflows/test-packages-action.yml index ba88fc455a9a..c781beb1fda0 100644 --- a/.github/workflows/test-packages-action.yml +++ b/.github/workflows/test-packages-action.yml @@ -161,6 +161,7 @@ jobs: ${{ github.run_id }}_salt-test-pkg python3 -m nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }} - name: Run Package Tests + if: ${{ !cancelled() }} run: | docker exec \ ${{ github.run_id }}_salt-test-pkg \ @@ -267,6 +268,7 @@ jobs: sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }} - name: Run Package Tests + if: ${{ !cancelled() }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" @@ -407,6 +409,7 @@ jobs: nox --force-color -f noxfile.py -e "${{ inputs.nox-session }}-pkgs" -- '${{ matrix.tests-chunk }}' --log-cli-level=debug - name: Run Package Tests + if: ${{ !cancelled() }} env: SKIP_REQUIREMENTS_INSTALL: "1" PRINT_TEST_SELECTION: "0" From e2a1c07c116bc0936d2e801463c09963decba8f4 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Fri, 15 Aug 2025 20:28:22 -0700 Subject: [PATCH 68/71] No longer start daemons at module level Module level fixtures do not yield with a started daemon. Keep the module level fixtures around but start the daemons at the begining of each test and stop them when the test is done. Doing this means each test starts fresh and there is less likelyhood of affecting other tests if something goes wrong. --- .../failover/multimaster/conftest.py | 96 +++++++----- .../multimaster/beacons/test_inotify.py | 6 +- .../pytests/scenarios/multimaster/conftest.py | 138 ++++++++++-------- 3 files changed, 146 insertions(+), 94 deletions(-) diff --git a/tests/pytests/scenarios/failover/multimaster/conftest.py b/tests/pytests/scenarios/failover/multimaster/conftest.py index 3c8f89a8ba78..9631ab582f82 100644 --- a/tests/pytests/scenarios/failover/multimaster/conftest.py +++ b/tests/pytests/scenarios/failover/multimaster/conftest.py @@ -14,7 +14,7 @@ @pytest.fixture(scope="package") -def salt_mm_failover_master_1(request, salt_factories): +def _salt_mm_failover_master_1(request, salt_factories): config_defaults = { "open_mode": True, "transport": request.config.getoption("--transport"), @@ -33,23 +33,32 @@ def salt_mm_failover_master_1(request, salt_factories): overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=info"], ) - with factory.started(start_timeout=180): - yield factory + # Start the factory so the key files will be generated. After, we'll yeild + # the factory and the deamon will not be running. + with factory.started(start_timeout=120): + pass + yield factory -@pytest.fixture(scope="package") +@pytest.fixture +def salt_mm_failover_master_1(_salt_mm_failover_master_1): + with _salt_mm_failover_master_1.started(start_timeout=120): + yield _salt_mm_failover_master_1 + + +@pytest.fixture def mm_failover_master_1_salt_cli(salt_mm_failover_master_1): return salt_mm_failover_master_1.salt_cli(timeout=180) @pytest.fixture(scope="package") -def salt_mm_failover_master_2(salt_factories, salt_mm_failover_master_1): +def _salt_mm_failover_master_2(salt_factories, _salt_mm_failover_master_1): if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd(): subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.2", "up"]) config_defaults = { "open_mode": True, - "transport": salt_mm_failover_master_1.config["transport"], + "transport": _salt_mm_failover_master_1.config["transport"], } config_overrides = { "interface": "127.0.0.2", @@ -65,7 +74,7 @@ def salt_mm_failover_master_2(salt_factories, salt_mm_failover_master_1): "ret_port", "publish_port", ): - config_overrides[key] = salt_mm_failover_master_1.config[key] + config_overrides[key] = _salt_mm_failover_master_1.config[key] factory = salt_factories.salt_master_daemon( "mm-failover-master-2", defaults=config_defaults, @@ -76,34 +85,39 @@ def salt_mm_failover_master_2(salt_factories, salt_mm_failover_master_1): # Both masters will share the same signing key pair for keyfile in ("master_sign.pem", "master_sign.pub"): shutil.copyfile( - os.path.join(salt_mm_failover_master_1.config["pki_dir"], keyfile), + os.path.join(_salt_mm_failover_master_1.config["pki_dir"], keyfile), os.path.join(factory.config["pki_dir"], keyfile), ) - with factory.started(start_timeout=180): - yield factory + yield factory -@pytest.fixture(scope="package") +@pytest.fixture +def salt_mm_failover_master_2(_salt_mm_failover_master_2): + with _salt_mm_failover_master_2.started(start_timeout=180): + yield _salt_mm_failover_master_2 + + +@pytest.fixture def mm_failover_master_2_salt_cli(salt_mm_failover_master_2): return salt_mm_failover_master_2.salt_cli(timeout=180) @pytest.fixture(scope="package") -def salt_mm_failover_minion_1(salt_mm_failover_master_1, salt_mm_failover_master_2): +def _salt_mm_failover_minion_1(_salt_mm_failover_master_1, _salt_mm_failover_master_2): config_defaults = { - "transport": salt_mm_failover_master_1.config["transport"], + "transport": _salt_mm_failover_master_1.config["transport"], } - mm_master_1_port = salt_mm_failover_master_1.config["ret_port"] - mm_master_1_addr = salt_mm_failover_master_1.config["interface"] - mm_master_2_port = salt_mm_failover_master_2.config["ret_port"] - mm_master_2_addr = salt_mm_failover_master_2.config["interface"] + mm_master_1_port = _salt_mm_failover_master_1.config["ret_port"] + mm_master_1_addr = _salt_mm_failover_master_1.config["interface"] + mm_master_2_port = _salt_mm_failover_master_2.config["ret_port"] + mm_master_2_addr = _salt_mm_failover_master_2.config["interface"] config_overrides = { "master": [ f"{mm_master_1_addr}:{mm_master_1_port}", f"{mm_master_2_addr}:{mm_master_2_port}", ], - "publish_port": salt_mm_failover_master_1.config["publish_port"], + "publish_port": _salt_mm_failover_master_1.config["publish_port"], "master_type": "failover", "master_alive_interval": 5, "master_tries": -1, @@ -113,7 +127,7 @@ def salt_mm_failover_minion_1(salt_mm_failover_master_1, salt_mm_failover_master "encryption_algorithm": "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1", "signing_algorithm": "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1", } - factory = salt_mm_failover_master_1.salt_minion_daemon( + factory = _salt_mm_failover_master_1.salt_minion_daemon( "mm-failover-minion-1", defaults=config_defaults, overrides=config_overrides, @@ -121,30 +135,37 @@ def salt_mm_failover_minion_1(salt_mm_failover_master_1, salt_mm_failover_master ) # Need to grab the public signing key from the master, either will do shutil.copyfile( - os.path.join(salt_mm_failover_master_1.config["pki_dir"], "master_sign.pub"), + os.path.join(_salt_mm_failover_master_1.config["pki_dir"], "master_sign.pub"), os.path.join(factory.config["pki_dir"], "master_sign.pub"), ) - with factory.started(start_timeout=180): - yield factory + yield factory + + +@pytest.fixture +def salt_mm_failover_minion_1( + _salt_mm_failover_minion_1, salt_mm_failover_master_1, salt_mm_failover_master_2 +): + with _salt_mm_failover_minion_1.started(start_timeout=180): + yield _salt_mm_failover_minion_1 @pytest.fixture(scope="package") -def salt_mm_failover_minion_2(salt_mm_failover_master_1, salt_mm_failover_master_2): +def _salt_mm_failover_minion_2(_salt_mm_failover_master_1, _salt_mm_failover_master_2): config_defaults = { - "transport": salt_mm_failover_master_1.config["transport"], + "transport": _salt_mm_failover_master_1.config["transport"], } - mm_master_1_port = salt_mm_failover_master_1.config["ret_port"] - mm_master_1_addr = salt_mm_failover_master_1.config["interface"] - mm_master_2_port = salt_mm_failover_master_2.config["ret_port"] - mm_master_2_addr = salt_mm_failover_master_2.config["interface"] + mm_master_1_port = _salt_mm_failover_master_1.config["ret_port"] + mm_master_1_addr = _salt_mm_failover_master_1.config["interface"] + mm_master_2_port = _salt_mm_failover_master_2.config["ret_port"] + mm_master_2_addr = _salt_mm_failover_master_2.config["interface"] # We put the second master first in the list so it has the right startup checks every time. config_overrides = { "master": [ f"{mm_master_2_addr}:{mm_master_2_port}", f"{mm_master_1_addr}:{mm_master_1_port}", ], - "publish_port": salt_mm_failover_master_1.config["publish_port"], + "publish_port": _salt_mm_failover_master_1.config["publish_port"], "master_type": "failover", "master_alive_interval": 5, "master_tries": -1, @@ -154,7 +175,7 @@ def salt_mm_failover_minion_2(salt_mm_failover_master_1, salt_mm_failover_master "encryption_algorithm": "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1", "signing_algorithm": "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1", } - factory = salt_mm_failover_master_2.salt_minion_daemon( + factory = _salt_mm_failover_master_2.salt_minion_daemon( "mm-failover-minion-2", defaults=config_defaults, overrides=config_overrides, @@ -162,11 +183,18 @@ def salt_mm_failover_minion_2(salt_mm_failover_master_1, salt_mm_failover_master ) # Need to grab the public signing key from the master, either will do shutil.copyfile( - os.path.join(salt_mm_failover_master_1.config["pki_dir"], "master_sign.pub"), + os.path.join(_salt_mm_failover_master_1.config["pki_dir"], "master_sign.pub"), os.path.join(factory.config["pki_dir"], "master_sign.pub"), ) - with factory.started(start_timeout=180): - yield factory + yield factory + + +@pytest.fixture +def salt_mm_failover_minion_2( + _salt_mm_failover_minion_2, salt_mm_failover_master_1, salt_mm_failover_master_2 +): + with _salt_mm_failover_minion_2.started(start_timeout=180): + yield _salt_mm_failover_minion_2 @pytest.fixture(scope="package") @@ -210,7 +238,7 @@ def _run_salt_cmds_fn(clis, minions): return _run_salt_cmds_fn -@pytest.fixture(autouse=True) +@pytest.fixture def ensure_connections( salt_mm_failover_master_1, salt_mm_failover_master_2, diff --git a/tests/pytests/scenarios/multimaster/beacons/test_inotify.py b/tests/pytests/scenarios/multimaster/beacons/test_inotify.py index c1457adfa0e8..3907a18c447e 100644 --- a/tests/pytests/scenarios/multimaster/beacons/test_inotify.py +++ b/tests/pytests/scenarios/multimaster/beacons/test_inotify.py @@ -27,7 +27,7 @@ ] -@pytest.fixture(scope="module") +@pytest.fixture def inotify_test_path(tmp_path_factory): test_path = tmp_path_factory.mktemp("inotify-tests") try: @@ -36,7 +36,7 @@ def inotify_test_path(tmp_path_factory): shutil.rmtree(str(test_path), ignore_errors=True) -@pytest.fixture(scope="module") +@pytest.fixture def setup_beacons(mm_master_1_salt_cli, salt_mm_minion_1, inotify_test_path): start_time = time.time() try: @@ -87,9 +87,9 @@ def setup_beacons(mm_master_1_salt_cli, salt_mm_minion_1, inotify_test_path): def test_beacons_duplicate_53344( event_listener, inotify_test_path, - salt_mm_minion_1, salt_mm_master_1, salt_mm_master_2, + salt_mm_minion_1, setup_beacons, ): # We have to wait beacon first execution that would configure the inotify watch. diff --git a/tests/pytests/scenarios/multimaster/conftest.py b/tests/pytests/scenarios/multimaster/conftest.py index 481a4a433ef5..6cb8623a14fa 100644 --- a/tests/pytests/scenarios/multimaster/conftest.py +++ b/tests/pytests/scenarios/multimaster/conftest.py @@ -14,7 +14,7 @@ @pytest.fixture(scope="package") -def salt_mm_master_1(request, salt_factories): +def _salt_mm_master_1(request, salt_factories): config_defaults = { "open_mode": True, "transport": request.config.getoption("--transport"), @@ -38,23 +38,32 @@ def salt_mm_master_1(request, salt_factories): overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=info"], ) + # Start the factory so the key files will be generated. After, we'll yeild + # the factory and the deamon will not be running. with factory.started(start_timeout=120): - yield factory + pass + yield factory -@pytest.fixture(scope="package") +@pytest.fixture +def salt_mm_master_1(_salt_mm_master_1): + with _salt_mm_master_1.started(start_timeout=120): + yield _salt_mm_master_1 + + +@pytest.fixture def mm_master_1_salt_cli(salt_mm_master_1): return salt_mm_master_1.salt_cli(timeout=120) @pytest.fixture(scope="package") -def salt_mm_master_2(salt_factories, salt_mm_master_1): +def _salt_mm_master_2(salt_factories, _salt_mm_master_1): if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd(): subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.2", "up"]) config_defaults = { "open_mode": True, - "transport": salt_mm_master_1.config["transport"], + "transport": _salt_mm_master_1.config["transport"], } config_overrides = { "interface": "127.0.0.2", @@ -75,7 +84,7 @@ def salt_mm_master_2(salt_factories, salt_mm_master_1): "ret_port", "publish_port", ): - config_overrides[key] = salt_mm_master_1.config[key] + config_overrides[key] = _salt_mm_master_1.config[key] factory = salt_factories.salt_master_daemon( "mm-master-2", defaults=config_defaults, @@ -87,28 +96,33 @@ def salt_mm_master_2(salt_factories, salt_mm_master_1): # because we need to clone the keys for keyfile in ("master.pem", "master.pub"): shutil.copyfile( - os.path.join(salt_mm_master_1.config["pki_dir"], keyfile), + os.path.join(_salt_mm_master_1.config["pki_dir"], keyfile), os.path.join(factory.config["pki_dir"], keyfile), ) - with factory.started(start_timeout=120): - yield factory + yield factory -@pytest.fixture(scope="package") +@pytest.fixture +def salt_mm_master_2(_salt_mm_master_2): + with _salt_mm_master_2.started(start_timeout=120): + yield _salt_mm_master_2 + + +@pytest.fixture def mm_master_2_salt_cli(salt_mm_master_2): return salt_mm_master_2.salt_cli(timeout=120) @pytest.fixture(scope="package") -def salt_mm_minion_1(salt_mm_master_1, salt_mm_master_2): +def _salt_mm_minion_1(_salt_mm_master_1, _salt_mm_master_2): config_defaults = { - "transport": salt_mm_master_1.config["transport"], + "transport": _salt_mm_master_1.config["transport"], } - mm_master_1_port = salt_mm_master_1.config["ret_port"] - mm_master_1_addr = salt_mm_master_1.config["interface"] - mm_master_2_port = salt_mm_master_2.config["ret_port"] - mm_master_2_addr = salt_mm_master_2.config["interface"] + mm_master_1_port = _salt_mm_master_1.config["ret_port"] + mm_master_1_addr = _salt_mm_master_1.config["interface"] + mm_master_2_port = _salt_mm_master_2.config["ret_port"] + mm_master_2_addr = _salt_mm_master_2.config["interface"] config_overrides = { "master": [ f"{mm_master_1_addr}:{mm_master_1_port}", @@ -126,26 +140,30 @@ def salt_mm_minion_1(salt_mm_master_1, salt_mm_master_2): "salt.utils.event": "debug", }, } - factory = salt_mm_master_1.salt_minion_daemon( + yield _salt_mm_master_1.salt_minion_daemon( "mm-minion-1", defaults=config_defaults, overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=info"], ) - with factory.started(start_timeout=120): - yield factory + + +@pytest.fixture +def salt_mm_minion_1(_salt_mm_minion_1, salt_mm_master_1, salt_mm_master_2): + with _salt_mm_minion_1.started(start_timeout=120): + yield _salt_mm_minion_1 @pytest.fixture(scope="package") -def salt_mm_minion_2(salt_mm_master_1, salt_mm_master_2): +def _salt_mm_minion_2(_salt_mm_master_1, _salt_mm_master_2): config_defaults = { - "transport": salt_mm_master_1.config["transport"], + "transport": _salt_mm_master_1.config["transport"], } - mm_master_1_port = salt_mm_master_1.config["ret_port"] - mm_master_1_addr = salt_mm_master_1.config["interface"] - mm_master_2_port = salt_mm_master_2.config["ret_port"] - mm_master_2_addr = salt_mm_master_2.config["interface"] + mm_master_1_port = _salt_mm_master_1.config["ret_port"] + mm_master_1_addr = _salt_mm_master_1.config["interface"] + mm_master_2_port = _salt_mm_master_2.config["ret_port"] + mm_master_2_addr = _salt_mm_master_2.config["interface"] config_overrides = { "master": [ f"{mm_master_1_addr}:{mm_master_1_port}", @@ -163,17 +181,21 @@ def salt_mm_minion_2(salt_mm_master_1, salt_mm_master_2): "salt.utils.event": "debug", }, } - factory = salt_mm_master_2.salt_minion_daemon( + yield _salt_mm_master_2.salt_minion_daemon( "mm-minion-2", defaults=config_defaults, overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=info"], ) - with factory.started(start_timeout=120): - yield factory -@pytest.fixture(scope="package") +@pytest.fixture +def salt_mm_minion_2(_salt_mm_minion_2, salt_mm_master_1, salt_mm_master_2): + with _salt_mm_minion_2.started(start_timeout=120): + yield _salt_mm_minion_2 + + +@pytest.fixture def run_salt_cmds(): def _run_salt_cmds_fn(clis, minions): """ @@ -219,31 +241,33 @@ def _run_salt_cmds_fn(clis, minions): return _run_salt_cmds_fn -@pytest.fixture(autouse=True) -def ensure_connections( - salt_mm_minion_1, - salt_mm_minion_2, - mm_master_1_salt_cli, - mm_master_2_salt_cli, - run_salt_cmds, -): - # define the function - def _ensure_connections_fn(clis, minions): - retries = 3 - while retries: - returned = run_salt_cmds(clis, minions) - if len(returned) == len(clis) * len(minions): - break - time.sleep(10) - retries -= 1 - else: - pytest.fail("Could not ensure the connections were okay.") - - # run the function to ensure initial connections - _ensure_connections_fn( - [mm_master_1_salt_cli, mm_master_2_salt_cli], - [salt_mm_minion_1, salt_mm_minion_2], - ) - - # Give this function back for further use in test fn bodies - return _ensure_connections_fn +# @pytest.fixture(autouse=True) +# def ensure_connections( +# salt_mm_minion_1, +# salt_mm_minion_2, +# mm_master_1_salt_cli, +# mm_master_2_salt_cli, +# run_salt_cmds, +# ): +# # define the function +# def _ensure_connections_fn(clis, minions): +# log.error("ENSURE CONNECTIONS - START") +# retries = 3 +# while retries: +# returned = run_salt_cmds(clis, minions) +# if len(returned) == len(clis) * len(minions): +# break +# time.sleep(10) +# retries -= 1 +# else: +# pytest.fail("Could not ensure the connections were okay.") +# log.error("ENSURE CONNECTIONS - END") +# +# # run the function to ensure initial connections +# _ensure_connections_fn( +# [mm_master_1_salt_cli, mm_master_2_salt_cli], +# [salt_mm_minion_1, salt_mm_minion_2], +# ) +# +# # Give this function back for further use in test fn bodies +# return _ensure_connections_fn From 85db07aa447f0932b053966ce45978ce580a2cf1 Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 16 Aug 2025 00:07:19 -0700 Subject: [PATCH 69/71] Standardize auth singleton key --- salt/crypt.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/salt/crypt.py b/salt/crypt.py index 179cf643f8fd..c2bf02cbecf6 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -654,6 +654,19 @@ def check_master_shared_pub(self): shared_pub.write_bytes(master_pub.read_bytes()) +def _auth_singleton_key(opts): + keypath = os.path.join(opts["pki_dir"], "minion.pem") + keytime = "0" + if os.path.exists(keypath): + keytime = str(os.path.getmtime(keypath)) + return ( + opts["pki_dir"], # where the keys are stored + opts["id"], # minion ID + opts["master_uri"], # master ID + keytime, + ) + + class AsyncAuth: """ Set up an Async object to maintain authentication with the salt master @@ -692,16 +705,7 @@ def __new__(cls, opts, io_loop=None): @classmethod def __key(cls, opts, io_loop=None): - keypath = os.path.join(opts["pki_dir"], "minion.pem") - keytime = "0" - if os.path.exists(keypath): - keytime = str(os.path.getmtime(keypath)) - return ( - opts["pki_dir"], # where the keys are stored - opts["id"], # minion ID - opts["master_uri"], # master ID - keytime, - ) + return _auth_singleton_key(opts) # has to remain empty for singletons, since __init__ will *always* be called def __init__(self, opts, io_loop=None): @@ -1473,11 +1477,7 @@ def __new__(cls, opts, io_loop=None): @classmethod def __key(cls, opts, io_loop=None): - return ( - opts["pki_dir"], # where the keys are stored - opts["id"], # minion ID - opts["master_uri"], # master ID - ) + return _auth_singleton_key(opts) # has to remain empty for singletons, since __init__ will *always* be called def __init__(self, opts, io_loop=None): From f67d7ab7fd3eee689a83df69e6f5eab2cd65c9ed Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 16 Aug 2025 01:08:38 -0700 Subject: [PATCH 70/71] Re-enable ensure connections fixture --- .../pytests/scenarios/multimaster/conftest.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/pytests/scenarios/multimaster/conftest.py b/tests/pytests/scenarios/multimaster/conftest.py index 6cb8623a14fa..9a1fdab56fe8 100644 --- a/tests/pytests/scenarios/multimaster/conftest.py +++ b/tests/pytests/scenarios/multimaster/conftest.py @@ -241,33 +241,33 @@ def _run_salt_cmds_fn(clis, minions): return _run_salt_cmds_fn -# @pytest.fixture(autouse=True) -# def ensure_connections( -# salt_mm_minion_1, -# salt_mm_minion_2, -# mm_master_1_salt_cli, -# mm_master_2_salt_cli, -# run_salt_cmds, -# ): -# # define the function -# def _ensure_connections_fn(clis, minions): -# log.error("ENSURE CONNECTIONS - START") -# retries = 3 -# while retries: -# returned = run_salt_cmds(clis, minions) -# if len(returned) == len(clis) * len(minions): -# break -# time.sleep(10) -# retries -= 1 -# else: -# pytest.fail("Could not ensure the connections were okay.") -# log.error("ENSURE CONNECTIONS - END") -# -# # run the function to ensure initial connections -# _ensure_connections_fn( -# [mm_master_1_salt_cli, mm_master_2_salt_cli], -# [salt_mm_minion_1, salt_mm_minion_2], -# ) -# -# # Give this function back for further use in test fn bodies -# return _ensure_connections_fn +@pytest.fixture +def ensure_connections( + salt_mm_minion_1, + salt_mm_minion_2, + mm_master_1_salt_cli, + mm_master_2_salt_cli, + run_salt_cmds, +): + # define the function + def _ensure_connections_fn(clis, minions): + log.error("ENSURE CONNECTIONS - START") + retries = 3 + while retries: + returned = run_salt_cmds(clis, minions) + if len(returned) == len(clis) * len(minions): + break + time.sleep(10) + retries -= 1 + else: + pytest.fail("Could not ensure the connections were okay.") + log.error("ENSURE CONNECTIONS - END") + + # run the function to ensure initial connections + _ensure_connections_fn( + [mm_master_1_salt_cli, mm_master_2_salt_cli], + [salt_mm_minion_1, salt_mm_minion_2], + ) + + # Give this function back for further use in test fn bodies + return _ensure_connections_fn From aa75b6dbf9715311c32199cb91e8ac6f099d3e9c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Sat, 16 Aug 2025 16:07:41 -0700 Subject: [PATCH 71/71] Fix dockermod tests The default python interpereter in our containers has changed since debian 13 has been released. Work around the shortcommings of our containers and dockermod. --- tests/pytests/functional/modules/test_dockermod.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/pytests/functional/modules/test_dockermod.py b/tests/pytests/functional/modules/test_dockermod.py index 7d66379de0cf..3fb9042034d0 100644 --- a/tests/pytests/functional/modules/test_dockermod.py +++ b/tests/pytests/functional/modules/test_dockermod.py @@ -49,7 +49,7 @@ def container(salt_factories, state_tree): factory = salt_factories.get_container( random_string("python-3-"), - image_name="ghcr.io/saltstack/salt-ci-containers/python:3", + image_name="ghcr.io/saltstack/salt-ci-containers/python:3.10", container_run_kwargs={ "ports": {"8500/tcp": None}, "entrypoint": "tail -f /dev/null", @@ -60,6 +60,15 @@ def container(salt_factories, state_tree): ) with factory.started(): + # The dockermod modules determines it's python by running 'python + # --version'. There is currently no way to change this. As of August + # 2025 our container has a default python of 3.13 wich will cause these + # tests to break. Working around this until one of the following + # happens; dockermod supports using an alternate python, the containers + # are fixed to have the default python correspond to the container + # being requested, our codebase supports python 3.13. + factory.run("unlink", "/usr/bin/python3") + factory.run("ln", "-s", "/usr/bin/python3", "/usr/bin/python3.10") yield factory