diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 624f6956b9..34ad5f2d4d 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -16,7 +16,7 @@ jobs: persist-credentials: false - name: Set up Python (oldest supported version) - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.9" cache: 'pip' @@ -55,7 +55,7 @@ jobs: persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -99,7 +99,7 @@ jobs: run: touch requirements.txt - name: Set up Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' cache: 'pip' diff --git a/.github/workflows/_test_sslib_main.yml b/.github/workflows/_test_sslib_main.yml index 86b4d946b7..c8cf3107d9 100644 --- a/.github/workflows/_test_sslib_main.yml +++ b/.github/workflows/_test_sslib_main.yml @@ -16,7 +16,7 @@ jobs: persist-credentials: false - name: Set up Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' cache: 'pip' diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 623fae02b2..68ccb087b4 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -24,7 +24,7 @@ jobs: ref: ${{ github.event.workflow_run.head_branch }} - name: Set up Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' @@ -37,7 +37,7 @@ jobs: awk "/## $GITHUB_REF_NAME/{flag=1; next} /## v/{flag=0} flag" docs/CHANGELOG.md > changelog - name: Store build artifacts - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: build-artifacts path: | @@ -54,7 +54,7 @@ jobs: release_id: ${{ steps.gh-release.outputs.result }} steps: - name: Fetch build artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: build-artifacts @@ -96,7 +96,7 @@ jobs: id-token: write # to authenticate as Trusted Publisher to pypi.org steps: - name: Fetch build artifacts - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: build-artifacts diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 7940418b33..955c0c11b4 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif # sarif format required by upload-sarif action diff --git a/.github/workflows/specification-version-check.yml b/.github/workflows/specification-version-check.yml index 9fcd5b4f88..ed4f6bbe1f 100644 --- a/.github/workflows/specification-version-check.yml +++ b/.github/workflows/specification-version-check.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.x" - id: get-version diff --git a/docs/index.rst b/docs/index.rst index a158b70422..6a5b50d9bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,5 @@ -TUF Developer Documentation -=========================== +Python-TUF |version| Developer Documentation +======================================================================= This documentation provides essential information for those developing software with the `Python reference implementation of The Update Framework (TUF) diff --git a/pyproject.toml b/pyproject.toml index a5c24fc987..266b2188f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,7 +135,6 @@ disable_error_code = ["attr-defined"] [[tool.mypy.overrides]] module = [ "requests.*", - "securesystemslib.*", ] ignore_missing_imports = "True" diff --git a/requirements/lint.txt b/requirements/lint.txt index b4ae77b517..d93b602728 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -6,9 +6,9 @@ # Lint tools # (We are not so interested in the specific versions of the tools: the versions # are pinned to prevent unexpected linting failures when tools update) -ruff==0.9.10 -mypy==1.15.0 -zizmor==1.4.1 +ruff==0.11.12 +mypy==1.16.0 +zizmor==1.9.0 # Required for type stubs -freezegun==1.5.1 +freezegun==1.5.2 diff --git a/requirements/pinned.txt b/requirements/pinned.txt index d73f7fc7cc..464dd5c641 100644 --- a/requirements/pinned.txt +++ b/requirements/pinned.txt @@ -6,11 +6,11 @@ # cffi==1.17.1 # via cryptography -cryptography==44.0.2 +cryptography==45.0.3 # via securesystemslib pycparser==2.22 # via cffi -securesystemslib==1.2.0 +securesystemslib==1.3.0 # via -r requirements/main.txt -urllib3==2.3.0 +urllib3==2.4.0 # via -r requirements/main.txt diff --git a/requirements/test.txt b/requirements/test.txt index 6a54f92051..7a299e5a75 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,5 +4,5 @@ -r pinned.txt # coverage measurement -coverage[toml]==7.6.12 -freezegun==1.5.1 +coverage[toml]==7.8.2 +freezegun==1.5.2 diff --git a/tests/repository_simulator.py b/tests/repository_simulator.py index 637ba42a54..d0c50bc424 100644 --- a/tests/repository_simulator.py +++ b/tests/repository_simulator.py @@ -45,6 +45,7 @@ from __future__ import annotations import datetime +import hashlib import logging import os import tempfile @@ -52,7 +53,6 @@ from typing import TYPE_CHECKING from urllib import parse -import securesystemslib.hash as sslib_hash from securesystemslib.signer import CryptoSigner, Signer from tuf.api.exceptions import DownloadHTTPError @@ -80,6 +80,8 @@ SPEC_VER = ".".join(SPECIFICATION_VERSION) +_HASH_ALGORITHM = "sha256" + @dataclass class FetchTracker: @@ -292,9 +294,9 @@ def _compute_hashes_and_length( self, role: str ) -> tuple[dict[str, str], int]: data = self.fetch_metadata(role) - digest_object = sslib_hash.digest(sslib_hash.DEFAULT_HASH_ALGORITHM) + digest_object = hashlib.new(_HASH_ALGORITHM) digest_object.update(data) - hashes = {sslib_hash.DEFAULT_HASH_ALGORITHM: digest_object.hexdigest()} + hashes = {_HASH_ALGORITHM: digest_object.hexdigest()} return hashes, len(data) def update_timestamp(self) -> None: diff --git a/tests/test_api.py b/tests/test_api.py index 7b80d36041..dabf50c86c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -17,12 +17,12 @@ from typing import ClassVar from securesystemslib import exceptions as sslib_exceptions -from securesystemslib import hash as sslib_hash from securesystemslib.signer import ( CryptoSigner, Key, SecretsHandler, Signer, + SSlibKey, ) from tests import utils @@ -245,11 +245,11 @@ class FailingSigner(Signer): @classmethod def from_priv_key_uri( cls, - priv_key_uri: str, - public_key: Key, - secrets_handler: SecretsHandler | None = None, + _priv_key_uri: str, + _public_key: Key, + _secrets_handler: SecretsHandler | None = None, ) -> Signer: - pass + raise RuntimeError("Not a real signer") @property def public_key(self) -> Key: @@ -470,43 +470,45 @@ def test_signed_verify_delegate(self) -> None: ) def test_verification_result(self) -> None: - vr = VerificationResult(3, {"a": None}, {"b": None}) + key = SSlibKey("", "", "", {"public": ""}) + vr = VerificationResult(3, {"a": key}, {"b": key}) self.assertEqual(vr.missing, 2) self.assertFalse(vr.verified) self.assertFalse(vr) # Add a signature - vr.signed["c"] = None + vr.signed["c"] = key self.assertEqual(vr.missing, 1) self.assertFalse(vr.verified) self.assertFalse(vr) # Add last missing signature - vr.signed["d"] = None + vr.signed["d"] = key self.assertEqual(vr.missing, 0) self.assertTrue(vr.verified) self.assertTrue(vr) # Add one more signature - vr.signed["e"] = None + vr.signed["e"] = key self.assertEqual(vr.missing, 0) self.assertTrue(vr.verified) self.assertTrue(vr) def test_root_verification_result(self) -> None: - vr1 = VerificationResult(3, {"a": None}, {"b": None}) - vr2 = VerificationResult(1, {"c": None}, {"b": None}) + key = SSlibKey("", "", "", {"public": ""}) + vr1 = VerificationResult(3, {"a": key}, {"b": key}) + vr2 = VerificationResult(1, {"c": key}, {"b": key}) vr = RootVerificationResult(vr1, vr2) - self.assertEqual(vr.signed, {"a": None, "c": None}) - self.assertEqual(vr.unsigned, {"b": None}) + self.assertEqual(vr.signed, {"a": key, "c": key}) + self.assertEqual(vr.unsigned, {"b": key}) self.assertFalse(vr.verified) self.assertFalse(vr) - vr1.signed["c"] = None - vr1.signed["f"] = None - self.assertEqual(vr.signed, {"a": None, "c": None, "f": None}) - self.assertEqual(vr.unsigned, {"b": None}) + vr1.signed["c"] = key + vr1.signed["f"] = key + self.assertEqual(vr.signed, {"a": key, "c": key, "f": key}) + self.assertEqual(vr.unsigned, {"b": key}) self.assertTrue(vr.verified) self.assertTrue(vr) @@ -679,7 +681,7 @@ def test_root_add_key_and_revoke_key(self) -> None: # Assert that add_key with old argument order will raise an error with self.assertRaises(ValueError): - root.signed.add_key(Root.type, key) + root.signed.add_key(Root.type, key) # type: ignore [arg-type] # Add new root key root.signed.add_key(key, Root.type) @@ -779,7 +781,7 @@ def test_targets_key_api(self) -> None: # Assert that add_key with old argument order will raise an error with self.assertRaises(ValueError): - targets.add_key("role1", key) + targets.add_key(Root.type, key) # type: ignore [arg-type] # Assert that delegated role "role1" does not contain the new key self.assertNotIn(key.keyid, targets.delegations.roles["role1"].keyids) @@ -896,6 +898,12 @@ def test_length_and_hash_validation(self) -> None: # test with data as bytes snapshot_metafile.verify_length_and_hashes(data) + # test with custom blake algorithm + snapshot_metafile.hashes = { + "blake2b-256": "963a3c31aad8e2a91cfc603fdba12555e48dd0312674ac48cce2c19c243236a1" + } + snapshot_metafile.verify_length_and_hashes(data) + # test exceptions expected_length = snapshot_metafile.length snapshot_metafile.length = 2345 @@ -958,9 +966,7 @@ def test_targetfile_from_file(self) -> None: # Test with a non-existing file file_path = os.path.join(self.repo_dir, Targets.type, "file123.txt") with self.assertRaises(FileNotFoundError): - TargetFile.from_file( - file_path, file_path, [sslib_hash.DEFAULT_HASH_ALGORITHM] - ) + TargetFile.from_file(file_path, file_path, ["sha256"]) # Test with an unsupported algorithm file_path = os.path.join(self.repo_dir, Targets.type, "file1.txt") @@ -990,6 +996,12 @@ def test_targetfile_from_data(self) -> None: targetfile_from_data = TargetFile.from_data(target_file_path, data) targetfile_from_data.verify_length_and_hashes(data) + # Test with custom blake hash algorithm + targetfile_from_data = TargetFile.from_data( + target_file_path, data, ["blake2b-256"] + ) + targetfile_from_data.verify_length_and_hashes(data) + def test_metafile_from_data(self) -> None: data = b"Inline test content" @@ -1013,6 +1025,10 @@ def test_metafile_from_data(self) -> None: ), ) + # Test with custom blake hash algorithm + metafile = MetaFile.from_data(1, data, ["blake2b-256"]) + metafile.verify_length_and_hashes(data) + def test_targetfile_get_prefixed_paths(self) -> None: target = TargetFile(100, {"sha256": "abc", "md5": "def"}, "a/b/f.ext") self.assertEqual( @@ -1165,7 +1181,7 @@ def test_serialization(self) -> None: self.assertEqual(metadata.signed, payload) def test_fail_envelope_serialization(self) -> None: - envelope = SimpleEnvelope(b"foo", "bar", ["baz"]) + envelope = SimpleEnvelope(b"foo", "bar", []) # type: ignore[arg-type] with self.assertRaises(SerializationError): envelope.to_bytes() @@ -1180,7 +1196,7 @@ def test_fail_payload_serialization(self) -> None: def test_fail_payload_deserialization(self) -> None: payloads = [b"[", b'{"_type": "foo"}'] for payload in payloads: - envelope = SimpleEnvelope(payload, "bar", []) + envelope = SimpleEnvelope(payload, "bar", {}) with self.assertRaises(DeserializationError): envelope.get_signed() diff --git a/tuf/api/_payload.py b/tuf/api/_payload.py index 56852082ea..89fcbfe812 100644 --- a/tuf/api/_payload.py +++ b/tuf/api/_payload.py @@ -8,8 +8,10 @@ import abc import fnmatch +import hashlib import io import logging +import sys from dataclasses import dataclass from datetime import datetime, timezone from typing import ( @@ -21,7 +23,6 @@ ) from securesystemslib import exceptions as sslib_exceptions -from securesystemslib import hash as sslib_hash from securesystemslib.signer import Key, Signature from tuf.api.exceptions import LengthOrHashMismatchError, UnsignedMetadataError @@ -34,6 +35,9 @@ _TARGETS = "targets" _TIMESTAMP = "timestamp" +_DEFAULT_HASH_ALGORITHM = "sha256" +_BLAKE_HASH_ALGORITHM = "blake2b-256" + # We aim to support SPECIFICATION_VERSION and require the input metadata # files to have the same major version (the first number) as ours. SPECIFICATION_VERSION = ["1", "0", "31"] @@ -45,6 +49,38 @@ T = TypeVar("T", "Root", "Timestamp", "Snapshot", "Targets") +def _get_digest(algo: str) -> Any: # noqa: ANN401 + """New digest helper to support custom "blake2b-256" algo name.""" + if algo == _BLAKE_HASH_ALGORITHM: + return hashlib.blake2b(digest_size=32) + + return hashlib.new(algo) + + +def _hash_bytes(data: bytes, algo: str) -> str: + """Returns hexdigest for data using algo.""" + digest = _get_digest(algo) + digest.update(data) + + return digest.hexdigest() + + +def _hash_file(f: IO[bytes], algo: str) -> str: + """Returns hexdigest for file using algo.""" + f.seek(0) + if sys.version_info >= (3, 11): + digest = hashlib.file_digest(f, lambda: _get_digest(algo)) # type: ignore[arg-type] + + else: + # Fallback for older Pythons. Chunk size is taken from the previously + # used and now deprecated `securesystemslib.hash.digest_fileobject`. + digest = _get_digest(algo) + for chunk in iter(lambda: f.read(4096), b""): + digest.update(chunk) + + return digest.hexdigest() + + class Signed(metaclass=abc.ABCMeta): """A base class for the signed part of TUF metadata. @@ -664,24 +700,18 @@ def _verify_hashes( data: bytes | IO[bytes], expected_hashes: dict[str, str] ) -> None: """Verify that the hash of ``data`` matches ``expected_hashes``.""" - is_bytes = isinstance(data, bytes) for algo, exp_hash in expected_hashes.items(): try: - if is_bytes: - digest_object = sslib_hash.digest(algo) - digest_object.update(data) + if isinstance(data, bytes): + observed_hash = _hash_bytes(data, algo) else: # if data is not bytes, assume it is a file object - digest_object = sslib_hash.digest_fileobject(data, algo) - except ( - sslib_exceptions.UnsupportedAlgorithmError, - sslib_exceptions.FormatError, - ) as e: + observed_hash = _hash_file(data, algo) + except (ValueError, TypeError) as e: raise LengthOrHashMismatchError( f"Unsupported algorithm '{algo}'" ) from e - observed_hash = digest_object.hexdigest() if observed_hash != exp_hash: raise LengthOrHashMismatchError( f"Observed hash {observed_hash} does not match " @@ -731,25 +761,17 @@ def _get_length_and_hashes( hashes = {} if hash_algorithms is None: - hash_algorithms = [sslib_hash.DEFAULT_HASH_ALGORITHM] + hash_algorithms = [_DEFAULT_HASH_ALGORITHM] for algorithm in hash_algorithms: try: if isinstance(data, bytes): - digest_object = sslib_hash.digest(algorithm) - digest_object.update(data) + hashes[algorithm] = _hash_bytes(data, algorithm) else: - digest_object = sslib_hash.digest_fileobject( - data, algorithm - ) - except ( - sslib_exceptions.UnsupportedAlgorithmError, - sslib_exceptions.FormatError, - ) as e: + hashes[algorithm] = _hash_file(data, algorithm) + except (ValueError, TypeError) as e: raise ValueError(f"Unsupported algorithm '{algorithm}'") from e - hashes[algorithm] = digest_object.hexdigest() - return (length, hashes) @@ -832,7 +854,7 @@ def from_data( version: Version of the metadata file. data: Metadata bytes that the metafile represents. hash_algorithms: Hash algorithms to create the hashes with. If not - specified, the securesystemslib default hash algorithm is used. + specified, "sha256" is used. Raises: ValueError: The hash algorithms list contains an unsupported @@ -1150,7 +1172,7 @@ def is_delegated_path(self, target_filepath: str) -> bool: if self.path_hash_prefixes is not None: # Calculate the hash of the filepath # to determine in which bin to find the target. - digest_object = sslib_hash.digest(algorithm="sha256") + digest_object = hashlib.new(name="sha256") digest_object.update(target_filepath.encode("utf-8")) target_filepath_hash = digest_object.hexdigest() @@ -1269,7 +1291,7 @@ def get_role_for_target(self, target_filepath: str) -> str: target_filepath: URL path to a target file, relative to a base targets URL. """ - hasher = sslib_hash.digest(algorithm="sha256") + hasher = hashlib.new(name="sha256") hasher.update(target_filepath.encode("utf-8")) # We can't ever need more than 4 bytes (32 bits). @@ -1542,7 +1564,7 @@ def from_file( targets URL. local_path: Local path to target file content. hash_algorithms: Hash algorithms to calculate hashes with. If not - specified the securesystemslib default hash algorithm is used. + specified, "sha256" is used. Raises: FileNotFoundError: The file doesn't exist. @@ -1566,7 +1588,7 @@ def from_data( targets URL. data: Target file content. hash_algorithms: Hash algorithms to create the hashes with. If not - specified the securesystemslib default hash algorithm is used. + specified, "sha256" is used. Raises: ValueError: The hash algorithms list contains an unsupported diff --git a/tuf/api/dsse.py b/tuf/api/dsse.py index 493fefd1d0..8f812d0741 100644 --- a/tuf/api/dsse.py +++ b/tuf/api/dsse.py @@ -81,7 +81,7 @@ def from_bytes(cls, data: bytes) -> SimpleEnvelope[T]: except Exception as e: raise DeserializationError from e - return cast(SimpleEnvelope[T], envelope) + return cast("SimpleEnvelope[T]", envelope) def to_bytes(self) -> bytes: """Return envelope as JSON bytes. @@ -150,4 +150,4 @@ def get_signed(self) -> T: except Exception as e: raise DeserializationError from e - return cast(T, inner_cls.from_dict(payload_dict)) + return cast("T", inner_cls.from_dict(payload_dict)) diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py index 76b5ce0fde..d03a501546 100644 --- a/tuf/api/metadata.py +++ b/tuf/api/metadata.py @@ -199,7 +199,7 @@ def from_dict(cls, metadata: dict[str, Any]) -> Metadata[T]: return cls( # Specific type T is not known at static type check time: use cast - signed=cast(T, inner_cls.from_dict(metadata.pop("signed"))), + signed=cast("T", inner_cls.from_dict(metadata.pop("signed"))), signatures=signatures, # All fields left in the metadata dict are unrecognized. unrecognized_fields=metadata, diff --git a/tuf/ngclient/_internal/trusted_metadata_set.py b/tuf/ngclient/_internal/trusted_metadata_set.py index 3678ddf3a1..179a65ed87 100644 --- a/tuf/ngclient/_internal/trusted_metadata_set.py +++ b/tuf/ngclient/_internal/trusted_metadata_set.py @@ -145,22 +145,22 @@ def __iter__(self) -> Iterator[Signed]: @property def root(self) -> Root: """Get current root.""" - return cast(Root, self._trusted_set[Root.type]) + return cast("Root", self._trusted_set[Root.type]) @property def timestamp(self) -> Timestamp: """Get current timestamp.""" - return cast(Timestamp, self._trusted_set[Timestamp.type]) + return cast("Timestamp", self._trusted_set[Timestamp.type]) @property def snapshot(self) -> Snapshot: """Get current snapshot.""" - return cast(Snapshot, self._trusted_set[Snapshot.type]) + return cast("Snapshot", self._trusted_set[Snapshot.type]) @property def targets(self) -> Targets: """Get current top-level targets.""" - return cast(Targets, self._trusted_set[Targets.type]) + return cast("Targets", self._trusted_set[Targets.type]) # Methods for updating metadata def update_root(self, data: bytes) -> Root: diff --git a/tuf/ngclient/updater.py b/tuf/ngclient/updater.py index 2504c86aa4..a98e799ce4 100644 --- a/tuf/ngclient/updater.py +++ b/tuf/ngclient/updater.py @@ -459,7 +459,7 @@ def _load_targets(self, role: str, parent_role: str) -> Targets: # Avoid loading 'role' more than once during "get_targetinfo" if role in self._trusted_set: - return cast(Targets, self._trusted_set[role]) + return cast("Targets", self._trusted_set[role]) try: data = self._load_local_metadata(role)