Merge branch 'sigstore:main' into offline-flag

emilejbm · web-flow · commit 31493a5356e2 · 2023-04-21T17:09:00.000-04:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -28,7 +28,7 @@ jobs:
           - { py: "3.11", os: "macos-latest" }
     runs-on: ${{ matrix.conf.os }}
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
 
       - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
         with:
@@ -82,7 +82,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.2.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.2.0
 
       - uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
         with:
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -18,7 +18,7 @@ jobs:
       id-token: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
 
       - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
         with:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -9,7 +9,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
 
       - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
         with:
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -10,7 +10,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
 
       # NOTE: We intentionally lint against our minimum supported Python.
       - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
@@ -28,7 +28,7 @@ jobs:
   check-readme:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
 
       # NOTE: We intentionally check `--help` rendering against our minimum Python,
       # since it changes slightly between Python versions.
@@ -47,7 +47,7 @@ jobs:
   licenses:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
       # adapted from Warehouse's bin/licenses
       - run: |
           for fn in $(find . -type f -name "*.py"); do
diff --git a/.github/workflows/pin-requirements.yml b/.github/workflows/pin-requirements.yml
@@ -30,7 +30,7 @@ jobs:
       contents: write # Branch creation for PR.
 
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.3.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.3.0
         with:
           ref: main
 
@@ -82,7 +82,7 @@ jobs:
       pull-requests: write # Pull Request creation.
 
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.3.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.3.0
         with:
           ref: ${{ env.SIGSTORE_NEW_BRANCH }}
 
@@ -92,7 +92,7 @@ jobs:
           git push -f origin "origin/main:${SIGSTORE_NEW_BRANCH}"
 
       - name: Open pull request
-        uses: peter-evans/create-pull-request@38e0b6e68b4c852a5500a94740f0e535e0d7ba54 # v4.2.4
+        uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5 # v5.0.0
         with:
           title: |
             Update pinned requirements for ${{ env.SIGSTORE_RELEASE_TAG }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -17,7 +17,7 @@ jobs:
     outputs:
       hashes: ${{ steps.hash.outputs.hashes }}
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
 
       - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
         with:
diff --git a/.github/workflows/requirements.yml b/.github/workflows/requirements.yml
@@ -31,7 +31,7 @@ jobs:
         run: |
           echo "SIGSTORE_REF=${{ github.ref }}" >> "${GITHUB_ENV}"
 
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
         with:
           ref: ${{ env.SIGSTORE_REF }}
 
diff --git a/.github/workflows/scorecards-analysis.yml b/.github/workflows/scorecards-analysis.yml
@@ -24,7 +24,7 @@ jobs:
       id-token: write
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+        uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
         with:
           persist-credentials: false
 
@@ -52,6 +52,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@04df1262e6247151b5ac09cd2c303ac36ad3f62b # v2.2.9
+        uses: github/codeql-action/upload-sarif@d186a2a36cc67bfa1b860e6170d37fb9634742c7 # v2.2.11
         with:
           sarif_file: results.sarif
diff --git a/.github/workflows/staging-tests.yml b/.github/workflows/staging-tests.yml
@@ -18,7 +18,7 @@ jobs:
   staging-tests:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+      - uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 # v3.5.1
 
       - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
         with:
diff --git a/pyproject.toml b/pyproject.toml
@@ -64,7 +64,7 @@ lint = [
   "mypy ~= 1.1",
   # NOTE(ww): ruff is under active development, so we pin conservatively here
   # and let Dependabot periodically perform this update.
-  "ruff < 0.0.261",
+  "ruff < 0.0.262",
   "types-requests",
   # Needed for protocol typing in 3.7; remove when our minimum Python is 3.8.
   "typing-extensions; python_version < '3.8'",
diff --git a/sigstore/_internal/tuf.py b/sigstore/_internal/tuf.py
@@ -24,7 +24,7 @@
 from datetime import datetime, timezone
 from functools import lru_cache
 from pathlib import Path
-from typing import Optional
+from typing import Iterable, Optional
 from urllib import parse
 
 import appdirs
@@ -34,7 +34,11 @@
     load_pem_x509_certificate,
 )
 from sigstore_protobuf_specs.dev.sigstore.common.v1 import TimeRange
-from sigstore_protobuf_specs.dev.sigstore.trustroot.v1 import TrustedRoot
+from sigstore_protobuf_specs.dev.sigstore.trustroot.v1 import (
+    CertificateAuthority,
+    TransparencyLogInstance,
+    TrustedRoot,
+)
 from tuf.api import exceptions as TUFExceptions
 from tuf.ngclient import RequestsFetcher, Updater, config
 
@@ -43,8 +47,8 @@
 
 logger = logging.getLogger(__name__)
 
-DEFAULT_TUF_URL = "https://sigstore-tuf-root.storage.googleapis.com/"
-STAGING_TUF_URL = "https://tuf-root-staging.storage.googleapis.com/"
+DEFAULT_TUF_URL = "https://tuf-repo-cdn.sigstore.dev"
+STAGING_TUF_URL = "https://tuf-repo-cdn.sigstage.dev"
 
 
 @lru_cache()
@@ -201,11 +205,11 @@ def _get_trusted_root(self) -> Optional[TrustedRoot]:
             ) as e:
                 raise TUFError("Failed to download trusted key bundle") from e
 
+        logger.debug("Found trusted root")
         return TrustedRoot().from_json(Path(path).read_bytes())
 
-    def _get(self, usage: str, statuses: list[str]) -> list[bytes]:
+    def _get(self, usage: str, statuses: list[str]) -> Iterable[bytes]:
         """Return all targets with given usage and any of the statuses"""
-        data = []
 
         targets = self._updater()._trusted_set.targets.signed.targets
         for target_info in targets.values():
@@ -234,29 +238,41 @@ def _get(self, usage: str, statuses: list[str]) -> list[bytes]:
                         f"TUF cache target {base_name}:\n"
                         f"{target_contents.decode('utf-8')}"
                     )
-                    data.append(target_contents)
+                    yield target_contents
+
+    def _get_tlog_keys(self, tlogs: list[TransparencyLogInstance]) -> Iterable[bytes]:
+        """Return public key contents given transparency log instances."""
+
+        for key in tlogs:
+            if not _is_timerange_valid(key.public_key.valid_for, allow_expired=False):
+                continue
+            key_bytes = key.public_key.raw_bytes
+            if key_bytes:
+                yield key_bytes
 
-        return data
+    def _get_ca_keys(
+        self, cas: list[CertificateAuthority], *, allow_expired: bool
+    ) -> Iterable[bytes]:
+        """Return public key contents given certificate authorities."""
+
+        for ca in cas:
+            if not _is_timerange_valid(ca.valid_for, allow_expired=allow_expired):
+                continue
+            for cert in ca.cert_chain.certificates:
+                yield cert.raw_bytes
 
     def get_ctfe_keys(self) -> list[bytes]:
         """Return the active CTFE public keys contents.
 
         May download files from the remote repository.
         """
-        ctfes = []
+        ctfes: list[bytes]
 
         trusted_root = self._get_trusted_root()
         if trusted_root:
-            for key in trusted_root.ctlogs:
-                if not _is_timerange_valid(
-                    key.public_key.valid_for, allow_expired=False
-                ):
-                    continue
-                key_bytes = key.public_key.raw_bytes
-                if key_bytes:
-                    ctfes.append(key_bytes)
+            ctfes = list(self._get_tlog_keys(trusted_root.ctlogs))
         else:
-            ctfes = self._get("CTFE", ["Active"])
+            ctfes = list(self._get("CTFE", ["Active"]))
 
         if not ctfes:
             raise MetadataError("CTFE keys not found in TUF metadata")
@@ -267,20 +283,13 @@ def get_rekor_keys(self) -> list[bytes]:
 
         May download files from the remote repository.
         """
-        keys = []
+        keys: list[bytes]
 
         trusted_root = self._get_trusted_root()
         if trusted_root:
-            for key in trusted_root.tlogs:
-                if not _is_timerange_valid(
-                    key.public_key.valid_for, allow_expired=False
-                ):
-                    continue
-                key_bytes = key.public_key.raw_bytes
-                if key_bytes:
-                    keys.append(key_bytes)
+            keys = list(self._get_tlog_keys(trusted_root.tlogs))
         else:
-            keys = self._get("Rekor", ["Active"])
+            keys = list(self._get("Rekor", ["Active"]))
 
         if len(keys) != 1:
             raise MetadataError("Did not find one active Rekor key in TUF metadata")
@@ -291,21 +300,18 @@ def get_fulcio_certs(self) -> list[Certificate]:
 
         May download files from the remote repository.
         """
-        certs = []
+        certs: list[Certificate]
 
         trusted_root = self._get_trusted_root()
         # Return expired certificates too: they are expired now but may have
         # been active when the certificate was used to sign.
         if trusted_root:
-            for ca in trusted_root.certificate_authorities:
-                if not _is_timerange_valid(ca.valid_for, allow_expired=True):
-                    continue
-                certs.extend(
-                    [
-                        load_der_x509_certificate(cert.raw_bytes)
-                        for cert in ca.cert_chain.certificates
-                    ]
+            certs = [
+                load_der_x509_certificate(c)
+                for c in self._get_ca_keys(
+                    trusted_root.certificate_authorities, allow_expired=True
                 )
+            ]
         else:
             certs = [
                 load_pem_x509_certificate(c)
diff --git a/test/unit/conftest.py b/test/unit/conftest.py
@@ -14,6 +14,7 @@
 
 import base64
 import os
+import re
 from collections import defaultdict
 from io import BytesIO
 from pathlib import Path
@@ -105,10 +106,30 @@ def _asset(name: str) -> Path:
 
 @pytest.fixture
 def tuf_asset():
-    def _tuf_asset(name: str) -> Path:
-        return _TUF_ASSETS / name
+    SHA256_TARGET_PATTERN = re.compile(r"[0-9a-f]{64}\.")
+
+    class TUFAsset:
+        def asset(self, name: str):
+            return (_TUF_ASSETS / name).read_bytes()
+
+        def target(self, name: str):
+            # Since TUF contains both sha256 and sha512 prefixed targets, filter
+            # out the sha512 ones.
+            matches = filter(
+                lambda path: SHA256_TARGET_PATTERN.match(path.name) is not None,
+                (_TUF_ASSETS / "targets").glob(f"*.{name}"),
+            )
+
+            try:
+                path = next(matches)
+            except StopIteration as e:
+                raise Exception(f"Unable to match {name} in targets/") from e
+
+            if next(matches, None) is None:
+                return path.read_bytes()
+            return None
 
-    return _tuf_asset
+    return TUFAsset()
 
 
 @pytest.fixture
@@ -174,8 +195,6 @@ def _fetch(self, url: str) -> Iterator[bytes]:
             if filepath.is_file():
                 success[filepath] += 1
                 return BytesIO(filepath.read_bytes())
-            else:
-                print(f"POO {filepath}")
 
             failure[filepath] += 1
             raise DownloadHTTPError("File not found", 404)
diff --git a/test/unit/internal/test_tuf.py b/test/unit/internal/test_tuf.py
