diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..095f9ec7a --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: 2023 spdx contributors +# +# SPDX-License-Identifier: Apache-2.0 +name: Generate API docs + +on: + push: + branches: + - main + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install dependencies + run: | + sudo apt-get install graphviz-dev + pip install -e ".[test,graph_generation]" + pip install pdoc + - name: Generate docs + run: pdoc spdx_tools -o docs/ + - name: Upload docs as artifact + uses: actions/upload-pages-artifact@v1 + with: + path: docs/ + + deploy: + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + name: Deploy docs to GitHub pages + uses: actions/deploy-pages@v2 diff --git a/README.md b/README.md index 8a0ee72e1..9516b5de2 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ This library implements SPDX parsers, convertors, validators and handlers in Pyt - Home: https://github.com/spdx/tools-python - Issues: https://github.com/spdx/tools-python/issues - PyPI: https://pypi.python.org/pypi/spdx-tools +- Browse the API: https://spdx.github.io/tools-python # License diff --git a/src/spdx_tools/spdx/parser/actor_parser.py b/src/spdx_tools/spdx/parser/actor_parser.py index 734b41386..14cc4ffb0 100644 --- a/src/spdx_tools/spdx/parser/actor_parser.py +++ b/src/spdx_tools/spdx/parser/actor_parser.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import re -from beartype.typing import Match, Optional, Pattern +from beartype.typing import Match, Pattern from spdx_tools.spdx.model import Actor, ActorType from spdx_tools.spdx.parser.error import SPDXParsingError @@ -14,8 +14,8 @@ class ActorParser: @staticmethod def parse_actor(actor: str) -> Actor: tool_re: Pattern = re.compile(r"^Tool:\s*(.+)", re.UNICODE) - person_re: Pattern = re.compile(r"^Person:\s*(([^(])+)(\((.*)\))?", re.UNICODE) - org_re: Pattern = re.compile(r"^Organization:\s*(([^(])+)(\((.*)\))?", re.UNICODE) + person_re: Pattern = re.compile(r"^Person:\s*(?:(.*)\((.*)\)|(.*))$", re.UNICODE) + org_re: Pattern = re.compile(r"^Organization:\s*(?:(.*)\((.*)\)|(.*))$", re.UNICODE) tool_match: Match = tool_re.match(actor) person_match: Match = person_re.match(actor) org_match: Match = org_re.match(actor) @@ -24,34 +24,30 @@ def parse_actor(actor: str) -> Actor: name: str = tool_match.group(1).strip() if not name: raise SPDXParsingError([f"No name for Tool provided: {actor}."]) - creator = construct_or_raise_parsing_error(Actor, dict(actor_type=ActorType.TOOL, name=name)) + return construct_or_raise_parsing_error(Actor, dict(actor_type=ActorType.TOOL, name=name)) - elif person_match: - name: str = person_match.group(1).strip() - if not name: - raise SPDXParsingError([f"No name for Person provided: {actor}."]) - email: Optional[str] = ActorParser.get_email_or_none(person_match) - creator = construct_or_raise_parsing_error( - Actor, dict(actor_type=ActorType.PERSON, name=name, email=email) - ) + if person_match: + actor_type = ActorType.PERSON + match = person_match elif org_match: - name: str = org_match.group(1).strip() - if not name: - raise SPDXParsingError([f"No name for Organization provided: {actor}."]) - email: Optional[str] = ActorParser.get_email_or_none(org_match) - creator = construct_or_raise_parsing_error( - Actor, dict(actor_type=ActorType.ORGANIZATION, name=name, email=email) - ) + actor_type = ActorType.ORGANIZATION + match = org_match else: raise SPDXParsingError([f"Actor {actor} doesn't match any of person, organization or tool."]) - return creator - - @staticmethod - def get_email_or_none(match: Match) -> Optional[str]: - email_match = match.group(4) - if email_match and email_match.strip(): - email = email_match.strip() + if match.group(3): + return construct_or_raise_parsing_error( + Actor, dict(actor_type=actor_type, name=match.group(3).strip(), email=None) + ) else: - email = None - return email + name = match.group(1) + if not name: + raise SPDXParsingError([f"No name for Actor provided: {actor}."]) + else: + name = name.strip() + + email = match.group(2).strip() + + return construct_or_raise_parsing_error( + Actor, dict(actor_type=actor_type, name=name, email=email if email else None) + ) diff --git a/src/spdx_tools/spdx/spdx_element_utils.py b/src/spdx_tools/spdx/spdx_element_utils.py index 49b466144..c3cb3f7fa 100644 --- a/src/spdx_tools/spdx/spdx_element_utils.py +++ b/src/spdx_tools/spdx/spdx_element_utils.py @@ -1,9 +1,18 @@ # SPDX-FileCopyrightText: 2022 spdx contributors # # SPDX-License-Identifier: Apache-2.0 +import hashlib + from beartype.typing import List, Union -from spdx_tools.spdx.model import ExternalDocumentRef, File, Package, Snippet +from spdx_tools.spdx.model import ( + ChecksumAlgorithm, + ExternalDocumentRef, + File, + Package, + PackageVerificationCode, + Snippet, +) def get_full_element_spdx_id( @@ -29,3 +38,41 @@ def get_full_element_spdx_id( raise ValueError(f"external id {external_id} not found in external document references") return external_uri + "#" + local_id + + +def calculate_package_verification_code(files: List[File]) -> PackageVerificationCode: + list_of_file_hashes = [] + for file in files: + file_checksum_value = None + for checksum in file.checksums: + if checksum.algorithm == ChecksumAlgorithm.SHA1: + file_checksum_value = checksum.value + if not file_checksum_value: + try: + file_checksum_value = calculate_file_checksum(file.name, ChecksumAlgorithm.SHA1) + except FileNotFoundError: + raise FileNotFoundError( + f"Cannot calculate package verification code because the file '{file.name}' " + f"provides no SHA1 checksum and can't be found at the specified location." + ) + list_of_file_hashes.append(file_checksum_value) + + list_of_file_hashes.sort() + hasher = hashlib.new("sha1") + hasher.update("".join(list_of_file_hashes).encode("utf-8")) + value = hasher.hexdigest() + return PackageVerificationCode(value) + + +def calculate_file_checksum(file_name: str, hash_algorithm=ChecksumAlgorithm.SHA1) -> str: + BUFFER_SIZE = 65536 + + file_hash = hashlib.new(hash_algorithm.name.lower()) + with open(file_name, "rb") as file_handle: + while True: + data = file_handle.read(BUFFER_SIZE) + if not data: + break + file_hash.update(data) + + return file_hash.hexdigest() diff --git a/src/spdx_tools/spdx3/bump_from_spdx2/creation_info.py b/src/spdx_tools/spdx3/bump_from_spdx2/creation_info.py index 54ac666eb..914d12226 100644 --- a/src/spdx_tools/spdx3/bump_from_spdx2/creation_info.py +++ b/src/spdx_tools/spdx3/bump_from_spdx2/creation_info.py @@ -7,7 +7,7 @@ from spdx_tools.spdx3.bump_from_spdx2.actor import bump_actor from spdx_tools.spdx3.bump_from_spdx2.external_document_ref import bump_external_document_ref from spdx_tools.spdx3.bump_from_spdx2.message import print_missing_conversion -from spdx_tools.spdx3.model import CreationInfo, ProfileIdentifier, SpdxDocument +from spdx_tools.spdx3.model import CreationInfo, ProfileIdentifierType, SpdxDocument from spdx_tools.spdx3.payload import Payload from spdx_tools.spdx.model.actor import ActorType from spdx_tools.spdx.model.document import CreationInfo as Spdx2_CreationInfo @@ -40,7 +40,7 @@ def bump_creation_info(spdx2_creation_info: Spdx2_CreationInfo, payload: Payload spec_version=Version("3.0.0"), created=spdx2_creation_info.created, created_by=[], - profile=[ProfileIdentifier.CORE, ProfileIdentifier.SOFTWARE, ProfileIdentifier.LICENSING], + profile=[ProfileIdentifierType.CORE, ProfileIdentifierType.SOFTWARE, ProfileIdentifierType.LICENSING], data_license="https://spdx.org/licenses/" + spdx2_creation_info.data_license, ) diff --git a/src/spdx_tools/spdx3/model/__init__.py b/src/spdx_tools/spdx3/model/__init__.py index 9bdc62b36..8fab45e9e 100644 --- a/src/spdx_tools/spdx3/model/__init__.py +++ b/src/spdx_tools/spdx3/model/__init__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2023 spdx contributors # # SPDX-License-Identifier: Apache-2.0 -from spdx_tools.spdx3.model.profile_identifier import ProfileIdentifier +from spdx_tools.spdx3.model.profile_identifier import ProfileIdentifierType from spdx_tools.spdx3.model.creation_info import CreationInfo from spdx_tools.spdx3.model.integrity_method import IntegrityMethod from spdx_tools.spdx3.model.hash import Hash, HashAlgorithm diff --git a/src/spdx_tools/spdx3/model/creation_info.py b/src/spdx_tools/spdx3/model/creation_info.py index 71b80f1a5..125d4d30d 100644 --- a/src/spdx_tools/spdx3/model/creation_info.py +++ b/src/spdx_tools/spdx3/model/creation_info.py @@ -9,7 +9,7 @@ from spdx_tools.common.typing.dataclass_with_properties import dataclass_with_properties from spdx_tools.common.typing.type_checks import check_types_and_set_values -from spdx_tools.spdx3.model import ProfileIdentifier +from spdx_tools.spdx3.model import ProfileIdentifierType @dataclass_with_properties @@ -17,7 +17,7 @@ class CreationInfo: spec_version: Version created: datetime created_by: List[str] # SPDXID of Agents - profile: List[ProfileIdentifier] + profile: List[ProfileIdentifierType] data_license: Optional[str] = "CC0-1.0" created_using: List[str] = field(default_factory=list) # SPDXID of Tools comment: Optional[str] = None @@ -27,7 +27,7 @@ def __init__( spec_version: Version, created: datetime, created_by: List[str], - profile: List[ProfileIdentifier], + profile: List[ProfileIdentifierType], data_license: Optional[str] = "CC0-1.0", created_using: List[str] = None, comment: Optional[str] = None, diff --git a/src/spdx_tools/spdx3/model/profile_identifier.py b/src/spdx_tools/spdx3/model/profile_identifier.py index 7295abfcb..40fe7ac41 100644 --- a/src/spdx_tools/spdx3/model/profile_identifier.py +++ b/src/spdx_tools/spdx3/model/profile_identifier.py @@ -4,7 +4,7 @@ from enum import Enum, auto -class ProfileIdentifier(Enum): +class ProfileIdentifierType(Enum): CORE = auto() SOFTWARE = auto() LICENSING = auto() diff --git a/tests/spdx/parser/tagvalue/test_annotation_parser.py b/tests/spdx/parser/tagvalue/test_annotation_parser.py index 204756187..629fe72e9 100644 --- a/tests/spdx/parser/tagvalue/test_annotation_parser.py +++ b/tests/spdx/parser/tagvalue/test_annotation_parser.py @@ -57,7 +57,7 @@ def test_parse_annotation(): "not match specified grammar rule. Line: 1', 'Error while parsing " "AnnotationDate: Token did not match specified grammar rule. Line: 2']", ), - ("Annotator: Person: ()", "Error while parsing Annotation: [['No name for Person provided: Person: ().']]"), + ("Annotator: Person: ()", "Error while parsing Annotation: [['No name for Actor provided: Person: ().']]"), ( "AnnotationType: REVIEW", "Element Annotation is not the current element in scope, probably the " diff --git a/tests/spdx/test_actor_parser.py b/tests/spdx/test_actor_parser.py index 17d12a296..24002794c 100644 --- a/tests/spdx/test_actor_parser.py +++ b/tests/spdx/test_actor_parser.py @@ -21,7 +21,16 @@ "organization@example.com", ), ("Organization: Example organization ( )", ActorType.ORGANIZATION, "Example organization", None), + ("Person: Example person ()", ActorType.PERSON, "Example person", None), + ("Person: Example person ", ActorType.PERSON, "Example person", None), ("Tool: Example tool ", ActorType.TOOL, "Example tool", None), + ("Tool: Example tool (email@mail.com)", ActorType.TOOL, "Example tool (email@mail.com)", None), + ( + "Organization: (c) Chris Sainty (chris@sainty.com)", + ActorType.ORGANIZATION, + "(c) Chris Sainty", + "chris@sainty.com", + ), ], ) def test_parse_actor(actor_string, expected_type, expected_name, expected_mail): @@ -42,6 +51,8 @@ def test_parse_actor(actor_string, expected_type, expected_name, expected_mail): ["Actor Perso: Jane Doe (jane.doe@example.com) doesn't match any of person, organization or tool."], ), ("Toole Example Tool ()", ["Actor Toole Example Tool () doesn't match any of person, organization or tool."]), + ("Organization:", ["No name for Actor provided: Organization:."]), + ("Person: ( )", ["No name for Actor provided: Person: ( )."]), ], ) def test_parse_invalid_actor(actor_string, expected_message): diff --git a/tests/spdx/test_checksum_calculation.py b/tests/spdx/test_checksum_calculation.py new file mode 100644 index 000000000..0e45b11c9 --- /dev/null +++ b/tests/spdx/test_checksum_calculation.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2023 spdx contributors +# +# SPDX-License-Identifier: Apache-2.0 +import pytest + +from spdx_tools.spdx.model import Checksum, ChecksumAlgorithm, File, PackageVerificationCode +from spdx_tools.spdx.spdx_element_utils import calculate_file_checksum, calculate_package_verification_code + + +@pytest.fixture +def generate_test_files(tmp_path): + file_path_1 = tmp_path.joinpath("file1") + file_path_2 = tmp_path.joinpath("file2") + + with open(file_path_1, "wb") as file: + file.write(bytes(111)) + with open(file_path_2, "wb") as file: + file.write(bytes(222)) + + yield str(file_path_1), str(file_path_2) + + +def test_file_checksum_calculation(generate_test_files): + filepath1, filepath2 = generate_test_files + checksum = calculate_file_checksum(filepath1, ChecksumAlgorithm.SHA1) + assert checksum == "dd90903d2f566a3922979dd5e18378a075c7ed33" + checksum = calculate_file_checksum(filepath2, ChecksumAlgorithm.SHA1) + assert checksum == "140dc52658e2eeee3fdc4d471cce84fec7253fe3" + + +def test_verification_code_calculation_with_predefined_checksums(generate_test_files): + filepath1, filepath2 = generate_test_files + file1 = File( + filepath1, + "SPDXRef-hello", + [Checksum(ChecksumAlgorithm.SHA1, "20862a6d08391d07d09344029533ec644fac6b21")], + ) + file2 = File( + filepath2, + "SPDXRef-Makefile", + [Checksum(ChecksumAlgorithm.SHA1, "69a2e85696fff1865c3f0686d6c3824b59915c80")], + ) + verification_code = calculate_package_verification_code([file1, file2]) + + assert verification_code == PackageVerificationCode("c6cb0949d7cd7439fce8690262a0946374824639") + + +def test_verification_code_calculation_with_calculated_checksums(generate_test_files): + filepath1, filepath2 = generate_test_files + file1 = File( + filepath1, + "SPDXRef-hello", + [Checksum(ChecksumAlgorithm.MD4, "20862a6d08391d07d09344029533ec644fac6b21")], + ) + file2 = File( + filepath2, + "SPDXRef-Makefile", + [Checksum(ChecksumAlgorithm.MD4, "69a2e85696fff1865c3f0686d6c3824b59915c80")], + ) + verification_code = calculate_package_verification_code([file1, file2]) + + assert verification_code == PackageVerificationCode("6f29d813abb63ee52a47dbcb691ea2e70f956328") + + +def test_verification_code_calculation_with_wrong_file_location(): + unknown_file_name = "./unknown_file_name" + file1 = File( + unknown_file_name, + "SPDXRef-unknown", + [Checksum(ChecksumAlgorithm.MD4, "20862a6d08391d07d09344029533ec644fac6b21")], + ) + + with pytest.raises(FileNotFoundError) as err: + calculate_package_verification_code([file1]) + + assert unknown_file_name in str(err.value) diff --git a/tests/spdx3/bump/test_actor_bump.py b/tests/spdx3/bump/test_actor_bump.py index c0a4a2af0..e6606134e 100644 --- a/tests/spdx3/bump/test_actor_bump.py +++ b/tests/spdx3/bump/test_actor_bump.py @@ -13,7 +13,7 @@ ExternalIdentifierType, Organization, Person, - ProfileIdentifier, + ProfileIdentifierType, Tool, ) from spdx_tools.spdx3.payload import Payload @@ -37,7 +37,7 @@ def test_bump_actor(actor_type, actor_name, actor_mail, element_type, new_spdx_id): payload = Payload() document_namespace = "https://doc.namespace" - creation_info = CreationInfo(Version("3.0.0"), datetime(2022, 1, 1), ["Creator"], [ProfileIdentifier.CORE]) + creation_info = CreationInfo(Version("3.0.0"), datetime(2022, 1, 1), ["Creator"], [ProfileIdentifierType.CORE]) actor = Actor(actor_type, actor_name, actor_mail) agent_or_tool_id = bump_actor(actor, payload, document_namespace, creation_info) @@ -54,8 +54,8 @@ def test_bump_actor(actor_type, actor_name, actor_mail, element_type, new_spdx_i def test_bump_actor_that_already_exists(): - creation_info_old = CreationInfo(Version("3.0.0"), datetime(2022, 1, 1), ["Creator"], [ProfileIdentifier.CORE]) - creation_info_new = CreationInfo(Version("3.0.0"), datetime(2023, 2, 2), ["Creator"], [ProfileIdentifier.CORE]) + creation_info_old = CreationInfo(Version("3.0.0"), datetime(2022, 1, 1), ["Creator"], [ProfileIdentifierType.CORE]) + creation_info_new = CreationInfo(Version("3.0.0"), datetime(2023, 2, 2), ["Creator"], [ProfileIdentifierType.CORE]) name = "some name" document_namespace = "https://doc.namespace" diff --git a/tests/spdx3/fixtures.py b/tests/spdx3/fixtures.py index 6c7e9db34..9d7e5fd24 100644 --- a/tests/spdx3/fixtures.py +++ b/tests/spdx3/fixtures.py @@ -25,7 +25,7 @@ NamespaceMap, Organization, Person, - ProfileIdentifier, + ProfileIdentifierType, Relationship, RelationshipCompleteness, RelationshipType, @@ -93,7 +93,7 @@ def creation_info_fixture( ["https://spdx.test/tools-python/creation_info_created_using"] if created_using is None else created_using ) profile = ( - [ProfileIdentifier.CORE, ProfileIdentifier.SOFTWARE, ProfileIdentifier.LICENSING] + [ProfileIdentifierType.CORE, ProfileIdentifierType.SOFTWARE, ProfileIdentifierType.LICENSING] if profile is None else profile ) diff --git a/tests/spdx3/model/test_creation_info.py b/tests/spdx3/model/test_creation_info.py index 40b6d4536..03b2fad58 100644 --- a/tests/spdx3/model/test_creation_info.py +++ b/tests/spdx3/model/test_creation_info.py @@ -6,7 +6,7 @@ import pytest from semantic_version import Version -from spdx_tools.spdx3.model import CreationInfo, ProfileIdentifier +from spdx_tools.spdx3.model import CreationInfo, ProfileIdentifierType from tests.spdx3.fixtures import creation_info_fixture from tests.spdx3.model.model_test_utils import get_property_names @@ -22,9 +22,9 @@ def test_correct_initialization(): assert creation_info.created_by == ["https://spdx.test/tools-python/creation_info_created_by"] assert creation_info.created_using == ["https://spdx.test/tools-python/creation_info_created_using"] assert creation_info.profile == [ - ProfileIdentifier.CORE, - ProfileIdentifier.SOFTWARE, - ProfileIdentifier.LICENSING, + ProfileIdentifierType.CORE, + ProfileIdentifierType.SOFTWARE, + ProfileIdentifierType.LICENSING, ] assert creation_info.data_license == "CC0-1.0" assert creation_info.comment == "creationInfoComment"