diff --git a/docs/cli-usage.rst b/docs/cli-usage.rst index 983b3e7f4..1a80bbc79 100644 --- a/docs/cli-usage.rst +++ b/docs/cli-usage.rst @@ -319,6 +319,20 @@ Delete a specific project package by id: $ gitlab -v project-package delete --id 1 --project-id 3 +Upload a generic package to a project: + +.. code-block:: console + + $ gitlab generic-package upload --project-id 1 --package-name hello-world \ + --package-version v1.0.0 --file-name hello.tar.gz --path /path/to/hello.tar.gz + +Download a project's generic package: + +.. code-block:: console + + $ gitlab generic-package download --project-id 1 --package-name hello-world \ + --package-version v1.0.0 --file-name hello.tar.gz > /path/to/hello.tar.gz + Get a list of issues for this project: .. code-block:: console diff --git a/docs/gl_objects/packages.rst b/docs/gl_objects/packages.rst index 60c4436d8..cc64e076c 100644 --- a/docs/gl_objects/packages.rst +++ b/docs/gl_objects/packages.rst @@ -3,7 +3,7 @@ Packages ######## Packages allow you to utilize GitLab as a private repository for a variety -of common package managers. +of common package managers, as well as GitLab's generic package registry. Project Packages ===================== @@ -88,3 +88,44 @@ List package files for package in project:: package = project.packages.get(1) package_files = package.package_files.list() + +Generic Packages +================ + +You can use python-gitlab to upload and download generic packages. + +Reference +--------- + +* v4 API: + + + :class:`gitlab.v4.objects.GenericPackage` + + :class:`gitlab.v4.objects.GenericPackageManager` + + :attr:`gitlab.v4.objects.Project.generic_packages` + +* GitLab API: https://docs.gitlab.com/ee/user/packages/generic_packages + +Examples +-------- + +Upload a generic package to a project:: + + project = gl.projects.get(1, lazy=True) + package = project.generic_packages.upload( + package_name="hello-world", + package_version="v1.0.0", + file_name="hello.tar.gz", + path="/path/to/local/hello.tar.gz" + ) + +Download a project's generic package:: + + project = gl.projects.get(1, lazy=True) + package = project.generic_packages.download( + package_name="hello-world", + package_version="v1.0.0", + file_name="hello.tar.gz", + ) + +.. hint:: You can use the Packages API described above to find packages and + retrieve the metadata you need download them. diff --git a/gitlab/client.py b/gitlab/client.py index fa9a394f3..e00f77688 100644 --- a/gitlab/client.py +++ b/gitlab/client.py @@ -393,15 +393,9 @@ def enable_debug(self) -> None: requests_log.setLevel(logging.DEBUG) requests_log.propagate = True - def _create_headers(self, content_type: Optional[str] = None) -> Dict[str, Any]: - request_headers = self.headers.copy() - if content_type is not None: - request_headers["Content-type"] = content_type - return request_headers - - def _get_session_opts(self, content_type: str) -> Dict[str, Any]: + def _get_session_opts(self) -> Dict[str, Any]: return { - "headers": self._create_headers(content_type), + "headers": self.headers.copy(), "auth": self._http_auth, "timeout": self.timeout, "verify": self.ssl_verify, @@ -441,12 +435,43 @@ def _check_redirects(self, result: requests.Response) -> None: if location and location.startswith("https://"): raise gitlab.exceptions.RedirectError(REDIRECT_MSG) + def _prepare_send_data( + self, + files: Optional[Dict[str, Any]] = None, + post_data: Optional[Dict[str, Any]] = None, + raw: bool = False, + ) -> Tuple[ + Optional[Dict[str, Any]], + Optional[Union[Dict[str, Any], MultipartEncoder]], + str, + ]: + if files: + if post_data is None: + post_data = {} + else: + # booleans does not exists for data (neither for MultipartEncoder): + # cast to string int to avoid: 'bool' object has no attribute 'encode' + for k, v in post_data.items(): + if isinstance(v, bool): + post_data[k] = str(int(v)) + post_data["file"] = files.get("file") + post_data["avatar"] = files.get("avatar") + + data = MultipartEncoder(post_data) + return (None, data, data.content_type) + + if raw and post_data: + return (None, post_data, "application/octet-stream") + + return (post_data, None, "application/json") + def http_request( self, verb: str, path: str, query_data: Optional[Dict[str, Any]] = None, post_data: Optional[Dict[str, Any]] = None, + raw: bool = False, streamed: bool = False, files: Optional[Dict[str, Any]] = None, timeout: Optional[float] = None, @@ -464,7 +489,8 @@ def http_request( 'http://whatever/v4/api/projecs') query_data (dict): Data to send as query parameters post_data (dict): Data to send in the body (will be converted to - json) + json by default) + raw (bool): If True, do not convert post_data to json streamed (bool): Whether the data should be streamed files (dict): The files to send to the server timeout (float): The timeout, in seconds, for the request @@ -503,7 +529,7 @@ def http_request( else: utils.copy_dict(params, kwargs) - opts = self._get_session_opts(content_type="application/json") + opts = self._get_session_opts() verify = opts.pop("verify") opts_timeout = opts.pop("timeout") @@ -512,23 +538,8 @@ def http_request( timeout = opts_timeout # We need to deal with json vs. data when uploading files - if files: - json = None - if post_data is None: - post_data = {} - else: - # booleans does not exists for data (neither for MultipartEncoder): - # cast to string int to avoid: 'bool' object has no attribute 'encode' - for k, v in post_data.items(): - if isinstance(v, bool): - post_data[k] = str(int(v)) - post_data["file"] = files.get("file") - post_data["avatar"] = files.get("avatar") - data = MultipartEncoder(post_data) - opts["headers"]["Content-type"] = data.content_type - else: - json = post_data - data = None + json, data, content_type = self._prepare_send_data(files, post_data, raw) + opts["headers"]["Content-type"] = content_type # Requests assumes that `.` should not be encoded as %2E and will make # changes to urls using this encoding. Using a prepped request we can @@ -683,6 +694,7 @@ def http_post( path: str, query_data: Optional[Dict[str, Any]] = None, post_data: Optional[Dict[str, Any]] = None, + raw: bool = False, files: Optional[Dict[str, Any]] = None, **kwargs: Any, ) -> Union[Dict[str, Any], requests.Response]: @@ -693,7 +705,8 @@ def http_post( 'http://whatever/v4/api/projecs') query_data (dict): Data to send as query parameters post_data (dict): Data to send in the body (will be converted to - json) + json by default) + raw (bool): If True, do not convert post_data to json files (dict): The files to send to the server **kwargs: Extra options to send to the server (e.g. sudo) @@ -730,6 +743,7 @@ def http_put( path: str, query_data: Optional[Dict[str, Any]] = None, post_data: Optional[Dict[str, Any]] = None, + raw: bool = False, files: Optional[Dict[str, Any]] = None, **kwargs: Any, ) -> Union[Dict[str, Any], requests.Response]: @@ -740,7 +754,8 @@ def http_put( 'http://whatever/v4/api/projecs') query_data (dict): Data to send as query parameters post_data (dict): Data to send in the body (will be converted to - json) + json by default) + raw (bool): If True, do not convert post_data to json files (dict): The files to send to the server **kwargs: Extra options to send to the server (e.g. sudo) @@ -760,6 +775,7 @@ def http_put( query_data=query_data, post_data=post_data, files=files, + raw=raw, **kwargs, ) try: diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index f5ca081c4..3e9d9f278 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -1,7 +1,17 @@ +from pathlib import Path +from typing import Any, Callable, Optional, TYPE_CHECKING, Union + +import requests + +from gitlab import cli +from gitlab import exceptions as exc +from gitlab import utils from gitlab.base import RESTManager, RESTObject from gitlab.mixins import DeleteMixin, GetMixin, ListMixin, ObjectDeleteMixin __all__ = [ + "GenericPackage", + "GenericPackageManager", "GroupPackage", "GroupPackageManager", "ProjectPackage", @@ -11,6 +21,110 @@ ] +class GenericPackage(RESTObject): + _id_attr = "package_name" + + +class GenericPackageManager(RESTManager): + _path = "/projects/%(project_id)s/packages/generic" + _obj_cls = GenericPackage + _from_parent_attrs = {"project_id": "id"} + + @cli.register_custom_action( + "GenericPackageManager", + ("package_name", "package_version", "file_name", "path"), + ) + @exc.on_http_error(exc.GitlabUploadError) + def upload( + self, + package_name: str, + package_version: str, + file_name: str, + path: Union[str, Path], + **kwargs, + ) -> GenericPackage: + """Upload a file as a generic package. + + Args: + package_name (str): The package name. Must follow generic package + name regex rules + package_version (str): The package version. Must follow semantic + version regex rules + file_name (str): The name of the file as uploaded in the registry + path (str): The path to a local file to upload + + Raises: + GitlabConnectionError: If the server cannot be reached + GitlabUploadError: If the file upload fails + GitlabUploadError: If ``filepath`` cannot be read + + Returns: + GenericPackage: An object storing the metadata of the uploaded package. + """ + + try: + with open(path, "rb") as f: + file_data = f.read() + except OSError: + raise exc.GitlabUploadError(f"Failed to read package file {path}") + + url = f"{self._computed_path}/{package_name}/{package_version}/{file_name}" + server_data = self.gitlab.http_put(url, post_data=file_data, raw=True, **kwargs) + + return self._obj_cls( + self, + attrs={ + "package_name": package_name, + "package_version": package_version, + "file_name": file_name, + "path": path, + "message": server_data["message"], + }, + ) + + @cli.register_custom_action( + "GenericPackageManager", + ("package_name", "package_version", "file_name"), + ) + @exc.on_http_error(exc.GitlabGetError) + def download( + self, + package_name: str, + package_version: str, + file_name: str, + streamed: bool = False, + action: Optional[Callable] = None, + chunk_size: int = 1024, + **kwargs: Any, + ) -> Optional[bytes]: + """Download a generic package. + + Args: + package_name (str): The package name. + package_version (str): The package version. + file_name (str): The name of the file in the registry + streamed (bool): If True the data will be processed by chunks of + `chunk_size` and each chunk is passed to `action` for + reatment + action (callable): Callable responsible of dealing with chunk of + data + chunk_size (int): Size of each chunk + **kwargs: Extra options to send to the server (e.g. sudo) + + Raises: + GitlabAuthenticationError: If authentication is not correct + GitlabGetError: If the server failed to perform the request + + Returns: + str: The package content if streamed is False, None otherwise + """ + path = f"{self._computed_path}/{package_name}/{package_version}/{file_name}" + result = self.gitlab.http_get(path, streamed=streamed, raw=True, **kwargs) + if TYPE_CHECKING: + assert isinstance(result, requests.Response) + return utils.response_content(result, streamed, action, chunk_size) + + class GroupPackage(RESTObject): pass diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index 8401c5c91..d1f58cb9a 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -41,7 +41,7 @@ from .milestones import ProjectMilestoneManager # noqa: F401 from .notes import ProjectNoteManager # noqa: F401 from .notification_settings import ProjectNotificationSettingsManager # noqa: F401 -from .packages import ProjectPackageManager # noqa: F401 +from .packages import GenericPackageManager, ProjectPackageManager # noqa: F401 from .pages import ProjectPagesDomainManager # noqa: F401 from .pipelines import ( # noqa: F401 ProjectPipeline, @@ -124,6 +124,7 @@ class Project(RefreshMixin, SaveMixin, ObjectDeleteMixin, RepositoryMixin, RESTO ("exports", "ProjectExportManager"), ("files", "ProjectFileManager"), ("forks", "ProjectForkManager"), + ("generic_packages", "GenericPackageManager"), ("hooks", "ProjectHookManager"), ("keys", "ProjectKeyManager"), ("imports", "ProjectImportManager"), diff --git a/tests/functional/api/test_packages.py b/tests/functional/api/test_packages.py index 9160a6820..64b57b827 100644 --- a/tests/functional/api/test_packages.py +++ b/tests/functional/api/test_packages.py @@ -1,6 +1,14 @@ """ -GitLab API: https://docs.gitlab.com/ce/api/packages.html +GitLab API: +https://docs.gitlab.com/ce/api/packages.html +https://docs.gitlab.com/ee/user/packages/generic_packages """ +from gitlab.v4.objects import GenericPackage + +package_name = "hello-world" +package_version = "v1.0.0" +file_name = "hello.tar.gz" +file_content = "package content" def test_list_project_packages(project): @@ -11,3 +19,44 @@ def test_list_project_packages(project): def test_list_group_packages(group): packages = group.packages.list() assert isinstance(packages, list) + + +def test_upload_generic_package(tmp_path, project): + path = tmp_path / file_name + path.write_text(file_content) + package = project.generic_packages.upload( + package_name=package_name, + package_version=package_version, + file_name=file_name, + path=path, + ) + + assert isinstance(package, GenericPackage) + assert package.message == "201 Created" + + +def test_download_generic_package(project): + package = project.generic_packages.download( + package_name=package_name, + package_version=package_version, + file_name=file_name, + ) + + assert isinstance(package, bytes) + assert package.decode("utf-8") == file_content + + +def test_download_generic_package_to_file(tmp_path, project): + path = tmp_path / file_name + + with open(path, "wb") as f: + project.generic_packages.download( + package_name=package_name, + package_version=package_version, + file_name=file_name, + streamed=True, + action=f.write, + ) + + with open(path, "r") as f: + assert f.read() == file_content diff --git a/tests/functional/cli/test_cli_packages.py b/tests/functional/cli/test_cli_packages.py index a3734a2fd..d7cdd18cb 100644 --- a/tests/functional/cli/test_cli_packages.py +++ b/tests/functional/cli/test_cli_packages.py @@ -1,3 +1,9 @@ +package_name = "hello-world" +package_version = "v1.0.0" +file_name = "hello.tar.gz" +file_content = "package content" + + def test_list_project_packages(gitlab_cli, project): cmd = ["project-package", "list", "--project-id", project.id] ret = gitlab_cli(cmd) @@ -10,3 +16,45 @@ def test_list_group_packages(gitlab_cli, group): ret = gitlab_cli(cmd) assert ret.success + + +def test_upload_generic_package(tmp_path, gitlab_cli, project): + path = tmp_path / file_name + path.write_text(file_content) + + cmd = [ + "-v", + "generic-package", + "upload", + "--project-id", + project.id, + "--package-name", + package_name, + "--path", + path, + "--package-version", + package_version, + "--file-name", + file_name, + ] + ret = gitlab_cli(cmd) + + assert "201 Created" in ret.stdout + + +def test_download_generic_package(gitlab_cli, project): + cmd = [ + "generic-package", + "download", + "--project-id", + project.id, + "--package-name", + package_name, + "--package-version", + package_version, + "--file-name", + file_name, + ] + ret = gitlab_cli(cmd) + + assert ret.stdout == file_content diff --git a/tests/unit/objects/test_packages.py b/tests/unit/objects/test_packages.py index 672eee01d..687054f27 100644 --- a/tests/unit/objects/test_packages.py +++ b/tests/unit/objects/test_packages.py @@ -2,11 +2,17 @@ GitLab API: https://docs.gitlab.com/ce/api/packages.html """ import re +from urllib.parse import quote_plus import pytest import responses -from gitlab.v4.objects import GroupPackage, ProjectPackage, ProjectPackageFile +from gitlab.v4.objects import ( + GenericPackage, + GroupPackage, + ProjectPackage, + ProjectPackageFile, +) package_content = { "id": 1, @@ -98,6 +104,17 @@ }, ] +package_name = "hello-world" +package_version = "v1.0.0" +file_name = "hello.tar.gz" +file_content = "package content" +package_url = "http://localhost/api/v4/projects/1/packages/generic/{}/{}/{}".format( + # https://datatracker.ietf.org/doc/html/rfc3986.html#section-2.3 :( + quote_plus(package_name).replace(".", "%2E"), + quote_plus(package_version).replace(".", "%2E"), + quote_plus(file_name).replace(".", "%2E"), +) + @pytest.fixture def resp_list_packages(): @@ -153,6 +170,32 @@ def resp_list_package_files(): yield rsps +@pytest.fixture +def resp_upload_generic_package(created_content): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.PUT, + url=package_url, + json=created_content, + content_type="application/json", + status=201, + ) + yield rsps + + +@pytest.fixture +def resp_download_generic_package(created_content): + with responses.RequestsMock() as rsps: + rsps.add( + method=responses.GET, + url=package_url, + body=file_content, + content_type="application/octet-stream", + status=200, + ) + yield rsps + + def test_list_project_packages(project, resp_list_packages): packages = project.packages.list() assert isinstance(packages, list) @@ -184,3 +227,26 @@ def test_list_project_package_files(project, resp_list_package_files): assert isinstance(package_files, list) assert isinstance(package_files[0], ProjectPackageFile) assert package_files[0].id == 25 + + +def test_upload_generic_package(tmp_path, project, resp_upload_generic_package): + path = tmp_path / file_name + path.write_text(file_content) + package = project.generic_packages.upload( + package_name=package_name, + package_version=package_version, + file_name=file_name, + path=path, + ) + + assert isinstance(package, GenericPackage) + + +def test_download_generic_package(project, resp_download_generic_package): + package = project.generic_packages.download( + package_name=package_name, + package_version=package_version, + file_name=file_name, + ) + + assert isinstance(package, bytes)