From 4f9807f2fc137b81ef79c6f7d9a7d67c16f73a17 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Wed, 30 Mar 2022 23:04:20 +0200 Subject: [PATCH 01/12] feat(downloads): allow streaming downloads access to response iterator Allow access to the underlying response iterator when downloading in streaming mode by specifying action="iterator" Update type annotations to support this change --- gitlab/mixins.py | 6 ++++-- gitlab/utils.py | 9 ++++++--- gitlab/v4/cli.py | 3 ++- gitlab/v4/objects/artifacts.py | 15 ++++++++------- gitlab/v4/objects/files.py | 17 ++++++++++++++--- gitlab/v4/objects/jobs.py | 20 +++++++++++++++----- gitlab/v4/objects/packages.py | 15 ++++++++++++--- gitlab/v4/objects/projects.py | 19 +++++++++++++++---- gitlab/v4/objects/repositories.py | 20 +++++++++++++++----- gitlab/v4/objects/snippets.py | 20 +++++++++++++++----- 10 files changed, 106 insertions(+), 38 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index 1a3ff4dbf..f8b5a8a30 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -20,7 +20,9 @@ Any, Callable, Dict, + Iterator, List, + Literal, Optional, Tuple, Type, @@ -657,10 +659,10 @@ class DownloadMixin(_RestObjectBase): def download( self, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download the archive of a resource export. Args: diff --git a/gitlab/utils.py b/gitlab/utils.py index 197935549..0193a7562 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -19,7 +19,7 @@ import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Optional, Type, Union +from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union import requests @@ -32,15 +32,18 @@ def __call__(self, chunk: Any) -> None: def response_content( response: requests.Response, streamed: bool, - action: Optional[Callable], + action: Optional[Union[Callable, Literal["iterator"]]], chunk_size: int, -) -> Optional[bytes]: +) -> Optional[Union[bytes, Iterator[Any]]]: if streamed is False: return response.content if action is None: action = _StdoutStream() + if action == "iterator": + return response.iter_content(chunk_size=chunk_size) + for chunk in response.iter_content(chunk_size=chunk_size): if chunk: action(chunk) diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py index 6830b0874..f9349bdbd 100644 --- a/gitlab/v4/cli.py +++ b/gitlab/v4/cli.py @@ -19,7 +19,7 @@ import argparse import operator import sys -from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING, Union +from typing import Any, Dict, Iterator, List, Optional, Type, TYPE_CHECKING, Union import gitlab import gitlab.base @@ -123,6 +123,7 @@ def do_project_export_download(self) -> None: data = export_status.download() if TYPE_CHECKING: assert data is not None + assert not isinstance(data, Iterator) sys.stdout.buffer.write(data) except Exception as e: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index 541e5e2f4..cdcfda7c1 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Optional, TYPE_CHECKING +from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union import requests @@ -32,7 +32,7 @@ def __call__( self, *args: Any, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: utils.warn( message=( "The project.artifacts() method is deprecated and will be removed in a " @@ -71,10 +71,10 @@ def download( ref_name: str, job: str, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts archive from a specific tag or branch. Args: @@ -86,7 +86,8 @@ def download( `chunk_size` and each chunk is passed to `action` for treatment action: Callable responsible of dealing with chunk of - data + data. May also be the string "iterator" to directly return + the response iterator chunk_size: Size of each chunk **kwargs: Extra options to send to the server (e.g. sudo) @@ -115,10 +116,10 @@ def raw( artifact_path: str, job: str, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a single artifact file from a specific tag or branch from within the job's artifacts archive. diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 435e71b55..1f4d98ccf 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,5 +1,16 @@ import base64 -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -217,10 +228,10 @@ def raw( file_path: str, ref: str, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a file for a commit. Args: diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index fbcb1fd40..db278287a 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,4 +1,14 @@ -from typing import Any, Callable, cast, Dict, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -116,10 +126,10 @@ def delete_artifacts(self, **kwargs: Any) -> None: def artifacts( self, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts. Args: @@ -152,10 +162,10 @@ def artifact( self, path: str, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get a single artifact file from within the job's artifacts archive. Args: diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 0461bdcd9..1f8539558 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -5,7 +5,16 @@ """ from pathlib import Path -from typing import Any, Callable, cast, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -103,10 +112,10 @@ def download( package_version: str, file_name: str, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a generic package. Args: diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index 81eb62496..cd4aafd6c 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,4 +1,15 @@ -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -457,10 +468,10 @@ def snapshot( self, wiki: bool = False, streamed: bool = False, - action: Optional[Callable] = None, + action: Optional[Union[Callable, Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return a snapshot of the repository. Args: @@ -562,7 +573,7 @@ def artifact( self, *args: Any, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: utils.warn( message=( "The project.artifact() method is deprecated and will be " diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index f2792b14e..9ea5e5237 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,7 +3,17 @@ Currently this module only contains repository-related methods for projects. """ -from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -107,10 +117,10 @@ def repository_raw_blob( self, sha: str, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the raw file contents for a blob. Args: @@ -192,11 +202,11 @@ def repository_archive( self, sha: str = None, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, format: Optional[str] = None, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return an archive of the repository. Args: diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 9d9dcc4e6..6d06fc7ed 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,4 +1,14 @@ -from typing import Any, Callable, cast, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -28,10 +38,10 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject): def content( self, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: @@ -102,10 +112,10 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj def content( self, streamed: bool = False, - action: Optional[Callable[..., Any]] = None, + action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: From efd8b486605bb38a5b0e8f5066eed675ed389836 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 9 Apr 2022 15:37:47 +0200 Subject: [PATCH 02/12] feat(downloads): add conditional dependency on literal for python 3.7 This supports commit c76b3b14b65a19b4d6b2a745745b3e493efa8c68 and makes sure it works on python 3.7 as well. --- gitlab/mixins.py | 7 ++++++- gitlab/utils.py | 8 +++++++- gitlab/v4/objects/artifacts.py | 8 +++++++- gitlab/v4/objects/files.py | 7 ++++++- gitlab/v4/objects/jobs.py | 18 +++++++----------- gitlab/v4/objects/packages.py | 17 +++++++---------- gitlab/v4/objects/projects.py | 7 ++++++- gitlab/v4/objects/repositories.py | 18 +++++++----------- gitlab/v4/objects/snippets.py | 18 +++++++----------- 9 files changed, 60 insertions(+), 48 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index f8b5a8a30..68388a410 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . +import sys from types import ModuleType from typing import ( Any, @@ -22,7 +23,6 @@ Dict, Iterator, List, - Literal, Optional, Tuple, Type, @@ -38,6 +38,11 @@ from gitlab import types as g_types from gitlab import utils +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "GetMixin", "GetWithoutIdMixin", diff --git a/gitlab/utils.py b/gitlab/utils.py index 0193a7562..46b4ea773 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -16,13 +16,19 @@ # along with this program. If not, see . import pathlib +import sys import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union +from typing import Any, Callable, Dict, Iterator, Optional, Type, Union import requests +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + class _StdoutStream: def __call__(self, chunk: Any) -> None: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index cdcfda7c1..d78efae21 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,8 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union +import sys +from typing import Any, Callable, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -11,6 +12,11 @@ from gitlab import utils from gitlab.base import RESTManager, RESTObject +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = ["ProjectArtifact", "ProjectArtifactManager"] diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 1f4d98ccf..3476a18c0 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,4 +1,5 @@ import base64 +import sys from typing import ( Any, Callable, @@ -6,7 +7,6 @@ Dict, Iterator, List, - Literal, Optional, TYPE_CHECKING, Union, @@ -27,6 +27,11 @@ UpdateMixin, ) +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "ProjectFile", "ProjectFileManager", diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index db278287a..ef018ca3f 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,14 +1,5 @@ -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +import sys +from typing import Any, Callable, cast, Dict, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -18,6 +9,11 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import RefreshMixin, RetrieveMixin +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "ProjectJob", "ProjectJobManager", diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 1f8539558..60a06d8d3 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -4,17 +4,9 @@ https://docs.gitlab.com/ee/user/packages/generic_packages/ """ +import sys from pathlib import Path -from typing import ( - Any, - Callable, - cast, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -24,6 +16,11 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import DeleteMixin, GetMixin, ListMixin, ObjectDeleteMixin +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "GenericPackage", "GenericPackageManager", diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index cd4aafd6c..74053c13e 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,3 +1,4 @@ +import sys from typing import ( Any, Callable, @@ -5,7 +6,6 @@ Dict, Iterator, List, - Literal, Optional, TYPE_CHECKING, Union, @@ -82,6 +82,11 @@ from .variables import ProjectVariableManager # noqa: F401 from .wikis import ProjectWikiManager # noqa: F401 +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "GroupProject", "GroupProjectManager", diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index 9ea5e5237..b8af26979 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,17 +3,8 @@ Currently this module only contains repository-related methods for projects. """ -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +import sys +from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -22,6 +13,11 @@ from gitlab import exceptions as exc from gitlab import utils +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + if TYPE_CHECKING: # When running mypy we use these as the base classes _RestObjectBase = gitlab.base.RESTObject diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 6d06fc7ed..8c4dd7159 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,14 +1,5 @@ -from typing import ( - Any, - Callable, - cast, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +import sys +from typing import Any, Callable, cast, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -22,6 +13,11 @@ from .discussions import ProjectSnippetDiscussionManager # noqa: F401 from .notes import ProjectSnippetNoteManager # noqa: F401 +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + __all__ = [ "Snippet", "SnippetManager", From 8cc4cfc0e78b8e1f430ed9e23089423cad3381ed Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 9 Apr 2022 15:41:35 +0200 Subject: [PATCH 03/12] chore(deps): add conditional dependency on typing_extensions This is used to support Literal from typing on python < 3.8 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index c94a1d220..cccf30dd7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests==2.27.1 requests-toolbelt==0.9.1 +typing_extensions; python_version<"3.8" \ No newline at end of file From 63fa05b8eb29ccd732b118b144e4812041a94b03 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 10 Apr 2022 10:16:11 +0200 Subject: [PATCH 04/12] docs(api-docs): add iterator example to artifact download Document the usage of the action="iterator" option when downloading artifacts --- docs/gl_objects/pipelines_and_jobs.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/gl_objects/pipelines_and_jobs.rst b/docs/gl_objects/pipelines_and_jobs.rst index a05d968a4..78cae51ba 100644 --- a/docs/gl_objects/pipelines_and_jobs.rst +++ b/docs/gl_objects/pipelines_and_jobs.rst @@ -274,6 +274,18 @@ You can also directly stream the output into a file, and unzip it afterwards:: subprocess.run(["unzip", "-bo", zipfn]) os.unlink(zipfn) +It is also possible to use the underlying iterator directly:: + + artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + +This can be used with FastAPI/Starlette StreamingResponse to forward a download from gitlab without having to download +the entire file server side first:: + + @app.get("/download_artifact") + def download_artifact(): + artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + return StreamingResponse(artifact_bytes_iterator, media_type="application/zip") + Delete all artifacts of a project that can be deleted:: project.artifacts.delete() From d7ee6f8e556b7829e41b3fa800cf07addfe9a564 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 10 Apr 2022 10:57:35 +0200 Subject: [PATCH 05/12] test(packages): add tests for streaming downloads Tests for the functionality implemented in 4f9807f2fc137b81ef79c6f7d9a7d67c16f73a17 --- tests/functional/api/test_packages.py | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/functional/api/test_packages.py b/tests/functional/api/test_packages.py index 64b57b827..e009572c4 100644 --- a/tests/functional/api/test_packages.py +++ b/tests/functional/api/test_packages.py @@ -3,6 +3,8 @@ https://docs.gitlab.com/ce/api/packages.html https://docs.gitlab.com/ee/user/packages/generic_packages """ +from collections.abc import Iterator + from gitlab.v4.objects import GenericPackage package_name = "hello-world" @@ -46,6 +48,25 @@ def test_download_generic_package(project): assert package.decode("utf-8") == file_content +def test_stream_generic_package(project): + bytes_iterator = project.generic_packages.download( + package_name=package_name, + package_version=package_version, + file_name=file_name, + streamed=True, + action="iterator", + ) + + assert isinstance(bytes_iterator, Iterator) + + package = bytes() + for chunk in bytes_iterator: + package += chunk + + assert isinstance(package, bytes) + assert package.decode("utf-8") == file_content + + def test_download_generic_package_to_file(tmp_path, project): path = tmp_path / file_name @@ -60,3 +81,22 @@ def test_download_generic_package_to_file(tmp_path, project): with open(path, "r") as f: assert f.read() == file_content + + +def test_stream_generic_package_to_file(tmp_path, project): + path = tmp_path / file_name + + bytes_iterator = project.generic_packages.download( + package_name=package_name, + package_version=package_version, + file_name=file_name, + streamed=True, + action="iterator", + ) + + with open(path, "wb") as f: + for chunk in bytes_iterator: + f.write(chunk) + + with open(path, "r") as f: + assert f.read() == file_content From af3eb0b0b12dda8f9e94a6fd29828831b2dc681a Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Thu, 14 Apr 2022 14:40:43 +0200 Subject: [PATCH 06/12] docs(api-docs): make iterator download documentation more generic Co-authored-by: Nejc Habjan --- docs/gl_objects/pipelines_and_jobs.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/gl_objects/pipelines_and_jobs.rst b/docs/gl_objects/pipelines_and_jobs.rst index 78cae51ba..901799ff6 100644 --- a/docs/gl_objects/pipelines_and_jobs.rst +++ b/docs/gl_objects/pipelines_and_jobs.rst @@ -274,12 +274,13 @@ You can also directly stream the output into a file, and unzip it afterwards:: subprocess.run(["unzip", "-bo", zipfn]) os.unlink(zipfn) -It is also possible to use the underlying iterator directly:: +Or, you can also use the underlying response iterator directly:: artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') -This can be used with FastAPI/Starlette StreamingResponse to forward a download from gitlab without having to download -the entire file server side first:: +This can be used with frameworks that expect an iterator (such as FastAPI/Starlette's +``StreamingResponse``) to forward a download from GitLab without having to download +the entire content server-side first:: @app.get("/download_artifact") def download_artifact(): From 4b31b517f107097253e47fb940c9705b53730481 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 25 Jun 2022 22:17:42 +0200 Subject: [PATCH 07/12] Revert "chore(deps): add conditional dependency on typing_extensions" This reverts commit 8cc4cfc0e78b8e1f430ed9e23089423cad3381ed. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cccf30dd7..c94a1d220 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ requests==2.27.1 requests-toolbelt==0.9.1 -typing_extensions; python_version<"3.8" \ No newline at end of file From 7e5f4eed642c1a7839ca98627062d8358be1554b Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 25 Jun 2022 22:17:42 +0200 Subject: [PATCH 08/12] Revert "feat(downloads): add conditional dependency on literal for python 3.7" This reverts commit efd8b486605bb38a5b0e8f5066eed675ed389836. --- gitlab/mixins.py | 7 +------ gitlab/utils.py | 8 +------- gitlab/v4/objects/artifacts.py | 8 +------- gitlab/v4/objects/files.py | 7 +------ gitlab/v4/objects/jobs.py | 18 +++++++++++------- gitlab/v4/objects/packages.py | 17 ++++++++++------- gitlab/v4/objects/projects.py | 7 +------ gitlab/v4/objects/repositories.py | 18 +++++++++++------- gitlab/v4/objects/snippets.py | 18 +++++++++++------- 9 files changed, 48 insertions(+), 60 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index 68388a410..f8b5a8a30 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -import sys from types import ModuleType from typing import ( Any, @@ -23,6 +22,7 @@ Dict, Iterator, List, + Literal, Optional, Tuple, Type, @@ -38,11 +38,6 @@ from gitlab import types as g_types from gitlab import utils -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "GetMixin", "GetWithoutIdMixin", diff --git a/gitlab/utils.py b/gitlab/utils.py index 46b4ea773..0193a7562 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -16,19 +16,13 @@ # along with this program. If not, see . import pathlib -import sys import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Iterator, Optional, Type, Union +from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union import requests -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - class _StdoutStream: def __call__(self, chunk: Any) -> None: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index d78efae21..cdcfda7c1 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,8 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -import sys -from typing import Any, Callable, Iterator, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union import requests @@ -12,11 +11,6 @@ from gitlab import utils from gitlab.base import RESTManager, RESTObject -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = ["ProjectArtifact", "ProjectArtifactManager"] diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 3476a18c0..1f4d98ccf 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,5 +1,4 @@ import base64 -import sys from typing import ( Any, Callable, @@ -7,6 +6,7 @@ Dict, Iterator, List, + Literal, Optional, TYPE_CHECKING, Union, @@ -27,11 +27,6 @@ UpdateMixin, ) -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "ProjectFile", "ProjectFileManager", diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index ef018ca3f..db278287a 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,5 +1,14 @@ -import sys -from typing import Any, Callable, cast, Dict, Iterator, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -9,11 +18,6 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import RefreshMixin, RetrieveMixin -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "ProjectJob", "ProjectJobManager", diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 60a06d8d3..1f8539558 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -4,9 +4,17 @@ https://docs.gitlab.com/ee/user/packages/generic_packages/ """ -import sys from pathlib import Path -from typing import Any, Callable, cast, Iterator, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -16,11 +24,6 @@ from gitlab.base import RESTManager, RESTObject from gitlab.mixins import DeleteMixin, GetMixin, ListMixin, ObjectDeleteMixin -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "GenericPackage", "GenericPackageManager", diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index 74053c13e..cd4aafd6c 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,4 +1,3 @@ -import sys from typing import ( Any, Callable, @@ -6,6 +5,7 @@ Dict, Iterator, List, + Literal, Optional, TYPE_CHECKING, Union, @@ -82,11 +82,6 @@ from .variables import ProjectVariableManager # noqa: F401 from .wikis import ProjectWikiManager # noqa: F401 -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "GroupProject", "GroupProjectManager", diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index b8af26979..9ea5e5237 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,8 +3,17 @@ Currently this module only contains repository-related methods for projects. """ -import sys -from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -13,11 +22,6 @@ from gitlab import exceptions as exc from gitlab import utils -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - if TYPE_CHECKING: # When running mypy we use these as the base classes _RestObjectBase = gitlab.base.RESTObject diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 8c4dd7159..6d06fc7ed 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,5 +1,14 @@ -import sys -from typing import Any, Callable, cast, Iterator, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Iterator, + List, + Literal, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -13,11 +22,6 @@ from .discussions import ProjectSnippetDiscussionManager # noqa: F401 from .notes import ProjectSnippetNoteManager # noqa: F401 -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - __all__ = [ "Snippet", "SnippetManager", From 5a97cdb1e4a343a15aa69352a4f07d320e37bf95 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sat, 25 Jun 2022 22:17:42 +0200 Subject: [PATCH 09/12] Revert "feat(downloads): allow streaming downloads access to response iterator" This reverts commit 4f9807f2fc137b81ef79c6f7d9a7d67c16f73a17. --- gitlab/mixins.py | 6 ++---- gitlab/utils.py | 9 +++------ gitlab/v4/cli.py | 3 +-- gitlab/v4/objects/artifacts.py | 15 +++++++-------- gitlab/v4/objects/files.py | 17 +++-------------- gitlab/v4/objects/jobs.py | 20 +++++--------------- gitlab/v4/objects/packages.py | 15 +++------------ gitlab/v4/objects/projects.py | 19 ++++--------------- gitlab/v4/objects/repositories.py | 20 +++++--------------- gitlab/v4/objects/snippets.py | 20 +++++--------------- 10 files changed, 38 insertions(+), 106 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index f8b5a8a30..1a3ff4dbf 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -20,9 +20,7 @@ Any, Callable, Dict, - Iterator, List, - Literal, Optional, Tuple, Type, @@ -659,10 +657,10 @@ class DownloadMixin(_RestObjectBase): def download( self, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Download the archive of a resource export. Args: diff --git a/gitlab/utils.py b/gitlab/utils.py index 0193a7562..197935549 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -19,7 +19,7 @@ import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Iterator, Literal, Optional, Type, Union +from typing import Any, Callable, Dict, Optional, Type, Union import requests @@ -32,18 +32,15 @@ def __call__(self, chunk: Any) -> None: def response_content( response: requests.Response, streamed: bool, - action: Optional[Union[Callable, Literal["iterator"]]], + action: Optional[Callable], chunk_size: int, -) -> Optional[Union[bytes, Iterator[Any]]]: +) -> Optional[bytes]: if streamed is False: return response.content if action is None: action = _StdoutStream() - if action == "iterator": - return response.iter_content(chunk_size=chunk_size) - for chunk in response.iter_content(chunk_size=chunk_size): if chunk: action(chunk) diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py index f9349bdbd..6830b0874 100644 --- a/gitlab/v4/cli.py +++ b/gitlab/v4/cli.py @@ -19,7 +19,7 @@ import argparse import operator import sys -from typing import Any, Dict, Iterator, List, Optional, Type, TYPE_CHECKING, Union +from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING, Union import gitlab import gitlab.base @@ -123,7 +123,6 @@ def do_project_export_download(self) -> None: data = export_status.download() if TYPE_CHECKING: assert data is not None - assert not isinstance(data, Iterator) sys.stdout.buffer.write(data) except Exception as e: diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index cdcfda7c1..541e5e2f4 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Optional, TYPE_CHECKING import requests @@ -32,7 +32,7 @@ def __call__( self, *args: Any, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: utils.warn( message=( "The project.artifacts() method is deprecated and will be removed in a " @@ -71,10 +71,10 @@ def download( ref_name: str, job: str, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Get the job artifacts archive from a specific tag or branch. Args: @@ -86,8 +86,7 @@ def download( `chunk_size` and each chunk is passed to `action` for treatment action: Callable responsible of dealing with chunk of - data. May also be the string "iterator" to directly return - the response iterator + data chunk_size: Size of each chunk **kwargs: Extra options to send to the server (e.g. sudo) @@ -116,10 +115,10 @@ def raw( artifact_path: str, job: str, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Download a single artifact file from a specific tag or branch from within the job's artifacts archive. diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index 1f4d98ccf..435e71b55 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,16 +1,5 @@ import base64 -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING import requests @@ -228,10 +217,10 @@ def raw( file_path: str, ref: str, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the content of a file for a commit. Args: diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index db278287a..fbcb1fd40 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,14 +1,4 @@ -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Dict, Optional, TYPE_CHECKING, Union import requests @@ -126,10 +116,10 @@ def delete_artifacts(self, **kwargs: Any) -> None: def artifacts( self, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Get the job artifacts. Args: @@ -162,10 +152,10 @@ def artifact( self, path: str, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Get a single artifact file from within the job's artifacts archive. Args: diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 1f8539558..0461bdcd9 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -5,16 +5,7 @@ """ from pathlib import Path -from typing import ( - Any, - Callable, - cast, - Iterator, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Optional, TYPE_CHECKING, Union import requests @@ -112,10 +103,10 @@ def download( package_version: str, file_name: str, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Download a generic package. Args: diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index cd4aafd6c..81eb62496 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,15 +1,4 @@ -from typing import ( - Any, - Callable, - cast, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING, Union import requests @@ -468,10 +457,10 @@ def snapshot( self, wiki: bool = False, streamed: bool = False, - action: Optional[Union[Callable, Literal["iterator"]]] = None, + action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return a snapshot of the repository. Args: @@ -573,7 +562,7 @@ def artifact( self, *args: Any, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: utils.warn( message=( "The project.artifact() method is deprecated and will be " diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index 9ea5e5237..f2792b14e 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,17 +3,7 @@ Currently this module only contains repository-related methods for projects. """ -from typing import ( - Any, - Callable, - Dict, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union import requests @@ -117,10 +107,10 @@ def repository_raw_blob( self, sha: str, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the raw file contents for a blob. Args: @@ -202,11 +192,11 @@ def repository_archive( self, sha: str = None, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, format: Optional[str] = None, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return an archive of the repository. Args: diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 6d06fc7ed..9d9dcc4e6 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,14 +1,4 @@ -from typing import ( - Any, - Callable, - cast, - Iterator, - List, - Literal, - Optional, - TYPE_CHECKING, - Union, -) +from typing import Any, Callable, cast, List, Optional, TYPE_CHECKING, Union import requests @@ -38,10 +28,10 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject): def content( self, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the content of a snippet. Args: @@ -112,10 +102,10 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj def content( self, streamed: bool = False, - action: Optional[Union[Callable[..., Any], Literal["iterator"]]] = None, + action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[Union[bytes, Iterator[Any]]]: + ) -> Optional[bytes]: """Return the content of a snippet. Args: From 8580a778aaf89f75c95f5bec727125f7c693d0b4 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 26 Jun 2022 00:17:18 +0200 Subject: [PATCH 10/12] feat(downloads): use iterator=True for returning response iterator for downloads Instead of having action="iterator", we can now do iterator=True to get the underlying response iterator when downloading things. --- gitlab/mixins.py | 8 ++++++-- gitlab/utils.py | 8 ++++++-- gitlab/v4/cli.py | 1 + gitlab/v4/objects/artifacts.py | 22 ++++++++++++++++------ gitlab/v4/objects/files.py | 19 ++++++++++++++++--- gitlab/v4/objects/jobs.py | 23 +++++++++++++++++------ gitlab/v4/objects/packages.py | 9 ++++++--- gitlab/v4/objects/projects.py | 25 +++++++++++++++++++++---- gitlab/v4/objects/repositories.py | 16 +++++++++++----- gitlab/v4/objects/snippets.py | 16 +++++++++++----- 10 files changed, 111 insertions(+), 36 deletions(-) diff --git a/gitlab/mixins.py b/gitlab/mixins.py index 71ba8210c..14542e0a6 100644 --- a/gitlab/mixins.py +++ b/gitlab/mixins.py @@ -20,6 +20,7 @@ Any, Callable, Dict, + Iterator, List, Optional, Tuple, @@ -614,16 +615,19 @@ class DownloadMixin(_RestObjectBase): def download( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download the archive of a resource export. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -642,7 +646,7 @@ def download( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class SubscribableMixin(_RestObjectBase): diff --git a/gitlab/utils.py b/gitlab/utils.py index bab670584..6acb86160 100644 --- a/gitlab/utils.py +++ b/gitlab/utils.py @@ -19,7 +19,7 @@ import traceback import urllib.parse import warnings -from typing import Any, Callable, Dict, Optional, Tuple, Type, Union +from typing import Any, Callable, Dict, Iterator, Optional, Tuple, Type, Union import requests @@ -34,9 +34,13 @@ def __call__(self, chunk: Any) -> None: def response_content( response: requests.Response, streamed: bool, + iterator: bool, action: Optional[Callable], chunk_size: int, -) -> Optional[bytes]: +) -> Optional[Union[bytes, Iterator[Any]]]: + if iterator: + return response.iter_content(chunk_size=chunk_size) + if streamed is False: return response.content diff --git a/gitlab/v4/cli.py b/gitlab/v4/cli.py index 2b0d4ce72..ba2e788b7 100644 --- a/gitlab/v4/cli.py +++ b/gitlab/v4/cli.py @@ -127,6 +127,7 @@ def do_project_export_download(self) -> None: data = export_status.download() if TYPE_CHECKING: assert data is not None + assert isinstance(data, bytes) sys.stdout.buffer.write(data) except Exception as e: # pragma: no cover, cli.die is unit-tested diff --git a/gitlab/v4/objects/artifacts.py b/gitlab/v4/objects/artifacts.py index 541e5e2f4..f5f106d8b 100644 --- a/gitlab/v4/objects/artifacts.py +++ b/gitlab/v4/objects/artifacts.py @@ -2,7 +2,7 @@ GitLab API: https://docs.gitlab.com/ee/api/job_artifacts.html """ -from typing import Any, Callable, Optional, TYPE_CHECKING +from typing import Any, Callable, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -40,10 +40,14 @@ def __call__( ), category=DeprecationWarning, ) - return self.download( + data = self.download( *args, **kwargs, ) + if TYPE_CHECKING: + assert data is not None + assert isinstance(data, bytes) + return data @exc.on_http_error(exc.GitlabDeleteError) def delete(self, **kwargs: Any) -> None: @@ -71,10 +75,11 @@ def download( ref_name: str, job: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts archive from a specific tag or branch. Args: @@ -85,6 +90,8 @@ def download( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -103,7 +110,7 @@ def download( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action( "ProjectArtifactManager", ("ref_name", "artifact_path", "job") @@ -115,10 +122,11 @@ def raw( artifact_path: str, job: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a single artifact file from a specific tag or branch from within the job's artifacts archive. @@ -130,6 +138,8 @@ def raw( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -148,4 +158,4 @@ def raw( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) diff --git a/gitlab/v4/objects/files.py b/gitlab/v4/objects/files.py index aa86704c9..2fd79fd54 100644 --- a/gitlab/v4/objects/files.py +++ b/gitlab/v4/objects/files.py @@ -1,5 +1,15 @@ import base64 -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -220,10 +230,11 @@ def raw( file_path: str, ref: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a file for a commit. Args: @@ -232,6 +243,8 @@ def raw( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -252,7 +265,7 @@ def raw( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("ProjectFileManager", ("file_path", "ref")) @exc.on_http_error(exc.GitlabListError) diff --git a/gitlab/v4/objects/jobs.py b/gitlab/v4/objects/jobs.py index fbcb1fd40..850227725 100644 --- a/gitlab/v4/objects/jobs.py +++ b/gitlab/v4/objects/jobs.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, cast, Dict, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, cast, Dict, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -116,16 +116,19 @@ def delete_artifacts(self, **kwargs: Any) -> None: def artifacts( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get the job artifacts. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -144,7 +147,7 @@ def artifacts( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("ProjectJob") @exc.on_http_error(exc.GitlabGetError) @@ -152,10 +155,11 @@ def artifact( self, path: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Get a single artifact file from within the job's artifacts archive. Args: @@ -163,6 +167,8 @@ def artifact( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -181,13 +187,14 @@ def artifact( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("ProjectJob") @exc.on_http_error(exc.GitlabGetError) def trace( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, @@ -198,6 +205,8 @@ def trace( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -216,7 +225,9 @@ def trace( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return_value = utils.response_content(result, streamed, action, chunk_size) + return_value = utils.response_content( + result, streamed, iterator, action, chunk_size + ) if TYPE_CHECKING: assert isinstance(return_value, dict) return return_value diff --git a/gitlab/v4/objects/packages.py b/gitlab/v4/objects/packages.py index 882cb1a5a..a82080167 100644 --- a/gitlab/v4/objects/packages.py +++ b/gitlab/v4/objects/packages.py @@ -5,7 +5,7 @@ """ from pathlib import Path -from typing import Any, Callable, cast, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, cast, Iterator, Optional, TYPE_CHECKING, Union import requests @@ -103,10 +103,11 @@ def download( package_version: str, file_name: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Download a generic package. Args: @@ -116,6 +117,8 @@ def download( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -132,7 +135,7 @@ def download( result = self.gitlab.http_get(path, streamed=streamed, raw=True, **kwargs) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class GroupPackage(RESTObject): diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py index f32cf2257..fc2aac3d5 100644 --- a/gitlab/v4/objects/projects.py +++ b/gitlab/v4/objects/projects.py @@ -1,4 +1,14 @@ -from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + cast, + Dict, + Iterator, + List, + Optional, + TYPE_CHECKING, + Union, +) import requests @@ -466,10 +476,11 @@ def snapshot( self, wiki: bool = False, streamed: bool = False, + iterator: bool = False, action: Optional[Callable] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return a snapshot of the repository. Args: @@ -477,6 +488,8 @@ def snapshot( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment. + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -495,7 +508,7 @@ def snapshot( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("Project", ("scope", "search")) @exc.on_http_error(exc.GitlabSearchError) @@ -579,7 +592,11 @@ def artifact( ), category=DeprecationWarning, ) - return self.artifacts.raw(*args, **kwargs) + data = self.artifacts.raw(*args, **kwargs) + if TYPE_CHECKING: + assert data is not None + assert isinstance(data, bytes) + return data class ProjectManager(CRUDMixin, RESTManager): diff --git a/gitlab/v4/objects/repositories.py b/gitlab/v4/objects/repositories.py index 5826d9d83..1f10473aa 100644 --- a/gitlab/v4/objects/repositories.py +++ b/gitlab/v4/objects/repositories.py @@ -3,7 +3,7 @@ Currently this module only contains repository-related methods for projects. """ -from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Dict, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -107,10 +107,11 @@ def repository_raw_blob( self, sha: str, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the raw file contents for a blob. Args: @@ -118,6 +119,8 @@ def repository_raw_blob( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -136,7 +139,7 @@ def repository_raw_blob( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("Project", ("from_", "to")) @exc.on_http_error(exc.GitlabGetError) @@ -192,11 +195,12 @@ def repository_archive( self, sha: str = None, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, format: Optional[str] = None, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return an archive of the repository. Args: @@ -204,6 +208,8 @@ def repository_archive( streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -228,7 +234,7 @@ def repository_archive( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) @cli.register_custom_action("Project") @exc.on_http_error(exc.GitlabDeleteError) diff --git a/gitlab/v4/objects/snippets.py b/gitlab/v4/objects/snippets.py index 597a3aaf0..aa46c7747 100644 --- a/gitlab/v4/objects/snippets.py +++ b/gitlab/v4/objects/snippets.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, cast, List, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, cast, Iterator, List, Optional, TYPE_CHECKING, Union import requests @@ -29,16 +29,19 @@ class Snippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObject): def content( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment. + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -57,7 +60,7 @@ def content( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class SnippetManager(CRUDMixin, RESTManager): @@ -103,16 +106,19 @@ class ProjectSnippet(UserAgentDetailMixin, SaveMixin, ObjectDeleteMixin, RESTObj def content( self, streamed: bool = False, + iterator: bool = False, action: Optional[Callable[..., Any]] = None, chunk_size: int = 1024, **kwargs: Any, - ) -> Optional[bytes]: + ) -> Optional[Union[bytes, Iterator[Any]]]: """Return the content of a snippet. Args: streamed: If True the data will be processed by chunks of `chunk_size` and each chunk is passed to `action` for treatment. + iterator: If True directly return the underlying response + iterator action: Callable responsible of dealing with chunk of data chunk_size: Size of each chunk @@ -131,7 +137,7 @@ def content( ) if TYPE_CHECKING: assert isinstance(result, requests.Response) - return utils.response_content(result, streamed, action, chunk_size) + return utils.response_content(result, streamed, iterator, action, chunk_size) class ProjectSnippetManager(CRUDMixin, RESTManager): From ec61e736d87b26bc3a90b0fabc4b21788986f3e7 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 26 Jun 2022 00:19:18 +0200 Subject: [PATCH 11/12] test(packages): update tests for downloading with response iterator Adapted the existing tests for the new iterator=True argument --- tests/functional/api/test_packages.py | 6 ++---- tests/unit/test_utils.py | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional/api/test_packages.py b/tests/functional/api/test_packages.py index e009572c4..9f06439b4 100644 --- a/tests/functional/api/test_packages.py +++ b/tests/functional/api/test_packages.py @@ -53,8 +53,7 @@ def test_stream_generic_package(project): package_name=package_name, package_version=package_version, file_name=file_name, - streamed=True, - action="iterator", + iterator=True, ) assert isinstance(bytes_iterator, Iterator) @@ -90,8 +89,7 @@ def test_stream_generic_package_to_file(tmp_path, project): package_name=package_name, package_version=package_version, file_name=file_name, - streamed=True, - action="iterator", + iterator=True, ) with open(path, "wb") as f: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c5f9f931d..74b48ae31 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -36,7 +36,9 @@ def test_response_content(capsys): ) resp = requests.get("https://example.com", stream=True) - utils.response_content(resp, streamed=True, action=None, chunk_size=1024) + utils.response_content( + resp, streamed=True, iterator=False, action=None, chunk_size=1024 + ) captured = capsys.readouterr() assert "test" in captured.out From 9f01ee59ef026758c7fe4ba0bb74bab4899e9353 Mon Sep 17 00:00:00 2001 From: Tom Catshoek Date: Sun, 26 Jun 2022 00:20:12 +0200 Subject: [PATCH 12/12] docs(api-docs): update artifact download iterator example Adapted the example for the new iterator=True argument --- docs/gl_objects/pipelines_and_jobs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gl_objects/pipelines_and_jobs.rst b/docs/gl_objects/pipelines_and_jobs.rst index 901799ff6..f0bdd3a68 100644 --- a/docs/gl_objects/pipelines_and_jobs.rst +++ b/docs/gl_objects/pipelines_and_jobs.rst @@ -276,7 +276,7 @@ You can also directly stream the output into a file, and unzip it afterwards:: Or, you can also use the underlying response iterator directly:: - artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + artifact_bytes_iterator = build_or_job.artifacts(iterator=True) This can be used with frameworks that expect an iterator (such as FastAPI/Starlette's ``StreamingResponse``) to forward a download from GitLab without having to download @@ -284,7 +284,7 @@ the entire content server-side first:: @app.get("/download_artifact") def download_artifact(): - artifact_bytes_iterator = build_or_job.artifacts(streamed=True, action='iterator') + artifact_bytes_iterator = build_or_job.artifacts(iterator=True) return StreamingResponse(artifact_bytes_iterator, media_type="application/zip") Delete all artifacts of a project that can be deleted::