diff --git a/.coveragerc b/.coveragerc
index d097511c..34417c3f 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -11,3 +11,5 @@ exclude_lines =
def __repr__
# Ignore abstract methods
raise NotImplementedError
+ # Ignore coverage for code specific to static type checkers
+ TYPE_CHECKING
diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml
index 5fc5daa3..51b21a62 100644
--- a/.github/.OwlBot.lock.yaml
+++ b/.github/.OwlBot.lock.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,4 +13,5 @@
# limitations under the License.
docker:
image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
- digest: sha256:8555f0e37e6261408f792bfd6635102d2da5ad73f8f09bcb24f25e6afb5fac97
+ digest: sha256:a7aef70df5f13313ddc027409fc8f3151422ec2a57ac8730fce8fa75c060d5bb
+# created: 2025-04-10T17:00:10.042601326Z
diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml
index 41bff0b5..21786a4e 100644
--- a/.github/auto-label.yaml
+++ b/.github/auto-label.yaml
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,3 +13,8 @@
# limitations under the License.
requestsize:
enabled: true
+
+path:
+ pullrequest: true
+ paths:
+ samples: "samples"
diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml
new file mode 100644
index 00000000..1618464d
--- /dev/null
+++ b/.github/blunderbuss.yml
@@ -0,0 +1,17 @@
+# Blunderbuss config
+#
+# This file controls who is assigned for pull requests and issues.
+# Note: This file is autogenerated. To make changes to the assignee
+# team, please update `codeowner_team` in `.repo-metadata.json`.
+assign_issues:
+ - googleapis/actools-python
+
+assign_issues_by:
+ - labels:
+ - "samples"
+ to:
+ - googleapis/python-samples-reviewers
+ - googleapis/actools-python
+
+assign_prs:
+ - googleapis/actools-python
diff --git a/.github/cherry-pick-bot.yml b/.github/cherry-pick-bot.yml
new file mode 100644
index 00000000..1e9cfcd3
--- /dev/null
+++ b/.github/cherry-pick-bot.yml
@@ -0,0 +1,2 @@
+enabled: true
+
diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml
index d4ca9418..50e8bd30 100644
--- a/.github/release-trigger.yml
+++ b/.github/release-trigger.yml
@@ -1 +1,2 @@
enabled: true
+multiScmName: python-api-core
diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml
index cdec3e9b..b724bada 100644
--- a/.github/sync-repo-settings.yaml
+++ b/.github/sync-repo-settings.yaml
@@ -16,13 +16,38 @@ branchProtectionRules:
- 'unit_grpc_gcp-3.9'
- 'unit_grpc_gcp-3.10'
- 'unit_grpc_gcp-3.11'
+ - 'unit_grpc_gcp-3.12'
+ - 'unit_grpc_gcp-3.13'
+ - 'unit_grpc_gcp-3.14'
- 'unit-3.7'
- 'unit-3.8'
- 'unit-3.9'
- 'unit-3.10'
- 'unit-3.11'
+ - 'unit-3.12'
+ - 'unit-3.13'
+ - 'unit-3.14'
- 'unit_wo_grpc-3.10'
- 'unit_wo_grpc-3.11'
+ - 'unit_wo_grpc-3.12'
+ - 'unit_wo_grpc-3.13'
+ - 'unit_wo_grpc-3.14'
+ - 'unit_w_prerelease_deps-3.7'
+ - 'unit_w_prerelease_deps-3.8'
+ - 'unit_w_prerelease_deps-3.9'
+ - 'unit_w_prerelease_deps-3.10'
+ - 'unit_w_prerelease_deps-3.11'
+ - 'unit_w_prerelease_deps-3.12'
+ - 'unit_w_prerelease_deps-3.13'
+ - 'unit_w_prerelease_deps-3.14'
+ - 'unit_w_async_rest_extra-3.7'
+ - 'unit_w_async_rest_extra-3.8'
+ - 'unit_w_async_rest_extra-3.9'
+ - 'unit_w_async_rest_extra-3.10'
+ - 'unit_w_async_rest_extra-3.11'
+ - 'unit_w_async_rest_extra-3.12'
+ - 'unit_w_async_rest_extra-3.13'
+ - 'unit_w_async_rest_extra-3.14'
- 'cover'
- 'docs'
- 'docfx'
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index e97d89e4..2833fe98 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -8,11 +8,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
- python-version: "3.9"
+ python-version: "3.10"
- name: Install nox
run: |
python -m pip install --upgrade setuptools pip wheel
@@ -24,11 +24,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
- python-version: "3.9"
+ python-version: "3.10"
- name: Install nox
run: |
python -m pip install --upgrade setuptools pip wheel
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index d2aee5b7..1051da0b 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -8,9 +8,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install nox
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
index a505525d..e6a79291 100644
--- a/.github/workflows/mypy.yml
+++ b/.github/workflows/mypy.yml
@@ -8,9 +8,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install nox
diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 0d7789b6..f260a6a5 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -8,16 +8,22 @@ on:
jobs:
run-unittests:
name: unit${{ matrix.option }}-${{ matrix.python }}
- runs-on: ubuntu-latest
+ # TODO(https://github.com/googleapis/gapic-generator-python/issues/2303): use `ubuntu-latest` once this bug is fixed.
+ # Use ubuntu-22.04 until Python 3.7 is removed from the test matrix
+ # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+ runs-on: ubuntu-22.04
strategy:
matrix:
- option: ["", "_grpc_gcp", "_wo_grpc"]
+ option: ["", "_grpc_gcp", "_wo_grpc", "_w_prerelease_deps", "_w_async_rest_extra"]
python:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
- "3.11"
+ - "3.12"
+ - "3.13"
+ - "3.14"
exclude:
- option: "_wo_grpc"
python: 3.7
@@ -27,11 +33,12 @@ jobs:
python: 3.9
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
+ allow-prereleases: true
- name: Install nox
run: |
python -m pip install --upgrade setuptools pip wheel
@@ -42,10 +49,11 @@ jobs:
run: |
nox -s unit${{ matrix.option }}-${{ matrix.python }}
- name: Upload coverage results
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
- name: coverage-artifacts
+ name: coverage-artifact-${{ matrix.option }}-${{ matrix.python }}
path: .coverage${{ matrix.option }}-${{ matrix.python }}
+ include-hidden-files: true
report-coverage:
name: cover
@@ -54,9 +62,9 @@ jobs:
- run-unittests
steps:
- name: Checkout
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- name: Setup Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install coverage
@@ -64,11 +72,11 @@ jobs:
python -m pip install --upgrade setuptools pip wheel
python -m pip install coverage
- name: Download coverage results
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
with:
- name: coverage-artifacts
path: .coverage-results/
- name: Report coverage results
run: |
- coverage combine .coverage-results/.coverage*
+ find .coverage-results -type f -name '*.zip' -exec unzip {} \;
+ coverage combine .coverage-results/**/.coverage*
coverage report --show-missing --fail-under=100
diff --git a/.gitignore b/.gitignore
index 99c3a144..168b201f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -51,6 +51,7 @@ docs.metadata
# Virtual environment
env/
+venv/
# Test logs
coverage.xml
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
index 0394c8aa..d41b45aa 100755
--- a/.kokoro/build.sh
+++ b/.kokoro/build.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2018 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -15,11 +15,13 @@
set -eo pipefail
+CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
if [[ -z "${PROJECT_ROOT:-}" ]]; then
- PROJECT_ROOT="github/python-api-core"
+ PROJECT_ROOT=$(realpath "${CURRENT_DIR}/..")
fi
-cd "${PROJECT_ROOT}"
+pushd "${PROJECT_ROOT}"
# Disable buffering, so that the logs stream through.
export PYTHONUNBUFFERED=1
@@ -28,17 +30,16 @@ export PYTHONUNBUFFERED=1
env | grep KOKORO
# Setup service account credentials.
-export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
+if [[ -f "${KOKORO_GFILE_DIR}/service-account.json" ]]
+then
+ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
+fi
# Setup project id.
-export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
-
-# Remove old nox
-python3 -m pip uninstall --yes --quiet nox-automation
-
-# Install nox
-python3 -m pip install --upgrade --quiet nox
-python3 -m nox --version
+if [[ -f "${KOKORO_GFILE_DIR}/project-id.json" ]]
+then
+ export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
+fi
# If this is a continuous build, send the test log to the FlakyBot.
# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
@@ -53,7 +54,7 @@ fi
# If NOX_SESSION is set, it only runs the specified session,
# otherwise run all the sessions.
if [[ -n "${NOX_SESSION:-}" ]]; then
- python3 -m nox -s ${NOX_SESSION:-}
+ python3 -m nox -s ${NOX_SESSION:-}
else
- python3 -m nox
+ python3 -m nox
fi
diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile
deleted file mode 100644
index f8137d0a..00000000
--- a/.kokoro/docker/docs/Dockerfile
+++ /dev/null
@@ -1,83 +0,0 @@
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from ubuntu:22.04
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Ensure local Python is preferred over distribution Python.
-ENV PATH /usr/local/bin:$PATH
-
-# Install dependencies.
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
- apt-transport-https \
- build-essential \
- ca-certificates \
- curl \
- dirmngr \
- git \
- gpg-agent \
- graphviz \
- libbz2-dev \
- libdb5.3-dev \
- libexpat1-dev \
- libffi-dev \
- liblzma-dev \
- libreadline-dev \
- libsnappy-dev \
- libssl-dev \
- libsqlite3-dev \
- portaudio19-dev \
- python3-distutils \
- redis-server \
- software-properties-common \
- ssh \
- sudo \
- tcl \
- tcl-dev \
- tk \
- tk-dev \
- uuid-dev \
- wget \
- zlib1g-dev \
- && add-apt-repository universe \
- && apt-get update \
- && apt-get -y install jq \
- && apt-get clean autoclean \
- && apt-get autoremove -y \
- && rm -rf /var/lib/apt/lists/* \
- && rm -f /var/cache/apt/archives/*.deb
-
-###################### Install python 3.9.13
-
-# Download python 3.9.13
-RUN wget https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tgz
-
-# Extract files
-RUN tar -xvf Python-3.9.13.tgz
-
-# Install python 3.9.13
-RUN ./Python-3.9.13/configure --enable-optimizations
-RUN make altinstall
-
-###################### Install pip
-RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
- && python3 /tmp/get-pip.py \
- && rm /tmp/get-pip.py
-
-# Test pip
-RUN python3 -m pip
-
-CMD ["python3.8"]
diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg
deleted file mode 100644
index 48e89855..00000000
--- a/.kokoro/docs/common.cfg
+++ /dev/null
@@ -1,67 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
-
-# Build logs will be here
-action {
- define_artifacts {
- regex: "**/*sponge_log.xml"
- }
-}
-
-# Download trampoline resources.
-gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
-
-# Use the trampoline script to run in docker.
-build_file: "python-api-core/.kokoro/trampoline_v2.sh"
-
-# Configure the docker image for kokoro-trampoline.
-env_vars: {
- key: "TRAMPOLINE_IMAGE"
- value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
-}
-env_vars: {
- key: "TRAMPOLINE_BUILD_FILE"
- value: "github/python-api-core/.kokoro/publish-docs.sh"
-}
-
-env_vars: {
- key: "STAGING_BUCKET"
- value: "docs-staging"
-}
-
-env_vars: {
- key: "V2_STAGING_BUCKET"
- # Push non-cloud library docs to `docs-staging-v2-staging` instead of the
- # Cloud RAD bucket `docs-staging-v2`
- value: "docs-staging-v2-staging"
-}
-
-# It will upload the docker image after successful builds.
-env_vars: {
- key: "TRAMPOLINE_IMAGE_UPLOAD"
- value: "true"
-}
-
-# It will always build the docker image.
-env_vars: {
- key: "TRAMPOLINE_DOCKERFILE"
- value: ".kokoro/docker/docs/Dockerfile"
-}
-
-# Fetch the token needed for reporting release status to GitHub
-before_action {
- fetch_keystore {
- keystore_resource {
- keystore_config_id: 73713
- keyname: "yoshi-automation-github-key"
- }
- }
-}
-
-before_action {
- fetch_keystore {
- keystore_resource {
- keystore_config_id: 73713
- keyname: "docuploader_service_account"
- }
- }
-}
\ No newline at end of file
diff --git a/.kokoro/docs/docs-presubmit.cfg b/.kokoro/docs/docs-presubmit.cfg
deleted file mode 100644
index d1ed51eb..00000000
--- a/.kokoro/docs/docs-presubmit.cfg
+++ /dev/null
@@ -1,28 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
-
-env_vars: {
- key: "STAGING_BUCKET"
- value: "gcloud-python-test"
-}
-
-env_vars: {
- key: "V2_STAGING_BUCKET"
- value: "gcloud-python-test"
-}
-
-# We only upload the image in the main `docs` build.
-env_vars: {
- key: "TRAMPOLINE_IMAGE_UPLOAD"
- value: "false"
-}
-
-env_vars: {
- key: "TRAMPOLINE_BUILD_FILE"
- value: "github/python-api-core/.kokoro/build.sh"
-}
-
-# Only run this nox session.
-env_vars: {
- key: "NOX_SESSION"
- value: "docs docfx"
-}
diff --git a/.kokoro/docs/docs.cfg b/.kokoro/docs/docs.cfg
deleted file mode 100644
index 8f43917d..00000000
--- a/.kokoro/docs/docs.cfg
+++ /dev/null
@@ -1 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh
index f5251425..c435402f 100755
--- a/.kokoro/populate-secrets.sh
+++ b/.kokoro/populate-secrets.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2020 Google LLC.
+# Copyright 2024 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh
deleted file mode 100755
index 1c4d6237..00000000
--- a/.kokoro/publish-docs.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/bash
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eo pipefail
-
-# Disable buffering, so that the logs stream through.
-export PYTHONUNBUFFERED=1
-
-export PATH="${HOME}/.local/bin:${PATH}"
-
-# Install nox
-python3 -m pip install --require-hashes -r .kokoro/requirements.txt
-python3 -m nox --version
-
-# build docs
-nox -s docs
-
-# create metadata
-python3 -m docuploader create-metadata \
- --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
- --version=$(python3 setup.py --version) \
- --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
- --distribution-name=$(python3 setup.py --name) \
- --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
- --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
- --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
-
-cat docs.metadata
-
-# upload docs
-python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
-
-
-# docfx yaml files
-nox -s docfx
-
-# create metadata.
-python3 -m docuploader create-metadata \
- --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
- --version=$(python3 setup.py --version) \
- --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
- --distribution-name=$(python3 setup.py --name) \
- --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
- --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
- --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
-
-cat docs.metadata
-
-# upload docs
-python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"
diff --git a/.kokoro/release.sh b/.kokoro/release.sh
deleted file mode 100755
index 697f7e6d..00000000
--- a/.kokoro/release.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -eo pipefail
-
-# Start the releasetool reporter
-python3 -m pip install --require-hashes -r github/python-api-core/.kokoro/requirements.txt
-python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
-
-# Disable buffering, so that the logs stream through.
-export PYTHONUNBUFFERED=1
-
-# Move into the package, build the distribution and upload.
-TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
-cd github/python-api-core
-python3 setup.py sdist bdist_wheel
-twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*
diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg
deleted file mode 100644
index de4f6f89..00000000
--- a/.kokoro/release/common.cfg
+++ /dev/null
@@ -1,40 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
-
-# Build logs will be here
-action {
- define_artifacts {
- regex: "**/*sponge_log.xml"
- }
-}
-
-# Download trampoline resources.
-gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
-
-# Use the trampoline script to run in docker.
-build_file: "python-api-core/.kokoro/trampoline.sh"
-
-# Configure the docker image for kokoro-trampoline.
-env_vars: {
- key: "TRAMPOLINE_IMAGE"
- value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
-}
-env_vars: {
- key: "TRAMPOLINE_BUILD_FILE"
- value: "github/python-api-core/.kokoro/release.sh"
-}
-
-# Fetch PyPI password
-before_action {
- fetch_keystore {
- keystore_resource {
- keystore_config_id: 73713
- keyname: "google-cloud-pypi-token-keystore-1"
- }
- }
-}
-
-# Tokens needed to report release status back to GitHub
-env_vars: {
- key: "SECRET_MANAGER_KEYS"
- value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"
-}
diff --git a/.kokoro/release/release.cfg b/.kokoro/release/release.cfg
deleted file mode 100644
index 8f43917d..00000000
--- a/.kokoro/release/release.cfg
+++ /dev/null
@@ -1 +0,0 @@
-# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in
deleted file mode 100644
index 882178ce..00000000
--- a/.kokoro/requirements.in
+++ /dev/null
@@ -1,10 +0,0 @@
-gcp-docuploader
-gcp-releasetool>=1.10.5 # required for compatibility with cryptography>=39.x
-importlib-metadata
-typing-extensions
-twine
-wheel
-setuptools
-nox
-charset-normalizer<3
-click<8.1.0
diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt
deleted file mode 100644
index fa99c129..00000000
--- a/.kokoro/requirements.txt
+++ /dev/null
@@ -1,498 +0,0 @@
-#
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
-#
-# pip-compile --allow-unsafe --generate-hashes requirements.in
-#
-argcomplete==2.0.0 \
- --hash=sha256:6372ad78c89d662035101418ae253668445b391755cfe94ea52f1b9d22425b20 \
- --hash=sha256:cffa11ea77999bb0dd27bb25ff6dc142a6796142f68d45b1a26b11f58724561e
- # via nox
-attrs==22.1.0 \
- --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \
- --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c
- # via gcp-releasetool
-bleach==5.0.1 \
- --hash=sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a \
- --hash=sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c
- # via readme-renderer
-cachetools==5.2.0 \
- --hash=sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757 \
- --hash=sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db
- # via google-auth
-certifi==2022.12.7 \
- --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \
- --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18
- # via requests
-cffi==1.15.1 \
- --hash=sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5 \
- --hash=sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef \
- --hash=sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104 \
- --hash=sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426 \
- --hash=sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405 \
- --hash=sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375 \
- --hash=sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a \
- --hash=sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e \
- --hash=sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc \
- --hash=sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf \
- --hash=sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185 \
- --hash=sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497 \
- --hash=sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3 \
- --hash=sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35 \
- --hash=sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c \
- --hash=sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83 \
- --hash=sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21 \
- --hash=sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca \
- --hash=sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984 \
- --hash=sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac \
- --hash=sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd \
- --hash=sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee \
- --hash=sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a \
- --hash=sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2 \
- --hash=sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192 \
- --hash=sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7 \
- --hash=sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585 \
- --hash=sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f \
- --hash=sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e \
- --hash=sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27 \
- --hash=sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b \
- --hash=sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e \
- --hash=sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e \
- --hash=sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d \
- --hash=sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c \
- --hash=sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415 \
- --hash=sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82 \
- --hash=sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02 \
- --hash=sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314 \
- --hash=sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325 \
- --hash=sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c \
- --hash=sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3 \
- --hash=sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914 \
- --hash=sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045 \
- --hash=sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d \
- --hash=sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9 \
- --hash=sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5 \
- --hash=sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2 \
- --hash=sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c \
- --hash=sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3 \
- --hash=sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2 \
- --hash=sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8 \
- --hash=sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d \
- --hash=sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d \
- --hash=sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9 \
- --hash=sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162 \
- --hash=sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76 \
- --hash=sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4 \
- --hash=sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e \
- --hash=sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9 \
- --hash=sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6 \
- --hash=sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b \
- --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \
- --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0
- # via cryptography
-charset-normalizer==2.1.1 \
- --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \
- --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f
- # via
- # -r requirements.in
- # requests
-click==8.0.4 \
- --hash=sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1 \
- --hash=sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb
- # via
- # -r requirements.in
- # gcp-docuploader
- # gcp-releasetool
-colorlog==6.7.0 \
- --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \
- --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5
- # via
- # gcp-docuploader
- # nox
-commonmark==0.9.1 \
- --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \
- --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9
- # via rich
-cryptography==39.0.1 \
- --hash=sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4 \
- --hash=sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f \
- --hash=sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502 \
- --hash=sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41 \
- --hash=sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965 \
- --hash=sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e \
- --hash=sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc \
- --hash=sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad \
- --hash=sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505 \
- --hash=sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388 \
- --hash=sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6 \
- --hash=sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2 \
- --hash=sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac \
- --hash=sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695 \
- --hash=sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6 \
- --hash=sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336 \
- --hash=sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0 \
- --hash=sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c \
- --hash=sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106 \
- --hash=sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a \
- --hash=sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8
- # via
- # gcp-releasetool
- # secretstorage
-distlib==0.3.6 \
- --hash=sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46 \
- --hash=sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e
- # via virtualenv
-docutils==0.19 \
- --hash=sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6 \
- --hash=sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc
- # via readme-renderer
-filelock==3.8.0 \
- --hash=sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc \
- --hash=sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4
- # via virtualenv
-gcp-docuploader==0.6.4 \
- --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \
- --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf
- # via -r requirements.in
-gcp-releasetool==1.10.5 \
- --hash=sha256:174b7b102d704b254f2a26a3eda2c684fd3543320ec239baf771542a2e58e109 \
- --hash=sha256:e29d29927fe2ca493105a82958c6873bb2b90d503acac56be2c229e74de0eec9
- # via -r requirements.in
-google-api-core==2.10.2 \
- --hash=sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320 \
- --hash=sha256:34f24bd1d5f72a8c4519773d99ca6bf080a6c4e041b4e9f024fe230191dda62e
- # via
- # google-cloud-core
- # google-cloud-storage
-google-auth==2.14.1 \
- --hash=sha256:ccaa901f31ad5cbb562615eb8b664b3dd0bf5404a67618e642307f00613eda4d \
- --hash=sha256:f5d8701633bebc12e0deea4df8abd8aff31c28b355360597f7f2ee60f2e4d016
- # via
- # gcp-releasetool
- # google-api-core
- # google-cloud-core
- # google-cloud-storage
-google-cloud-core==2.3.2 \
- --hash=sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe \
- --hash=sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a
- # via google-cloud-storage
-google-cloud-storage==2.6.0 \
- --hash=sha256:104ca28ae61243b637f2f01455cc8a05e8f15a2a18ced96cb587241cdd3820f5 \
- --hash=sha256:4ad0415ff61abdd8bb2ae81c1f8f7ec7d91a1011613f2db87c614c550f97bfe9
- # via gcp-docuploader
-google-crc32c==1.5.0 \
- --hash=sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a \
- --hash=sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876 \
- --hash=sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c \
- --hash=sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289 \
- --hash=sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298 \
- --hash=sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02 \
- --hash=sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f \
- --hash=sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2 \
- --hash=sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a \
- --hash=sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb \
- --hash=sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210 \
- --hash=sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5 \
- --hash=sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee \
- --hash=sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c \
- --hash=sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a \
- --hash=sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314 \
- --hash=sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd \
- --hash=sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65 \
- --hash=sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37 \
- --hash=sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4 \
- --hash=sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13 \
- --hash=sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894 \
- --hash=sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31 \
- --hash=sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e \
- --hash=sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709 \
- --hash=sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740 \
- --hash=sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc \
- --hash=sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d \
- --hash=sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c \
- --hash=sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c \
- --hash=sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d \
- --hash=sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906 \
- --hash=sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61 \
- --hash=sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57 \
- --hash=sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c \
- --hash=sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a \
- --hash=sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438 \
- --hash=sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946 \
- --hash=sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7 \
- --hash=sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96 \
- --hash=sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091 \
- --hash=sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae \
- --hash=sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d \
- --hash=sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88 \
- --hash=sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2 \
- --hash=sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd \
- --hash=sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541 \
- --hash=sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728 \
- --hash=sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178 \
- --hash=sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968 \
- --hash=sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346 \
- --hash=sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8 \
- --hash=sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93 \
- --hash=sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7 \
- --hash=sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273 \
- --hash=sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462 \
- --hash=sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94 \
- --hash=sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd \
- --hash=sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e \
- --hash=sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57 \
- --hash=sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b \
- --hash=sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9 \
- --hash=sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a \
- --hash=sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100 \
- --hash=sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325 \
- --hash=sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183 \
- --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \
- --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4
- # via google-resumable-media
-google-resumable-media==2.4.0 \
- --hash=sha256:2aa004c16d295c8f6c33b2b4788ba59d366677c0a25ae7382436cb30f776deaa \
- --hash=sha256:8d5518502f92b9ecc84ac46779bd4f09694ecb3ba38a3e7ca737a86d15cbca1f
- # via google-cloud-storage
-googleapis-common-protos==1.57.0 \
- --hash=sha256:27a849d6205838fb6cc3c1c21cb9800707a661bb21c6ce7fb13e99eb1f8a0c46 \
- --hash=sha256:a9f4a1d7f6d9809657b7f1316a1aa527f6664891531bcfcc13b6696e685f443c
- # via google-api-core
-idna==3.4 \
- --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \
- --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2
- # via requests
-importlib-metadata==5.0.0 \
- --hash=sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab \
- --hash=sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43
- # via
- # -r requirements.in
- # keyring
- # twine
-jaraco-classes==3.2.3 \
- --hash=sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158 \
- --hash=sha256:89559fa5c1d3c34eff6f631ad80bb21f378dbcbb35dd161fd2c6b93f5be2f98a
- # via keyring
-jeepney==0.8.0 \
- --hash=sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806 \
- --hash=sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755
- # via
- # keyring
- # secretstorage
-jinja2==3.1.2 \
- --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \
- --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61
- # via gcp-releasetool
-keyring==23.11.0 \
- --hash=sha256:3dd30011d555f1345dec2c262f0153f2f0ca6bca041fb1dc4588349bb4c0ac1e \
- --hash=sha256:ad192263e2cdd5f12875dedc2da13534359a7e760e77f8d04b50968a821c2361
- # via
- # gcp-releasetool
- # twine
-markupsafe==2.1.1 \
- --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \
- --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \
- --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \
- --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \
- --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \
- --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \
- --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \
- --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \
- --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \
- --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \
- --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \
- --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \
- --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \
- --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \
- --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \
- --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \
- --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \
- --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \
- --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \
- --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \
- --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \
- --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \
- --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \
- --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \
- --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \
- --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \
- --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \
- --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \
- --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \
- --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \
- --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \
- --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \
- --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \
- --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \
- --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \
- --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \
- --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \
- --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \
- --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \
- --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7
- # via jinja2
-more-itertools==9.0.0 \
- --hash=sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41 \
- --hash=sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab
- # via jaraco-classes
-nox==2022.8.7 \
- --hash=sha256:1b894940551dc5c389f9271d197ca5d655d40bdc6ccf93ed6880e4042760a34b \
- --hash=sha256:96cca88779e08282a699d672258ec01eb7c792d35bbbf538c723172bce23212c
- # via -r requirements.in
-packaging==21.3 \
- --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \
- --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522
- # via
- # gcp-releasetool
- # nox
-pkginfo==1.8.3 \
- --hash=sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594 \
- --hash=sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c
- # via twine
-platformdirs==2.5.4 \
- --hash=sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7 \
- --hash=sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10
- # via virtualenv
-protobuf==3.20.3 \
- --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \
- --hash=sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c \
- --hash=sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2 \
- --hash=sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b \
- --hash=sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050 \
- --hash=sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9 \
- --hash=sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7 \
- --hash=sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454 \
- --hash=sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480 \
- --hash=sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469 \
- --hash=sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c \
- --hash=sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e \
- --hash=sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db \
- --hash=sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905 \
- --hash=sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b \
- --hash=sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86 \
- --hash=sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4 \
- --hash=sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402 \
- --hash=sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7 \
- --hash=sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4 \
- --hash=sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99 \
- --hash=sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee
- # via
- # gcp-docuploader
- # gcp-releasetool
- # google-api-core
-py==1.11.0 \
- --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \
- --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378
- # via nox
-pyasn1==0.4.8 \
- --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
- --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba
- # via
- # pyasn1-modules
- # rsa
-pyasn1-modules==0.2.8 \
- --hash=sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e \
- --hash=sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74
- # via google-auth
-pycparser==2.21 \
- --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \
- --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206
- # via cffi
-pygments==2.13.0 \
- --hash=sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1 \
- --hash=sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42
- # via
- # readme-renderer
- # rich
-pyjwt==2.6.0 \
- --hash=sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd \
- --hash=sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14
- # via gcp-releasetool
-pyparsing==3.0.9 \
- --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \
- --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc
- # via packaging
-pyperclip==1.8.2 \
- --hash=sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57
- # via gcp-releasetool
-python-dateutil==2.8.2 \
- --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
- --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
- # via gcp-releasetool
-readme-renderer==37.3 \
- --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \
- --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343
- # via twine
-requests==2.28.1 \
- --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \
- --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349
- # via
- # gcp-releasetool
- # google-api-core
- # google-cloud-storage
- # requests-toolbelt
- # twine
-requests-toolbelt==0.10.1 \
- --hash=sha256:18565aa58116d9951ac39baa288d3adb5b3ff975c4f25eee78555d89e8f247f7 \
- --hash=sha256:62e09f7ff5ccbda92772a29f394a49c3ad6cb181d568b1337626b2abb628a63d
- # via twine
-rfc3986==2.0.0 \
- --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \
- --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c
- # via twine
-rich==12.6.0 \
- --hash=sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e \
- --hash=sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0
- # via twine
-rsa==4.9 \
- --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \
- --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21
- # via google-auth
-secretstorage==3.3.3 \
- --hash=sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77 \
- --hash=sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99
- # via keyring
-six==1.16.0 \
- --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
- --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
- # via
- # bleach
- # gcp-docuploader
- # google-auth
- # python-dateutil
-twine==4.0.1 \
- --hash=sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e \
- --hash=sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0
- # via -r requirements.in
-typing-extensions==4.4.0 \
- --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \
- --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e
- # via -r requirements.in
-urllib3==1.26.12 \
- --hash=sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e \
- --hash=sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997
- # via
- # requests
- # twine
-virtualenv==20.16.7 \
- --hash=sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e \
- --hash=sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29
- # via nox
-webencodings==0.5.1 \
- --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \
- --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923
- # via bleach
-wheel==0.38.4 \
- --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \
- --hash=sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8
- # via -r requirements.in
-zipp==3.10.0 \
- --hash=sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1 \
- --hash=sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8
- # via importlib-metadata
-
-# The following packages are considered to be unsafe in a requirements file:
-setuptools==65.5.1 \
- --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \
- --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f
- # via -r requirements.in
diff --git a/.kokoro/samples/python3.12/common.cfg b/.kokoro/samples/python3.12/common.cfg
new file mode 100644
index 00000000..8a5840a7
--- /dev/null
+++ b/.kokoro/samples/python3.12/common.cfg
@@ -0,0 +1,40 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+ define_artifacts {
+ regex: "**/*sponge_log.xml"
+ }
+}
+
+# Specify which tests to run
+env_vars: {
+ key: "RUN_TESTS_SESSION"
+ value: "py-3.12"
+}
+
+# Declare build specific Cloud project.
+env_vars: {
+ key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
+ value: "python-docs-samples-tests-312"
+}
+
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-api-core/.kokoro/test-samples.sh"
+}
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+ key: "TRAMPOLINE_IMAGE"
+ value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
+}
+
+# Download secrets for samples
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-api-core/.kokoro/trampoline_v2.sh"
\ No newline at end of file
diff --git a/.kokoro/samples/python3.12/continuous.cfg b/.kokoro/samples/python3.12/continuous.cfg
new file mode 100644
index 00000000..a1c8d975
--- /dev/null
+++ b/.kokoro/samples/python3.12/continuous.cfg
@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "True"
+}
\ No newline at end of file
diff --git a/.kokoro/samples/python3.12/periodic-head.cfg b/.kokoro/samples/python3.12/periodic-head.cfg
new file mode 100644
index 00000000..a18c0cfc
--- /dev/null
+++ b/.kokoro/samples/python3.12/periodic-head.cfg
@@ -0,0 +1,11 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "True"
+}
+
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-api-core/.kokoro/test-samples-against-head.sh"
+}
diff --git a/.kokoro/samples/python3.12/periodic.cfg b/.kokoro/samples/python3.12/periodic.cfg
new file mode 100644
index 00000000..71cd1e59
--- /dev/null
+++ b/.kokoro/samples/python3.12/periodic.cfg
@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "False"
+}
diff --git a/.kokoro/samples/python3.12/presubmit.cfg b/.kokoro/samples/python3.12/presubmit.cfg
new file mode 100644
index 00000000..a1c8d975
--- /dev/null
+++ b/.kokoro/samples/python3.12/presubmit.cfg
@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "True"
+}
\ No newline at end of file
diff --git a/.kokoro/samples/python3.13/common.cfg b/.kokoro/samples/python3.13/common.cfg
new file mode 100644
index 00000000..2a4199f4
--- /dev/null
+++ b/.kokoro/samples/python3.13/common.cfg
@@ -0,0 +1,40 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+ define_artifacts {
+ regex: "**/*sponge_log.xml"
+ }
+}
+
+# Specify which tests to run
+env_vars: {
+ key: "RUN_TESTS_SESSION"
+ value: "py-3.13"
+}
+
+# Declare build specific Cloud project.
+env_vars: {
+ key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
+ value: "python-docs-samples-tests-313"
+}
+
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-api-core/.kokoro/test-samples.sh"
+}
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+ key: "TRAMPOLINE_IMAGE"
+ value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker"
+}
+
+# Download secrets for samples
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-api-core/.kokoro/trampoline_v2.sh"
diff --git a/.kokoro/samples/python3.13/continuous.cfg b/.kokoro/samples/python3.13/continuous.cfg
new file mode 100644
index 00000000..a1c8d975
--- /dev/null
+++ b/.kokoro/samples/python3.13/continuous.cfg
@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "True"
+}
\ No newline at end of file
diff --git a/.kokoro/samples/python3.13/periodic-head.cfg b/.kokoro/samples/python3.13/periodic-head.cfg
new file mode 100644
index 00000000..a18c0cfc
--- /dev/null
+++ b/.kokoro/samples/python3.13/periodic-head.cfg
@@ -0,0 +1,11 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "True"
+}
+
+env_vars: {
+ key: "TRAMPOLINE_BUILD_FILE"
+ value: "github/python-api-core/.kokoro/test-samples-against-head.sh"
+}
diff --git a/.kokoro/samples/python3.13/periodic.cfg b/.kokoro/samples/python3.13/periodic.cfg
new file mode 100644
index 00000000..71cd1e59
--- /dev/null
+++ b/.kokoro/samples/python3.13/periodic.cfg
@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "False"
+}
diff --git a/.kokoro/samples/python3.13/presubmit.cfg b/.kokoro/samples/python3.13/presubmit.cfg
new file mode 100644
index 00000000..a1c8d975
--- /dev/null
+++ b/.kokoro/samples/python3.13/presubmit.cfg
@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+ key: "INSTALL_LIBRARY_FROM_SOURCE"
+ value: "True"
+}
\ No newline at end of file
diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh
index ba3a707b..e9d8bd79 100755
--- a/.kokoro/test-samples-against-head.sh
+++ b/.kokoro/test-samples-against-head.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh
index 2c6500ca..53e365bc 100755
--- a/.kokoro/test-samples-impl.sh
+++ b/.kokoro/test-samples-impl.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2021 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -33,7 +33,8 @@ export PYTHONUNBUFFERED=1
env | grep KOKORO
# Install nox
-python3.9 -m pip install --upgrade --quiet nox
+# `virtualenv==20.26.6` is added for Python 3.7 compatibility
+python3.9 -m pip install --upgrade --quiet nox virtualenv==20.26.6
# Use secrets acessor service account to get secrets
if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then
diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh
index 11c042d3..7933d820 100755
--- a/.kokoro/test-samples.sh
+++ b/.kokoro/test-samples.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh
index f39236e9..48f79699 100755
--- a/.kokoro/trampoline.sh
+++ b/.kokoro/trampoline.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2017 Google Inc.
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 4af6cdc2..35fa5292 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5405cc8f..1d74695f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -22,10 +22,10 @@ repos:
- id: end-of-file-fixer
- id: check-yaml
- repo: https://github.com/psf/black
- rev: 22.3.0
+ rev: 23.7.0
hooks:
- id: black
- repo: https://github.com/pycqa/flake8
- rev: 3.9.2
+ rev: 6.1.0
hooks:
- id: flake8
diff --git a/.trampolinerc b/.trampolinerc
index 0eee72ab..00801523 100644
--- a/.trampolinerc
+++ b/.trampolinerc
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,8 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Template for .trampolinerc
-
# Add required env vars here.
required_envvars+=(
)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 403b6359..98a6b8d7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,230 @@
[1]: https://pypi.org/project/google-api-core/#history
+## [2.25.0](https://github.com/googleapis/python-api-core/compare/v2.24.2...v2.25.0) (2025-05-06)
+
+
+### Features
+
+* Add protobuf runtime version to `x-goog-api-client` header ([#812](https://github.com/googleapis/python-api-core/issues/812)) ([118bd96](https://github.com/googleapis/python-api-core/commit/118bd96f3907234351972409834ab5309cdfcee4))
+* Support dynamic retry backoff values ([#793](https://github.com/googleapis/python-api-core/issues/793)) ([70697a3](https://github.com/googleapis/python-api-core/commit/70697a3e39c389768e724fddacb3c9b97d609384))
+
+
+### Bug Fixes
+
+* Resolve issue where pre-release versions of dependencies are installed ([#808](https://github.com/googleapis/python-api-core/issues/808)) ([1ca7973](https://github.com/googleapis/python-api-core/commit/1ca7973a395099403be1a99c7c4583a8f22d5d8e))
+
+## [2.24.2](https://github.com/googleapis/python-api-core/compare/v2.24.1...v2.24.2) (2025-03-06)
+
+
+### Bug Fixes
+
+* **deps:** Allow protobuf 6.x ([#804](https://github.com/googleapis/python-api-core/issues/804)) ([687be7c](https://github.com/googleapis/python-api-core/commit/687be7cbf629a61feb43ef37d3d920fa32b2d636))
+
+## [2.24.1](https://github.com/googleapis/python-api-core/compare/v2.24.0...v2.24.1) (2025-01-24)
+
+
+### Bug Fixes
+
+* Memory leak in bidi classes ([#770](https://github.com/googleapis/python-api-core/issues/770)) ([c1b8afa](https://github.com/googleapis/python-api-core/commit/c1b8afa4e2abe256e70651defccdc285f104ed19))
+* Resolve the issue where rpc timeout of 0 is used when timeout expires ([#776](https://github.com/googleapis/python-api-core/issues/776)) ([a5604a5](https://github.com/googleapis/python-api-core/commit/a5604a55070c6d92618d078191bf99f4c168d5f6))
+
+
+### Documentation
+
+* Add warnings regarding consuming externally sourced credentials ([#783](https://github.com/googleapis/python-api-core/issues/783)) ([0ec1825](https://github.com/googleapis/python-api-core/commit/0ec18254b90721684679a98bcacef4615467a227))
+
+## [2.24.0](https://github.com/googleapis/python-api-core/compare/v2.23.0...v2.24.0) (2024-12-06)
+
+
+### Features
+
+* Add automatic logging config to support debug logging ([#754](https://github.com/googleapis/python-api-core/issues/754)) ([d18d9b5](https://github.com/googleapis/python-api-core/commit/d18d9b5131162b44eebcc0859a7aca1198a2ac06))
+* Update recognized logging fields ([#766](https://github.com/googleapis/python-api-core/issues/766)) ([5f80f77](https://github.com/googleapis/python-api-core/commit/5f80f778bc25d878b3187c6138077ad8c6bcd35f))
+
+## [2.23.0](https://github.com/googleapis/python-api-core/compare/v2.22.0...v2.23.0) (2024-11-11)
+
+
+### Features
+
+* Migrate to pyproject.toml ([#736](https://github.com/googleapis/python-api-core/issues/736)) ([159e9a4](https://github.com/googleapis/python-api-core/commit/159e9a49525937f18a55c38136aae32575424d55))
+
+## [2.22.0](https://github.com/googleapis/python-api-core/compare/v2.21.0...v2.22.0) (2024-10-25)
+
+
+### Features
+
+* Add support for python 3.13 ([#696](https://github.com/googleapis/python-api-core/issues/696)) ([46b3d3a](https://github.com/googleapis/python-api-core/commit/46b3d3abaa1bae28e9d788d7c3006224cd6f74d5))
+
+
+### Bug Fixes
+
+* Add type hints to ClientOptions ([#735](https://github.com/googleapis/python-api-core/issues/735)) ([b91ed19](https://github.com/googleapis/python-api-core/commit/b91ed19210148dfa49ec790c4dd5f4a7bff80954))
+* Improve `Any` decode error ([#712](https://github.com/googleapis/python-api-core/issues/712)) ([0d5ed37](https://github.com/googleapis/python-api-core/commit/0d5ed37c96f9b40bccae98e228163a88abeb1763))
+* Require proto-plus >= 1.25.0 for Python 3.13 ([#740](https://github.com/googleapis/python-api-core/issues/740)) ([a26313e](https://github.com/googleapis/python-api-core/commit/a26313e1cb12e44aa498f12622edccc0c83ba0c3))
+* Switch to unittest.mock from mock ([#713](https://github.com/googleapis/python-api-core/issues/713)) ([8c53381](https://github.com/googleapis/python-api-core/commit/8c533819b7e212aa2f1d695a7ce08629f4fb2daf))
+
+## [2.21.0](https://github.com/googleapis/python-api-core/compare/v2.20.0...v2.21.0) (2024-10-07)
+
+
+### Features
+
+* Add support for asynchronous long running operations ([#724](https://github.com/googleapis/python-api-core/issues/724)) ([aaed69b](https://github.com/googleapis/python-api-core/commit/aaed69b6f1d694cd7e561e2aa03fdd8d6cfb369a))
+
+
+### Bug Fixes
+
+* Set chunk size for async stream content ([#702](https://github.com/googleapis/python-api-core/issues/702)) ([45b8a6d](https://github.com/googleapis/python-api-core/commit/45b8a6db5a5c75acdd8be896d0152f11608c7e51))
+
+## [2.20.0](https://github.com/googleapis/python-api-core/compare/v2.19.2...v2.20.0) (2024-09-18)
+
+
+### Features
+
+* Add async unsupported paramater exception ([#694](https://github.com/googleapis/python-api-core/issues/694)) ([8c137fe](https://github.com/googleapis/python-api-core/commit/8c137feb6e880fdd93d1248d9b6c10002dc3c096))
+* Add support for asynchronous rest streaming ([#686](https://github.com/googleapis/python-api-core/issues/686)) ([1b7bb6d](https://github.com/googleapis/python-api-core/commit/1b7bb6d1b721e4ee1561e8e4a347846d7fdd7c27))
+* Add support for creating exceptions from an asynchronous response ([#688](https://github.com/googleapis/python-api-core/issues/688)) ([1c4b0d0](https://github.com/googleapis/python-api-core/commit/1c4b0d079f2103a7b5562371a7bd1ada92528de3))
+
+## [2.19.2](https://github.com/googleapis/python-api-core/compare/v2.19.1...v2.19.2) (2024-08-16)
+
+
+### Bug Fixes
+
+* Fail gracefully if could not import `rpc_status` module ([#680](https://github.com/googleapis/python-api-core/issues/680)) ([7ccbf57](https://github.com/googleapis/python-api-core/commit/7ccbf5738fa236649f9a155055c71789362b5c4c))
+
+## [2.19.1](https://github.com/googleapis/python-api-core/compare/v2.19.0...v2.19.1) (2024-06-19)
+
+
+### Bug Fixes
+
+* Add support for protobuf 5.x ([#644](https://github.com/googleapis/python-api-core/issues/644)) ([fda0ca6](https://github.com/googleapis/python-api-core/commit/fda0ca6f0664ac5044671591ed62618175a7393f))
+* Ignore unknown fields in rest streaming. ([#651](https://github.com/googleapis/python-api-core/issues/651)) ([1203fb9](https://github.com/googleapis/python-api-core/commit/1203fb97d2685535f89113e944c4764c1deb595e))
+
+## [2.19.0](https://github.com/googleapis/python-api-core/compare/v2.18.0...v2.19.0) (2024-04-29)
+
+
+### Features
+
+* Add google.api_core.version_header ([#638](https://github.com/googleapis/python-api-core/issues/638)) ([a7b53e9](https://github.com/googleapis/python-api-core/commit/a7b53e9e9a7deb88baf92a2827958429e3677069))
+
+## [2.18.0](https://github.com/googleapis/python-api-core/compare/v2.17.1...v2.18.0) (2024-03-20)
+
+
+### Features
+
+* Add common logic for supporting universe domain ([#621](https://github.com/googleapis/python-api-core/issues/621)) ([94f2ca3](https://github.com/googleapis/python-api-core/commit/94f2ca3b4d094e6e10154634d3463d07ebea2035))
+
+
+### Bug Fixes
+
+* Add _registered_method to grpc ChannelStub ([#614](https://github.com/googleapis/python-api-core/issues/614)) ([5eaaea8](https://github.com/googleapis/python-api-core/commit/5eaaea8a989f8bdbdb5fbc95a155a20837c87f42))
+* **deps:** Require proto-plus >= 1.22.3 ([#626](https://github.com/googleapis/python-api-core/issues/626)) ([4fed37c](https://github.com/googleapis/python-api-core/commit/4fed37cbc32122f156e38250b5fa8b2b08a787a1))
+
+## [2.17.1](https://github.com/googleapis/python-api-core/compare/v2.17.0...v2.17.1) (2024-02-13)
+
+
+### Bug Fixes
+
+* Resolve issue handling protobuf responses in rest streaming ([#604](https://github.com/googleapis/python-api-core/issues/604)) ([bcebc92](https://github.com/googleapis/python-api-core/commit/bcebc92eca69dae81c5e546d526c92b164a6b3b4))
+
+## [2.17.0](https://github.com/googleapis/python-api-core/compare/v2.16.2...v2.17.0) (2024-02-06)
+
+
+### Features
+
+* Add attempt_direct_path argument to create_channel ([#583](https://github.com/googleapis/python-api-core/issues/583)) ([94726e7](https://github.com/googleapis/python-api-core/commit/94726e739698035b00667983f854c600252abd28))
+
+
+### Bug Fixes
+
+* Retry constructors methods support None ([#592](https://github.com/googleapis/python-api-core/issues/592)) ([416203c](https://github.com/googleapis/python-api-core/commit/416203c1888934670bfeccafe5f5469f87314512))
+
+## [2.16.2](https://github.com/googleapis/python-api-core/compare/v2.16.1...v2.16.2) (2024-02-02)
+
+
+### Bug Fixes
+
+* Spelling error `a,out` -> `amount` ([#596](https://github.com/googleapis/python-api-core/issues/596)) ([88688b1](https://github.com/googleapis/python-api-core/commit/88688b1625c4dab0df6124a0560f550eb322500f))
+
+## [2.16.1](https://github.com/googleapis/python-api-core/compare/v2.16.0...v2.16.1) (2024-01-30)
+
+
+### Bug Fixes
+
+* Fix broken import for google.api_core.retry_async.AsyncRetry ([#587](https://github.com/googleapis/python-api-core/issues/587)) ([ac012c0](https://github.com/googleapis/python-api-core/commit/ac012c04c69b8bbe72962f0d0d9e9536c0b4a524))
+
+## [2.16.0](https://github.com/googleapis/python-api-core/compare/v2.15.0...v2.16.0) (2024-01-29)
+
+
+### Features
+
+* Retry and retry_async support streaming rpcs ([#495](https://github.com/googleapis/python-api-core/issues/495)) ([17ff5f1](https://github.com/googleapis/python-api-core/commit/17ff5f1d83a9a6f50a0226fb0e794634bd584f17))
+
+## [2.15.0](https://github.com/googleapis/python-api-core/compare/v2.14.0...v2.15.0) (2023-12-07)
+
+
+### Features
+
+* Add support for Python 3.12 ([#557](https://github.com/googleapis/python-api-core/issues/557)) ([091b4f1](https://github.com/googleapis/python-api-core/commit/091b4f1c7fcc59c3f2a02ee44fd3c30b78423f12))
+* Add type annotations to wrapped grpc calls ([#554](https://github.com/googleapis/python-api-core/issues/554)) ([fc12b40](https://github.com/googleapis/python-api-core/commit/fc12b40bfc6e0c4bb313196e2e3a9c9374ce1c45))
+* Add universe_domain argument to ClientOptions ([3069ef4](https://github.com/googleapis/python-api-core/commit/3069ef4b9123ddb64841cbb7bbb183b53d502e0a))
+* Introduce compatibility with native namespace packages ([#561](https://github.com/googleapis/python-api-core/issues/561)) ([bd82827](https://github.com/googleapis/python-api-core/commit/bd82827108f1eeb6c05cfacf6c044b2afacc18a2))
+
+
+### Bug Fixes
+
+* Fix regression in `bidi` causing `Thread-ConsumeBidirectionalStream caught unexpected exception and will exit` ([#562](https://github.com/googleapis/python-api-core/issues/562)) ([40c8ae0](https://github.com/googleapis/python-api-core/commit/40c8ae0cf1f797e31e106461164e22db4fb2d3d9))
+* Replace deprecated `datetime.datetime.utcnow()` ([#552](https://github.com/googleapis/python-api-core/issues/552)) ([448923a](https://github.com/googleapis/python-api-core/commit/448923acf277a70e8704c949311bf4feaef8cab6)), closes [#540](https://github.com/googleapis/python-api-core/issues/540)
+
+## [2.14.0](https://github.com/googleapis/python-api-core/compare/v2.13.1...v2.14.0) (2023-11-09)
+
+
+### Features
+
+* Support with_call for wrapped rpcs ([#550](https://github.com/googleapis/python-api-core/issues/550)) ([01a57a7](https://github.com/googleapis/python-api-core/commit/01a57a745f4c8345c9c93412c27dd416b49f5953))
+
+## [2.13.1](https://github.com/googleapis/python-api-core/compare/v2.13.0...v2.13.1) (2023-11-09)
+
+
+### Bug Fixes
+
+* Update async client to use async retry ([#544](https://github.com/googleapis/python-api-core/issues/544)) ([f21bb32](https://github.com/googleapis/python-api-core/commit/f21bb32b8e6310116a642a6e6b6dd8e44e30e656))
+
+## [2.13.0](https://github.com/googleapis/python-api-core/compare/v2.12.0...v2.13.0) (2023-11-03)
+
+
+### Features
+
+* Add caching to routing header calculation ([#526](https://github.com/googleapis/python-api-core/issues/526)) ([6251eab](https://github.com/googleapis/python-api-core/commit/6251eab3fca5f7e509cb9b6e476ce1184094b711))
+
+
+### Bug Fixes
+
+* Add warning to retry target to avoid incorrect usage ([#543](https://github.com/googleapis/python-api-core/issues/543)) ([bfb40e6](https://github.com/googleapis/python-api-core/commit/bfb40e6929ef47be7a6464d2f1e0d06595736b8d))
+* Drop usage of distutils ([#541](https://github.com/googleapis/python-api-core/issues/541)) ([4bd9e10](https://github.com/googleapis/python-api-core/commit/4bd9e10f20eea227c88e3e1496010cca6dd8a270))
+* Ensure exception is available when BackgroundConsumer open stream fails ([#357](https://github.com/googleapis/python-api-core/issues/357)) ([405272c](https://github.com/googleapis/python-api-core/commit/405272c05f8c6d20e242c6172b01f78f0fd3bf32))
+
+## [2.12.0](https://github.com/googleapis/python-api-core/compare/v2.11.1...v2.12.0) (2023-09-07)
+
+
+### Features
+
+* Add a little bit of typing to google.api_core.retry ([#453](https://github.com/googleapis/python-api-core/issues/453)) ([2477ab9](https://github.com/googleapis/python-api-core/commit/2477ab9ea5c2e863a493fb7ebebaa429a44ea096))
+* Add grpc Compression argument to channels and methods ([#451](https://github.com/googleapis/python-api-core/issues/451)) ([bdebd63](https://github.com/googleapis/python-api-core/commit/bdebd6331f9c0d3d1a8ceaf274f07d2ed75bfe92))
+
+
+### Documentation
+
+* Fix a typo in google/api_core/page_iterator.py ([#511](https://github.com/googleapis/python-api-core/issues/511)) ([c0ce73c](https://github.com/googleapis/python-api-core/commit/c0ce73c4de53ad694fe36d17408998aa1230398f))
+
+## [2.11.1](https://github.com/googleapis/python-api-core/compare/v2.11.0...v2.11.1) (2023-06-12)
+
+
+### Bug Fixes
+
+* Add actionable errors for GCE long running operations ([#498](https://github.com/googleapis/python-api-core/issues/498)) ([7dfc3a7](https://github.com/googleapis/python-api-core/commit/7dfc3a7a439243f05238a11b68a31720fde1769e))
+* Invalid `dev` version identifiers in `setup.py` ([#505](https://github.com/googleapis/python-api-core/issues/505)) ([8844edb](https://github.com/googleapis/python-api-core/commit/8844edb1e802040810918a12bc9ff89104da38d4))
+
## [2.11.0](https://github.com/googleapis/python-api-core/compare/v2.10.2...v2.11.0) (2022-11-10)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 13d7a516..1a1f608b 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -21,7 +21,7 @@ In order to add a feature:
documentation.
- The feature must work fully on the following CPython versions:
- 3.7, 3.8, 3.9, 3.10, and 3.11 on both UNIX and Windows.
+ 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows.
- The feature must not add unnecessary dependencies (where
"unnecessary" is of course subjective, but new dependencies should
@@ -71,7 +71,7 @@ We use `nox `__ to instrument our tests.
- To run a single unit test::
- $ nox -s unit-3.11 -- -k
+ $ nox -s unit-3.13 -- -k
.. note::
@@ -202,12 +202,16 @@ We support:
- `Python 3.9`_
- `Python 3.10`_
- `Python 3.11`_
+- `Python 3.12`_
+- `Python 3.13`_
.. _Python 3.7: https://docs.python.org/3.7/
.. _Python 3.8: https://docs.python.org/3.8/
.. _Python 3.9: https://docs.python.org/3.9/
.. _Python 3.10: https://docs.python.org/3.10/
.. _Python 3.11: https://docs.python.org/3.11/
+.. _Python 3.12: https://docs.python.org/3.12/
+.. _Python 3.13: https://docs.python.org/3.13/
Supported versions can be found in our ``noxfile.py`` `config`_.
diff --git a/MANIFEST.in b/MANIFEST.in
index e783f4c6..d6814cd6 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docs/conf.py b/docs/conf.py
index 9a80171b..ad4723c0 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2021 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docs/retry.rst b/docs/retry.rst
index 97a7f2ca..6e165f56 100644
--- a/docs/retry.rst
+++ b/docs/retry.rst
@@ -10,4 +10,5 @@ Retry in AsyncIO
.. automodule:: google.api_core.retry_async
:members:
+ :noindex:
:show-inheritance:
diff --git a/google/api_core/_rest_streaming_base.py b/google/api_core/_rest_streaming_base.py
new file mode 100644
index 00000000..3bc87a96
--- /dev/null
+++ b/google/api_core/_rest_streaming_base.py
@@ -0,0 +1,118 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for server-side streaming in REST."""
+
+from collections import deque
+import string
+from typing import Deque, Union
+import types
+
+import proto
+import google.protobuf.message
+from google.protobuf.json_format import Parse
+
+
+class BaseResponseIterator:
+ """Base Iterator over REST API responses. This class should not be used directly.
+
+ Args:
+ response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response
+ class expected to be returned from an API.
+
+ Raises:
+ ValueError: If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`.
+ """
+
+ def __init__(
+ self,
+ response_message_cls: Union[proto.Message, google.protobuf.message.Message],
+ ):
+ self._response_message_cls = response_message_cls
+ # Contains a list of JSON responses ready to be sent to user.
+ self._ready_objs: Deque[str] = deque()
+ # Current JSON response being built.
+ self._obj = ""
+ # Keeps track of the nesting level within a JSON object.
+ self._level = 0
+ # Keeps track whether HTTP response is currently sending values
+ # inside of a string value.
+ self._in_string = False
+ # Whether an escape symbol "\" was encountered.
+ self._escape_next = False
+
+ self._grab = types.MethodType(self._create_grab(), self)
+
+ def _process_chunk(self, chunk: str):
+ if self._level == 0:
+ if chunk[0] != "[":
+ raise ValueError(
+ "Can only parse array of JSON objects, instead got %s" % chunk
+ )
+ for char in chunk:
+ if char == "{":
+ if self._level == 1:
+ # Level 1 corresponds to the outermost JSON object
+ # (i.e. the one we care about).
+ self._obj = ""
+ if not self._in_string:
+ self._level += 1
+ self._obj += char
+ elif char == "}":
+ self._obj += char
+ if not self._in_string:
+ self._level -= 1
+ if not self._in_string and self._level == 1:
+ self._ready_objs.append(self._obj)
+ elif char == '"':
+ # Helps to deal with an escaped quotes inside of a string.
+ if not self._escape_next:
+ self._in_string = not self._in_string
+ self._obj += char
+ elif char in string.whitespace:
+ if self._in_string:
+ self._obj += char
+ elif char == "[":
+ if self._level == 0:
+ self._level += 1
+ else:
+ self._obj += char
+ elif char == "]":
+ if self._level == 1:
+ self._level -= 1
+ else:
+ self._obj += char
+ else:
+ self._obj += char
+ self._escape_next = not self._escape_next if char == "\\" else False
+
+ def _create_grab(self):
+ if issubclass(self._response_message_cls, proto.Message):
+
+ def grab(this):
+ return this._response_message_cls.from_json(
+ this._ready_objs.popleft(), ignore_unknown_fields=True
+ )
+
+ return grab
+ elif issubclass(self._response_message_cls, google.protobuf.message.Message):
+
+ def grab(this):
+ return Parse(this._ready_objs.popleft(), this._response_message_cls())
+
+ return grab
+ else:
+ raise ValueError(
+ "Response message class must be a subclass of proto.Message or google.protobuf.message.Message."
+ )
diff --git a/google/api_core/bidi.py b/google/api_core/bidi.py
index 57f5f9dd..b002b409 100644
--- a/google/api_core/bidi.py
+++ b/google/api_core/bidi.py
@@ -91,11 +91,9 @@ def __init__(self, queue, period=1, initial_request=None):
def _is_active(self):
# Note: there is a possibility that this starts *before* the call
# property is set. So we have to check if self.call is set before
- # seeing if it's active.
- if self.call is not None and not self.call.is_active():
- return False
- else:
- return True
+ # seeing if it's active. We need to return True if self.call is None.
+ # See https://github.com/googleapis/python-api-core/issues/560.
+ return self.call is None or self.call.is_active()
def __iter__(self):
if self._initial_request is not None:
@@ -265,6 +263,10 @@ def add_done_callback(self, callback):
self._callbacks.append(callback)
def _on_call_done(self, future):
+ # This occurs when the RPC errors or is successfully terminated.
+ # Note that grpc's "future" here can also be a grpc.RpcError.
+ # See note in https://github.com/grpc/grpc/issues/10885#issuecomment-302651331
+ # that `grpc.RpcError` is also `grpc.call`.
for callback in self._callbacks:
callback(future)
@@ -276,7 +278,13 @@ def open(self):
request_generator = _RequestQueueGenerator(
self._request_queue, initial_request=self._initial_request
)
- call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata)
+ try:
+ call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata)
+ except exceptions.GoogleAPICallError as exc:
+ # The original `grpc.RpcError` (which is usually also a `grpc.Call`) is
+ # available from the ``response`` property on the mapped exception.
+ self._on_call_done(exc.response)
+ raise
request_generator.call = call
@@ -298,6 +306,8 @@ def close(self):
self._request_queue.put(None)
self.call.cancel()
self._request_generator = None
+ self._initial_request = None
+ self._callbacks = []
# Don't set self.call to None. Keep it around so that send/recv can
# raise the error.
@@ -654,7 +664,8 @@ def _thread_main(self, ready):
_LOGGER.debug("waiting for recv.")
response = self._bidi_rpc.recv()
_LOGGER.debug("recved response.")
- self._on_response(response)
+ if self._on_response is not None:
+ self._on_response(response)
except exceptions.GoogleAPICallError as exc:
_LOGGER.debug(
@@ -709,6 +720,7 @@ def stop(self):
_LOGGER.warning("Background thread did not exit.")
self._thread = None
+ self._on_response = None
@property
def is_active(self):
diff --git a/google/api_core/client_info.py b/google/api_core/client_info.py
index 48326799..f0678d24 100644
--- a/google/api_core/client_info.py
+++ b/google/api_core/client_info.py
@@ -57,7 +57,9 @@ class ClientInfo(object):
user_agent (Optional[str]): Prefix to the user agent header. This is
used to supply information such as application name or partner tool.
Recommended format: ``application-or-tool-ID/major.minor.version``.
- rest_version (Optional[str]): The requests library version.
+ rest_version (Optional[str]): A string with labeled versions of the
+ dependencies used for REST transport.
+ protobuf_runtime_version (Optional[str]): The protobuf runtime version.
"""
def __init__(
@@ -69,6 +71,7 @@ def __init__(
client_library_version=None,
user_agent=None,
rest_version=None,
+ protobuf_runtime_version=None,
):
self.python_version = python_version
self.grpc_version = grpc_version
@@ -77,6 +80,7 @@ def __init__(
self.client_library_version = client_library_version
self.user_agent = user_agent
self.rest_version = rest_version
+ self.protobuf_runtime_version = protobuf_runtime_version
def to_user_agent(self):
"""Returns the user-agent string for this client info."""
@@ -104,4 +108,7 @@ def to_user_agent(self):
if self.client_library_version is not None:
ua += "gccl/{client_library_version} "
+ if self.protobuf_runtime_version is not None:
+ ua += "pb/{protobuf_runtime_version} "
+
return ua.format(**self.__dict__).strip()
diff --git a/google/api_core/client_logging.py b/google/api_core/client_logging.py
new file mode 100644
index 00000000..837e3e0c
--- /dev/null
+++ b/google/api_core/client_logging.py
@@ -0,0 +1,144 @@
+import logging
+import json
+import os
+
+from typing import List, Optional
+
+_LOGGING_INITIALIZED = False
+_BASE_LOGGER_NAME = "google"
+
+# Fields to be included in the StructuredLogFormatter.
+#
+# TODO(https://github.com/googleapis/python-api-core/issues/761): Update this list to support additional logging fields.
+_recognized_logging_fields = [
+ "httpRequest",
+ "rpcName",
+ "serviceName",
+ "credentialsType",
+ "credentialsInfo",
+ "universeDomain",
+ "request",
+ "response",
+ "metadata",
+ "retryAttempt",
+ "httpResponse",
+] # Additional fields to be Logged.
+
+
+def logger_configured(logger) -> bool:
+ """Determines whether `logger` has non-default configuration
+
+ Args:
+ logger: The logger to check.
+
+ Returns:
+ bool: Whether the logger has any non-default configuration.
+ """
+ return (
+ logger.handlers != [] or logger.level != logging.NOTSET or not logger.propagate
+ )
+
+
+def initialize_logging():
+ """Initializes "google" loggers, partly based on the environment variable
+
+ Initializes the "google" logger and any loggers (at the "google"
+ level or lower) specified by the environment variable
+ GOOGLE_SDK_PYTHON_LOGGING_SCOPE, as long as none of these loggers
+ were previously configured. If any such loggers (including the
+ "google" logger) are initialized, they are set to NOT propagate
+ log events up to their parent loggers.
+
+ This initialization is executed only once, and hence the
+ environment variable is only processed the first time this
+ function is called.
+ """
+ global _LOGGING_INITIALIZED
+ if _LOGGING_INITIALIZED:
+ return
+ scopes = os.getenv("GOOGLE_SDK_PYTHON_LOGGING_SCOPE", "")
+ setup_logging(scopes)
+ _LOGGING_INITIALIZED = True
+
+
+def parse_logging_scopes(scopes: Optional[str] = None) -> List[str]:
+ """Returns a list of logger names.
+
+ Splits the single string of comma-separated logger names into a list of individual logger name strings.
+
+ Args:
+ scopes: The name of a single logger. (In the future, this will be a comma-separated list of multiple loggers.)
+
+ Returns:
+ A list of all the logger names in scopes.
+ """
+ if not scopes:
+ return []
+ # TODO(https://github.com/googleapis/python-api-core/issues/759): check if the namespace is a valid namespace.
+ # TODO(b/380481951): Support logging multiple scopes.
+ # TODO(b/380483756): Raise or log a warning for an invalid scope.
+ namespaces = [scopes]
+ return namespaces
+
+
+def configure_defaults(logger):
+ """Configures `logger` to emit structured info to stdout."""
+ if not logger_configured(logger):
+ console_handler = logging.StreamHandler()
+ logger.setLevel("DEBUG")
+ logger.propagate = False
+ formatter = StructuredLogFormatter()
+ console_handler.setFormatter(formatter)
+ logger.addHandler(console_handler)
+
+
+def setup_logging(scopes: str = ""):
+ """Sets up logging for the specified `scopes`.
+
+ If the loggers specified in `scopes` have not been previously
+ configured, this will configure them to emit structured log
+ entries to stdout, and to not propagate their log events to their
+ parent loggers. Additionally, if the "google" logger (whether it
+ was specified in `scopes` or not) was not previously configured,
+ it will also configure it to not propagate log events to the root
+ logger.
+
+ Args:
+ scopes: The name of a single logger. (In the future, this will be a comma-separated list of multiple loggers.)
+
+ """
+
+ # only returns valid logger scopes (namespaces)
+ # this list has at most one element.
+ logger_names = parse_logging_scopes(scopes)
+
+ for namespace in logger_names:
+ # This will either create a module level logger or get the reference of the base logger instantiated above.
+ logger = logging.getLogger(namespace)
+
+ # Configure default settings.
+ configure_defaults(logger)
+
+ # disable log propagation at base logger level to the root logger only if a base logger is not already configured via code changes.
+ base_logger = logging.getLogger(_BASE_LOGGER_NAME)
+ if not logger_configured(base_logger):
+ base_logger.propagate = False
+
+
+# TODO(https://github.com/googleapis/python-api-core/issues/763): Expand documentation.
+class StructuredLogFormatter(logging.Formatter):
+ # TODO(https://github.com/googleapis/python-api-core/issues/761): ensure that additional fields such as
+ # function name, file name, and line no. appear in a log output.
+ def format(self, record: logging.LogRecord):
+ log_obj = {
+ "timestamp": self.formatTime(record),
+ "severity": record.levelname,
+ "name": record.name,
+ "message": record.getMessage(),
+ }
+
+ for field_name in _recognized_logging_fields:
+ value = getattr(record, field_name, None)
+ if value is not None:
+ log_obj[field_name] = value
+ return json.dumps(log_obj)
diff --git a/google/api_core/client_options.py b/google/api_core/client_options.py
index ee9f28a9..d11665d2 100644
--- a/google/api_core/client_options.py
+++ b/google/api_core/client_options.py
@@ -48,6 +48,8 @@ def get_client_cert():
"""
+from typing import Callable, Mapping, Optional, Sequence, Tuple
+
class ClientOptions(object):
"""Client Options used to set options on clients.
@@ -55,11 +57,11 @@ class ClientOptions(object):
Args:
api_endpoint (Optional[str]): The desired API endpoint, e.g.,
compute.googleapis.com
- client_cert_source (Optional[Callable[[], (bytes, bytes)]]): A callback
+ client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): A callback
which returns client certificate bytes and private key bytes both in
PEM format. ``client_cert_source`` and ``client_encrypted_cert_source``
are mutually exclusive.
- client_encrypted_cert_source (Optional[Callable[[], (str, str, bytes)]]):
+ client_encrypted_cert_source (Optional[Callable[[], Tuple[str, str, bytes]]]):
A callback which returns client certificate file path, encrypted
private key file path, and the passphrase bytes.``client_cert_source``
and ``client_encrypted_cert_source`` are mutually exclusive.
@@ -67,6 +69,18 @@ class ClientOptions(object):
quota belongs to.
credentials_file (Optional[str]): A path to a file storing credentials.
``credentials_file` and ``api_key`` are mutually exclusive.
+
+ .. warning::
+ Important: If you accept a credential configuration (credential JSON/File/Stream)
+ from an external source for authentication to Google Cloud Platform, you must
+ validate it before providing it to any Google API or client library. Providing an
+ unvalidated credential configuration to Google APIs or libraries can compromise
+ the security of your systems and data. For more information, refer to
+ `Validate credential configurations from external sources`_.
+
+ .. _Validate credential configurations from external sources:
+
+ https://cloud.google.com/docs/authentication/external/externally-sourced-credentials
scopes (Optional[Sequence[str]]): OAuth access token override scopes.
api_key (Optional[str]): Google API key. ``credentials_file`` and
``api_key`` are mutually exclusive.
@@ -75,6 +89,11 @@ class ClientOptions(object):
authentication flows. Audience is typically a resource identifier.
If not set, the service endpoint value will be used as a default.
An example of a valid ``api_audience`` is: "https://language.googleapis.com".
+ universe_domain (Optional[str]): The desired universe domain. This must match
+ the one in credentials. If not set, the default universe domain is
+ `googleapis.com`. If both `api_endpoint` and `universe_domain` are set,
+ then `api_endpoint` is used as the service endpoint. If `api_endpoint` is
+ not specified, the format will be `{service}.{universe_domain}`.
Raises:
ValueError: If both ``client_cert_source`` and ``client_encrypted_cert_source``
@@ -83,14 +102,17 @@ class ClientOptions(object):
def __init__(
self,
- api_endpoint=None,
- client_cert_source=None,
- client_encrypted_cert_source=None,
- quota_project_id=None,
- credentials_file=None,
- scopes=None,
- api_key=None,
- api_audience=None,
+ api_endpoint: Optional[str] = None,
+ client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None,
+ client_encrypted_cert_source: Optional[
+ Callable[[], Tuple[str, str, bytes]]
+ ] = None,
+ quota_project_id: Optional[str] = None,
+ credentials_file: Optional[str] = None,
+ scopes: Optional[Sequence[str]] = None,
+ api_key: Optional[str] = None,
+ api_audience: Optional[str] = None,
+ universe_domain: Optional[str] = None,
):
if client_cert_source and client_encrypted_cert_source:
raise ValueError(
@@ -106,12 +128,13 @@ def __init__(
self.scopes = scopes
self.api_key = api_key
self.api_audience = api_audience
+ self.universe_domain = universe_domain
- def __repr__(self):
+ def __repr__(self) -> str:
return "ClientOptions: " + repr(self.__dict__)
-def from_dict(options):
+def from_dict(options: Mapping[str, object]) -> ClientOptions:
"""Construct a client options object from a mapping object.
Args:
diff --git a/google/api_core/datetime_helpers.py b/google/api_core/datetime_helpers.py
index c584c8cd..c3792300 100644
--- a/google/api_core/datetime_helpers.py
+++ b/google/api_core/datetime_helpers.py
@@ -42,7 +42,7 @@
def utcnow():
"""A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests."""
- return datetime.datetime.utcnow()
+ return datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)
def to_milliseconds(value):
diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py
index 35f2a6f8..e3eb696c 100644
--- a/google/api_core/exceptions.py
+++ b/google/api_core/exceptions.py
@@ -22,22 +22,27 @@
from __future__ import unicode_literals
import http.client
-from typing import Dict
+from typing import Optional, Dict
from typing import Union
import warnings
from google.rpc import error_details_pb2
+
+def _warn_could_not_import_grpcio_status():
+ warnings.warn(
+ "Please install grpcio-status to obtain helpful grpc error messages.",
+ ImportWarning,
+ ) # pragma: NO COVER
+
+
try:
import grpc
try:
from grpc_status import rpc_status
except ImportError: # pragma: NO COVER
- warnings.warn(
- "Please install grpcio-status to obtain helpful grpc error messages.",
- ImportWarning,
- )
+ _warn_could_not_import_grpcio_status()
rpc_status = None
except ImportError: # pragma: NO COVER
grpc = None
@@ -142,10 +147,21 @@ def __init__(self, message, errors=(), details=(), response=None, error_info=Non
self._error_info = error_info
def __str__(self):
+ error_msg = "{} {}".format(self.code, self.message)
if self.details:
- return "{} {} {}".format(self.code, self.message, self.details)
+ error_msg = "{} {}".format(error_msg, self.details)
+ # Note: This else condition can be removed once proposal A from
+ # b/284179390 is implemented.
else:
- return "{} {}".format(self.code, self.message)
+ if self.errors:
+ errors = [
+ f"{error.code}: {error.message}"
+ for error in self.errors
+ if hasattr(error, "code") and hasattr(error, "message")
+ ]
+ if errors:
+ error_msg = "{} {}".format(error_msg, "\n".join(errors))
+ return error_msg
@property
def reason(self):
@@ -426,6 +442,12 @@ class DeadlineExceeded(GatewayTimeout):
grpc_status_code = grpc.StatusCode.DEADLINE_EXCEEDED if grpc is not None else None
+class AsyncRestUnsupportedParameterError(NotImplementedError):
+ """Raised when an unsupported parameter is configured against async rest transport."""
+
+ pass
+
+
def exception_class_for_http_status(status_code):
"""Return the exception class for a specific HTTP status code.
@@ -460,40 +482,50 @@ def from_http_status(status_code, message, **kwargs):
return error
-def from_http_response(response):
- """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`.
+def _format_rest_error_message(error, method, url):
+ method = method.upper() if method else None
+ message = "{method} {url}: {error}".format(
+ method=method,
+ url=url,
+ error=error,
+ )
+ return message
+
+
+# NOTE: We're moving away from `from_http_status` because it expects an aiohttp response compared
+# to `format_http_response_error` which expects a more abstract response from google.auth and is
+# compatible with both sync and async response types.
+# TODO(https://github.com/googleapis/python-api-core/issues/691): Add type hint for response.
+def format_http_response_error(
+ response, method: str, url: str, payload: Optional[Dict] = None
+):
+ """Create a :class:`GoogleAPICallError` from a google auth rest response.
Args:
- response (requests.Response): The HTTP response.
+ response Union[google.auth.transport.Response, google.auth.aio.transport.Response]: The HTTP response.
+ method Optional(str): The HTTP request method.
+ url Optional(str): The HTTP request url.
+ payload Optional(dict): The HTTP response payload. If not passed in, it is read from response for a response type of google.auth.transport.Response.
Returns:
GoogleAPICallError: An instance of the appropriate subclass of
:class:`GoogleAPICallError`, with the message and errors populated
from the response.
"""
- try:
- payload = response.json()
- except ValueError:
- payload = {"error": {"message": response.text or "unknown error"}}
-
+ payload = {} if not payload else payload
error_message = payload.get("error", {}).get("message", "unknown error")
errors = payload.get("error", {}).get("errors", ())
# In JSON, details are already formatted in developer-friendly way.
details = payload.get("error", {}).get("details", ())
- error_info = list(
+ error_info_list = list(
filter(
lambda detail: detail.get("@type", "")
== "type.googleapis.com/google.rpc.ErrorInfo",
details,
)
)
- error_info = error_info[0] if error_info else None
-
- message = "{method} {url}: {error}".format(
- method=response.request.method,
- url=response.request.url,
- error=error_message,
- )
+ error_info = error_info_list[0] if error_info_list else None
+ message = _format_rest_error_message(error_message, method, url)
exception = from_http_status(
response.status_code,
@@ -506,6 +538,26 @@ def from_http_response(response):
return exception
+def from_http_response(response):
+ """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`.
+
+ Args:
+ response (requests.Response): The HTTP response.
+
+ Returns:
+ GoogleAPICallError: An instance of the appropriate subclass of
+ :class:`GoogleAPICallError`, with the message and errors populated
+ from the response.
+ """
+ try:
+ payload = response.json()
+ except ValueError:
+ payload = {"error": {"message": response.text or "unknown error"}}
+ return format_http_response_error(
+ response, response.request.method, response.request.url, payload
+ )
+
+
def exception_class_for_grpc_status(status_code):
"""Return the exception class for a specific :class:`grpc.StatusCode`.
@@ -549,6 +601,9 @@ def _is_informative_grpc_error(rpc_exc):
def _parse_grpc_error_details(rpc_exc):
+ if not rpc_status: # pragma: NO COVER
+ _warn_could_not_import_grpcio_status()
+ return [], None
try:
status = rpc_status.from_call(rpc_exc)
except NotImplementedError: # workaround
diff --git a/google/api_core/extended_operation.py b/google/api_core/extended_operation.py
index 79d47f0d..d474632b 100644
--- a/google/api_core/extended_operation.py
+++ b/google/api_core/extended_operation.py
@@ -158,10 +158,16 @@ def _handle_refreshed_operation(self):
return
if self.error_code and self.error_message:
+ # Note: `errors` can be removed once proposal A from
+ # b/284179390 is implemented.
+ errors = []
+ if hasattr(self, "error") and hasattr(self.error, "errors"):
+ errors = self.error.errors
exception = exceptions.from_http_status(
status_code=self.error_code,
message=self.error_message,
response=self._extended_operation,
+ errors=errors,
)
self.set_exception(exception)
elif self.error_code or self.error_message:
diff --git a/google/api_core/gapic_v1/client_info.py b/google/api_core/gapic_v1/client_info.py
index 2de1be7f..4b3b5649 100644
--- a/google/api_core/gapic_v1/client_info.py
+++ b/google/api_core/gapic_v1/client_info.py
@@ -45,6 +45,9 @@ class ClientInfo(client_info.ClientInfo):
user_agent (Optional[str]): Prefix to the user agent header. This is
used to supply information such as application name or partner tool.
Recommended format: ``application-or-tool-ID/major.minor.version``.
+ rest_version (Optional[str]): A string with labeled versions of the
+ dependencies used for REST transport.
+ protobuf_runtime_version (Optional[str]): The protobuf runtime version.
"""
def to_grpc_metadata(self):
diff --git a/google/api_core/gapic_v1/method.py b/google/api_core/gapic_v1/method.py
index 0c1624a3..0f14ea9c 100644
--- a/google/api_core/gapic_v1/method.py
+++ b/google/api_core/gapic_v1/method.py
@@ -15,7 +15,7 @@
"""Helpers for wrapping low-level gRPC methods with common functionality.
This is used by gapic clients to provide common error mapping, retry, timeout,
-pagination, and long-running operations to gRPC methods.
+compression, pagination, and long-running operations to gRPC methods.
"""
import enum
@@ -38,7 +38,7 @@ class _MethodDefault(enum.Enum):
DEFAULT = _MethodDefault._DEFAULT_VALUE
-"""Sentinel value indicating that a retry or timeout argument was unspecified,
+"""Sentinel value indicating that a retry, timeout, or compression argument was unspecified,
so the default should be used."""
@@ -72,20 +72,33 @@ class _GapicCallable(object):
after its start, not to be confused with deadline). If ``None``,
this callable will not specify a timeout argument to the low-level
RPC method.
+ compression (grpc.Compression): The default compression for the callable.
+ If ``None``, this callable will not specify a compression argument
+ to the low-level RPC method.
metadata (Sequence[Tuple[str, str]]): Additional metadata that is
provided to the RPC method on every invocation. This is merged with
any metadata specified during invocation. If ``None``, no
additional metadata will be passed to the RPC method.
"""
- def __init__(self, target, retry, timeout, metadata=None):
+ def __init__(
+ self,
+ target,
+ retry,
+ timeout,
+ compression,
+ metadata=None,
+ ):
self._target = target
self._retry = retry
self._timeout = timeout
+ self._compression = compression
self._metadata = metadata
- def __call__(self, *args, timeout=DEFAULT, retry=DEFAULT, **kwargs):
- """Invoke the low-level RPC with retry, timeout, and metadata."""
+ def __call__(
+ self, *args, timeout=DEFAULT, retry=DEFAULT, compression=DEFAULT, **kwargs
+ ):
+ """Invoke the low-level RPC with retry, timeout, compression, and metadata."""
if retry is DEFAULT:
retry = self._retry
@@ -93,6 +106,9 @@ def __call__(self, *args, timeout=DEFAULT, retry=DEFAULT, **kwargs):
if timeout is DEFAULT:
timeout = self._timeout
+ if compression is DEFAULT:
+ compression = self._compression
+
if isinstance(timeout, (int, float)):
timeout = TimeToDeadlineTimeout(timeout=timeout)
@@ -109,6 +125,8 @@ def __call__(self, *args, timeout=DEFAULT, retry=DEFAULT, **kwargs):
metadata = list(metadata)
metadata.extend(self._metadata)
kwargs["metadata"] = metadata
+ if self._compression is not None:
+ kwargs["compression"] = compression
return wrapped_func(*args, **kwargs)
@@ -117,12 +135,15 @@ def wrap_method(
func,
default_retry=None,
default_timeout=None,
+ default_compression=None,
client_info=client_info.DEFAULT_CLIENT_INFO,
+ *,
+ with_call=False,
):
"""Wrap an RPC method with common behavior.
- This applies common error wrapping, retry, and timeout behavior a function.
- The wrapped function will take optional ``retry`` and ``timeout``
+ This applies common error wrapping, retry, timeout, and compression behavior to a function.
+ The wrapped function will take optional ``retry``, ``timeout``, and ``compression``
arguments.
For example::
@@ -130,6 +151,7 @@ def wrap_method(
import google.api_core.gapic_v1.method
from google.api_core import retry
from google.api_core import timeout
+ from grpc import Compression
# The original RPC method.
def get_topic(name, timeout=None):
@@ -138,6 +160,7 @@ def get_topic(name, timeout=None):
default_retry = retry.Retry(deadline=60)
default_timeout = timeout.Timeout(deadline=60)
+ default_compression = Compression.NoCompression
wrapped_get_topic = google.api_core.gapic_v1.method.wrap_method(
get_topic, default_retry)
@@ -186,20 +209,34 @@ def get_topic(name, timeout=None):
default_timeout (Optional[google.api_core.Timeout]): The default
timeout strategy. Can also be specified as an int or float. If
``None``, the method will not have timeout specified by default.
+ default_compression (Optional[grpc.Compression]): The default
+ grpc.Compression. If ``None``, the method will not have
+ compression specified by default.
client_info
(Optional[google.api_core.gapic_v1.client_info.ClientInfo]):
Client information used to create a user-agent string that's
passed as gRPC metadata to the method. If unspecified, then
a sane default will be used. If ``None``, then no user agent
metadata will be provided to the RPC method.
+ with_call (bool): If True, wrapped grpc.UnaryUnaryMulticallables will
+ return a tuple of (response, grpc.Call) instead of just the response.
+ This is useful for extracting trailing metadata from unary calls.
+ Defaults to False.
Returns:
- Callable: A new callable that takes optional ``retry`` and ``timeout``
- arguments and applies the common error mapping, retry, timeout,
+ Callable: A new callable that takes optional ``retry``, ``timeout``,
+ and ``compression``
+ arguments and applies the common error mapping, retry, timeout, compression,
and metadata behavior to the low-level RPC method.
"""
+ if with_call:
+ try:
+ func = func.with_call
+ except AttributeError as exc:
+ raise ValueError(
+ "with_call=True is only supported for unary calls."
+ ) from exc
func = grpc_helpers.wrap_errors(func)
-
if client_info is not None:
user_agent_metadata = [client_info.to_grpc_metadata()]
else:
@@ -207,6 +244,10 @@ def get_topic(name, timeout=None):
return functools.wraps(func)(
_GapicCallable(
- func, default_retry, default_timeout, metadata=user_agent_metadata
+ func,
+ default_retry,
+ default_timeout,
+ default_compression,
+ metadata=user_agent_metadata,
)
)
diff --git a/google/api_core/gapic_v1/method_async.py b/google/api_core/gapic_v1/method_async.py
index 84c99aa2..c0f38c0e 100644
--- a/google/api_core/gapic_v1/method_async.py
+++ b/google/api_core/gapic_v1/method_async.py
@@ -14,7 +14,7 @@
"""AsyncIO helpers for wrapping gRPC methods with common functionality.
This is used by gapic clients to provide common error mapping, retry, timeout,
-pagination, and long-running operations to gRPC methods.
+compression, pagination, and long-running operations to gRPC methods.
"""
import functools
@@ -25,24 +25,35 @@
from google.api_core.gapic_v1.method import DEFAULT # noqa: F401
from google.api_core.gapic_v1.method import USE_DEFAULT_METADATA # noqa: F401
+_DEFAULT_ASYNC_TRANSPORT_KIND = "grpc_asyncio"
+
def wrap_method(
func,
default_retry=None,
default_timeout=None,
+ default_compression=None,
client_info=client_info.DEFAULT_CLIENT_INFO,
+ kind=_DEFAULT_ASYNC_TRANSPORT_KIND,
):
"""Wrap an async RPC method with common behavior.
Returns:
- Callable: A new callable that takes optional ``retry`` and ``timeout``
- arguments and applies the common error mapping, retry, timeout,
- and metadata behavior to the low-level RPC method.
+ Callable: A new callable that takes optional ``retry``, ``timeout``,
+ and ``compression`` arguments and applies the common error mapping,
+ retry, timeout, metadata, and compression behavior to the low-level RPC method.
"""
- func = grpc_helpers_async.wrap_errors(func)
+ if kind == _DEFAULT_ASYNC_TRANSPORT_KIND:
+ func = grpc_helpers_async.wrap_errors(func)
metadata = [client_info.to_grpc_metadata()] if client_info is not None else None
return functools.wraps(func)(
- _GapicCallable(func, default_retry, default_timeout, metadata=metadata)
+ _GapicCallable(
+ func,
+ default_retry,
+ default_timeout,
+ default_compression,
+ metadata=metadata,
+ )
)
diff --git a/google/api_core/gapic_v1/routing_header.py b/google/api_core/gapic_v1/routing_header.py
index 28b13abf..c0c6f648 100644
--- a/google/api_core/gapic_v1/routing_header.py
+++ b/google/api_core/gapic_v1/routing_header.py
@@ -20,17 +20,22 @@
Generally, these headers are specified as gRPC metadata.
"""
+import functools
from enum import Enum
from urllib.parse import urlencode
ROUTING_METADATA_KEY = "x-goog-request-params"
+# This is the value for the `maxsize` argument of @functools.lru_cache
+# https://docs.python.org/3/library/functools.html#functools.lru_cache
+# This represents the number of recent function calls to store.
+ROUTING_PARAM_CACHE_SIZE = 32
def to_routing_header(params, qualified_enums=True):
"""Returns a routing header string for the given request parameters.
Args:
- params (Mapping[str, Any]): A dictionary containing the request
+ params (Mapping[str, str | bytes | Enum]): A dictionary containing the request
parameters used for routing.
qualified_enums (bool): Whether to represent enum values
as their type-qualified symbol names instead of as their
@@ -38,19 +43,11 @@ def to_routing_header(params, qualified_enums=True):
Returns:
str: The routing header string.
-
"""
+ tuples = params.items() if isinstance(params, dict) else params
if not qualified_enums:
- if isinstance(params, dict):
- tuples = params.items()
- else:
- tuples = params
- params = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples]
- return urlencode(
- params,
- # Per Google API policy (go/api-url-encoding), / is not encoded.
- safe="/",
- )
+ tuples = [(x[0], x[1].name) if isinstance(x[1], Enum) else x for x in tuples]
+ return "&".join([_urlencode_param(*t) for t in tuples])
def to_grpc_metadata(params, qualified_enums=True):
@@ -58,7 +55,7 @@ def to_grpc_metadata(params, qualified_enums=True):
request parameters.
Args:
- params (Mapping[str, Any]): A dictionary containing the request
+ params (Mapping[str, str | bytes | Enum]): A dictionary containing the request
parameters used for routing.
qualified_enums (bool): Whether to represent enum values
as their type-qualified symbol names instead of as their
@@ -69,3 +66,22 @@ def to_grpc_metadata(params, qualified_enums=True):
and value.
"""
return (ROUTING_METADATA_KEY, to_routing_header(params, qualified_enums))
+
+
+# use caching to avoid repeated computation
+@functools.lru_cache(maxsize=ROUTING_PARAM_CACHE_SIZE)
+def _urlencode_param(key, value):
+ """Cacheable wrapper over urlencode
+
+ Args:
+ key (str): The key of the parameter to encode.
+ value (str | bytes | Enum): The value of the parameter to encode.
+
+ Returns:
+ str: The encoded parameter.
+ """
+ return urlencode(
+ {key: value},
+ # Per Google API policy (go/api-url-encoding), / is not encoded.
+ safe="/",
+ )
diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py
index 102dc0a0..07963024 100644
--- a/google/api_core/grpc_helpers.py
+++ b/google/api_core/grpc_helpers.py
@@ -13,6 +13,7 @@
# limitations under the License.
"""Helpers for :mod:`grpc`."""
+from typing import Generic, Iterator, Optional, TypeVar
import collections
import functools
@@ -51,6 +52,9 @@
# The list of gRPC Callable interfaces that return iterators.
_STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable)
+# denotes the proto response type for grpc calls
+P = TypeVar("P")
+
def _patch_callable_name(callable_):
"""Fix-up gRPC callable attributes.
@@ -76,7 +80,7 @@ def error_remapped_callable(*args, **kwargs):
return error_remapped_callable
-class _StreamingResponseIterator(grpc.Call):
+class _StreamingResponseIterator(Generic[P], grpc.Call):
def __init__(self, wrapped, prefetch_first_result=True):
self._wrapped = wrapped
@@ -94,11 +98,11 @@ def __init__(self, wrapped, prefetch_first_result=True):
# ignore stop iteration at this time. This should be handled outside of retry.
pass
- def __iter__(self):
+ def __iter__(self) -> Iterator[P]:
"""This iterator is also an iterable that returns itself."""
return self
- def __next__(self):
+ def __next__(self) -> P:
"""Get the next response from the stream.
Returns:
@@ -141,6 +145,10 @@ def trailing_metadata(self):
return self._wrapped.trailing_metadata()
+# public type alias denoting the return type of streaming gapic calls
+GrpcStream = _StreamingResponseIterator[P]
+
+
def _wrap_stream_errors(callable_):
"""Wrap errors for Unary-Stream and Stream-Stream gRPC callables.
@@ -208,6 +216,18 @@ def _create_composite_credentials(
credentials_file (str): A file with credentials that can be loaded with
:func:`google.auth.load_credentials_from_file`. This argument is
mutually exclusive with credentials.
+
+ .. warning::
+ Important: If you accept a credential configuration (credential JSON/File/Stream)
+ from an external source for authentication to Google Cloud Platform, you must
+ validate it before providing it to any Google API or client library. Providing an
+ unvalidated credential configuration to Google APIs or libraries can compromise
+ the security of your systems and data. For more information, refer to
+ `Validate credential configurations from external sources`_.
+
+ .. _Validate credential configurations from external sources:
+
+ https://cloud.google.com/docs/authentication/external/externally-sourced-credentials
default_scopes (Sequence[str]): A optional list of scopes needed for this
service. These are only used when credentials are not specified and
are passed to :func:`google.auth.default`.
@@ -260,11 +280,24 @@ def _create_composite_credentials(
# Create a set of grpc.CallCredentials using the metadata plugin.
google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin)
- if ssl_credentials is None:
- ssl_credentials = grpc.ssl_channel_credentials()
-
- # Combine the ssl credentials and the authorization credentials.
- return grpc.composite_channel_credentials(ssl_credentials, google_auth_credentials)
+ # if `ssl_credentials` is set, use `grpc.composite_channel_credentials` instead of
+ # `grpc.compute_engine_channel_credentials` as the former supports passing
+ # `ssl_credentials` via `channel_credentials` which is needed for mTLS.
+ if ssl_credentials:
+ # Combine the ssl credentials and the authorization credentials.
+ # See https://grpc.github.io/grpc/python/grpc.html#grpc.composite_channel_credentials
+ return grpc.composite_channel_credentials(
+ ssl_credentials, google_auth_credentials
+ )
+ else:
+ # Use grpc.compute_engine_channel_credentials in order to support Direct Path.
+ # See https://grpc.github.io/grpc/python/grpc.html#grpc.compute_engine_channel_credentials
+ # TODO(https://github.com/googleapis/python-api-core/issues/598):
+ # Although `grpc.compute_engine_channel_credentials` returns channel credentials
+ # outside of a Google Compute Engine environment (GCE), we should determine if
+ # there is a way to reliably detect a GCE environment so that
+ # `grpc.compute_engine_channel_credentials` is not called outside of GCE.
+ return grpc.compute_engine_channel_credentials(google_auth_credentials)
def create_channel(
@@ -276,7 +309,9 @@ def create_channel(
quota_project_id=None,
default_scopes=None,
default_host=None,
- **kwargs
+ compression=None,
+ attempt_direct_path: Optional[bool] = False,
+ **kwargs,
):
"""Create a secure channel with credentials.
@@ -293,10 +328,40 @@ def create_channel(
credentials_file (str): A file with credentials that can be loaded with
:func:`google.auth.load_credentials_from_file`. This argument is
mutually exclusive with credentials.
+
+ .. warning::
+ Important: If you accept a credential configuration (credential JSON/File/Stream)
+ from an external source for authentication to Google Cloud Platform, you must
+ validate it before providing it to any Google API or client library. Providing an
+ unvalidated credential configuration to Google APIs or libraries can compromise
+ the security of your systems and data. For more information, refer to
+ `Validate credential configurations from external sources`_.
+
+ .. _Validate credential configurations from external sources:
+
+ https://cloud.google.com/docs/authentication/external/externally-sourced-credentials
quota_project_id (str): An optional project to use for billing and quota.
default_scopes (Sequence[str]): Default scopes passed by a Google client
library. Use 'scopes' for user-defined scopes.
default_host (str): The default endpoint. e.g., "pubsub.googleapis.com".
+ compression (grpc.Compression): An optional value indicating the
+ compression method to be used over the lifetime of the channel.
+ attempt_direct_path (Optional[bool]): If set, Direct Path will be attempted
+ when the request is made. Direct Path is only available within a Google
+ Compute Engine (GCE) environment and provides a proxyless connection
+ which increases the available throughput, reduces latency, and increases
+ reliability. Note:
+
+ - This argument should only be set in a GCE environment and for Services
+ that are known to support Direct Path.
+ - If this argument is set outside of GCE, then this request will fail
+ unless the back-end service happens to have configured fall-back to DNS.
+ - If the request causes a `ServiceUnavailable` response, it is recommended
+ that the client repeat the request with `attempt_direct_path` set to
+ `False` as the Service may not support Direct Path.
+ - Using `ssl_credentials` with `attempt_direct_path` set to `True` will
+ result in `ValueError` as this combination is not yet supported.
+
kwargs: Additional key-word args passed to
:func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`.
Note: `grpc_gcp` is only supported in environments with protobuf < 4.0.0.
@@ -306,8 +371,15 @@ def create_channel(
Raises:
google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed.
+ ValueError: If `ssl_credentials` is set and `attempt_direct_path` is set to `True`.
"""
+ # If `ssl_credentials` is set and `attempt_direct_path` is set to `True`,
+ # raise ValueError as this is not yet supported.
+ # See https://github.com/googleapis/python-api-core/issues/590
+ if ssl_credentials and attempt_direct_path:
+ raise ValueError("Using ssl_credentials with Direct Path is not supported")
+
composite_credentials = _create_composite_credentials(
credentials=credentials,
credentials_file=credentials_file,
@@ -318,13 +390,60 @@ def create_channel(
default_host=default_host,
)
+ # Note that grpcio-gcp is deprecated
if HAS_GRPC_GCP: # pragma: NO COVER
+ if compression is not None and compression != grpc.Compression.NoCompression:
+ warnings.warn(
+ "The `compression` argument is ignored for grpc_gcp.secure_channel creation.",
+ DeprecationWarning,
+ )
+ if attempt_direct_path:
+ warnings.warn(
+ """The `attempt_direct_path` argument is ignored for grpc_gcp.secure_channel creation.""",
+ DeprecationWarning,
+ )
return grpc_gcp.secure_channel(target, composite_credentials, **kwargs)
- return grpc.secure_channel(target, composite_credentials, **kwargs)
+
+ if attempt_direct_path:
+ target = _modify_target_for_direct_path(target)
+
+ return grpc.secure_channel(
+ target, composite_credentials, compression=compression, **kwargs
+ )
+
+
+def _modify_target_for_direct_path(target: str) -> str:
+ """
+ Given a target, return a modified version which is compatible with Direct Path.
+
+ Args:
+ target (str): The target service address in the format 'hostname[:port]' or
+ 'dns://hostname[:port]'.
+
+ Returns:
+ target (str): The target service address which is converted into a format compatible with Direct Path.
+ If the target contains `dns:///` or does not contain `:///`, the target will be converted in
+ a format compatible with Direct Path; otherwise the original target will be returned as the
+ original target may already denote Direct Path.
+ """
+
+ # A DNS prefix may be included with the target to indicate the endpoint is living in the Internet,
+ # outside of Google Cloud Platform.
+ dns_prefix = "dns:///"
+ # Remove "dns:///" if `attempt_direct_path` is set to True as
+ # the Direct Path prefix `google-c2p:///` will be used instead.
+ target = target.replace(dns_prefix, "")
+
+ direct_path_separator = ":///"
+ if direct_path_separator not in target:
+ target_without_port = target.split(":")[0]
+ # Modify the target to use Direct Path by adding the `google-c2p:///` prefix
+ target = f"google-c2p{direct_path_separator}{target_without_port}"
+ return target
_MethodCall = collections.namedtuple(
- "_MethodCall", ("request", "timeout", "metadata", "credentials")
+ "_MethodCall", ("request", "timeout", "metadata", "credentials", "compression")
)
_ChannelRequest = collections.namedtuple("_ChannelRequest", ("method", "request"))
@@ -351,11 +470,15 @@ def __init__(self, method, channel):
"""List[protobuf.Message]: All requests sent to this callable."""
self.calls = []
"""List[Tuple]: All invocations of this callable. Each tuple is the
- request, timeout, metadata, and credentials."""
+ request, timeout, metadata, compression, and credentials."""
- def __call__(self, request, timeout=None, metadata=None, credentials=None):
+ def __call__(
+ self, request, timeout=None, metadata=None, credentials=None, compression=None
+ ):
self._channel.requests.append(_ChannelRequest(self._method, request))
- self.calls.append(_MethodCall(request, timeout, metadata, credentials))
+ self.calls.append(
+ _MethodCall(request, timeout, metadata, credentials, compression)
+ )
self.requests.append(request)
response = self.response
@@ -470,20 +593,42 @@ def __getattr__(self, key):
except KeyError:
raise AttributeError
- def unary_unary(self, method, request_serializer=None, response_deserializer=None):
+ def unary_unary(
+ self,
+ method,
+ request_serializer=None,
+ response_deserializer=None,
+ _registered_method=False,
+ ):
"""grpc.Channel.unary_unary implementation."""
return self._stub_for_method(method)
- def unary_stream(self, method, request_serializer=None, response_deserializer=None):
+ def unary_stream(
+ self,
+ method,
+ request_serializer=None,
+ response_deserializer=None,
+ _registered_method=False,
+ ):
"""grpc.Channel.unary_stream implementation."""
return self._stub_for_method(method)
- def stream_unary(self, method, request_serializer=None, response_deserializer=None):
+ def stream_unary(
+ self,
+ method,
+ request_serializer=None,
+ response_deserializer=None,
+ _registered_method=False,
+ ):
"""grpc.Channel.stream_unary implementation."""
return self._stub_for_method(method)
def stream_stream(
- self, method, request_serializer=None, response_deserializer=None
+ self,
+ method,
+ request_serializer=None,
+ response_deserializer=None,
+ _registered_method=False,
):
"""grpc.Channel.stream_stream implementation."""
return self._stub_for_method(method)
diff --git a/google/api_core/grpc_helpers_async.py b/google/api_core/grpc_helpers_async.py
index 5a5bf2a6..af661430 100644
--- a/google/api_core/grpc_helpers_async.py
+++ b/google/api_core/grpc_helpers_async.py
@@ -21,11 +21,15 @@
import asyncio
import functools
+from typing import AsyncGenerator, Generic, Iterator, Optional, TypeVar
+
import grpc
from grpc import aio
from google.api_core import exceptions, grpc_helpers
+# denotes the proto response type for grpc calls
+P = TypeVar("P")
# NOTE(lidiz) Alternatively, we can hack "__getattribute__" to perform
# automatic patching for us. But that means the overhead of creating an
@@ -75,8 +79,8 @@ async def wait_for_connection(self):
raise exceptions.from_grpc_error(rpc_error) from rpc_error
-class _WrappedUnaryResponseMixin(_WrappedCall):
- def __await__(self):
+class _WrappedUnaryResponseMixin(Generic[P], _WrappedCall):
+ def __await__(self) -> Iterator[P]:
try:
response = yield from self._call.__await__()
return response
@@ -84,17 +88,17 @@ def __await__(self):
raise exceptions.from_grpc_error(rpc_error) from rpc_error
-class _WrappedStreamResponseMixin(_WrappedCall):
+class _WrappedStreamResponseMixin(Generic[P], _WrappedCall):
def __init__(self):
self._wrapped_async_generator = None
- async def read(self):
+ async def read(self) -> P:
try:
return await self._call.read()
except grpc.RpcError as rpc_error:
raise exceptions.from_grpc_error(rpc_error) from rpc_error
- async def _wrapped_aiter(self):
+ async def _wrapped_aiter(self) -> AsyncGenerator[P, None]:
try:
# NOTE(lidiz) coverage doesn't understand the exception raised from
# __anext__ method. It is covered by test case:
@@ -104,7 +108,7 @@ async def _wrapped_aiter(self):
except grpc.RpcError as rpc_error:
raise exceptions.from_grpc_error(rpc_error) from rpc_error
- def __aiter__(self):
+ def __aiter__(self) -> AsyncGenerator[P, None]:
if not self._wrapped_async_generator:
self._wrapped_async_generator = self._wrapped_aiter()
return self._wrapped_async_generator
@@ -127,29 +131,34 @@ async def done_writing(self):
# NOTE(lidiz) Implementing each individual class separately, so we don't
# expose any API that should not be seen. E.g., __aiter__ in unary-unary
# RPC, or __await__ in stream-stream RPC.
-class _WrappedUnaryUnaryCall(_WrappedUnaryResponseMixin, aio.UnaryUnaryCall):
+class _WrappedUnaryUnaryCall(_WrappedUnaryResponseMixin[P], aio.UnaryUnaryCall):
"""Wrapped UnaryUnaryCall to map exceptions."""
-class _WrappedUnaryStreamCall(_WrappedStreamResponseMixin, aio.UnaryStreamCall):
+class _WrappedUnaryStreamCall(_WrappedStreamResponseMixin[P], aio.UnaryStreamCall):
"""Wrapped UnaryStreamCall to map exceptions."""
class _WrappedStreamUnaryCall(
- _WrappedUnaryResponseMixin, _WrappedStreamRequestMixin, aio.StreamUnaryCall
+ _WrappedUnaryResponseMixin[P], _WrappedStreamRequestMixin, aio.StreamUnaryCall
):
"""Wrapped StreamUnaryCall to map exceptions."""
class _WrappedStreamStreamCall(
- _WrappedStreamRequestMixin, _WrappedStreamResponseMixin, aio.StreamStreamCall
+ _WrappedStreamRequestMixin, _WrappedStreamResponseMixin[P], aio.StreamStreamCall
):
"""Wrapped StreamStreamCall to map exceptions."""
+# public type alias denoting the return type of async streaming gapic calls
+GrpcAsyncStream = _WrappedStreamResponseMixin
+# public type alias denoting the return type of unary gapic calls
+AwaitableGrpcCall = _WrappedUnaryResponseMixin
+
+
def _wrap_unary_errors(callable_):
"""Map errors for Unary-Unary async callables."""
- grpc_helpers._patch_callable_name(callable_)
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
@@ -159,23 +168,13 @@ def error_remapped_callable(*args, **kwargs):
return error_remapped_callable
-def _wrap_stream_errors(callable_):
+def _wrap_stream_errors(callable_, wrapper_type):
"""Map errors for streaming RPC async callables."""
- grpc_helpers._patch_callable_name(callable_)
@functools.wraps(callable_)
async def error_remapped_callable(*args, **kwargs):
call = callable_(*args, **kwargs)
-
- if isinstance(call, aio.UnaryStreamCall):
- call = _WrappedUnaryStreamCall().with_call(call)
- elif isinstance(call, aio.StreamUnaryCall):
- call = _WrappedStreamUnaryCall().with_call(call)
- elif isinstance(call, aio.StreamStreamCall):
- call = _WrappedStreamStreamCall().with_call(call)
- else:
- raise TypeError("Unexpected type of call %s" % type(call))
-
+ call = wrapper_type().with_call(call)
await call.wait_for_connection()
return call
@@ -197,10 +196,16 @@ def wrap_errors(callable_):
Returns: Callable: The wrapped gRPC callable.
"""
- if isinstance(callable_, aio.UnaryUnaryMultiCallable):
- return _wrap_unary_errors(callable_)
+ grpc_helpers._patch_callable_name(callable_)
+
+ if isinstance(callable_, aio.UnaryStreamMultiCallable):
+ return _wrap_stream_errors(callable_, _WrappedUnaryStreamCall)
+ elif isinstance(callable_, aio.StreamUnaryMultiCallable):
+ return _wrap_stream_errors(callable_, _WrappedStreamUnaryCall)
+ elif isinstance(callable_, aio.StreamStreamMultiCallable):
+ return _wrap_stream_errors(callable_, _WrappedStreamStreamCall)
else:
- return _wrap_stream_errors(callable_)
+ return _wrap_unary_errors(callable_)
def create_channel(
@@ -212,6 +217,8 @@ def create_channel(
quota_project_id=None,
default_scopes=None,
default_host=None,
+ compression=None,
+ attempt_direct_path: Optional[bool] = False,
**kwargs
):
"""Create an AsyncIO secure channel with credentials.
@@ -229,10 +236,40 @@ def create_channel(
credentials_file (str): A file with credentials that can be loaded with
:func:`google.auth.load_credentials_from_file`. This argument is
mutually exclusive with credentials.
+
+ .. warning::
+ Important: If you accept a credential configuration (credential JSON/File/Stream)
+ from an external source for authentication to Google Cloud Platform, you must
+ validate it before providing it to any Google API or client library. Providing an
+ unvalidated credential configuration to Google APIs or libraries can compromise
+ the security of your systems and data. For more information, refer to
+ `Validate credential configurations from external sources`_.
+
+ .. _Validate credential configurations from external sources:
+
+ https://cloud.google.com/docs/authentication/external/externally-sourced-credentials
quota_project_id (str): An optional project to use for billing and quota.
default_scopes (Sequence[str]): Default scopes passed by a Google client
library. Use 'scopes' for user-defined scopes.
default_host (str): The default endpoint. e.g., "pubsub.googleapis.com".
+ compression (grpc.Compression): An optional value indicating the
+ compression method to be used over the lifetime of the channel.
+ attempt_direct_path (Optional[bool]): If set, Direct Path will be attempted
+ when the request is made. Direct Path is only available within a Google
+ Compute Engine (GCE) environment and provides a proxyless connection
+ which increases the available throughput, reduces latency, and increases
+ reliability. Note:
+
+ - This argument should only be set in a GCE environment and for Services
+ that are known to support Direct Path.
+ - If this argument is set outside of GCE, then this request will fail
+ unless the back-end service happens to have configured fall-back to DNS.
+ - If the request causes a `ServiceUnavailable` response, it is recommended
+ that the client repeat the request with `attempt_direct_path` set to
+ `False` as the Service may not support Direct Path.
+ - Using `ssl_credentials` with `attempt_direct_path` set to `True` will
+ result in `ValueError` as this combination is not yet supported.
+
kwargs: Additional key-word args passed to :func:`aio.secure_channel`.
Returns:
@@ -240,8 +277,15 @@ def create_channel(
Raises:
google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed.
+ ValueError: If `ssl_credentials` is set and `attempt_direct_path` is set to `True`.
"""
+ # If `ssl_credentials` is set and `attempt_direct_path` is set to `True`,
+ # raise ValueError as this is not yet supported.
+ # See https://github.com/googleapis/python-api-core/issues/590
+ if ssl_credentials and attempt_direct_path:
+ raise ValueError("Using ssl_credentials with Direct Path is not supported")
+
composite_credentials = grpc_helpers._create_composite_credentials(
credentials=credentials,
credentials_file=credentials_file,
@@ -252,7 +296,12 @@ def create_channel(
default_host=default_host,
)
- return aio.secure_channel(target, composite_credentials, **kwargs)
+ if attempt_direct_path:
+ target = grpc_helpers._modify_target_for_direct_path(target)
+
+ return aio.secure_channel(
+ target, composite_credentials, compression=compression, **kwargs
+ )
class FakeUnaryUnaryCall(_WrappedUnaryUnaryCall):
diff --git a/google/api_core/operation.py b/google/api_core/operation.py
index 90cbdc99..4b9c9a58 100644
--- a/google/api_core/operation.py
+++ b/google/api_core/operation.py
@@ -315,10 +315,16 @@ def from_grpc(operation, operations_stub, result_type, grpc_metadata=None, **kwa
operation.
"""
refresh = functools.partial(
- _refresh_grpc, operations_stub, operation.name, metadata=grpc_metadata
+ _refresh_grpc,
+ operations_stub,
+ operation.name,
+ metadata=grpc_metadata,
)
cancel = functools.partial(
- _cancel_grpc, operations_stub, operation.name, metadata=grpc_metadata
+ _cancel_grpc,
+ operations_stub,
+ operation.name,
+ metadata=grpc_metadata,
)
return Operation(operation, refresh, cancel, result_type, **kwargs)
@@ -347,9 +353,13 @@ def from_gapic(operation, operations_client, result_type, grpc_metadata=None, **
operation.
"""
refresh = functools.partial(
- operations_client.get_operation, operation.name, metadata=grpc_metadata
+ operations_client.get_operation,
+ operation.name,
+ metadata=grpc_metadata,
)
cancel = functools.partial(
- operations_client.cancel_operation, operation.name, metadata=grpc_metadata
+ operations_client.cancel_operation,
+ operation.name,
+ metadata=grpc_metadata,
)
return Operation(operation, refresh, cancel, result_type, **kwargs)
diff --git a/google/api_core/operation_async.py b/google/api_core/operation_async.py
index 6bae8654..2fd341d9 100644
--- a/google/api_core/operation_async.py
+++ b/google/api_core/operation_async.py
@@ -213,9 +213,13 @@ def from_gapic(operation, operations_client, result_type, grpc_metadata=None, **
operation.
"""
refresh = functools.partial(
- operations_client.get_operation, operation.name, metadata=grpc_metadata
+ operations_client.get_operation,
+ operation.name,
+ metadata=grpc_metadata,
)
cancel = functools.partial(
- operations_client.cancel_operation, operation.name, metadata=grpc_metadata
+ operations_client.cancel_operation,
+ operation.name,
+ metadata=grpc_metadata,
)
return AsyncOperation(operation, refresh, cancel, result_type, **kwargs)
diff --git a/google/api_core/operations_v1/__init__.py b/google/api_core/operations_v1/__init__.py
index 61186451..4db32a4c 100644
--- a/google/api_core/operations_v1/__init__.py
+++ b/google/api_core/operations_v1/__init__.py
@@ -25,3 +25,16 @@
"OperationsClient",
"OperationsRestTransport"
]
+
+try:
+ from google.api_core.operations_v1.transports.rest_asyncio import (
+ AsyncOperationsRestTransport,
+ )
+ from google.api_core.operations_v1.operations_rest_client_async import AsyncOperationsRestClient
+
+ __all__ += ["AsyncOperationsRestClient", "AsyncOperationsRestTransport"]
+except ImportError:
+ # This import requires the `async_rest` extra.
+ # Don't raise an exception if `AsyncOperationsRestTransport` cannot be imported
+ # as other transports are still available.
+ pass
diff --git a/google/api_core/operations_v1/abstract_operations_base_client.py b/google/api_core/operations_v1/abstract_operations_base_client.py
new file mode 100644
index 00000000..160c2a88
--- /dev/null
+++ b/google/api_core/operations_v1/abstract_operations_base_client.py
@@ -0,0 +1,370 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+import os
+import re
+from typing import Dict, Optional, Type, Union
+
+from google.api_core import client_options as client_options_lib # type: ignore
+from google.api_core import gapic_v1 # type: ignore
+from google.api_core.operations_v1.transports.base import (
+ DEFAULT_CLIENT_INFO,
+ OperationsTransport,
+)
+from google.api_core.operations_v1.transports.rest import OperationsRestTransport
+
+try:
+ from google.api_core.operations_v1.transports.rest_asyncio import (
+ AsyncOperationsRestTransport,
+ )
+
+ HAS_ASYNC_REST_DEPENDENCIES = True
+except ImportError as e:
+ HAS_ASYNC_REST_DEPENDENCIES = False
+ ASYNC_REST_EXCEPTION = e
+
+from google.auth import credentials as ga_credentials # type: ignore
+from google.auth.exceptions import MutualTLSChannelError # type: ignore
+from google.auth.transport import mtls # type: ignore
+
+
+class AbstractOperationsBaseClientMeta(type):
+ """Metaclass for the Operations Base client.
+
+ This provides base class-level methods for building and retrieving
+ support objects (e.g. transport) without polluting the client instance
+ objects.
+ """
+
+ _transport_registry = OrderedDict() # type: Dict[str, Type[OperationsTransport]]
+ _transport_registry["rest"] = OperationsRestTransport
+ if HAS_ASYNC_REST_DEPENDENCIES:
+ _transport_registry["rest_asyncio"] = AsyncOperationsRestTransport
+
+ def get_transport_class(
+ cls,
+ label: Optional[str] = None,
+ ) -> Type[OperationsTransport]:
+ """Returns an appropriate transport class.
+
+ Args:
+ label: The name of the desired transport. If none is
+ provided, then the first transport in the registry is used.
+
+ Returns:
+ The transport class to use.
+ """
+ # If a specific transport is requested, return that one.
+ if (
+ label == "rest_asyncio" and not HAS_ASYNC_REST_DEPENDENCIES
+ ): # pragma: NO COVER
+ raise ASYNC_REST_EXCEPTION
+
+ if label:
+ return cls._transport_registry[label]
+
+ # No transport is requested; return the default (that is, the first one
+ # in the dictionary).
+ return next(iter(cls._transport_registry.values()))
+
+
+class AbstractOperationsBaseClient(metaclass=AbstractOperationsBaseClientMeta):
+ """Manages long-running operations with an API service.
+
+ When an API method normally takes long time to complete, it can be
+ designed to return [Operation][google.api_core.operations_v1.Operation] to the
+ client, and the client can use this interface to receive the real
+ response asynchronously by polling the operation resource, or pass
+ the operation resource to another API (such as Google Cloud Pub/Sub
+ API) to receive the response. Any API service that returns
+ long-running operations should implement the ``Operations``
+ interface so developers can have a consistent client experience.
+ """
+
+ @staticmethod
+ def _get_default_mtls_endpoint(api_endpoint):
+ """Converts api endpoint to mTLS endpoint.
+
+ Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to
+ "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively.
+ Args:
+ api_endpoint (Optional[str]): the api endpoint to convert.
+ Returns:
+ str: converted mTLS api endpoint.
+ """
+ if not api_endpoint:
+ return api_endpoint
+
+ mtls_endpoint_re = re.compile(
+ r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?"
+ )
+
+ m = mtls_endpoint_re.match(api_endpoint)
+ name, mtls, sandbox, googledomain = m.groups()
+ if mtls or not googledomain:
+ return api_endpoint
+
+ if sandbox:
+ return api_endpoint.replace(
+ "sandbox.googleapis.com", "mtls.sandbox.googleapis.com"
+ )
+
+ return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com")
+
+ DEFAULT_ENDPOINT = "longrunning.googleapis.com"
+ DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore
+ DEFAULT_ENDPOINT
+ )
+
+ @classmethod
+ def from_service_account_info(cls, info: dict, *args, **kwargs):
+ """
+ This class method should be overridden by the subclasses.
+
+ Args:
+ info (dict): The service account private key info.
+ args: Additional arguments to pass to the constructor.
+ kwargs: Additional arguments to pass to the constructor.
+
+ Raises:
+ NotImplementedError: If the method is called on the base class.
+ """
+ raise NotImplementedError("`from_service_account_info` is not implemented.")
+
+ @classmethod
+ def from_service_account_file(cls, filename: str, *args, **kwargs):
+ """
+ This class method should be overridden by the subclasses.
+
+ Args:
+ filename (str): The path to the service account private key json
+ file.
+ args: Additional arguments to pass to the constructor.
+ kwargs: Additional arguments to pass to the constructor.
+
+ Raises:
+ NotImplementedError: If the method is called on the base class.
+ """
+ raise NotImplementedError("`from_service_account_file` is not implemented.")
+
+ from_service_account_json = from_service_account_file
+
+ @property
+ def transport(self) -> OperationsTransport:
+ """Returns the transport used by the client instance.
+
+ Returns:
+ OperationsTransport: The transport used by the client
+ instance.
+ """
+ return self._transport
+
+ @staticmethod
+ def common_billing_account_path(
+ billing_account: str,
+ ) -> str:
+ """Returns a fully-qualified billing_account string."""
+ return "billingAccounts/{billing_account}".format(
+ billing_account=billing_account,
+ )
+
+ @staticmethod
+ def parse_common_billing_account_path(path: str) -> Dict[str, str]:
+ """Parse a billing_account path into its component segments."""
+ m = re.match(r"^billingAccounts/(?P.+?)$", path)
+ return m.groupdict() if m else {}
+
+ @staticmethod
+ def common_folder_path(
+ folder: str,
+ ) -> str:
+ """Returns a fully-qualified folder string."""
+ return "folders/{folder}".format(
+ folder=folder,
+ )
+
+ @staticmethod
+ def parse_common_folder_path(path: str) -> Dict[str, str]:
+ """Parse a folder path into its component segments."""
+ m = re.match(r"^folders/(?P.+?)$", path)
+ return m.groupdict() if m else {}
+
+ @staticmethod
+ def common_organization_path(
+ organization: str,
+ ) -> str:
+ """Returns a fully-qualified organization string."""
+ return "organizations/{organization}".format(
+ organization=organization,
+ )
+
+ @staticmethod
+ def parse_common_organization_path(path: str) -> Dict[str, str]:
+ """Parse a organization path into its component segments."""
+ m = re.match(r"^organizations/(?P.+?)$", path)
+ return m.groupdict() if m else {}
+
+ @staticmethod
+ def common_project_path(
+ project: str,
+ ) -> str:
+ """Returns a fully-qualified project string."""
+ return "projects/{project}".format(
+ project=project,
+ )
+
+ @staticmethod
+ def parse_common_project_path(path: str) -> Dict[str, str]:
+ """Parse a project path into its component segments."""
+ m = re.match(r"^projects/(?P.+?)$", path)
+ return m.groupdict() if m else {}
+
+ @staticmethod
+ def common_location_path(
+ project: str,
+ location: str,
+ ) -> str:
+ """Returns a fully-qualified location string."""
+ return "projects/{project}/locations/{location}".format(
+ project=project,
+ location=location,
+ )
+
+ @staticmethod
+ def parse_common_location_path(path: str) -> Dict[str, str]:
+ """Parse a location path into its component segments."""
+ m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path)
+ return m.groupdict() if m else {}
+
+ def __init__(
+ self,
+ *,
+ credentials: Optional[ga_credentials.Credentials] = None,
+ transport: Union[str, OperationsTransport, None] = None,
+ client_options: Optional[client_options_lib.ClientOptions] = None,
+ client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
+ ) -> None:
+ """Instantiates the operations client.
+
+ Args:
+ credentials (Optional[google.auth.credentials.Credentials]): The
+ authorization credentials to attach to requests. These
+ credentials identify the application to the service; if none
+ are specified, the client will attempt to ascertain the
+ credentials from the environment.
+ transport (Union[str, OperationsTransport]): The
+ transport to use. If set to None, a transport is chosen
+ automatically.
+ client_options (google.api_core.client_options.ClientOptions): Custom options for the
+ client. It won't take effect if a ``transport`` instance is provided.
+ (1) The ``api_endpoint`` property can be used to override the
+ default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT
+ environment variable can also be used to override the endpoint:
+ "always" (always use the default mTLS endpoint), "never" (always
+ use the default regular endpoint) and "auto" (auto switch to the
+ default mTLS endpoint if client certificate is present, this is
+ the default value). However, the ``api_endpoint`` property takes
+ precedence if provided.
+ (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable
+ is "true", then the ``client_cert_source`` property can be used
+ to provide client certificate for mutual TLS transport. If
+ not provided, the default SSL client certificate will be used if
+ present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not
+ set, no client certificate will be used.
+ client_info (google.api_core.gapic_v1.client_info.ClientInfo):
+ The client info used to send a user-agent string along with
+ API requests. If ``None``, then default info will be used.
+ Generally, you only need to set this if you're developing
+ your own client library.
+
+ Raises:
+ google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport
+ creation failed for any reason.
+ """
+ if isinstance(client_options, dict):
+ client_options = client_options_lib.from_dict(client_options)
+ if client_options is None:
+ client_options = client_options_lib.ClientOptions()
+
+ # Create SSL credentials for mutual TLS if needed.
+ use_client_cert = os.getenv(
+ "GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"
+ ).lower()
+ if use_client_cert not in ("true", "false"):
+ raise ValueError(
+ "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`"
+ )
+ client_cert_source_func = None
+ is_mtls = False
+ if use_client_cert == "true":
+ if client_options.client_cert_source:
+ is_mtls = True
+ client_cert_source_func = client_options.client_cert_source
+ else:
+ is_mtls = mtls.has_default_client_cert_source()
+ if is_mtls:
+ client_cert_source_func = mtls.default_client_cert_source()
+ else:
+ client_cert_source_func = None
+
+ # Figure out which api endpoint to use.
+ if client_options.api_endpoint is not None:
+ api_endpoint = client_options.api_endpoint
+ else:
+ use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto")
+ if use_mtls_env == "never":
+ api_endpoint = self.DEFAULT_ENDPOINT
+ elif use_mtls_env == "always":
+ api_endpoint = self.DEFAULT_MTLS_ENDPOINT
+ elif use_mtls_env == "auto":
+ if is_mtls:
+ api_endpoint = self.DEFAULT_MTLS_ENDPOINT
+ else:
+ api_endpoint = self.DEFAULT_ENDPOINT
+ else:
+ raise MutualTLSChannelError(
+ "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted "
+ "values: never, auto, always"
+ )
+
+ # Save or instantiate the transport.
+ # Ordinarily, we provide the transport, but allowing a custom transport
+ # instance provides an extensibility point for unusual situations.
+ if isinstance(transport, OperationsTransport):
+ # transport is a OperationsTransport instance.
+ if credentials or client_options.credentials_file:
+ raise ValueError(
+ "When providing a transport instance, "
+ "provide its credentials directly."
+ )
+ if client_options.scopes:
+ raise ValueError(
+ "When providing a transport instance, provide its scopes "
+ "directly."
+ )
+ self._transport = transport
+ else:
+ Transport = type(self).get_transport_class(transport)
+ self._transport = Transport(
+ credentials=credentials,
+ credentials_file=client_options.credentials_file,
+ host=api_endpoint,
+ scopes=client_options.scopes,
+ client_cert_source_for_mtls=client_cert_source_func,
+ quota_project_id=client_options.quota_project_id,
+ client_info=client_info,
+ always_use_jwt_access=True,
+ )
diff --git a/google/api_core/operations_v1/abstract_operations_client.py b/google/api_core/operations_v1/abstract_operations_client.py
index 78a61746..fc445362 100644
--- a/google/api_core/operations_v1/abstract_operations_client.py
+++ b/google/api_core/operations_v1/abstract_operations_client.py
@@ -13,11 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from collections import OrderedDict
-from distutils import util
-import os
-import re
-from typing import Dict, Optional, Sequence, Tuple, Type, Union
+from typing import Optional, Sequence, Tuple, Union
from google.api_core import client_options as client_options_lib # type: ignore
from google.api_core import gapic_v1 # type: ignore
@@ -27,50 +23,18 @@
DEFAULT_CLIENT_INFO,
OperationsTransport,
)
-from google.api_core.operations_v1.transports.rest import OperationsRestTransport
+from google.api_core.operations_v1.abstract_operations_base_client import (
+ AbstractOperationsBaseClient,
+)
from google.auth import credentials as ga_credentials # type: ignore
-from google.auth.exceptions import MutualTLSChannelError # type: ignore
-from google.auth.transport import mtls # type: ignore
from google.longrunning import operations_pb2
from google.oauth2 import service_account # type: ignore
+import grpc
OptionalRetry = Union[retries.Retry, object]
-class AbstractOperationsClientMeta(type):
- """Metaclass for the Operations client.
-
- This provides class-level methods for building and retrieving
- support objects (e.g. transport) without polluting the client instance
- objects.
- """
-
- _transport_registry = OrderedDict() # type: Dict[str, Type[OperationsTransport]]
- _transport_registry["rest"] = OperationsRestTransport
-
- def get_transport_class(
- cls,
- label: Optional[str] = None,
- ) -> Type[OperationsTransport]:
- """Returns an appropriate transport class.
-
- Args:
- label: The name of the desired transport. If none is
- provided, then the first transport in the registry is used.
-
- Returns:
- The transport class to use.
- """
- # If a specific transport is requested, return that one.
- if label:
- return cls._transport_registry[label]
-
- # No transport is requested; return the default (that is, the first one
- # in the dictionary).
- return next(iter(cls._transport_registry.values()))
-
-
-class AbstractOperationsClient(metaclass=AbstractOperationsClientMeta):
+class AbstractOperationsClient(AbstractOperationsBaseClient):
"""Manages long-running operations with an API service.
When an API method normally takes long time to complete, it can be
@@ -83,165 +47,6 @@ class AbstractOperationsClient(metaclass=AbstractOperationsClientMeta):
interface so developers can have a consistent client experience.
"""
- @staticmethod
- def _get_default_mtls_endpoint(api_endpoint):
- """Converts api endpoint to mTLS endpoint.
-
- Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to
- "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively.
- Args:
- api_endpoint (Optional[str]): the api endpoint to convert.
- Returns:
- str: converted mTLS api endpoint.
- """
- if not api_endpoint:
- return api_endpoint
-
- mtls_endpoint_re = re.compile(
- r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?"
- )
-
- m = mtls_endpoint_re.match(api_endpoint)
- name, mtls, sandbox, googledomain = m.groups()
- if mtls or not googledomain:
- return api_endpoint
-
- if sandbox:
- return api_endpoint.replace(
- "sandbox.googleapis.com", "mtls.sandbox.googleapis.com"
- )
-
- return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com")
-
- DEFAULT_ENDPOINT = "longrunning.googleapis.com"
- DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore
- DEFAULT_ENDPOINT
- )
-
- @classmethod
- def from_service_account_info(cls, info: dict, *args, **kwargs):
- """Creates an instance of this client using the provided credentials
- info.
-
- Args:
- info (dict): The service account private key info.
- args: Additional arguments to pass to the constructor.
- kwargs: Additional arguments to pass to the constructor.
-
- Returns:
- AbstractOperationsClient: The constructed client.
- """
- credentials = service_account.Credentials.from_service_account_info(info)
- kwargs["credentials"] = credentials
- return cls(*args, **kwargs)
-
- @classmethod
- def from_service_account_file(cls, filename: str, *args, **kwargs):
- """Creates an instance of this client using the provided credentials
- file.
-
- Args:
- filename (str): The path to the service account private key json
- file.
- args: Additional arguments to pass to the constructor.
- kwargs: Additional arguments to pass to the constructor.
-
- Returns:
- AbstractOperationsClient: The constructed client.
- """
- credentials = service_account.Credentials.from_service_account_file(filename)
- kwargs["credentials"] = credentials
- return cls(*args, **kwargs)
-
- from_service_account_json = from_service_account_file
-
- @property
- def transport(self) -> OperationsTransport:
- """Returns the transport used by the client instance.
-
- Returns:
- OperationsTransport: The transport used by the client
- instance.
- """
- return self._transport
-
- @staticmethod
- def common_billing_account_path(
- billing_account: str,
- ) -> str:
- """Returns a fully-qualified billing_account string."""
- return "billingAccounts/{billing_account}".format(
- billing_account=billing_account,
- )
-
- @staticmethod
- def parse_common_billing_account_path(path: str) -> Dict[str, str]:
- """Parse a billing_account path into its component segments."""
- m = re.match(r"^billingAccounts/(?P.+?)$", path)
- return m.groupdict() if m else {}
-
- @staticmethod
- def common_folder_path(
- folder: str,
- ) -> str:
- """Returns a fully-qualified folder string."""
- return "folders/{folder}".format(
- folder=folder,
- )
-
- @staticmethod
- def parse_common_folder_path(path: str) -> Dict[str, str]:
- """Parse a folder path into its component segments."""
- m = re.match(r"^folders/(?P.+?)$", path)
- return m.groupdict() if m else {}
-
- @staticmethod
- def common_organization_path(
- organization: str,
- ) -> str:
- """Returns a fully-qualified organization string."""
- return "organizations/{organization}".format(
- organization=organization,
- )
-
- @staticmethod
- def parse_common_organization_path(path: str) -> Dict[str, str]:
- """Parse a organization path into its component segments."""
- m = re.match(r"^organizations/(?P.+?)$", path)
- return m.groupdict() if m else {}
-
- @staticmethod
- def common_project_path(
- project: str,
- ) -> str:
- """Returns a fully-qualified project string."""
- return "projects/{project}".format(
- project=project,
- )
-
- @staticmethod
- def parse_common_project_path(path: str) -> Dict[str, str]:
- """Parse a project path into its component segments."""
- m = re.match(r"^projects/(?P.+?)$", path)
- return m.groupdict() if m else {}
-
- @staticmethod
- def common_location_path(
- project: str,
- location: str,
- ) -> str:
- """Returns a fully-qualified location string."""
- return "projects/{project}/locations/{location}".format(
- project=project,
- location=location,
- )
-
- @staticmethod
- def parse_common_location_path(path: str) -> Dict[str, str]:
- """Parse a location path into its component segments."""
- m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path)
- return m.groupdict() if m else {}
-
def __init__(
self,
*,
@@ -287,77 +92,49 @@ def __init__(
google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport
creation failed for any reason.
"""
- if isinstance(client_options, dict):
- client_options = client_options_lib.from_dict(client_options)
- if client_options is None:
- client_options = client_options_lib.ClientOptions()
-
- # Create SSL credentials for mutual TLS if needed.
- use_client_cert = bool(
- util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"))
+ super().__init__(
+ credentials=credentials,
+ transport=transport,
+ client_options=client_options,
+ client_info=client_info,
)
- client_cert_source_func = None
- is_mtls = False
- if use_client_cert:
- if client_options.client_cert_source:
- is_mtls = True
- client_cert_source_func = client_options.client_cert_source
- else:
- is_mtls = mtls.has_default_client_cert_source()
- if is_mtls:
- client_cert_source_func = mtls.default_client_cert_source()
- else:
- client_cert_source_func = None
-
- # Figure out which api endpoint to use.
- if client_options.api_endpoint is not None:
- api_endpoint = client_options.api_endpoint
- else:
- use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto")
- if use_mtls_env == "never":
- api_endpoint = self.DEFAULT_ENDPOINT
- elif use_mtls_env == "always":
- api_endpoint = self.DEFAULT_MTLS_ENDPOINT
- elif use_mtls_env == "auto":
- if is_mtls:
- api_endpoint = self.DEFAULT_MTLS_ENDPOINT
- else:
- api_endpoint = self.DEFAULT_ENDPOINT
- else:
- raise MutualTLSChannelError(
- "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted "
- "values: never, auto, always"
- )
-
- # Save or instantiate the transport.
- # Ordinarily, we provide the transport, but allowing a custom transport
- # instance provides an extensibility point for unusual situations.
- if isinstance(transport, OperationsTransport):
- # transport is a OperationsTransport instance.
- if credentials or client_options.credentials_file:
- raise ValueError(
- "When providing a transport instance, "
- "provide its credentials directly."
- )
- if client_options.scopes:
- raise ValueError(
- "When providing a transport instance, provide its scopes "
- "directly."
- )
- self._transport = transport
- else:
- Transport = type(self).get_transport_class(transport)
- self._transport = Transport(
- credentials=credentials,
- credentials_file=client_options.credentials_file,
- host=api_endpoint,
- scopes=client_options.scopes,
- client_cert_source_for_mtls=client_cert_source_func,
- quota_project_id=client_options.quota_project_id,
- client_info=client_info,
- always_use_jwt_access=True,
- )
+ @classmethod
+ def from_service_account_info(cls, info: dict, *args, **kwargs):
+ """Creates an instance of this client using the provided credentials
+ info.
+
+ Args:
+ info (dict): The service account private key info.
+ args: Additional arguments to pass to the constructor.
+ kwargs: Additional arguments to pass to the constructor.
+
+ Returns:
+ AbstractOperationsClient: The constructed client.
+ """
+ credentials = service_account.Credentials.from_service_account_info(info)
+ kwargs["credentials"] = credentials
+ return cls(*args, **kwargs)
+
+ @classmethod
+ def from_service_account_file(cls, filename: str, *args, **kwargs):
+ """Creates an instance of this client using the provided credentials
+ file.
+
+ Args:
+ filename (str): The path to the service account private key json
+ file.
+ args: Additional arguments to pass to the constructor.
+ kwargs: Additional arguments to pass to the constructor.
+
+ Returns:
+ AbstractOperationsClient: The constructed client.
+ """
+ credentials = service_account.Credentials.from_service_account_file(filename)
+ kwargs["credentials"] = credentials
+ return cls(*args, **kwargs)
+
+ from_service_account_json = from_service_account_file
def list_operations(
self,
@@ -368,6 +145,7 @@ def list_operations(
page_token: Optional[str] = None,
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> pagers.ListOperationsPager:
r"""Lists operations that match the specified filter in the request.
@@ -429,6 +207,7 @@ def list_operations(
request,
retry=retry,
timeout=timeout,
+ compression=compression,
metadata=metadata,
)
@@ -450,6 +229,7 @@ def get_operation(
*,
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> operations_pb2.Operation:
r"""Gets the latest state of a long-running operation.
@@ -490,6 +270,7 @@ def get_operation(
request,
retry=retry,
timeout=timeout,
+ compression=compression,
metadata=metadata,
)
@@ -502,6 +283,7 @@ def delete_operation(
*,
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> None:
r"""Deletes a long-running operation. This method indicates that the
@@ -541,6 +323,7 @@ def delete_operation(
request,
retry=retry,
timeout=timeout,
+ compression=compression,
metadata=metadata,
)
@@ -550,6 +333,7 @@ def cancel_operation(
*,
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> None:
r"""Starts asynchronous cancellation on a long-running operation.
@@ -598,5 +382,6 @@ def cancel_operation(
request,
retry=retry,
timeout=timeout,
+ compression=compression,
metadata=metadata,
)
diff --git a/google/api_core/operations_v1/operations_async_client.py b/google/api_core/operations_v1/operations_async_client.py
index 81c4513c..a60c7177 100644
--- a/google/api_core/operations_v1/operations_async_client.py
+++ b/google/api_core/operations_v1/operations_async_client.py
@@ -26,9 +26,10 @@
from google.api_core import exceptions as core_exceptions
from google.api_core import gapic_v1, page_iterator_async
-from google.api_core import retry as retries
+from google.api_core import retry_async as retries
from google.api_core import timeout as timeouts
from google.longrunning import operations_pb2
+from grpc import Compression
class OperationsAsyncClient:
@@ -47,7 +48,7 @@ def __init__(self, channel, client_config=None):
# Create the gRPC client stub with gRPC AsyncIO channel.
self.operations_stub = operations_pb2.OperationsStub(channel)
- default_retry = retries.Retry(
+ default_retry = retries.AsyncRetry(
initial=0.1, # seconds
maximum=60.0, # seconds
multiplier=1.3,
@@ -59,28 +60,34 @@ def __init__(self, channel, client_config=None):
)
default_timeout = timeouts.TimeToDeadlineTimeout(timeout=600.0)
+ default_compression = Compression.NoCompression
+
self._get_operation = gapic_v1.method_async.wrap_method(
self.operations_stub.GetOperation,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
self._list_operations = gapic_v1.method_async.wrap_method(
self.operations_stub.ListOperations,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
self._cancel_operation = gapic_v1.method_async.wrap_method(
self.operations_stub.CancelOperation,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
self._delete_operation = gapic_v1.method_async.wrap_method(
self.operations_stub.DeleteOperation,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
async def get_operation(
@@ -88,6 +95,7 @@ async def get_operation(
name,
retry=gapic_v1.method_async.DEFAULT,
timeout=gapic_v1.method_async.DEFAULT,
+ compression=gapic_v1.method_async.DEFAULT,
metadata=None,
):
"""Gets the latest state of a long-running operation.
@@ -114,6 +122,8 @@ async def get_operation(
unspecified, the the default timeout in the client
configuration is used. If ``None``, then the RPC method will
not time out.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]):
Additional gRPC metadata.
@@ -133,7 +143,11 @@ async def get_operation(
metadata.append(gapic_v1.routing_header.to_grpc_metadata({"name": name}))
return await self._get_operation(
- request, retry=retry, timeout=timeout, metadata=metadata
+ request,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
)
async def list_operations(
@@ -142,6 +156,7 @@ async def list_operations(
filter_,
retry=gapic_v1.method_async.DEFAULT,
timeout=gapic_v1.method_async.DEFAULT,
+ compression=gapic_v1.method_async.DEFAULT,
metadata=None,
):
"""
@@ -178,6 +193,8 @@ async def list_operations(
unspecified, the the default timeout in the client
configuration is used. If ``None``, then the RPC method will
not time out.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]): Additional gRPC
metadata.
@@ -202,7 +219,11 @@ async def list_operations(
# Create the method used to fetch pages
method = functools.partial(
- self._list_operations, retry=retry, timeout=timeout, metadata=metadata
+ self._list_operations,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
)
iterator = page_iterator_async.AsyncGRPCIterator(
@@ -221,6 +242,7 @@ async def cancel_operation(
name,
retry=gapic_v1.method_async.DEFAULT,
timeout=gapic_v1.method_async.DEFAULT,
+ compression=gapic_v1.method_async.DEFAULT,
metadata=None,
):
"""Starts asynchronous cancellation on a long-running operation.
@@ -261,6 +283,8 @@ async def cancel_operation(
google.api_core.exceptions.GoogleAPICallError: If an error occurred
while invoking the RPC, the appropriate ``GoogleAPICallError``
subclass will be raised.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]): Additional gRPC
metadata.
"""
@@ -272,7 +296,11 @@ async def cancel_operation(
metadata.append(gapic_v1.routing_header.to_grpc_metadata({"name": name}))
await self._cancel_operation(
- request, retry=retry, timeout=timeout, metadata=metadata
+ request,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
)
async def delete_operation(
@@ -280,6 +308,7 @@ async def delete_operation(
name,
retry=gapic_v1.method_async.DEFAULT,
timeout=gapic_v1.method_async.DEFAULT,
+ compression=gapic_v1.method_async.DEFAULT,
metadata=None,
):
"""Deletes a long-running operation.
@@ -306,6 +335,8 @@ async def delete_operation(
unspecified, the the default timeout in the client
configuration is used. If ``None``, then the RPC method will
not time out.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]): Additional gRPC
metadata.
@@ -325,5 +356,9 @@ async def delete_operation(
metadata.append(gapic_v1.routing_header.to_grpc_metadata({"name": name}))
await self._delete_operation(
- request, retry=retry, timeout=timeout, metadata=metadata
+ request,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
)
diff --git a/google/api_core/operations_v1/operations_client.py b/google/api_core/operations_v1/operations_client.py
index 3ddd3c47..d1d3fd55 100644
--- a/google/api_core/operations_v1/operations_client.py
+++ b/google/api_core/operations_v1/operations_client.py
@@ -43,6 +43,7 @@
from google.api_core import retry as retries
from google.api_core import timeout as timeouts
from google.longrunning import operations_pb2
+from grpc import Compression
class OperationsClient(object):
@@ -72,28 +73,34 @@ def __init__(self, channel, client_config=None):
)
default_timeout = timeouts.TimeToDeadlineTimeout(timeout=600.0)
+ default_compression = Compression.NoCompression
+
self._get_operation = gapic_v1.method.wrap_method(
self.operations_stub.GetOperation,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
self._list_operations = gapic_v1.method.wrap_method(
self.operations_stub.ListOperations,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
self._cancel_operation = gapic_v1.method.wrap_method(
self.operations_stub.CancelOperation,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
self._delete_operation = gapic_v1.method.wrap_method(
self.operations_stub.DeleteOperation,
default_retry=default_retry,
default_timeout=default_timeout,
+ default_compression=default_compression,
)
# Service calls
@@ -102,6 +109,7 @@ def get_operation(
name,
retry=gapic_v1.method.DEFAULT,
timeout=gapic_v1.method.DEFAULT,
+ compression=gapic_v1.method.DEFAULT,
metadata=None,
):
"""Gets the latest state of a long-running operation.
@@ -128,6 +136,8 @@ def get_operation(
unspecified, the the default timeout in the client
configuration is used. If ``None``, then the RPC method will
not time out.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]):
Additional gRPC metadata.
@@ -147,7 +157,11 @@ def get_operation(
metadata.append(gapic_v1.routing_header.to_grpc_metadata({"name": name}))
return self._get_operation(
- request, retry=retry, timeout=timeout, metadata=metadata
+ request,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
)
def list_operations(
@@ -156,6 +170,7 @@ def list_operations(
filter_,
retry=gapic_v1.method.DEFAULT,
timeout=gapic_v1.method.DEFAULT,
+ compression=gapic_v1.method.DEFAULT,
metadata=None,
):
"""
@@ -192,6 +207,8 @@ def list_operations(
unspecified, the the default timeout in the client
configuration is used. If ``None``, then the RPC method will
not time out.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]): Additional gRPC
metadata.
@@ -216,7 +233,11 @@ def list_operations(
# Create the method used to fetch pages
method = functools.partial(
- self._list_operations, retry=retry, timeout=timeout, metadata=metadata
+ self._list_operations,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
)
iterator = page_iterator.GRPCIterator(
@@ -235,6 +256,7 @@ def cancel_operation(
name,
retry=gapic_v1.method.DEFAULT,
timeout=gapic_v1.method.DEFAULT,
+ compression=gapic_v1.method.DEFAULT,
metadata=None,
):
"""Starts asynchronous cancellation on a long-running operation.
@@ -267,6 +289,8 @@ def cancel_operation(
unspecified, the the default timeout in the client
configuration is used. If ``None``, then the RPC method will
not time out.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]): Additional gRPC
metadata.
@@ -285,13 +309,20 @@ def cancel_operation(
metadata = metadata or []
metadata.append(gapic_v1.routing_header.to_grpc_metadata({"name": name}))
- self._cancel_operation(request, retry=retry, timeout=timeout, metadata=metadata)
+ self._cancel_operation(
+ request,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
+ )
def delete_operation(
self,
name,
retry=gapic_v1.method.DEFAULT,
timeout=gapic_v1.method.DEFAULT,
+ compression=gapic_v1.method.DEFAULT,
metadata=None,
):
"""Deletes a long-running operation.
@@ -318,6 +349,8 @@ def delete_operation(
unspecified, the the default timeout in the client
configuration is used. If ``None``, then the RPC method will
not time out.
+ compression (grpc.Compression): An element of grpc.compression
+ e.g. grpc.compression.Gzip.
metadata (Optional[List[Tuple[str, str]]]): Additional gRPC
metadata.
@@ -336,4 +369,10 @@ def delete_operation(
metadata = metadata or []
metadata.append(gapic_v1.routing_header.to_grpc_metadata({"name": name}))
- self._delete_operation(request, retry=retry, timeout=timeout, metadata=metadata)
+ self._delete_operation(
+ request,
+ retry=retry,
+ timeout=timeout,
+ compression=compression,
+ metadata=metadata,
+ )
diff --git a/google/api_core/operations_v1/operations_rest_client_async.py b/google/api_core/operations_v1/operations_rest_client_async.py
new file mode 100644
index 00000000..7ab0cd36
--- /dev/null
+++ b/google/api_core/operations_v1/operations_rest_client_async.py
@@ -0,0 +1,345 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import Optional, Sequence, Tuple, Union
+
+from google.api_core import client_options as client_options_lib # type: ignore
+from google.api_core import gapic_v1 # type: ignore
+from google.api_core.operations_v1 import pagers_async as pagers
+from google.api_core.operations_v1.transports.base import (
+ DEFAULT_CLIENT_INFO,
+ OperationsTransport,
+)
+from google.api_core.operations_v1.abstract_operations_base_client import (
+ AbstractOperationsBaseClient,
+)
+from google.longrunning import operations_pb2
+
+try:
+ from google.auth.aio import credentials as ga_credentials # type: ignore
+except ImportError as e: # pragma: NO COVER
+ raise ImportError(
+ "The `async_rest` extra of `google-api-core` is required to use long-running operations. Install it by running "
+ "`pip install google-api-core[async_rest]`."
+ ) from e
+
+
+class AsyncOperationsRestClient(AbstractOperationsBaseClient):
+ """Manages long-running operations with a REST API service for the asynchronous client.
+
+ When an API method normally takes long time to complete, it can be
+ designed to return [Operation][google.api_core.operations_v1.Operation] to the
+ client, and the client can use this interface to receive the real
+ response asynchronously by polling the operation resource, or pass
+ the operation resource to another API (such as Google Cloud Pub/Sub
+ API) to receive the response. Any API service that returns
+ long-running operations should implement the ``Operations``
+ interface so developers can have a consistent client experience.
+ """
+
+ def __init__(
+ self,
+ *,
+ credentials: Optional[ga_credentials.Credentials] = None,
+ transport: Union[str, OperationsTransport, None] = None,
+ client_options: Optional[client_options_lib.ClientOptions] = None,
+ client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
+ ) -> None:
+ """Instantiates the operations client.
+
+ Args:
+ credentials (Optional[google.auth.aio.credentials.Credentials]): The
+ authorization credentials to attach to requests. These
+ credentials identify the application to the service; if none
+ are specified, the client will attempt to ascertain the
+ credentials from the environment.
+ transport (Union[str, OperationsTransport]): The
+ transport to use. If set to None, this defaults to 'rest_asyncio'.
+ client_options (google.api_core.client_options.ClientOptions): Custom options for the
+ client. It won't take effect if a ``transport`` instance is provided.
+ (1) The ``api_endpoint`` property can be used to override the
+ default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT
+ environment variable can also be used to override the endpoint:
+ "always" (always use the default mTLS endpoint), "never" (always
+ use the default regular endpoint) and "auto" (auto switch to the
+ default mTLS endpoint if client certificate is present, this is
+ the default value). However, the ``api_endpoint`` property takes
+ precedence if provided.
+ (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable
+ is "true", then the ``client_cert_source`` property can be used
+ to provide client certificate for mutual TLS transport. If
+ not provided, the default SSL client certificate will be used if
+ present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not
+ set, no client certificate will be used.
+ client_info (google.api_core.gapic_v1.client_info.ClientInfo):
+ The client info used to send a user-agent string along with
+ API requests. If ``None``, then default info will be used.
+ Generally, you only need to set this if you're developing
+ your own client library.
+
+ Raises:
+ google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport
+ creation failed for any reason.
+ """
+ super().__init__(
+ credentials=credentials, # type: ignore
+ # NOTE: If a transport is not provided, we force the client to use the async
+ # REST transport.
+ transport=transport or "rest_asyncio",
+ client_options=client_options,
+ client_info=client_info,
+ )
+
+ async def get_operation(
+ self,
+ name: str,
+ *,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ ) -> operations_pb2.Operation:
+ r"""Gets the latest state of a long-running operation.
+ Clients can use this method to poll the operation result
+ at intervals as recommended by the API service.
+
+ Args:
+ name (str):
+ The name of the operation resource.
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+
+ Returns:
+ google.longrunning.operations_pb2.Operation:
+ This resource represents a long-
+ running operation that is the result of a
+ network API call.
+
+ """
+
+ request = operations_pb2.GetOperationRequest(name=name)
+
+ # Wrap the RPC method; this adds retry and timeout information,
+ # and friendly error handling.
+ rpc = self._transport._wrapped_methods[self._transport.get_operation]
+
+ # Certain fields should be provided within the metadata header;
+ # add these here.
+ metadata = tuple(metadata or ()) + (
+ gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)),
+ )
+
+ # Send the request.
+ response = await rpc(
+ request,
+ retry=retry,
+ timeout=timeout,
+ metadata=metadata,
+ )
+
+ # Done; return the response.
+ return response
+
+ async def list_operations(
+ self,
+ name: str,
+ filter_: Optional[str] = None,
+ *,
+ page_size: Optional[int] = None,
+ page_token: Optional[str] = None,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ ) -> pagers.ListOperationsAsyncPager:
+ r"""Lists operations that match the specified filter in the request.
+ If the server doesn't support this method, it returns
+ ``UNIMPLEMENTED``.
+
+ NOTE: the ``name`` binding allows API services to override the
+ binding to use different resource name schemes, such as
+ ``users/*/operations``. To override the binding, API services
+ can add a binding such as ``"/v1/{name=users/*}/operations"`` to
+ their service configuration. For backwards compatibility, the
+ default name includes the operations collection id, however
+ overriding users must ensure the name binding is the parent
+ resource, without the operations collection id.
+
+ Args:
+ name (str):
+ The name of the operation's parent
+ resource.
+ filter_ (str):
+ The standard list filter.
+ This corresponds to the ``filter`` field
+ on the ``request`` instance; if ``request`` is provided, this
+ should not be set.
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+
+ Returns:
+ google.api_core.operations_v1.pagers.ListOperationsPager:
+ The response message for
+ [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations].
+
+ Iterating over this object will yield results and
+ resolve additional pages automatically.
+
+ """
+ # Create a protobuf request object.
+ request = operations_pb2.ListOperationsRequest(name=name, filter=filter_)
+ if page_size is not None:
+ request.page_size = page_size
+ if page_token is not None:
+ request.page_token = page_token
+
+ # Wrap the RPC method; this adds retry and timeout information,
+ # and friendly error handling.
+ rpc = self._transport._wrapped_methods[self._transport.list_operations]
+
+ # Certain fields should be provided within the metadata header;
+ # add these here.
+ metadata = tuple(metadata or ()) + (
+ gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)),
+ )
+
+ # Send the request.
+ response = await rpc(
+ request,
+ retry=retry,
+ timeout=timeout,
+ metadata=metadata,
+ )
+
+ # This method is paged; wrap the response in a pager, which provides
+ # an `__iter__` convenience method.
+ response = pagers.ListOperationsAsyncPager(
+ method=rpc,
+ request=request,
+ response=response,
+ metadata=metadata,
+ )
+
+ # Done; return the response.
+ return response
+
+ async def delete_operation(
+ self,
+ name: str,
+ *,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ ) -> None:
+ r"""Deletes a long-running operation. This method indicates that the
+ client is no longer interested in the operation result. It does
+ not cancel the operation. If the server doesn't support this
+ method, it returns ``google.rpc.Code.UNIMPLEMENTED``.
+
+ Args:
+ name (str):
+ The name of the operation resource to
+ be deleted.
+
+ This corresponds to the ``name`` field
+ on the ``request`` instance; if ``request`` is provided, this
+ should not be set.
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+ """
+ # Create the request object.
+ request = operations_pb2.DeleteOperationRequest(name=name)
+
+ # Wrap the RPC method; this adds retry and timeout information,
+ # and friendly error handling.
+ rpc = self._transport._wrapped_methods[self._transport.delete_operation]
+
+ # Certain fields should be provided within the metadata header;
+ # add these here.
+ metadata = tuple(metadata or ()) + (
+ gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)),
+ )
+
+ # Send the request.
+ await rpc(
+ request,
+ retry=retry,
+ timeout=timeout,
+ metadata=metadata,
+ )
+
+ async def cancel_operation(
+ self,
+ name: Optional[str] = None,
+ *,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ ) -> None:
+ r"""Starts asynchronous cancellation on a long-running operation.
+ The server makes a best effort to cancel the operation, but
+ success is not guaranteed. If the server doesn't support this
+ method, it returns ``google.rpc.Code.UNIMPLEMENTED``. Clients
+ can use
+ [Operations.GetOperation][google.api_core.operations_v1.Operations.GetOperation]
+ or other methods to check whether the cancellation succeeded or
+ whether the operation completed despite cancellation. On
+ successful cancellation, the operation is not deleted; instead,
+ it becomes an operation with an
+ [Operation.error][google.api_core.operations_v1.Operation.error] value with
+ a [google.rpc.Status.code][google.rpc.Status.code] of 1,
+ corresponding to ``Code.CANCELLED``.
+
+ Args:
+ name (str):
+ The name of the operation resource to
+ be cancelled.
+
+ This corresponds to the ``name`` field
+ on the ``request`` instance; if ``request`` is provided, this
+ should not be set.
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+ """
+ # Create the request object.
+ request = operations_pb2.CancelOperationRequest(name=name)
+
+ # Wrap the RPC method; this adds retry and timeout information,
+ # and friendly error handling.
+ rpc = self._transport._wrapped_methods[self._transport.cancel_operation]
+
+ # Certain fields should be provided within the metadata header;
+ # add these here.
+ metadata = tuple(metadata or ()) + (
+ gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)),
+ )
+
+ # Send the request.
+ await rpc(
+ request,
+ retry=retry,
+ timeout=timeout,
+ metadata=metadata,
+ )
diff --git a/google/api_core/operations_v1/pagers.py b/google/api_core/operations_v1/pagers.py
index b8a47757..132f1c66 100644
--- a/google/api_core/operations_v1/pagers.py
+++ b/google/api_core/operations_v1/pagers.py
@@ -14,7 +14,6 @@
# limitations under the License.
#
from typing import (
- Any,
Callable,
Iterator,
Sequence,
@@ -22,9 +21,10 @@
)
from google.longrunning import operations_pb2
+from google.api_core.operations_v1.pagers_base import ListOperationsPagerBase
-class ListOperationsPager:
+class ListOperationsPager(ListOperationsPagerBase):
"""A pager for iterating through ``list_operations`` requests.
This class thinly wraps an initial
@@ -50,25 +50,9 @@ def __init__(
*,
metadata: Sequence[Tuple[str, str]] = ()
):
- """Instantiate the pager.
-
- Args:
- method (Callable): The method that was originally called, and
- which instantiated this pager.
- request (google.longrunning.operations_pb2.ListOperationsRequest):
- The initial request object.
- response (google.longrunning.operations_pb2.ListOperationsResponse):
- The initial response object.
- metadata (Sequence[Tuple[str, str]]): Strings which should be
- sent along with the request as metadata.
- """
- self._method = method
- self._request = request
- self._response = response
- self._metadata = metadata
-
- def __getattr__(self, name: str) -> Any:
- return getattr(self._response, name)
+ super().__init__(
+ method=method, request=request, response=response, metadata=metadata
+ )
@property
def pages(self) -> Iterator[operations_pb2.ListOperationsResponse]:
@@ -81,6 +65,3 @@ def pages(self) -> Iterator[operations_pb2.ListOperationsResponse]:
def __iter__(self) -> Iterator[operations_pb2.Operation]:
for page in self.pages:
yield from page.operations
-
- def __repr__(self) -> str:
- return "{0}<{1!r}>".format(self.__class__.__name__, self._response)
diff --git a/google/api_core/operations_v1/pagers_async.py b/google/api_core/operations_v1/pagers_async.py
new file mode 100644
index 00000000..e2909dd5
--- /dev/null
+++ b/google/api_core/operations_v1/pagers_async.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import (
+ Callable,
+ AsyncIterator,
+ Sequence,
+ Tuple,
+)
+
+from google.longrunning import operations_pb2
+from google.api_core.operations_v1.pagers_base import ListOperationsPagerBase
+
+
+class ListOperationsAsyncPager(ListOperationsPagerBase):
+ """A pager for iterating through ``list_operations`` requests.
+
+ This class thinly wraps an initial
+ :class:`google.longrunning.operations_pb2.ListOperationsResponse` object, and
+ provides an ``__iter__`` method to iterate through its
+ ``operations`` field.
+
+ If there are more pages, the ``__iter__`` method will make additional
+ ``ListOperations`` requests and continue to iterate
+ through the ``operations`` field on the
+ corresponding responses.
+
+ All the usual :class:`google.longrunning.operations_pb2.ListOperationsResponse`
+ attributes are available on the pager. If multiple requests are made, only
+ the most recent response is retained, and thus used for attribute lookup.
+ """
+
+ def __init__(
+ self,
+ method: Callable[..., operations_pb2.ListOperationsResponse],
+ request: operations_pb2.ListOperationsRequest,
+ response: operations_pb2.ListOperationsResponse,
+ *,
+ metadata: Sequence[Tuple[str, str]] = ()
+ ):
+ super().__init__(
+ method=method, request=request, response=response, metadata=metadata
+ )
+
+ @property
+ async def pages(self) -> AsyncIterator[operations_pb2.ListOperationsResponse]:
+ yield self._response
+ while self._response.next_page_token:
+ self._request.page_token = self._response.next_page_token
+ self._response = await self._method(self._request, metadata=self._metadata)
+ yield self._response
+
+ def __aiter__(self) -> AsyncIterator[operations_pb2.Operation]:
+ async def async_generator():
+ async for page in self.pages:
+ for operation in page.operations:
+ yield operation
+
+ return async_generator()
diff --git a/google/api_core/operations_v1/pagers_base.py b/google/api_core/operations_v1/pagers_base.py
new file mode 100644
index 00000000..24caf74f
--- /dev/null
+++ b/google/api_core/operations_v1/pagers_base.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import (
+ Any,
+ Callable,
+ Sequence,
+ Tuple,
+)
+
+from google.longrunning import operations_pb2
+
+
+class ListOperationsPagerBase:
+ """A pager for iterating through ``list_operations`` requests.
+
+ This class thinly wraps an initial
+ :class:`google.longrunning.operations_pb2.ListOperationsResponse` object, and
+ provides an ``__iter__`` method to iterate through its
+ ``operations`` field.
+
+ If there are more pages, the ``__iter__`` method will make additional
+ ``ListOperations`` requests and continue to iterate
+ through the ``operations`` field on the
+ corresponding responses.
+
+ All the usual :class:`google.longrunning.operations_pb2.ListOperationsResponse`
+ attributes are available on the pager. If multiple requests are made, only
+ the most recent response is retained, and thus used for attribute lookup.
+ """
+
+ def __init__(
+ self,
+ method: Callable[..., operations_pb2.ListOperationsResponse],
+ request: operations_pb2.ListOperationsRequest,
+ response: operations_pb2.ListOperationsResponse,
+ *,
+ metadata: Sequence[Tuple[str, str]] = ()
+ ):
+ """Instantiate the pager.
+
+ Args:
+ method (Callable): The method that was originally called, and
+ which instantiated this pager.
+ request (google.longrunning.operations_pb2.ListOperationsRequest):
+ The initial request object.
+ response (google.longrunning.operations_pb2.ListOperationsResponse):
+ The initial response object.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+ """
+ self._method = method
+ self._request = request
+ self._response = response
+ self._metadata = metadata
+
+ def __getattr__(self, name: str) -> Any:
+ return getattr(self._response, name)
+
+ def __repr__(self) -> str:
+ return "{0}<{1!r}>".format(self.__class__.__name__, self._response)
diff --git a/google/api_core/operations_v1/transports/__init__.py b/google/api_core/operations_v1/transports/__init__.py
index df53e15e..8c24ce6e 100644
--- a/google/api_core/operations_v1/transports/__init__.py
+++ b/google/api_core/operations_v1/transports/__init__.py
@@ -14,16 +14,26 @@
# limitations under the License.
#
from collections import OrderedDict
+from typing import cast, Dict, Tuple
from .base import OperationsTransport
from .rest import OperationsRestTransport
-
# Compile a registry of transports.
-_transport_registry = OrderedDict()
-_transport_registry["rest"] = OperationsRestTransport
+_transport_registry: Dict[str, OperationsTransport] = OrderedDict()
+_transport_registry["rest"] = cast(OperationsTransport, OperationsRestTransport)
+
+__all__: Tuple[str, ...] = ("OperationsTransport", "OperationsRestTransport")
+
+try:
+ from .rest_asyncio import AsyncOperationsRestTransport
-__all__ = (
- "OperationsTransport",
- "OperationsRestTransport",
-)
+ __all__ += ("AsyncOperationsRestTransport",)
+ _transport_registry["rest_asyncio"] = cast(
+ OperationsTransport, AsyncOperationsRestTransport
+ )
+except ImportError:
+ # This import requires the `async_rest` extra.
+ # Don't raise an exception if `AsyncOperationsRestTransport` cannot be imported
+ # as other transports are still available.
+ pass
diff --git a/google/api_core/operations_v1/transports/base.py b/google/api_core/operations_v1/transports/base.py
index e19bc3e8..71764c1e 100644
--- a/google/api_core/operations_v1/transports/base.py
+++ b/google/api_core/operations_v1/transports/base.py
@@ -14,6 +14,7 @@
# limitations under the License.
#
import abc
+import re
from typing import Awaitable, Callable, Optional, Sequence, Union
import google.api_core # type: ignore
@@ -25,9 +26,13 @@
from google.auth import credentials as ga_credentials # type: ignore
from google.longrunning import operations_pb2
from google.oauth2 import service_account # type: ignore
-from google.protobuf import empty_pb2 # type: ignore
+import google.protobuf
+from google.protobuf import empty_pb2, json_format # type: ignore
+from grpc import Compression
+PROTOBUF_VERSION = google.protobuf.__version__
+
DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(
gapic_version=version.__version__,
)
@@ -44,12 +49,14 @@ def __init__(
self,
*,
host: str = DEFAULT_HOST,
- credentials: ga_credentials.Credentials = None,
+ # TODO(https://github.com/googleapis/python-api-core/issues/709): update type hint for credentials to include `google.auth.aio.Credentials`.
+ credentials: Optional[ga_credentials.Credentials] = None,
credentials_file: Optional[str] = None,
scopes: Optional[Sequence[str]] = None,
quota_project_id: Optional[str] = None,
client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
always_use_jwt_access: Optional[bool] = False,
+ url_scheme="https",
**kwargs,
) -> None:
"""Instantiate the transport.
@@ -65,6 +72,18 @@ def __init__(
credentials_file (Optional[str]): A file with credentials that can
be loaded with :func:`google.auth.load_credentials_from_file`.
This argument is mutually exclusive with credentials.
+
+ .. warning::
+ Important: If you accept a credential configuration (credential JSON/File/Stream)
+ from an external source for authentication to Google Cloud Platform, you must
+ validate it before providing it to any Google API or client library. Providing an
+ unvalidated credential configuration to Google APIs or libraries can compromise
+ the security of your systems and data. For more information, refer to
+ `Validate credential configurations from external sources`_.
+
+ .. _Validate credential configurations from external sources:
+
+ https://cloud.google.com/docs/authentication/external/externally-sourced-credentials
scopes (Optional[Sequence[str]]): A list of scopes.
quota_project_id (Optional[str]): An optional project to use for billing
and quota.
@@ -75,10 +94,23 @@ def __init__(
your own client library.
always_use_jwt_access (Optional[bool]): Whether self signed JWT should
be used for service account credentials.
+ url_scheme: the protocol scheme for the API endpoint. Normally
+ "https", but for testing or local servers,
+ "http" can be specified.
"""
+ maybe_url_match = re.match("^(?Phttp(?:s)?://)?(?P.*)$", host)
+ if maybe_url_match is None:
+ raise ValueError(
+ f"Unexpected hostname structure: {host}"
+ ) # pragma: NO COVER
+
+ url_match_items = maybe_url_match.groupdict()
+
+ host = f"{url_scheme}://{host}" if not url_match_items["scheme"] else host
+
# Save the hostname. Default to port 443 (HTTPS) if none is specified.
if ":" not in host:
- host += ":443"
+ host += ":443" # pragma: NO COVER
self._host = host
scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES}
@@ -129,6 +161,7 @@ def _prep_wrapped_messages(self, client_info):
deadline=10.0,
),
default_timeout=10.0,
+ default_compression=Compression.NoCompression,
client_info=client_info,
),
self.get_operation: gapic_v1.method.wrap_method(
@@ -143,6 +176,7 @@ def _prep_wrapped_messages(self, client_info):
deadline=10.0,
),
default_timeout=10.0,
+ default_compression=Compression.NoCompression,
client_info=client_info,
),
self.delete_operation: gapic_v1.method.wrap_method(
@@ -157,6 +191,7 @@ def _prep_wrapped_messages(self, client_info):
deadline=10.0,
),
default_timeout=10.0,
+ default_compression=Compression.NoCompression,
client_info=client_info,
),
self.cancel_operation: gapic_v1.method.wrap_method(
@@ -171,6 +206,7 @@ def _prep_wrapped_messages(self, client_info):
deadline=10.0,
),
default_timeout=10.0,
+ default_compression=Compression.NoCompression,
client_info=client_info,
),
}
@@ -184,6 +220,37 @@ def close(self):
"""
raise NotImplementedError()
+ def _convert_protobuf_message_to_dict(
+ self, message: google.protobuf.message.Message
+ ):
+ r"""Converts protobuf message to a dictionary.
+
+ When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.
+
+ Args:
+ message(google.protobuf.message.Message): The protocol buffers message
+ instance to serialize.
+
+ Returns:
+ A dict representation of the protocol buffer message.
+ """
+ # TODO(https://github.com/googleapis/python-api-core/issues/643): For backwards compatibility
+ # with protobuf 3.x 4.x, Remove once support for protobuf 3.x and 4.x is dropped.
+ if PROTOBUF_VERSION[0:2] in ["3.", "4."]:
+ result = json_format.MessageToDict(
+ message,
+ preserving_proto_field_name=True,
+ including_default_value_fields=True, # type: ignore # backward compatibility
+ )
+ else:
+ result = json_format.MessageToDict(
+ message,
+ preserving_proto_field_name=True,
+ always_print_fields_with_no_presence=True,
+ )
+
+ return result
+
@property
def list_operations(
self,
diff --git a/google/api_core/operations_v1/transports/rest.py b/google/api_core/operations_v1/transports/rest.py
index bb9000f4..0705c518 100644
--- a/google/api_core/operations_v1/transports/rest.py
+++ b/google/api_core/operations_v1/transports/rest.py
@@ -14,7 +14,6 @@
# limitations under the License.
#
-import re
from typing import Callable, Dict, Optional, Sequence, Tuple, Union
from requests import __version__ as requests_version
@@ -29,14 +28,19 @@
from google.longrunning import operations_pb2 # type: ignore
from google.protobuf import empty_pb2 # type: ignore
from google.protobuf import json_format # type: ignore
+import google.protobuf
+
+import grpc
from .base import DEFAULT_CLIENT_INFO as BASE_DEFAULT_CLIENT_INFO, OperationsTransport
+PROTOBUF_VERSION = google.protobuf.__version__
+
OptionalRetry = Union[retries.Retry, object]
DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(
gapic_version=BASE_DEFAULT_CLIENT_INFO.gapic_version,
grpc_version=None,
- rest_version=requests_version,
+ rest_version=f"requests@{requests_version}",
)
@@ -65,7 +69,7 @@ def __init__(
self,
*,
host: str = "longrunning.googleapis.com",
- credentials: ga_credentials.Credentials = None,
+ credentials: Optional[ga_credentials.Credentials] = None,
credentials_file: Optional[str] = None,
scopes: Optional[Sequence[str]] = None,
client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None,
@@ -90,6 +94,18 @@ def __init__(
credentials_file (Optional[str]): A file with credentials that can
be loaded with :func:`google.auth.load_credentials_from_file`.
This argument is ignored if ``channel`` is provided.
+
+ .. warning::
+ Important: If you accept a credential configuration (credential JSON/File/Stream)
+ from an external source for authentication to Google Cloud Platform, you must
+ validate it before providing it to any Google API or client library. Providing an
+ unvalidated credential configuration to Google APIs or libraries can compromise
+ the security of your systems and data. For more information, refer to
+ `Validate credential configurations from external sources`_.
+
+ .. _Validate credential configurations from external sources:
+
+ https://cloud.google.com/docs/authentication/external/externally-sourced-credentials
scopes (Optional(Sequence[str])): A list of scopes. This argument is
ignored if ``channel`` is provided.
client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client
@@ -118,16 +134,6 @@ def __init__(
# TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc.
# TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the
# credentials object
- maybe_url_match = re.match("^(?Phttp(?:s)?://)?(?P.*)$", host)
- if maybe_url_match is None:
- raise ValueError(
- f"Unexpected hostname structure: {host}"
- ) # pragma: NO COVER
-
- url_match_items = maybe_url_match.groupdict()
-
- host = f"{url_scheme}://{host}" if not url_match_items["scheme"] else host
-
super().__init__(
host=host,
credentials=credentials,
@@ -139,6 +145,7 @@ def __init__(
)
if client_cert_source_for_mtls:
self._session.configure_mtls_channel(client_cert_source_for_mtls)
+ # TODO(https://github.com/googleapis/python-api-core/issues/720): Add wrap logic directly to the property methods for callables.
self._prep_wrapped_messages(client_info)
self._http_options = http_options or {}
self._path_prefix = path_prefix
@@ -147,8 +154,11 @@ def _list_operations(
self,
request: operations_pb2.ListOperationsRequest,
*,
+ # TODO(https://github.com/googleapis/python-api-core/issues/723): Leverage `retry`
+ # to allow configuring retryable error codes.
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> operations_pb2.ListOperationsResponse:
r"""Call the list operations method over HTTP.
@@ -182,11 +192,7 @@ def _list_operations(
"google.longrunning.Operations.ListOperations"
]
- request_kwargs = json_format.MessageToDict(
- request,
- preserving_proto_field_name=True,
- including_default_value_fields=True,
- )
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
transcoded_request = path_template.transcode(http_options, **request_kwargs)
uri = transcoded_request["uri"]
@@ -197,7 +203,6 @@ def _list_operations(
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
query_params = json_format.MessageToDict(
query_params_request,
- including_default_value_fields=False,
preserving_proto_field_name=False,
use_integers_for_enums=False,
)
@@ -205,6 +210,7 @@ def _list_operations(
# Send the request
headers = dict(metadata)
headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
response = getattr(self._session, method)(
"{host}{uri}".format(host=self._host, uri=uri),
timeout=timeout,
@@ -226,8 +232,11 @@ def _get_operation(
self,
request: operations_pb2.GetOperationRequest,
*,
+ # TODO(https://github.com/googleapis/python-api-core/issues/723): Leverage `retry`
+ # to allow configuring retryable error codes.
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> operations_pb2.Operation:
r"""Call the get operation method over HTTP.
@@ -262,11 +271,7 @@ def _get_operation(
"google.longrunning.Operations.GetOperation"
]
- request_kwargs = json_format.MessageToDict(
- request,
- preserving_proto_field_name=True,
- including_default_value_fields=True,
- )
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
transcoded_request = path_template.transcode(http_options, **request_kwargs)
uri = transcoded_request["uri"]
@@ -277,7 +282,6 @@ def _get_operation(
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
query_params = json_format.MessageToDict(
query_params_request,
- including_default_value_fields=False,
preserving_proto_field_name=False,
use_integers_for_enums=False,
)
@@ -285,6 +289,7 @@ def _get_operation(
# Send the request
headers = dict(metadata)
headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
response = getattr(self._session, method)(
"{host}{uri}".format(host=self._host, uri=uri),
timeout=timeout,
@@ -306,8 +311,11 @@ def _delete_operation(
self,
request: operations_pb2.DeleteOperationRequest,
*,
+ # TODO(https://github.com/googleapis/python-api-core/issues/723): Leverage `retry`
+ # to allow configuring retryable error codes.
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> empty_pb2.Empty:
r"""Call the delete operation method over HTTP.
@@ -335,11 +343,7 @@ def _delete_operation(
"google.longrunning.Operations.DeleteOperation"
]
- request_kwargs = json_format.MessageToDict(
- request,
- preserving_proto_field_name=True,
- including_default_value_fields=True,
- )
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
transcoded_request = path_template.transcode(http_options, **request_kwargs)
uri = transcoded_request["uri"]
@@ -350,7 +354,6 @@ def _delete_operation(
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
query_params = json_format.MessageToDict(
query_params_request,
- including_default_value_fields=False,
preserving_proto_field_name=False,
use_integers_for_enums=False,
)
@@ -358,6 +361,7 @@ def _delete_operation(
# Send the request
headers = dict(metadata)
headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
response = getattr(self._session, method)(
"{host}{uri}".format(host=self._host, uri=uri),
timeout=timeout,
@@ -376,8 +380,11 @@ def _cancel_operation(
self,
request: operations_pb2.CancelOperationRequest,
*,
+ # TODO(https://github.com/googleapis/python-api-core/issues/723): Leverage `retry`
+ # to allow configuring retryable error codes.
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
+ compression: Optional[grpc.Compression] = gapic_v1.method.DEFAULT,
metadata: Sequence[Tuple[str, str]] = (),
) -> empty_pb2.Empty:
r"""Call the cancel operation method over HTTP.
@@ -406,11 +413,7 @@ def _cancel_operation(
"google.longrunning.Operations.CancelOperation"
]
- request_kwargs = json_format.MessageToDict(
- request,
- preserving_proto_field_name=True,
- including_default_value_fields=True,
- )
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
transcoded_request = path_template.transcode(http_options, **request_kwargs)
# Jsonify the request body
@@ -418,7 +421,6 @@ def _cancel_operation(
json_format.ParseDict(transcoded_request["body"], body_request)
body = json_format.MessageToDict(
body_request,
- including_default_value_fields=False,
preserving_proto_field_name=False,
use_integers_for_enums=False,
)
@@ -430,7 +432,6 @@ def _cancel_operation(
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
query_params = json_format.MessageToDict(
query_params_request,
- including_default_value_fields=False,
preserving_proto_field_name=False,
use_integers_for_enums=False,
)
@@ -438,6 +439,7 @@ def _cancel_operation(
# Send the request
headers = dict(metadata)
headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
response = getattr(self._session, method)(
"{host}{uri}".format(host=self._host, uri=uri),
timeout=timeout,
diff --git a/google/api_core/operations_v1/transports/rest_asyncio.py b/google/api_core/operations_v1/transports/rest_asyncio.py
new file mode 100644
index 00000000..71c20eb8
--- /dev/null
+++ b/google/api_core/operations_v1/transports/rest_asyncio.py
@@ -0,0 +1,560 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import json
+from typing import Any, Callable, Coroutine, Dict, Optional, Sequence, Tuple
+
+from google.auth import __version__ as auth_version
+
+try:
+ from google.auth.aio.transport.sessions import AsyncAuthorizedSession # type: ignore
+except ImportError as e: # pragma: NO COVER
+ raise ImportError(
+ "The `async_rest` extra of `google-api-core` is required to use long-running operations. Install it by running "
+ "`pip install google-api-core[async_rest]`."
+ ) from e
+
+from google.api_core import exceptions as core_exceptions # type: ignore
+from google.api_core import gapic_v1 # type: ignore
+from google.api_core import path_template # type: ignore
+from google.api_core import rest_helpers # type: ignore
+from google.api_core import retry_async as retries_async # type: ignore
+from google.auth.aio import credentials as ga_credentials_async # type: ignore
+from google.longrunning import operations_pb2 # type: ignore
+from google.protobuf import empty_pb2 # type: ignore
+from google.protobuf import json_format # type: ignore
+
+from .base import DEFAULT_CLIENT_INFO as BASE_DEFAULT_CLIENT_INFO, OperationsTransport
+
+DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(
+ gapic_version=BASE_DEFAULT_CLIENT_INFO.gapic_version,
+ grpc_version=None,
+ rest_version=f"google-auth@{auth_version}",
+)
+
+
+class AsyncOperationsRestTransport(OperationsTransport):
+ """Asynchronous REST backend transport for Operations.
+
+ Manages async long-running operations with an API service.
+
+ When an API method normally takes long time to complete, it can be
+ designed to return [Operation][google.api_core.operations_v1.Operation] to the
+ client, and the client can use this interface to receive the real
+ response asynchronously by polling the operation resource, or pass
+ the operation resource to another API (such as Google Cloud Pub/Sub
+ API) to receive the response. Any API service that returns
+ long-running operations should implement the ``Operations``
+ interface so developers can have a consistent client experience.
+
+ This class defines the same methods as the primary client, so the
+ primary client can load the underlying transport implementation
+ and call it.
+
+ It sends JSON representations of protocol buffers over HTTP/1.1
+ """
+
+ def __init__(
+ self,
+ *,
+ host: str = "longrunning.googleapis.com",
+ credentials: Optional[ga_credentials_async.Credentials] = None,
+ credentials_file: Optional[str] = None,
+ scopes: Optional[Sequence[str]] = None,
+ client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None,
+ quota_project_id: Optional[str] = None,
+ client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
+ always_use_jwt_access: Optional[bool] = False,
+ url_scheme: str = "https",
+ http_options: Optional[Dict] = None,
+ path_prefix: str = "v1",
+ # TODO(https://github.com/googleapis/python-api-core/issues/715): Add docstring for `credentials_file` to async REST transport.
+ # TODO(https://github.com/googleapis/python-api-core/issues/716): Add docstring for `scopes` to async REST transport.
+ # TODO(https://github.com/googleapis/python-api-core/issues/717): Add docstring for `quota_project_id` to async REST transport.
+ # TODO(https://github.com/googleapis/python-api-core/issues/718): Add docstring for `client_cert_source` to async REST transport.
+ ) -> None:
+ """Instantiate the transport.
+
+ Args:
+ host (Optional[str]):
+ The hostname to connect to.
+ credentials (Optional[google.auth.aio.credentials.Credentials]): The
+ authorization credentials to attach to requests. These
+ credentials identify the application to the service; if none
+ are specified, the client will attempt to ascertain the
+ credentials from the environment.
+ client_info (google.api_core.gapic_v1.client_info.ClientInfo):
+ The client info used to send a user-agent string along with
+ API requests. If ``None``, then default info will be used.
+ Generally, you only need to set this if you're developing
+ your own client library.
+ always_use_jwt_access (Optional[bool]): Whether self signed JWT should
+ be used for service account credentials.
+ url_scheme: the protocol scheme for the API endpoint. Normally
+ "https", but for testing or local servers,
+ "http" can be specified.
+ http_options: a dictionary of http_options for transcoding, to override
+ the defaults from operations.proto. Each method has an entry
+ with the corresponding http rules as value.
+ path_prefix: path prefix (usually represents API version). Set to
+ "v1" by default.
+
+ """
+ unsupported_params = {
+ # TODO(https://github.com/googleapis/python-api-core/issues/715): Add support for `credentials_file` to async REST transport.
+ "google.api_core.client_options.ClientOptions.credentials_file": credentials_file,
+ # TODO(https://github.com/googleapis/python-api-core/issues/716): Add support for `scopes` to async REST transport.
+ "google.api_core.client_options.ClientOptions.scopes": scopes,
+ # TODO(https://github.com/googleapis/python-api-core/issues/717): Add support for `quota_project_id` to async REST transport.
+ "google.api_core.client_options.ClientOptions.quota_project_id": quota_project_id,
+ # TODO(https://github.com/googleapis/python-api-core/issues/718): Add support for `client_cert_source` to async REST transport.
+ "google.api_core.client_options.ClientOptions.client_cert_source": client_cert_source_for_mtls,
+ # TODO(https://github.com/googleapis/python-api-core/issues/718): Add support for `client_cert_source` to async REST transport.
+ "google.api_core.client_options.ClientOptions.client_cert_source": client_cert_source_for_mtls,
+ }
+ provided_unsupported_params = [
+ name for name, value in unsupported_params.items() if value is not None
+ ]
+ if provided_unsupported_params:
+ raise core_exceptions.AsyncRestUnsupportedParameterError(
+ f"The following provided parameters are not supported for `transport=rest_asyncio`: {', '.join(provided_unsupported_params)}"
+ )
+
+ super().__init__(
+ host=host,
+ # TODO(https://github.com/googleapis/python-api-core/issues/709): Remove `type: ignore` when the linked issue is resolved.
+ credentials=credentials, # type: ignore
+ client_info=client_info,
+ # TODO(https://github.com/googleapis/python-api-core/issues/725): Set always_use_jwt_access token when supported.
+ always_use_jwt_access=False,
+ )
+ # TODO(https://github.com/googleapis/python-api-core/issues/708): add support for
+ # `default_host` in AsyncAuthorizedSession for feature parity with the synchronous
+ # code.
+ # TODO(https://github.com/googleapis/python-api-core/issues/709): Remove `type: ignore` when the linked issue is resolved.
+ self._session = AsyncAuthorizedSession(self._credentials) # type: ignore
+ # TODO(https://github.com/googleapis/python-api-core/issues/720): Add wrap logic directly to the property methods for callables.
+ self._prep_wrapped_messages(client_info)
+ self._http_options = http_options or {}
+ self._path_prefix = path_prefix
+
+ def _prep_wrapped_messages(self, client_info):
+ # Precompute the wrapped methods.
+ self._wrapped_methods = {
+ self.list_operations: gapic_v1.method_async.wrap_method(
+ self.list_operations,
+ default_retry=retries_async.AsyncRetry(
+ initial=0.5,
+ maximum=10.0,
+ multiplier=2.0,
+ predicate=retries_async.if_exception_type(
+ core_exceptions.ServiceUnavailable,
+ ),
+ deadline=10.0,
+ ),
+ default_timeout=10.0,
+ client_info=client_info,
+ kind="rest_asyncio",
+ ),
+ self.get_operation: gapic_v1.method_async.wrap_method(
+ self.get_operation,
+ default_retry=retries_async.AsyncRetry(
+ initial=0.5,
+ maximum=10.0,
+ multiplier=2.0,
+ predicate=retries_async.if_exception_type(
+ core_exceptions.ServiceUnavailable,
+ ),
+ deadline=10.0,
+ ),
+ default_timeout=10.0,
+ client_info=client_info,
+ kind="rest_asyncio",
+ ),
+ self.delete_operation: gapic_v1.method_async.wrap_method(
+ self.delete_operation,
+ default_retry=retries_async.AsyncRetry(
+ initial=0.5,
+ maximum=10.0,
+ multiplier=2.0,
+ predicate=retries_async.if_exception_type(
+ core_exceptions.ServiceUnavailable,
+ ),
+ deadline=10.0,
+ ),
+ default_timeout=10.0,
+ client_info=client_info,
+ kind="rest_asyncio",
+ ),
+ self.cancel_operation: gapic_v1.method_async.wrap_method(
+ self.cancel_operation,
+ default_retry=retries_async.AsyncRetry(
+ initial=0.5,
+ maximum=10.0,
+ multiplier=2.0,
+ predicate=retries_async.if_exception_type(
+ core_exceptions.ServiceUnavailable,
+ ),
+ deadline=10.0,
+ ),
+ default_timeout=10.0,
+ client_info=client_info,
+ kind="rest_asyncio",
+ ),
+ }
+
+ async def _list_operations(
+ self,
+ request: operations_pb2.ListOperationsRequest,
+ *,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ ) -> operations_pb2.ListOperationsResponse:
+ r"""Asynchronously call the list operations method over HTTP.
+
+ Args:
+ request (~.operations_pb2.ListOperationsRequest):
+ The request object. The request message for
+ [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations].
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+
+ Returns:
+ ~.operations_pb2.ListOperationsResponse:
+ The response message for
+ [Operations.ListOperations][google.api_core.operations_v1.Operations.ListOperations].
+
+ """
+
+ http_options = [
+ {
+ "method": "get",
+ "uri": "/{}/{{name=**}}/operations".format(self._path_prefix),
+ },
+ ]
+ if "google.longrunning.Operations.ListOperations" in self._http_options:
+ http_options = self._http_options[
+ "google.longrunning.Operations.ListOperations"
+ ]
+
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
+ transcoded_request = path_template.transcode(http_options, **request_kwargs)
+
+ uri = transcoded_request["uri"]
+ method = transcoded_request["method"]
+
+ # Jsonify the query params
+ query_params_request = operations_pb2.ListOperationsRequest()
+ json_format.ParseDict(transcoded_request["query_params"], query_params_request)
+ query_params = json_format.MessageToDict(
+ query_params_request,
+ preserving_proto_field_name=False,
+ use_integers_for_enums=False,
+ )
+
+ # Send the request
+ headers = dict(metadata)
+ headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
+ response = await getattr(self._session, method)(
+ "{host}{uri}".format(host=self._host, uri=uri),
+ timeout=timeout,
+ headers=headers,
+ params=rest_helpers.flatten_query_params(query_params),
+ )
+ content = await response.read()
+
+ # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception
+ # subclass.
+ if response.status_code >= 400:
+ payload = json.loads(content.decode("utf-8"))
+ request_url = "{host}{uri}".format(host=self._host, uri=uri)
+ raise core_exceptions.format_http_response_error(response, method, request_url, payload) # type: ignore
+
+ # Return the response
+ api_response = operations_pb2.ListOperationsResponse()
+ json_format.Parse(content, api_response, ignore_unknown_fields=False)
+ return api_response
+
+ async def _get_operation(
+ self,
+ request: operations_pb2.GetOperationRequest,
+ *,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ ) -> operations_pb2.Operation:
+ r"""Asynchronously call the get operation method over HTTP.
+
+ Args:
+ request (~.operations_pb2.GetOperationRequest):
+ The request object. The request message for
+ [Operations.GetOperation][google.api_core.operations_v1.Operations.GetOperation].
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+
+ Returns:
+ ~.operations_pb2.Operation:
+ This resource represents a long-
+ running operation that is the result of a
+ network API call.
+
+ """
+
+ http_options = [
+ {
+ "method": "get",
+ "uri": "/{}/{{name=**/operations/*}}".format(self._path_prefix),
+ },
+ ]
+ if "google.longrunning.Operations.GetOperation" in self._http_options:
+ http_options = self._http_options[
+ "google.longrunning.Operations.GetOperation"
+ ]
+
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
+ transcoded_request = path_template.transcode(http_options, **request_kwargs)
+
+ uri = transcoded_request["uri"]
+ method = transcoded_request["method"]
+
+ # Jsonify the query params
+ query_params_request = operations_pb2.GetOperationRequest()
+ json_format.ParseDict(transcoded_request["query_params"], query_params_request)
+ query_params = json_format.MessageToDict(
+ query_params_request,
+ preserving_proto_field_name=False,
+ use_integers_for_enums=False,
+ )
+
+ # Send the request
+ headers = dict(metadata)
+ headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
+ response = await getattr(self._session, method)(
+ "{host}{uri}".format(host=self._host, uri=uri),
+ timeout=timeout,
+ headers=headers,
+ params=rest_helpers.flatten_query_params(query_params),
+ )
+ content = await response.read()
+
+ # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception
+ # subclass.
+ if response.status_code >= 400:
+ payload = json.loads(content.decode("utf-8"))
+ request_url = "{host}{uri}".format(host=self._host, uri=uri)
+ raise core_exceptions.format_http_response_error(response, method, request_url, payload) # type: ignore
+
+ # Return the response
+ api_response = operations_pb2.Operation()
+ json_format.Parse(content, api_response, ignore_unknown_fields=False)
+ return api_response
+
+ async def _delete_operation(
+ self,
+ request: operations_pb2.DeleteOperationRequest,
+ *,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ ) -> empty_pb2.Empty:
+ r"""Asynchronously call the delete operation method over HTTP.
+
+ Args:
+ request (~.operations_pb2.DeleteOperationRequest):
+ The request object. The request message for
+ [Operations.DeleteOperation][google.api_core.operations_v1.Operations.DeleteOperation].
+
+ retry (google.api_core.retry.Retry): Designation of what errors, if any,
+ should be retried.
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+ """
+
+ http_options = [
+ {
+ "method": "delete",
+ "uri": "/{}/{{name=**/operations/*}}".format(self._path_prefix),
+ },
+ ]
+ if "google.longrunning.Operations.DeleteOperation" in self._http_options:
+ http_options = self._http_options[
+ "google.longrunning.Operations.DeleteOperation"
+ ]
+
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
+ transcoded_request = path_template.transcode(http_options, **request_kwargs)
+
+ uri = transcoded_request["uri"]
+ method = transcoded_request["method"]
+
+ # Jsonify the query params
+ query_params_request = operations_pb2.DeleteOperationRequest()
+ json_format.ParseDict(transcoded_request["query_params"], query_params_request)
+ query_params = json_format.MessageToDict(
+ query_params_request,
+ preserving_proto_field_name=False,
+ use_integers_for_enums=False,
+ )
+
+ # Send the request
+ headers = dict(metadata)
+ headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
+ response = await getattr(self._session, method)(
+ "{host}{uri}".format(host=self._host, uri=uri),
+ timeout=timeout,
+ headers=headers,
+ params=rest_helpers.flatten_query_params(query_params),
+ )
+
+ # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception
+ # subclass.
+ if response.status_code >= 400:
+ content = await response.read()
+ payload = json.loads(content.decode("utf-8"))
+ request_url = "{host}{uri}".format(host=self._host, uri=uri)
+ raise core_exceptions.format_http_response_error(response, method, request_url, payload) # type: ignore
+
+ return empty_pb2.Empty()
+
+ async def _cancel_operation(
+ self,
+ request: operations_pb2.CancelOperationRequest,
+ *,
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Leverage `retry`
+ # to allow configuring retryable error codes.
+ retry=gapic_v1.method_async.DEFAULT,
+ timeout: Optional[float] = None,
+ metadata: Sequence[Tuple[str, str]] = (),
+ # TODO(https://github.com/googleapis/python-api-core/issues/722): Add `retry` parameter
+ # to allow configuring retryable error codes.
+ ) -> empty_pb2.Empty:
+ r"""Asynchronously call the cancel operation method over HTTP.
+
+ Args:
+ request (~.operations_pb2.CancelOperationRequest):
+ The request object. The request message for
+ [Operations.CancelOperation][google.api_core.operations_v1.Operations.CancelOperation].
+ timeout (float): The timeout for this request.
+ metadata (Sequence[Tuple[str, str]]): Strings which should be
+ sent along with the request as metadata.
+ """
+
+ http_options = [
+ {
+ "method": "post",
+ "uri": "/{}/{{name=**/operations/*}}:cancel".format(self._path_prefix),
+ "body": "*",
+ },
+ ]
+ if "google.longrunning.Operations.CancelOperation" in self._http_options:
+ http_options = self._http_options[
+ "google.longrunning.Operations.CancelOperation"
+ ]
+
+ request_kwargs = self._convert_protobuf_message_to_dict(request)
+ transcoded_request = path_template.transcode(http_options, **request_kwargs)
+
+ # Jsonify the request body
+ body_request = operations_pb2.CancelOperationRequest()
+ json_format.ParseDict(transcoded_request["body"], body_request)
+ body = json_format.MessageToDict(
+ body_request,
+ preserving_proto_field_name=False,
+ use_integers_for_enums=False,
+ )
+ uri = transcoded_request["uri"]
+ method = transcoded_request["method"]
+
+ # Jsonify the query params
+ query_params_request = operations_pb2.CancelOperationRequest()
+ json_format.ParseDict(transcoded_request["query_params"], query_params_request)
+ query_params = json_format.MessageToDict(
+ query_params_request,
+ preserving_proto_field_name=False,
+ use_integers_for_enums=False,
+ )
+
+ # Send the request
+ headers = dict(metadata)
+ headers["Content-Type"] = "application/json"
+ # TODO(https://github.com/googleapis/python-api-core/issues/721): Update incorrect use of `uri`` variable name.
+ response = await getattr(self._session, method)(
+ "{host}{uri}".format(host=self._host, uri=uri),
+ timeout=timeout,
+ headers=headers,
+ params=rest_helpers.flatten_query_params(query_params),
+ data=body,
+ )
+
+ # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception
+ # subclass.
+ if response.status_code >= 400:
+ content = await response.read()
+ payload = json.loads(content.decode("utf-8"))
+ request_url = "{host}{uri}".format(host=self._host, uri=uri)
+ raise core_exceptions.format_http_response_error(response, method, request_url, payload) # type: ignore
+
+ return empty_pb2.Empty()
+
+ @property
+ def list_operations(
+ self,
+ ) -> Callable[
+ [operations_pb2.ListOperationsRequest],
+ Coroutine[Any, Any, operations_pb2.ListOperationsResponse],
+ ]:
+ return self._list_operations
+
+ @property
+ def get_operation(
+ self,
+ ) -> Callable[
+ [operations_pb2.GetOperationRequest],
+ Coroutine[Any, Any, operations_pb2.Operation],
+ ]:
+ return self._get_operation
+
+ @property
+ def delete_operation(
+ self,
+ ) -> Callable[
+ [operations_pb2.DeleteOperationRequest], Coroutine[Any, Any, empty_pb2.Empty]
+ ]:
+ return self._delete_operation
+
+ @property
+ def cancel_operation(
+ self,
+ ) -> Callable[
+ [operations_pb2.CancelOperationRequest], Coroutine[Any, Any, empty_pb2.Empty]
+ ]:
+ return self._cancel_operation
+
+
+__all__ = ("AsyncOperationsRestTransport",)
diff --git a/google/api_core/page_iterator.py b/google/api_core/page_iterator.py
index 7ddc5cbc..23761ec4 100644
--- a/google/api_core/page_iterator.py
+++ b/google/api_core/page_iterator.py
@@ -448,7 +448,7 @@ class _GAXIterator(Iterator):
page_iter (google.gax.PageIterator): A GAX page iterator to be wrapped
to conform to the :class:`Iterator` interface.
item_to_value (Callable[Iterator, Any]): Callable to convert an item
- from the the protobuf response into a native object. Will
+ from the protobuf response into a native object. Will
be called with the iterator and a single item.
max_results (int): The maximum number of results to fetch.
diff --git a/google/api_core/protobuf_helpers.py b/google/api_core/protobuf_helpers.py
index f1abf068..30cd7c85 100644
--- a/google/api_core/protobuf_helpers.py
+++ b/google/api_core/protobuf_helpers.py
@@ -63,9 +63,7 @@ def from_any_pb(pb_type, any_pb):
# Unpack the Any object and populate the protobuf message instance.
if not any_pb.Unpack(msg_pb):
raise TypeError(
- "Could not convert {} to {}".format(
- any_pb.__class__.__name__, pb_type.__name__
- )
+ f"Could not convert `{any_pb.TypeName()}` with underlying type `google.protobuf.any_pb2.Any` to `{msg_pb.DESCRIPTOR.full_name}`"
)
# Done; return the message.
@@ -313,7 +311,7 @@ def field_mask(original, modified):
modified = copy.deepcopy(original)
modified.Clear()
- if type(original) != type(modified):
+ if not isinstance(original, type(modified)):
raise ValueError(
"expected that both original and modified should be of the "
'same type, received "{!r}" and "{!r}".'.format(
diff --git a/google/api_core/rest_streaming.py b/google/api_core/rest_streaming.py
index f91381c1..84aa270c 100644
--- a/google/api_core/rest_streaming.py
+++ b/google/api_core/rest_streaming.py
@@ -14,86 +14,43 @@
"""Helpers for server-side streaming in REST."""
-from collections import deque
-import string
-from typing import Deque
+from typing import Union
+import proto
import requests
+import google.protobuf.message
+from google.api_core._rest_streaming_base import BaseResponseIterator
-class ResponseIterator:
+class ResponseIterator(BaseResponseIterator):
"""Iterator over REST API responses.
Args:
response (requests.Response): An API response object.
- response_message_cls (Callable[proto.Message]): A proto
+ response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response
class expected to be returned from an API.
+
+ Raises:
+ ValueError:
+ - If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`.
"""
- def __init__(self, response: requests.Response, response_message_cls):
+ def __init__(
+ self,
+ response: requests.Response,
+ response_message_cls: Union[proto.Message, google.protobuf.message.Message],
+ ):
self._response = response
- self._response_message_cls = response_message_cls
# Inner iterator over HTTP response's content.
self._response_itr = self._response.iter_content(decode_unicode=True)
- # Contains a list of JSON responses ready to be sent to user.
- self._ready_objs: Deque[str] = deque()
- # Current JSON response being built.
- self._obj = ""
- # Keeps track of the nesting level within a JSON object.
- self._level = 0
- # Keeps track whether HTTP response is currently sending values
- # inside of a string value.
- self._in_string = False
- # Whether an escape symbol "\" was encountered.
- self._escape_next = False
+ super(ResponseIterator, self).__init__(
+ response_message_cls=response_message_cls
+ )
def cancel(self):
"""Cancel existing streaming operation."""
self._response.close()
- def _process_chunk(self, chunk: str):
- if self._level == 0:
- if chunk[0] != "[":
- raise ValueError(
- "Can only parse array of JSON objects, instead got %s" % chunk
- )
- for char in chunk:
- if char == "{":
- if self._level == 1:
- # Level 1 corresponds to the outermost JSON object
- # (i.e. the one we care about).
- self._obj = ""
- if not self._in_string:
- self._level += 1
- self._obj += char
- elif char == "}":
- self._obj += char
- if not self._in_string:
- self._level -= 1
- if not self._in_string and self._level == 1:
- self._ready_objs.append(self._obj)
- elif char == '"':
- # Helps to deal with an escaped quotes inside of a string.
- if not self._escape_next:
- self._in_string = not self._in_string
- self._obj += char
- elif char in string.whitespace:
- if self._in_string:
- self._obj += char
- elif char == "[":
- if self._level == 0:
- self._level += 1
- else:
- self._obj += char
- elif char == "]":
- if self._level == 1:
- self._level -= 1
- else:
- self._obj += char
- else:
- self._obj += char
- self._escape_next = not self._escape_next if char == "\\" else False
-
def __next__(self):
while not self._ready_objs:
try:
@@ -105,9 +62,5 @@ def __next__(self):
raise e
return self._grab()
- def _grab(self):
- # Add extra quotes to make json.loads happy.
- return self._response_message_cls.from_json(self._ready_objs.popleft())
-
def __iter__(self):
return self
diff --git a/google/api_core/rest_streaming_async.py b/google/api_core/rest_streaming_async.py
new file mode 100644
index 00000000..370c2b53
--- /dev/null
+++ b/google/api_core/rest_streaming_async.py
@@ -0,0 +1,89 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for asynchronous server-side streaming in REST."""
+
+from typing import Union
+
+import proto
+
+try:
+ import google.auth.aio.transport
+except ImportError as e: # pragma: NO COVER
+ raise ImportError(
+ "`google-api-core[async_rest]` is required to use asynchronous rest streaming. "
+ "Install the `async_rest` extra of `google-api-core` using "
+ "`pip install google-api-core[async_rest]`."
+ ) from e
+
+import google.protobuf.message
+from google.api_core._rest_streaming_base import BaseResponseIterator
+
+
+class AsyncResponseIterator(BaseResponseIterator):
+ """Asynchronous Iterator over REST API responses.
+
+ Args:
+ response (google.auth.aio.transport.Response): An API response object.
+ response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response
+ class expected to be returned from an API.
+
+ Raises:
+ ValueError:
+ - If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`.
+ """
+
+ def __init__(
+ self,
+ response: google.auth.aio.transport.Response,
+ response_message_cls: Union[proto.Message, google.protobuf.message.Message],
+ ):
+ self._response = response
+ self._chunk_size = 1024
+ # TODO(https://github.com/googleapis/python-api-core/issues/703): mypy does not recognize the abstract content
+ # method as an async generator as it looks for the `yield` keyword in the implementation.
+ # Given that the abstract method is not implemented, mypy fails to recognize it as an async generator.
+ # mypy warnings are silenced until the linked issue is resolved.
+ self._response_itr = self._response.content(self._chunk_size).__aiter__() # type: ignore
+ super(AsyncResponseIterator, self).__init__(
+ response_message_cls=response_message_cls
+ )
+
+ async def __aenter__(self):
+ return self
+
+ async def cancel(self):
+ """Cancel existing streaming operation."""
+ await self._response.close()
+
+ async def __anext__(self):
+ while not self._ready_objs:
+ try:
+ chunk = await self._response_itr.__anext__()
+ chunk = chunk.decode("utf-8")
+ self._process_chunk(chunk)
+ except StopAsyncIteration as e:
+ if self._level > 0:
+ raise ValueError("i Unfinished stream: %s" % self._obj)
+ raise e
+ except ValueError as e:
+ raise e
+ return self._grab()
+
+ def __aiter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc, tb):
+ """Cancel existing async streaming operation."""
+ await self._response.close()
diff --git a/google/api_core/retry.py b/google/api_core/retry.py
deleted file mode 100644
index df1e65e0..00000000
--- a/google/api_core/retry.py
+++ /dev/null
@@ -1,454 +0,0 @@
-# Copyright 2017 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Helpers for retrying functions with exponential back-off.
-
-The :class:`Retry` decorator can be used to retry functions that raise
-exceptions using exponential backoff. Because a exponential sleep algorithm is
-used, the retry is limited by a `deadline`. The deadline is the maximum amount
-of time a method can block. This is used instead of total number of retries
-because it is difficult to ascertain the amount of time a function can block
-when using total number of retries and exponential backoff.
-
-By default, this decorator will retry transient
-API errors (see :func:`if_transient_error`). For example:
-
-.. code-block:: python
-
- @retry.Retry()
- def call_flaky_rpc():
- return client.flaky_rpc()
-
- # Will retry flaky_rpc() if it raises transient API errors.
- result = call_flaky_rpc()
-
-You can pass a custom predicate to retry on different exceptions, such as
-waiting for an eventually consistent item to be available:
-
-.. code-block:: python
-
- @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
- def check_if_exists():
- return client.does_thing_exist()
-
- is_available = check_if_exists()
-
-Some client library methods apply retry automatically. These methods can accept
-a ``retry`` parameter that allows you to configure the behavior:
-
-.. code-block:: python
-
- my_retry = retry.Retry(deadline=60)
- result = client.some_method(retry=my_retry)
-
-"""
-
-from __future__ import unicode_literals
-
-import datetime
-import functools
-import logging
-import random
-import time
-
-import requests.exceptions
-
-from google.api_core import datetime_helpers
-from google.api_core import exceptions
-from google.auth import exceptions as auth_exceptions
-
-_LOGGER = logging.getLogger(__name__)
-_DEFAULT_INITIAL_DELAY = 1.0 # seconds
-_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
-_DEFAULT_DELAY_MULTIPLIER = 2.0
-_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
-
-
-def if_exception_type(*exception_types):
- """Creates a predicate to check if the exception is of a given type.
-
- Args:
- exception_types (Sequence[:func:`type`]): The exception types to check
- for.
-
- Returns:
- Callable[Exception]: A predicate that returns True if the provided
- exception is of the given type(s).
- """
-
- def if_exception_type_predicate(exception):
- """Bound predicate for checking an exception type."""
- return isinstance(exception, exception_types)
-
- return if_exception_type_predicate
-
-
-# pylint: disable=invalid-name
-# Pylint sees this as a constant, but it is also an alias that should be
-# considered a function.
-if_transient_error = if_exception_type(
- exceptions.InternalServerError,
- exceptions.TooManyRequests,
- exceptions.ServiceUnavailable,
- requests.exceptions.ConnectionError,
- requests.exceptions.ChunkedEncodingError,
- auth_exceptions.TransportError,
-)
-"""A predicate that checks if an exception is a transient API error.
-
-The following server errors are considered transient:
-
-- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
- ``INTERNAL(13)`` and its subclasses.
-- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
-- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503
-- :class:`requests.exceptions.ConnectionError`
-- :class:`requests.exceptions.ChunkedEncodingError` - The server declared
- chunked encoding but sent an invalid chunk.
-- :class:`google.auth.exceptions.TransportError` - Used to indicate an
- error occurred during an HTTP request.
-"""
-# pylint: enable=invalid-name
-
-
-def exponential_sleep_generator(initial, maximum, multiplier=_DEFAULT_DELAY_MULTIPLIER):
- """Generates sleep intervals based on the exponential back-off algorithm.
-
- This implements the `Truncated Exponential Back-off`_ algorithm.
-
- .. _Truncated Exponential Back-off:
- https://cloud.google.com/storage/docs/exponential-backoff
-
- Args:
- initial (float): The minimum amount of time to delay. This must
- be greater than 0.
- maximum (float): The maximum amount of time to delay.
- multiplier (float): The multiplier applied to the delay.
-
- Yields:
- float: successive sleep intervals.
- """
- delay = min(initial, maximum)
- while True:
- yield random.uniform(0.0, delay)
- delay = min(delay * multiplier, maximum)
-
-
-def retry_target(
- target, predicate, sleep_generator, timeout=None, on_error=None, **kwargs
-):
- """Call a function and retry if it fails.
-
- This is the lowest-level retry helper. Generally, you'll use the
- higher-level retry helper :class:`Retry`.
-
- Args:
- target(Callable): The function to call and retry. This must be a
- nullary function - apply arguments with `functools.partial`.
- predicate (Callable[Exception]): A callable used to determine if an
- exception raised by the target should be considered retryable.
- It should return True to retry or False otherwise.
- sleep_generator (Iterable[float]): An infinite iterator that determines
- how long to sleep between retries.
- timeout (float): How long to keep retrying the target.
- on_error (Callable[Exception]): A function to call while processing a
- retryable exception. Any error raised by this function will *not*
- be caught.
- deadline (float): DEPRECATED: use ``timeout`` instead. For backward
- compatibility, if specified it will override ``timeout`` parameter.
-
- Returns:
- Any: the return value of the target function.
-
- Raises:
- google.api_core.RetryError: If the deadline is exceeded while retrying.
- ValueError: If the sleep generator stops yielding values.
- Exception: If the target raises a method that isn't retryable.
- """
-
- timeout = kwargs.get("deadline", timeout)
-
- if timeout is not None:
- deadline = datetime_helpers.utcnow() + datetime.timedelta(seconds=timeout)
- else:
- deadline = None
-
- last_exc = None
-
- for sleep in sleep_generator:
- try:
- return target()
-
- # pylint: disable=broad-except
- # This function explicitly must deal with broad exceptions.
- except Exception as exc:
- if not predicate(exc):
- raise
- last_exc = exc
- if on_error is not None:
- on_error(exc)
-
- if deadline is not None:
- next_attempt_time = datetime_helpers.utcnow() + datetime.timedelta(
- seconds=sleep
- )
- if deadline < next_attempt_time:
- raise exceptions.RetryError(
- "Deadline of {:.1f}s exceeded while calling target function".format(
- timeout
- ),
- last_exc,
- ) from last_exc
-
- _LOGGER.debug(
- "Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep)
- )
- time.sleep(sleep)
-
- raise ValueError("Sleep generator stopped yielding sleep values.")
-
-
-class Retry(object):
- """Exponential retry decorator.
-
- This class is a decorator used to add retry or polling behavior to an RPC
- call.
-
- Although the default behavior is to retry transient API errors, a
- different predicate can be provided to retry other exceptions.
-
- There two important concepts that retry/polling behavior may operate on,
- Deadline and Timeout, which need to be properly defined for the correct
- usage of this class and the rest of the library.
-
- Deadline: a fixed point in time by which a certain operation must
- terminate. For example, if a certain operation has a deadline
- "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an
- error) by that time, regardless of when it was started or whether it
- was started at all.
-
- Timeout: the maximum duration of time after which a certain operation
- must terminate (successfully or with an error). The countdown begins right
- after an operation was started. For example, if an operation was started at
- 09:24:00 with timeout of 75 seconds, it must terminate no later than
- 09:25:15.
-
- Unfortunately, in the past this class (and the api-core library as a whole) has not been
- properly distinguishing the concepts of "timeout" and "deadline", and the
- ``deadline`` parameter has meant ``timeout``. That is why
- ``deadline`` has been deprecated and ``timeout`` should be used instead. If the
- ``deadline`` parameter is set, it will override the ``timeout`` parameter. In other words,
- ``retry.deadline`` should be treated as just a deprecated alias for
- ``retry.timeout``.
-
- Said another way, it is safe to assume that this class and the rest of this
- library operate in terms of timeouts (not deadlines) unless explicitly
- noted the usage of deadline semantics.
-
- It is also important to
- understand the three most common applications of the Timeout concept in the
- context of this library.
-
- Usually the generic Timeout term may stand for one of the following actual
- timeouts: RPC Timeout, Retry Timeout, or Polling Timeout.
-
- RPC Timeout: a value supplied by the client to the server so
- that the server side knows the maximum amount of time it is expected to
- spend handling that specific RPC. For example, in the case of gRPC transport,
- RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2
- request. The `timeout` property of this class normally never represents the
- RPC Timeout as it is handled separately by the ``google.api_core.timeout``
- module of this library.
-
- Retry Timeout: this is the most common meaning of the ``timeout`` property
- of this class, and defines how long a certain RPC may be retried in case
- the server returns an error.
-
- Polling Timeout: defines how long the
- client side is allowed to call the polling RPC repeatedly to check a status of a
- long-running operation. Each polling RPC is
- expected to succeed (its errors are supposed to be handled by the retry
- logic). The decision as to whether a new polling attempt needs to be made is based
- not on the RPC status code but on the status of the returned
- status of an operation. In other words: we will poll a long-running operation until the operation is done or the polling timeout expires. Each poll will inform us of the status of the operation. The poll consists of an RPC to the server that may itself be retried as per the poll-specific retry settings in case of errors. The operation-level retry settings do NOT apply to polling-RPC retries.
-
- With the actual timeout types being defined above, the client libraries
- often refer to just Timeout without clarifying which type specifically
- that is. In that case the actual timeout type (sometimes also referred to as
- Logical Timeout) can be determined from the context. If it is a unary rpc
- call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if
- provided directly as a standalone value) or Retry Timeout (if provided as
- ``retry.timeout`` property of the unary RPC's retry config). For
- ``Operation`` or ``PollingFuture`` in general Timeout stands for
- Polling Timeout.
-
- Args:
- predicate (Callable[Exception]): A callable that should return ``True``
- if the given exception is retryable.
- initial (float): The minimum amount of time to delay in seconds. This
- must be greater than 0.
- maximum (float): The maximum amount of time to delay in seconds.
- multiplier (float): The multiplier applied to the delay.
- timeout (float): How long to keep retrying, in seconds.
- deadline (float): DEPRECATED: use `timeout` instead. For backward
- compatibility, if specified it will override the ``timeout`` parameter.
- """
-
- def __init__(
- self,
- predicate=if_transient_error,
- initial=_DEFAULT_INITIAL_DELAY,
- maximum=_DEFAULT_MAXIMUM_DELAY,
- multiplier=_DEFAULT_DELAY_MULTIPLIER,
- timeout=_DEFAULT_DEADLINE,
- on_error=None,
- **kwargs
- ):
- self._predicate = predicate
- self._initial = initial
- self._multiplier = multiplier
- self._maximum = maximum
- self._timeout = kwargs.get("deadline", timeout)
- self._deadline = self._timeout
- self._on_error = on_error
-
- def __call__(self, func, on_error=None):
- """Wrap a callable with retry behavior.
-
- Args:
- func (Callable): The callable to add retry behavior to.
- on_error (Callable[Exception]): A function to call while processing
- a retryable exception. Any error raised by this function will
- *not* be caught.
-
- Returns:
- Callable: A callable that will invoke ``func`` with retry
- behavior.
- """
- if self._on_error is not None:
- on_error = self._on_error
-
- @functools.wraps(func)
- def retry_wrapped_func(*args, **kwargs):
- """A wrapper that calls target function with retry."""
- target = functools.partial(func, *args, **kwargs)
- sleep_generator = exponential_sleep_generator(
- self._initial, self._maximum, multiplier=self._multiplier
- )
- return retry_target(
- target,
- self._predicate,
- sleep_generator,
- self._timeout,
- on_error=on_error,
- )
-
- return retry_wrapped_func
-
- @property
- def deadline(self):
- """
- DEPRECATED: use ``timeout`` instead. Refer to the ``Retry`` class
- documentation for details.
- """
- return self._timeout
-
- @property
- def timeout(self):
- return self._timeout
-
- def with_deadline(self, deadline):
- """Return a copy of this retry with the given timeout.
-
- DEPRECATED: use :meth:`with_timeout` instead. Refer to the ``Retry`` class
- documentation for details.
-
- Args:
- deadline (float): How long to keep retrying in seconds.
-
- Returns:
- Retry: A new retry instance with the given timeout.
- """
- return self.with_timeout(timeout=deadline)
-
- def with_timeout(self, timeout):
- """Return a copy of this retry with the given timeout.
-
- Args:
- timeout (float): How long to keep retrying, in seconds.
-
- Returns:
- Retry: A new retry instance with the given timeout.
- """
- return Retry(
- predicate=self._predicate,
- initial=self._initial,
- maximum=self._maximum,
- multiplier=self._multiplier,
- timeout=timeout,
- on_error=self._on_error,
- )
-
- def with_predicate(self, predicate):
- """Return a copy of this retry with the given predicate.
-
- Args:
- predicate (Callable[Exception]): A callable that should return
- ``True`` if the given exception is retryable.
-
- Returns:
- Retry: A new retry instance with the given predicate.
- """
- return Retry(
- predicate=predicate,
- initial=self._initial,
- maximum=self._maximum,
- multiplier=self._multiplier,
- timeout=self._timeout,
- on_error=self._on_error,
- )
-
- def with_delay(self, initial=None, maximum=None, multiplier=None):
- """Return a copy of this retry with the given delay options.
-
- Args:
- initial (float): The minimum amount of time to delay. This must
- be greater than 0.
- maximum (float): The maximum amount of time to delay.
- multiplier (float): The multiplier applied to the delay.
-
- Returns:
- Retry: A new retry instance with the given predicate.
- """
- return Retry(
- predicate=self._predicate,
- initial=initial if initial is not None else self._initial,
- maximum=maximum if maximum is not None else self._maximum,
- multiplier=multiplier if multiplier is not None else self._multiplier,
- timeout=self._timeout,
- on_error=self._on_error,
- )
-
- def __str__(self):
- return (
- "".format(
- self._predicate,
- self._initial,
- self._maximum,
- self._multiplier,
- self._timeout, # timeout can be None, thus no {:.1f}
- self._on_error,
- )
- )
diff --git a/google/api_core/retry/__init__.py b/google/api_core/retry/__init__.py
new file mode 100644
index 00000000..1724fdbd
--- /dev/null
+++ b/google/api_core/retry/__init__.py
@@ -0,0 +1,52 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Retry implementation for Google API client libraries."""
+
+from .retry_base import exponential_sleep_generator
+from .retry_base import if_exception_type
+from .retry_base import if_transient_error
+from .retry_base import build_retry_error
+from .retry_base import RetryFailureReason
+from .retry_unary import Retry
+from .retry_unary import retry_target
+from .retry_unary_async import AsyncRetry
+from .retry_unary_async import retry_target as retry_target_async
+from .retry_streaming import StreamingRetry
+from .retry_streaming import retry_target_stream
+from .retry_streaming_async import AsyncStreamingRetry
+from .retry_streaming_async import retry_target_stream as retry_target_stream_async
+
+# The following imports are for backwards compatibility with https://github.com/googleapis/python-api-core/blob/4d7d2edee2c108d43deb151e6e0fdceb56b73275/google/api_core/retry.py
+#
+# TODO: Revert these imports on the next major version release (https://github.com/googleapis/python-api-core/issues/576)
+from google.api_core import datetime_helpers # noqa: F401
+from google.api_core import exceptions # noqa: F401
+from google.auth import exceptions as auth_exceptions # noqa: F401
+
+__all__ = (
+ "exponential_sleep_generator",
+ "if_exception_type",
+ "if_transient_error",
+ "build_retry_error",
+ "RetryFailureReason",
+ "Retry",
+ "AsyncRetry",
+ "StreamingRetry",
+ "AsyncStreamingRetry",
+ "retry_target",
+ "retry_target_async",
+ "retry_target_stream",
+ "retry_target_stream_async",
+)
diff --git a/google/api_core/retry/retry_base.py b/google/api_core/retry/retry_base.py
new file mode 100644
index 00000000..263b4ccf
--- /dev/null
+++ b/google/api_core/retry/retry_base.py
@@ -0,0 +1,370 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared classes and functions for retrying requests.
+
+:class:`_BaseRetry` is the base class for :class:`Retry`,
+:class:`AsyncRetry`, :class:`StreamingRetry`, and :class:`AsyncStreamingRetry`.
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+import time
+
+from enum import Enum
+from typing import Any, Callable, Optional, Iterator, TYPE_CHECKING
+
+import requests.exceptions
+
+from google.api_core import exceptions
+from google.auth import exceptions as auth_exceptions
+
+if TYPE_CHECKING:
+ import sys
+
+ if sys.version_info >= (3, 11):
+ from typing import Self
+ else:
+ from typing_extensions import Self
+
+_DEFAULT_INITIAL_DELAY = 1.0 # seconds
+_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
+_DEFAULT_DELAY_MULTIPLIER = 2.0
+_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
+
+_LOGGER = logging.getLogger("google.api_core.retry")
+
+
+def if_exception_type(
+ *exception_types: type[Exception],
+) -> Callable[[Exception], bool]:
+ """Creates a predicate to check if the exception is of a given type.
+
+ Args:
+ exception_types (Sequence[:func:`type`]): The exception types to check
+ for.
+
+ Returns:
+ Callable[Exception]: A predicate that returns True if the provided
+ exception is of the given type(s).
+ """
+
+ def if_exception_type_predicate(exception: Exception) -> bool:
+ """Bound predicate for checking an exception type."""
+ return isinstance(exception, exception_types)
+
+ return if_exception_type_predicate
+
+
+# pylint: disable=invalid-name
+# Pylint sees this as a constant, but it is also an alias that should be
+# considered a function.
+if_transient_error = if_exception_type(
+ exceptions.InternalServerError,
+ exceptions.TooManyRequests,
+ exceptions.ServiceUnavailable,
+ requests.exceptions.ConnectionError,
+ requests.exceptions.ChunkedEncodingError,
+ auth_exceptions.TransportError,
+)
+"""A predicate that checks if an exception is a transient API error.
+
+The following server errors are considered transient:
+
+- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
+ ``INTERNAL(13)`` and its subclasses.
+- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
+- :class:`google.api_core.exceptions.ServiceUnavailable` - HTTP 503
+- :class:`requests.exceptions.ConnectionError`
+- :class:`requests.exceptions.ChunkedEncodingError` - The server declared
+ chunked encoding but sent an invalid chunk.
+- :class:`google.auth.exceptions.TransportError` - Used to indicate an
+ error occurred during an HTTP request.
+"""
+# pylint: enable=invalid-name
+
+
+def exponential_sleep_generator(
+ initial: float, maximum: float, multiplier: float = _DEFAULT_DELAY_MULTIPLIER
+):
+ """Generates sleep intervals based on the exponential back-off algorithm.
+
+ This implements the `Truncated Exponential Back-off`_ algorithm.
+
+ .. _Truncated Exponential Back-off:
+ https://cloud.google.com/storage/docs/exponential-backoff
+
+ Args:
+ initial (float): The minimum amount of time to delay. This must
+ be greater than 0.
+ maximum (float): The maximum amount of time to delay.
+ multiplier (float): The multiplier applied to the delay.
+
+ Yields:
+ float: successive sleep intervals.
+ """
+ max_delay = min(initial, maximum)
+ while True:
+ yield random.uniform(0.0, max_delay)
+ max_delay = min(max_delay * multiplier, maximum)
+
+
+class RetryFailureReason(Enum):
+ """
+ The cause of a failed retry, used when building exceptions
+ """
+
+ TIMEOUT = 0
+ NON_RETRYABLE_ERROR = 1
+
+
+def build_retry_error(
+ exc_list: list[Exception],
+ reason: RetryFailureReason,
+ timeout_val: float | None,
+ **kwargs: Any,
+) -> tuple[Exception, Exception | None]:
+ """
+ Default exception_factory implementation.
+
+ Returns a RetryError if the failure is due to a timeout, otherwise
+ returns the last exception encountered.
+
+ Args:
+ - exc_list: list of exceptions that occurred during the retry
+ - reason: reason for the retry failure.
+ Can be TIMEOUT or NON_RETRYABLE_ERROR
+ - timeout_val: the original timeout value for the retry (in seconds), for use in the exception message
+
+ Returns:
+ - tuple: a tuple of the exception to be raised, and the cause exception if any
+ """
+ if reason == RetryFailureReason.TIMEOUT:
+ # return RetryError with the most recent exception as the cause
+ src_exc = exc_list[-1] if exc_list else None
+ timeout_val_str = f"of {timeout_val:0.1f}s " if timeout_val is not None else ""
+ return (
+ exceptions.RetryError(
+ f"Timeout {timeout_val_str}exceeded",
+ src_exc,
+ ),
+ src_exc,
+ )
+ elif exc_list:
+ # return most recent exception encountered
+ return exc_list[-1], None
+ else:
+ # no exceptions were given in exc_list. Raise generic RetryError
+ return exceptions.RetryError("Unknown error", None), None
+
+
+def _retry_error_helper(
+ exc: Exception,
+ deadline: float | None,
+ sleep_iterator: Iterator[float],
+ error_list: list[Exception],
+ predicate_fn: Callable[[Exception], bool],
+ on_error_fn: Callable[[Exception], None] | None,
+ exc_factory_fn: Callable[
+ [list[Exception], RetryFailureReason, float | None],
+ tuple[Exception, Exception | None],
+ ],
+ original_timeout: float | None,
+) -> float:
+ """
+ Shared logic for handling an error for all retry implementations
+
+ - Raises an error on timeout or non-retryable error
+ - Calls on_error_fn if provided
+ - Logs the error
+
+ Args:
+ - exc: the exception that was raised
+ - deadline: the deadline for the retry, calculated as a diff from time.monotonic()
+ - sleep_iterator: iterator to draw the next backoff value from
+ - error_list: the list of exceptions that have been raised so far
+ - predicate_fn: takes `exc` and returns true if the operation should be retried
+ - on_error_fn: callback to execute when a retryable error occurs
+ - exc_factory_fn: callback used to build the exception to be raised on terminal failure
+ - original_timeout_val: the original timeout value for the retry (in seconds),
+ to be passed to the exception factory for building an error message
+ Returns:
+ - the sleep value chosen before the next attempt
+ """
+ error_list.append(exc)
+ if not predicate_fn(exc):
+ final_exc, source_exc = exc_factory_fn(
+ error_list,
+ RetryFailureReason.NON_RETRYABLE_ERROR,
+ original_timeout,
+ )
+ raise final_exc from source_exc
+ if on_error_fn is not None:
+ on_error_fn(exc)
+ # next_sleep is fetched after the on_error callback, to allow clients
+ # to update sleep_iterator values dynamically in response to errors
+ try:
+ next_sleep = next(sleep_iterator)
+ except StopIteration:
+ raise ValueError("Sleep generator stopped yielding sleep values.") from exc
+ if deadline is not None and time.monotonic() + next_sleep > deadline:
+ final_exc, source_exc = exc_factory_fn(
+ error_list,
+ RetryFailureReason.TIMEOUT,
+ original_timeout,
+ )
+ raise final_exc from source_exc
+ _LOGGER.debug(
+ "Retrying due to {}, sleeping {:.1f}s ...".format(error_list[-1], next_sleep)
+ )
+ return next_sleep
+
+
+class _BaseRetry(object):
+ """
+ Base class for retry configuration objects. This class is intended to capture retry
+ and backoff configuration that is common to both synchronous and asynchronous retries,
+ for both unary and streaming RPCs. It is not intended to be instantiated directly,
+ but rather to be subclassed by the various retry configuration classes.
+ """
+
+ def __init__(
+ self,
+ predicate: Callable[[Exception], bool] = if_transient_error,
+ initial: float = _DEFAULT_INITIAL_DELAY,
+ maximum: float = _DEFAULT_MAXIMUM_DELAY,
+ multiplier: float = _DEFAULT_DELAY_MULTIPLIER,
+ timeout: Optional[float] = _DEFAULT_DEADLINE,
+ on_error: Optional[Callable[[Exception], Any]] = None,
+ **kwargs: Any,
+ ) -> None:
+ self._predicate = predicate
+ self._initial = initial
+ self._multiplier = multiplier
+ self._maximum = maximum
+ self._timeout = kwargs.get("deadline", timeout)
+ self._deadline = self._timeout
+ self._on_error = on_error
+
+ def __call__(self, *args, **kwargs) -> Any:
+ raise NotImplementedError("Not implemented in base class")
+
+ @property
+ def deadline(self) -> float | None:
+ """
+ DEPRECATED: use ``timeout`` instead. Refer to the ``Retry`` class
+ documentation for details.
+ """
+ return self._timeout
+
+ @property
+ def timeout(self) -> float | None:
+ return self._timeout
+
+ def with_deadline(self, deadline: float | None) -> Self:
+ """Return a copy of this retry with the given timeout.
+
+ DEPRECATED: use :meth:`with_timeout` instead. Refer to the ``Retry`` class
+ documentation for details.
+
+ Args:
+ deadline (float|None): How long to keep retrying, in seconds. If None,
+ no timeout is enforced.
+
+ Returns:
+ Retry: A new retry instance with the given timeout.
+ """
+ return self.with_timeout(deadline)
+
+ def with_timeout(self, timeout: float | None) -> Self:
+ """Return a copy of this retry with the given timeout.
+
+ Args:
+ timeout (float): How long to keep retrying, in seconds. If None,
+ no timeout will be enforced.
+
+ Returns:
+ Retry: A new retry instance with the given timeout.
+ """
+ return type(self)(
+ predicate=self._predicate,
+ initial=self._initial,
+ maximum=self._maximum,
+ multiplier=self._multiplier,
+ timeout=timeout,
+ on_error=self._on_error,
+ )
+
+ def with_predicate(self, predicate: Callable[[Exception], bool]) -> Self:
+ """Return a copy of this retry with the given predicate.
+
+ Args:
+ predicate (Callable[Exception]): A callable that should return
+ ``True`` if the given exception is retryable.
+
+ Returns:
+ Retry: A new retry instance with the given predicate.
+ """
+ return type(self)(
+ predicate=predicate,
+ initial=self._initial,
+ maximum=self._maximum,
+ multiplier=self._multiplier,
+ timeout=self._timeout,
+ on_error=self._on_error,
+ )
+
+ def with_delay(
+ self,
+ initial: Optional[float] = None,
+ maximum: Optional[float] = None,
+ multiplier: Optional[float] = None,
+ ) -> Self:
+ """Return a copy of this retry with the given delay options.
+
+ Args:
+ initial (float): The minimum amount of time to delay (in seconds). This must
+ be greater than 0. If None, the current value is used.
+ maximum (float): The maximum amount of time to delay (in seconds). If None, the
+ current value is used.
+ multiplier (float): The multiplier applied to the delay. If None, the current
+ value is used.
+
+ Returns:
+ Retry: A new retry instance with the given delay options.
+ """
+ return type(self)(
+ predicate=self._predicate,
+ initial=initial if initial is not None else self._initial,
+ maximum=maximum if maximum is not None else self._maximum,
+ multiplier=multiplier if multiplier is not None else self._multiplier,
+ timeout=self._timeout,
+ on_error=self._on_error,
+ )
+
+ def __str__(self) -> str:
+ return (
+ "<{} predicate={}, initial={:.1f}, maximum={:.1f}, "
+ "multiplier={:.1f}, timeout={}, on_error={}>".format(
+ type(self).__name__,
+ self._predicate,
+ self._initial,
+ self._maximum,
+ self._multiplier,
+ self._timeout, # timeout can be None, thus no {:.1f}
+ self._on_error,
+ )
+ )
diff --git a/google/api_core/retry/retry_streaming.py b/google/api_core/retry/retry_streaming.py
new file mode 100644
index 00000000..e4474c8a
--- /dev/null
+++ b/google/api_core/retry/retry_streaming.py
@@ -0,0 +1,264 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generator wrapper for retryable streaming RPCs.
+"""
+from __future__ import annotations
+
+from typing import (
+ Callable,
+ Optional,
+ List,
+ Tuple,
+ Iterable,
+ Generator,
+ TypeVar,
+ Any,
+ TYPE_CHECKING,
+)
+
+import sys
+import time
+import functools
+
+from google.api_core.retry.retry_base import _BaseRetry
+from google.api_core.retry.retry_base import _retry_error_helper
+from google.api_core.retry import exponential_sleep_generator
+from google.api_core.retry import build_retry_error
+from google.api_core.retry import RetryFailureReason
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 10):
+ from typing import ParamSpec
+ else:
+ from typing_extensions import ParamSpec
+
+ _P = ParamSpec("_P") # target function call parameters
+ _Y = TypeVar("_Y") # yielded values
+
+
+def retry_target_stream(
+ target: Callable[_P, Iterable[_Y]],
+ predicate: Callable[[Exception], bool],
+ sleep_generator: Iterable[float],
+ timeout: Optional[float] = None,
+ on_error: Optional[Callable[[Exception], None]] = None,
+ exception_factory: Callable[
+ [List[Exception], RetryFailureReason, Optional[float]],
+ Tuple[Exception, Optional[Exception]],
+ ] = build_retry_error,
+ init_args: tuple = (),
+ init_kwargs: dict = {},
+ **kwargs,
+) -> Generator[_Y, Any, None]:
+ """Create a generator wrapper that retries the wrapped stream if it fails.
+
+ This is the lowest-level retry helper. Generally, you'll use the
+ higher-level retry helper :class:`Retry`.
+
+ Args:
+ target: The generator function to call and retry.
+ predicate: A callable used to determine if an
+ exception raised by the target should be considered retryable.
+ It should return True to retry or False otherwise.
+ sleep_generator: An infinite iterator that determines
+ how long to sleep between retries.
+ timeout: How long to keep retrying the target.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error: If given, the on_error callback will be called with each
+ retryable exception raised by the target. Any error raised by this
+ function will *not* be caught.
+ exception_factory: A function that is called when the retryable reaches
+ a terminal failure state, used to construct an exception to be raised.
+ It takes a list of all exceptions encountered, a retry.RetryFailureReason
+ enum indicating the failure cause, and the original timeout value
+ as arguments. It should return a tuple of the exception to be raised,
+ along with the cause exception if any. The default implementation will raise
+ a RetryError on timeout, or the last exception encountered otherwise.
+ init_args: Positional arguments to pass to the target function.
+ init_kwargs: Keyword arguments to pass to the target function.
+
+ Returns:
+ Generator: A retryable generator that wraps the target generator function.
+
+ Raises:
+ ValueError: If the sleep generator stops yielding values.
+ Exception: a custom exception specified by the exception_factory if provided.
+ If no exception_factory is provided:
+ google.api_core.RetryError: If the timeout is exceeded while retrying.
+ Exception: If the target raises an error that isn't retryable.
+ """
+
+ timeout = kwargs.get("deadline", timeout)
+ deadline: Optional[float] = (
+ time.monotonic() + timeout if timeout is not None else None
+ )
+ error_list: list[Exception] = []
+ sleep_iter = iter(sleep_generator)
+
+ # continue trying until an attempt completes, or a terminal exception is raised in _retry_error_helper
+ # TODO: support max_attempts argument: https://github.com/googleapis/python-api-core/issues/535
+ while True:
+ # Start a new retry loop
+ try:
+ # Note: in the future, we can add a ResumptionStrategy object
+ # to generate new args between calls. For now, use the same args
+ # for each attempt.
+ subgenerator = target(*init_args, **init_kwargs)
+ return (yield from subgenerator)
+ # handle exceptions raised by the subgenerator
+ # pylint: disable=broad-except
+ # This function explicitly must deal with broad exceptions.
+ except Exception as exc:
+ # defer to shared logic for handling errors
+ next_sleep = _retry_error_helper(
+ exc,
+ deadline,
+ sleep_iter,
+ error_list,
+ predicate,
+ on_error,
+ exception_factory,
+ timeout,
+ )
+ # if exception not raised, sleep before next attempt
+ time.sleep(next_sleep)
+
+
+class StreamingRetry(_BaseRetry):
+ """Exponential retry decorator for streaming synchronous RPCs.
+
+ This class returns a Generator when called, which wraps the target
+ stream in retry logic. If any exception is raised by the target, the
+ entire stream will be retried within the wrapper.
+
+ Although the default behavior is to retry transient API errors, a
+ different predicate can be provided to retry other exceptions.
+
+ Important Note: when a stream encounters a retryable error, it will
+ silently construct a fresh iterator instance in the background
+ and continue yielding (likely duplicate) values as if no error occurred.
+ This is the most general way to retry a stream, but it often is not the
+ desired behavior. Example: iter([1, 2, 1/0]) -> [1, 2, 1, 2, ...]
+
+ There are two ways to build more advanced retry logic for streams:
+
+ 1. Wrap the target
+ Use a ``target`` that maintains state between retries, and creates a
+ different generator on each retry call. For example, you can wrap a
+ network call in a function that modifies the request based on what has
+ already been returned:
+
+ .. code-block:: python
+
+ def attempt_with_modified_request(target, request, seen_items=[]):
+ # remove seen items from request on each attempt
+ new_request = modify_request(request, seen_items)
+ new_generator = target(new_request)
+ for item in new_generator:
+ yield item
+ seen_items.append(item)
+
+ retry_wrapped_fn = StreamingRetry()(attempt_with_modified_request)
+ retryable_generator = retry_wrapped_fn(target, request)
+
+ 2. Wrap the retry generator
+ Alternatively, you can wrap the retryable generator itself before
+ passing it to the end-user to add a filter on the stream. For
+ example, you can keep track of the items that were successfully yielded
+ in previous retry attempts, and only yield new items when the
+ new attempt surpasses the previous ones:
+
+ .. code-block:: python
+
+ def retryable_with_filter(target):
+ stream_idx = 0
+ # reset stream_idx when the stream is retried
+ def on_error(e):
+ nonlocal stream_idx
+ stream_idx = 0
+ # build retryable
+ retryable_gen = StreamingRetry(...)(target)
+ # keep track of what has been yielded out of filter
+ seen_items = []
+ for item in retryable_gen():
+ if stream_idx >= len(seen_items):
+ seen_items.append(item)
+ yield item
+ elif item != seen_items[stream_idx]:
+ raise ValueError("Stream differs from last attempt")
+ stream_idx += 1
+
+ filter_retry_wrapped = retryable_with_filter(target)
+
+ Args:
+ predicate (Callable[Exception]): A callable that should return ``True``
+ if the given exception is retryable.
+ initial (float): The minimum amount of time to delay in seconds. This
+ must be greater than 0.
+ maximum (float): The maximum amount of time to delay in seconds.
+ multiplier (float): The multiplier applied to the delay.
+ timeout (float): How long to keep retrying, in seconds.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error (Callable[Exception]): A function to call while processing
+ a retryable exception. Any error raised by this function will
+ *not* be caught.
+ deadline (float): DEPRECATED: use `timeout` instead. For backward
+ compatibility, if specified it will override the ``timeout`` parameter.
+ """
+
+ def __call__(
+ self,
+ func: Callable[_P, Iterable[_Y]],
+ on_error: Callable[[Exception], Any] | None = None,
+ ) -> Callable[_P, Generator[_Y, Any, None]]:
+ """Wrap a callable with retry behavior.
+
+ Args:
+ func (Callable): The callable to add retry behavior to.
+ on_error (Optional[Callable[Exception]]): If given, the
+ on_error callback will be called with each retryable exception
+ raised by the wrapped function. Any error raised by this
+ function will *not* be caught. If on_error was specified in the
+ constructor, this value will be ignored.
+
+ Returns:
+ Callable: A callable that will invoke ``func`` with retry
+ behavior.
+ """
+ if self._on_error is not None:
+ on_error = self._on_error
+
+ @functools.wraps(func)
+ def retry_wrapped_func(
+ *args: _P.args, **kwargs: _P.kwargs
+ ) -> Generator[_Y, Any, None]:
+ """A wrapper that calls target function with retry."""
+ sleep_generator = exponential_sleep_generator(
+ self._initial, self._maximum, multiplier=self._multiplier
+ )
+ return retry_target_stream(
+ func,
+ predicate=self._predicate,
+ sleep_generator=sleep_generator,
+ timeout=self._timeout,
+ on_error=on_error,
+ init_args=args,
+ init_kwargs=kwargs,
+ )
+
+ return retry_wrapped_func
diff --git a/google/api_core/retry/retry_streaming_async.py b/google/api_core/retry/retry_streaming_async.py
new file mode 100644
index 00000000..5e5fa240
--- /dev/null
+++ b/google/api_core/retry/retry_streaming_async.py
@@ -0,0 +1,328 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Generator wrapper for retryable async streaming RPCs.
+"""
+from __future__ import annotations
+
+from typing import (
+ cast,
+ Any,
+ Callable,
+ Iterable,
+ AsyncIterator,
+ AsyncIterable,
+ Awaitable,
+ TypeVar,
+ AsyncGenerator,
+ TYPE_CHECKING,
+)
+
+import asyncio
+import time
+import sys
+import functools
+
+from google.api_core.retry.retry_base import _BaseRetry
+from google.api_core.retry.retry_base import _retry_error_helper
+from google.api_core.retry import exponential_sleep_generator
+from google.api_core.retry import build_retry_error
+from google.api_core.retry import RetryFailureReason
+
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 10):
+ from typing import ParamSpec
+ else:
+ from typing_extensions import ParamSpec
+
+ _P = ParamSpec("_P") # target function call parameters
+ _Y = TypeVar("_Y") # yielded values
+
+
+async def retry_target_stream(
+ target: Callable[_P, AsyncIterable[_Y] | Awaitable[AsyncIterable[_Y]]],
+ predicate: Callable[[Exception], bool],
+ sleep_generator: Iterable[float],
+ timeout: float | None = None,
+ on_error: Callable[[Exception], None] | None = None,
+ exception_factory: Callable[
+ [list[Exception], RetryFailureReason, float | None],
+ tuple[Exception, Exception | None],
+ ] = build_retry_error,
+ init_args: tuple = (),
+ init_kwargs: dict = {},
+ **kwargs,
+) -> AsyncGenerator[_Y, None]:
+ """Create a generator wrapper that retries the wrapped stream if it fails.
+
+ This is the lowest-level retry helper. Generally, you'll use the
+ higher-level retry helper :class:`AsyncRetry`.
+
+ Args:
+ target: The generator function to call and retry.
+ predicate: A callable used to determine if an
+ exception raised by the target should be considered retryable.
+ It should return True to retry or False otherwise.
+ sleep_generator: An infinite iterator that determines
+ how long to sleep between retries.
+ timeout: How long to keep retrying the target.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error: If given, the on_error callback will be called with each
+ retryable exception raised by the target. Any error raised by this
+ function will *not* be caught.
+ exception_factory: A function that is called when the retryable reaches
+ a terminal failure state, used to construct an exception to be raised.
+ It takes a list of all exceptions encountered, a retry.RetryFailureReason
+ enum indicating the failure cause, and the original timeout value
+ as arguments. It should return a tuple of the exception to be raised,
+ along with the cause exception if any. The default implementation will raise
+ a RetryError on timeout, or the last exception encountered otherwise.
+ init_args: Positional arguments to pass to the target function.
+ init_kwargs: Keyword arguments to pass to the target function.
+
+ Returns:
+ AsyncGenerator: A retryable generator that wraps the target generator function.
+
+ Raises:
+ ValueError: If the sleep generator stops yielding values.
+ Exception: a custom exception specified by the exception_factory if provided.
+ If no exception_factory is provided:
+ google.api_core.RetryError: If the timeout is exceeded while retrying.
+ Exception: If the target raises an error that isn't retryable.
+ """
+ target_iterator: AsyncIterator[_Y] | None = None
+ timeout = kwargs.get("deadline", timeout)
+ deadline = time.monotonic() + timeout if timeout else None
+ # keep track of retryable exceptions we encounter to pass in to exception_factory
+ error_list: list[Exception] = []
+ sleep_iter = iter(sleep_generator)
+ target_is_generator: bool | None = None
+
+ # continue trying until an attempt completes, or a terminal exception is raised in _retry_error_helper
+ # TODO: support max_attempts argument: https://github.com/googleapis/python-api-core/issues/535
+ while True:
+ # Start a new retry loop
+ try:
+ # Note: in the future, we can add a ResumptionStrategy object
+ # to generate new args between calls. For now, use the same args
+ # for each attempt.
+ target_output: AsyncIterable[_Y] | Awaitable[AsyncIterable[_Y]] = target(
+ *init_args, **init_kwargs
+ )
+ try:
+ # gapic functions return the generator behind an awaitable
+ # unwrap the awaitable so we can work with the generator directly
+ target_output = await target_output # type: ignore
+ except TypeError:
+ # was not awaitable, continue
+ pass
+ target_iterator = cast(AsyncIterable["_Y"], target_output).__aiter__()
+
+ if target_is_generator is None:
+ # Check if target supports generator features (asend, athrow, aclose)
+ target_is_generator = bool(getattr(target_iterator, "asend", None))
+
+ sent_in = None
+ while True:
+ ## Read from target_iterator
+ # If the target is a generator, we will advance it with `asend`
+ # otherwise, we will use `anext`
+ if target_is_generator:
+ next_value = await target_iterator.asend(sent_in) # type: ignore
+ else:
+ next_value = await target_iterator.__anext__()
+ ## Yield from Wrapper to caller
+ try:
+ # yield latest value from target
+ # exceptions from `athrow` and `aclose` are injected here
+ sent_in = yield next_value
+ except GeneratorExit:
+ # if wrapper received `aclose` while waiting on yield,
+ # it will raise GeneratorExit here
+ if target_is_generator:
+ # pass to inner target_iterator for handling
+ await cast(AsyncGenerator["_Y", None], target_iterator).aclose()
+ else:
+ raise
+ return
+ except: # noqa: E722
+ # bare except catches any exception passed to `athrow`
+ if target_is_generator:
+ # delegate error handling to target_iterator
+ await cast(AsyncGenerator["_Y", None], target_iterator).athrow(
+ cast(BaseException, sys.exc_info()[1])
+ )
+ else:
+ raise
+ return
+ except StopAsyncIteration:
+ # if iterator exhausted, return
+ return
+ # handle exceptions raised by the target_iterator
+ # pylint: disable=broad-except
+ # This function explicitly must deal with broad exceptions.
+ except Exception as exc:
+ # defer to shared logic for handling errors
+ next_sleep = _retry_error_helper(
+ exc,
+ deadline,
+ sleep_iter,
+ error_list,
+ predicate,
+ on_error,
+ exception_factory,
+ timeout,
+ )
+ # if exception not raised, sleep before next attempt
+ await asyncio.sleep(next_sleep)
+
+ finally:
+ if target_is_generator and target_iterator is not None:
+ await cast(AsyncGenerator["_Y", None], target_iterator).aclose()
+
+
+class AsyncStreamingRetry(_BaseRetry):
+ """Exponential retry decorator for async streaming rpcs.
+
+ This class returns an AsyncGenerator when called, which wraps the target
+ stream in retry logic. If any exception is raised by the target, the
+ entire stream will be retried within the wrapper.
+
+ Although the default behavior is to retry transient API errors, a
+ different predicate can be provided to retry other exceptions.
+
+ Important Note: when a stream is encounters a retryable error, it will
+ silently construct a fresh iterator instance in the background
+ and continue yielding (likely duplicate) values as if no error occurred.
+ This is the most general way to retry a stream, but it often is not the
+ desired behavior. Example: iter([1, 2, 1/0]) -> [1, 2, 1, 2, ...]
+
+ There are two ways to build more advanced retry logic for streams:
+
+ 1. Wrap the target
+ Use a ``target`` that maintains state between retries, and creates a
+ different generator on each retry call. For example, you can wrap a
+ grpc call in a function that modifies the request based on what has
+ already been returned:
+
+ .. code-block:: python
+
+ async def attempt_with_modified_request(target, request, seen_items=[]):
+ # remove seen items from request on each attempt
+ new_request = modify_request(request, seen_items)
+ new_generator = await target(new_request)
+ async for item in new_generator:
+ yield item
+ seen_items.append(item)
+
+ retry_wrapped = AsyncRetry(is_stream=True,...)(attempt_with_modified_request, target, request, [])
+
+ 2. Wrap the retry generator
+ Alternatively, you can wrap the retryable generator itself before
+ passing it to the end-user to add a filter on the stream. For
+ example, you can keep track of the items that were successfully yielded
+ in previous retry attempts, and only yield new items when the
+ new attempt surpasses the previous ones:
+
+ .. code-block:: python
+
+ async def retryable_with_filter(target):
+ stream_idx = 0
+ # reset stream_idx when the stream is retried
+ def on_error(e):
+ nonlocal stream_idx
+ stream_idx = 0
+ # build retryable
+ retryable_gen = AsyncRetry(is_stream=True, ...)(target)
+ # keep track of what has been yielded out of filter
+ seen_items = []
+ async for item in retryable_gen:
+ if stream_idx >= len(seen_items):
+ yield item
+ seen_items.append(item)
+ elif item != previous_stream[stream_idx]:
+ raise ValueError("Stream differs from last attempt")"
+ stream_idx += 1
+
+ filter_retry_wrapped = retryable_with_filter(target)
+
+ Args:
+ predicate (Callable[Exception]): A callable that should return ``True``
+ if the given exception is retryable.
+ initial (float): The minimum amount of time to delay in seconds. This
+ must be greater than 0.
+ maximum (float): The maximum amount of time to delay in seconds.
+ multiplier (float): The multiplier applied to the delay.
+ timeout (Optional[float]): How long to keep retrying in seconds.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error (Optional[Callable[Exception]]): A function to call while processing
+ a retryable exception. Any error raised by this function will
+ *not* be caught.
+ is_stream (bool): Indicates whether the input function
+ should be treated as a stream function (i.e. an AsyncGenerator,
+ or function or coroutine that returns an AsyncIterable).
+ If True, the iterable will be wrapped with retry logic, and any
+ failed outputs will restart the stream. If False, only the input
+ function call itself will be retried. Defaults to False.
+ To avoid duplicate values, retryable streams should typically be
+ wrapped in additional filter logic before use.
+ deadline (float): DEPRECATED use ``timeout`` instead. If set it will
+ override ``timeout`` parameter.
+ """
+
+ def __call__(
+ self,
+ func: Callable[..., AsyncIterable[_Y] | Awaitable[AsyncIterable[_Y]]],
+ on_error: Callable[[Exception], Any] | None = None,
+ ) -> Callable[_P, Awaitable[AsyncGenerator[_Y, None]]]:
+ """Wrap a callable with retry behavior.
+
+ Args:
+ func (Callable): The callable or stream to add retry behavior to.
+ on_error (Optional[Callable[Exception]]): If given, the
+ on_error callback will be called with each retryable exception
+ raised by the wrapped function. Any error raised by this
+ function will *not* be caught. If on_error was specified in the
+ constructor, this value will be ignored.
+
+ Returns:
+ Callable: A callable that will invoke ``func`` with retry
+ behavior.
+ """
+ if self._on_error is not None:
+ on_error = self._on_error
+
+ @functools.wraps(func)
+ async def retry_wrapped_func(
+ *args: _P.args, **kwargs: _P.kwargs
+ ) -> AsyncGenerator[_Y, None]:
+ """A wrapper that calls target function with retry."""
+ sleep_generator = exponential_sleep_generator(
+ self._initial, self._maximum, multiplier=self._multiplier
+ )
+ return retry_target_stream(
+ func,
+ self._predicate,
+ sleep_generator,
+ self._timeout,
+ on_error,
+ init_args=args,
+ init_kwargs=kwargs,
+ )
+
+ return retry_wrapped_func
diff --git a/google/api_core/retry/retry_unary.py b/google/api_core/retry/retry_unary.py
new file mode 100644
index 00000000..6d36bc7d
--- /dev/null
+++ b/google/api_core/retry/retry_unary.py
@@ -0,0 +1,302 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for retrying functions with exponential back-off.
+
+The :class:`Retry` decorator can be used to retry functions that raise
+exceptions using exponential backoff. Because a exponential sleep algorithm is
+used, the retry is limited by a `timeout`. The timeout determines the window
+in which retries will be attempted. This is used instead of total number of retries
+because it is difficult to ascertain the amount of time a function can block
+when using total number of retries and exponential backoff.
+
+By default, this decorator will retry transient
+API errors (see :func:`if_transient_error`). For example:
+
+.. code-block:: python
+
+ @retry.Retry()
+ def call_flaky_rpc():
+ return client.flaky_rpc()
+
+ # Will retry flaky_rpc() if it raises transient API errors.
+ result = call_flaky_rpc()
+
+You can pass a custom predicate to retry on different exceptions, such as
+waiting for an eventually consistent item to be available:
+
+.. code-block:: python
+
+ @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
+ def check_if_exists():
+ return client.does_thing_exist()
+
+ is_available = check_if_exists()
+
+Some client library methods apply retry automatically. These methods can accept
+a ``retry`` parameter that allows you to configure the behavior:
+
+.. code-block:: python
+
+ my_retry = retry.Retry(timeout=60)
+ result = client.some_method(retry=my_retry)
+
+"""
+
+from __future__ import annotations
+
+import functools
+import sys
+import time
+import inspect
+import warnings
+from typing import Any, Callable, Iterable, TypeVar, TYPE_CHECKING
+
+from google.api_core.retry.retry_base import _BaseRetry
+from google.api_core.retry.retry_base import _retry_error_helper
+from google.api_core.retry.retry_base import exponential_sleep_generator
+from google.api_core.retry.retry_base import build_retry_error
+from google.api_core.retry.retry_base import RetryFailureReason
+
+
+if TYPE_CHECKING:
+ if sys.version_info >= (3, 10):
+ from typing import ParamSpec
+ else:
+ from typing_extensions import ParamSpec
+
+ _P = ParamSpec("_P") # target function call parameters
+ _R = TypeVar("_R") # target function returned value
+
+_ASYNC_RETRY_WARNING = "Using the synchronous google.api_core.retry.Retry with asynchronous calls may lead to unexpected results. Please use google.api_core.retry_async.AsyncRetry instead."
+
+
+def retry_target(
+ target: Callable[[], _R],
+ predicate: Callable[[Exception], bool],
+ sleep_generator: Iterable[float],
+ timeout: float | None = None,
+ on_error: Callable[[Exception], None] | None = None,
+ exception_factory: Callable[
+ [list[Exception], RetryFailureReason, float | None],
+ tuple[Exception, Exception | None],
+ ] = build_retry_error,
+ **kwargs,
+):
+ """Call a function and retry if it fails.
+
+ This is the lowest-level retry helper. Generally, you'll use the
+ higher-level retry helper :class:`Retry`.
+
+ Args:
+ target(Callable): The function to call and retry. This must be a
+ nullary function - apply arguments with `functools.partial`.
+ predicate (Callable[Exception]): A callable used to determine if an
+ exception raised by the target should be considered retryable.
+ It should return True to retry or False otherwise.
+ sleep_generator (Iterable[float]): An infinite iterator that determines
+ how long to sleep between retries.
+ timeout (Optional[float]): How long to keep retrying the target.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error (Optional[Callable[Exception]]): If given, the on_error
+ callback will be called with each retryable exception raised by the
+ target. Any error raised by this function will *not* be caught.
+ exception_factory: A function that is called when the retryable reaches
+ a terminal failure state, used to construct an exception to be raised.
+ It takes a list of all exceptions encountered, a retry.RetryFailureReason
+ enum indicating the failure cause, and the original timeout value
+ as arguments. It should return a tuple of the exception to be raised,
+ along with the cause exception if any. The default implementation will raise
+ a RetryError on timeout, or the last exception encountered otherwise.
+ deadline (float): DEPRECATED: use ``timeout`` instead. For backward
+ compatibility, if specified it will override ``timeout`` parameter.
+
+ Returns:
+ Any: the return value of the target function.
+
+ Raises:
+ ValueError: If the sleep generator stops yielding values.
+ Exception: a custom exception specified by the exception_factory if provided.
+ If no exception_factory is provided:
+ google.api_core.RetryError: If the timeout is exceeded while retrying.
+ Exception: If the target raises an error that isn't retryable.
+ """
+
+ timeout = kwargs.get("deadline", timeout)
+
+ deadline = time.monotonic() + timeout if timeout is not None else None
+ error_list: list[Exception] = []
+ sleep_iter = iter(sleep_generator)
+
+ # continue trying until an attempt completes, or a terminal exception is raised in _retry_error_helper
+ # TODO: support max_attempts argument: https://github.com/googleapis/python-api-core/issues/535
+ while True:
+ try:
+ result = target()
+ if inspect.isawaitable(result):
+ warnings.warn(_ASYNC_RETRY_WARNING)
+ return result
+
+ # pylint: disable=broad-except
+ # This function explicitly must deal with broad exceptions.
+ except Exception as exc:
+ # defer to shared logic for handling errors
+ next_sleep = _retry_error_helper(
+ exc,
+ deadline,
+ sleep_iter,
+ error_list,
+ predicate,
+ on_error,
+ exception_factory,
+ timeout,
+ )
+ # if exception not raised, sleep before next attempt
+ time.sleep(next_sleep)
+
+
+class Retry(_BaseRetry):
+ """Exponential retry decorator for unary synchronous RPCs.
+
+ This class is a decorator used to add retry or polling behavior to an RPC
+ call.
+
+ Although the default behavior is to retry transient API errors, a
+ different predicate can be provided to retry other exceptions.
+
+ There are two important concepts that retry/polling behavior may operate on,
+ Deadline and Timeout, which need to be properly defined for the correct
+ usage of this class and the rest of the library.
+
+ Deadline: a fixed point in time by which a certain operation must
+ terminate. For example, if a certain operation has a deadline
+ "2022-10-18T23:30:52.123Z" it must terminate (successfully or with an
+ error) by that time, regardless of when it was started or whether it
+ was started at all.
+
+ Timeout: the maximum duration of time after which a certain operation
+ must terminate (successfully or with an error). The countdown begins right
+ after an operation was started. For example, if an operation was started at
+ 09:24:00 with timeout of 75 seconds, it must terminate no later than
+ 09:25:15.
+
+ Unfortunately, in the past this class (and the api-core library as a whole) has not
+ been properly distinguishing the concepts of "timeout" and "deadline", and the
+ ``deadline`` parameter has meant ``timeout``. That is why
+ ``deadline`` has been deprecated and ``timeout`` should be used instead. If the
+ ``deadline`` parameter is set, it will override the ``timeout`` parameter.
+ In other words, ``retry.deadline`` should be treated as just a deprecated alias for
+ ``retry.timeout``.
+
+ Said another way, it is safe to assume that this class and the rest of this
+ library operate in terms of timeouts (not deadlines) unless explicitly
+ noted the usage of deadline semantics.
+
+ It is also important to
+ understand the three most common applications of the Timeout concept in the
+ context of this library.
+
+ Usually the generic Timeout term may stand for one of the following actual
+ timeouts: RPC Timeout, Retry Timeout, or Polling Timeout.
+
+ RPC Timeout: a value supplied by the client to the server so
+ that the server side knows the maximum amount of time it is expected to
+ spend handling that specific RPC. For example, in the case of gRPC transport,
+ RPC Timeout is represented by setting "grpc-timeout" header in the HTTP2
+ request. The `timeout` property of this class normally never represents the
+ RPC Timeout as it is handled separately by the ``google.api_core.timeout``
+ module of this library.
+
+ Retry Timeout: this is the most common meaning of the ``timeout`` property
+ of this class, and defines how long a certain RPC may be retried in case
+ the server returns an error.
+
+ Polling Timeout: defines how long the
+ client side is allowed to call the polling RPC repeatedly to check a status of a
+ long-running operation. Each polling RPC is
+ expected to succeed (its errors are supposed to be handled by the retry
+ logic). The decision as to whether a new polling attempt needs to be made is based
+ not on the RPC status code but on the status of the returned
+ status of an operation. In other words: we will poll a long-running operation until
+ the operation is done or the polling timeout expires. Each poll will inform us of
+ the status of the operation. The poll consists of an RPC to the server that may
+ itself be retried as per the poll-specific retry settings in case of errors. The
+ operation-level retry settings do NOT apply to polling-RPC retries.
+
+ With the actual timeout types being defined above, the client libraries
+ often refer to just Timeout without clarifying which type specifically
+ that is. In that case the actual timeout type (sometimes also referred to as
+ Logical Timeout) can be determined from the context. If it is a unary rpc
+ call (i.e. a regular one) Timeout usually stands for the RPC Timeout (if
+ provided directly as a standalone value) or Retry Timeout (if provided as
+ ``retry.timeout`` property of the unary RPC's retry config). For
+ ``Operation`` or ``PollingFuture`` in general Timeout stands for
+ Polling Timeout.
+
+ Args:
+ predicate (Callable[Exception]): A callable that should return ``True``
+ if the given exception is retryable.
+ initial (float): The minimum amount of time to delay in seconds. This
+ must be greater than 0.
+ maximum (float): The maximum amount of time to delay in seconds.
+ multiplier (float): The multiplier applied to the delay.
+ timeout (Optional[float]): How long to keep retrying, in seconds.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error (Callable[Exception]): A function to call while processing
+ a retryable exception. Any error raised by this function will
+ *not* be caught.
+ deadline (float): DEPRECATED: use `timeout` instead. For backward
+ compatibility, if specified it will override the ``timeout`` parameter.
+ """
+
+ def __call__(
+ self,
+ func: Callable[_P, _R],
+ on_error: Callable[[Exception], Any] | None = None,
+ ) -> Callable[_P, _R]:
+ """Wrap a callable with retry behavior.
+
+ Args:
+ func (Callable): The callable to add retry behavior to.
+ on_error (Optional[Callable[Exception]]): If given, the
+ on_error callback will be called with each retryable exception
+ raised by the wrapped function. Any error raised by this
+ function will *not* be caught. If on_error was specified in the
+ constructor, this value will be ignored.
+
+ Returns:
+ Callable: A callable that will invoke ``func`` with retry
+ behavior.
+ """
+ if self._on_error is not None:
+ on_error = self._on_error
+
+ @functools.wraps(func)
+ def retry_wrapped_func(*args: _P.args, **kwargs: _P.kwargs) -> _R:
+ """A wrapper that calls target function with retry."""
+ target = functools.partial(func, *args, **kwargs)
+ sleep_generator = exponential_sleep_generator(
+ self._initial, self._maximum, multiplier=self._multiplier
+ )
+ return retry_target(
+ target,
+ self._predicate,
+ sleep_generator,
+ timeout=self._timeout,
+ on_error=on_error,
+ )
+
+ return retry_wrapped_func
diff --git a/google/api_core/retry/retry_unary_async.py b/google/api_core/retry/retry_unary_async.py
new file mode 100644
index 00000000..1f72476a
--- /dev/null
+++ b/google/api_core/retry/retry_unary_async.py
@@ -0,0 +1,239 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for retrying coroutine functions with exponential back-off.
+
+The :class:`AsyncRetry` decorator shares most functionality and behavior with
+:class:`Retry`, but supports coroutine functions. Please refer to description
+of :class:`Retry` for more details.
+
+By default, this decorator will retry transient
+API errors (see :func:`if_transient_error`). For example:
+
+.. code-block:: python
+
+ @retry_async.AsyncRetry()
+ async def call_flaky_rpc():
+ return await client.flaky_rpc()
+
+ # Will retry flaky_rpc() if it raises transient API errors.
+ result = await call_flaky_rpc()
+
+You can pass a custom predicate to retry on different exceptions, such as
+waiting for an eventually consistent item to be available:
+
+.. code-block:: python
+
+ @retry_async.AsyncRetry(predicate=retry_async.if_exception_type(exceptions.NotFound))
+ async def check_if_exists():
+ return await client.does_thing_exist()
+
+ is_available = await check_if_exists()
+
+Some client library methods apply retry automatically. These methods can accept
+a ``retry`` parameter that allows you to configure the behavior:
+
+.. code-block:: python
+
+ my_retry = retry_async.AsyncRetry(timeout=60)
+ result = await client.some_method(retry=my_retry)
+
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+import functools
+from typing import (
+ Awaitable,
+ Any,
+ Callable,
+ Iterable,
+ TypeVar,
+ TYPE_CHECKING,
+)
+
+from google.api_core.retry.retry_base import _BaseRetry
+from google.api_core.retry.retry_base import _retry_error_helper
+from google.api_core.retry.retry_base import exponential_sleep_generator
+from google.api_core.retry.retry_base import build_retry_error
+from google.api_core.retry.retry_base import RetryFailureReason
+
+# for backwards compatibility, expose helpers in this module
+from google.api_core.retry.retry_base import if_exception_type # noqa
+from google.api_core.retry.retry_base import if_transient_error # noqa
+
+if TYPE_CHECKING:
+ import sys
+
+ if sys.version_info >= (3, 10):
+ from typing import ParamSpec
+ else:
+ from typing_extensions import ParamSpec
+
+ _P = ParamSpec("_P") # target function call parameters
+ _R = TypeVar("_R") # target function returned value
+
+_DEFAULT_INITIAL_DELAY = 1.0 # seconds
+_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
+_DEFAULT_DELAY_MULTIPLIER = 2.0
+_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
+_DEFAULT_TIMEOUT = 60.0 * 2.0 # seconds
+
+
+async def retry_target(
+ target: Callable[[], Awaitable[_R]],
+ predicate: Callable[[Exception], bool],
+ sleep_generator: Iterable[float],
+ timeout: float | None = None,
+ on_error: Callable[[Exception], None] | None = None,
+ exception_factory: Callable[
+ [list[Exception], RetryFailureReason, float | None],
+ tuple[Exception, Exception | None],
+ ] = build_retry_error,
+ **kwargs,
+):
+ """Await a coroutine and retry if it fails.
+
+ This is the lowest-level retry helper. Generally, you'll use the
+ higher-level retry helper :class:`Retry`.
+
+ Args:
+ target(Callable[[], Any]): The function to call and retry. This must be a
+ nullary function - apply arguments with `functools.partial`.
+ predicate (Callable[Exception]): A callable used to determine if an
+ exception raised by the target should be considered retryable.
+ It should return True to retry or False otherwise.
+ sleep_generator (Iterable[float]): An infinite iterator that determines
+ how long to sleep between retries.
+ timeout (Optional[float]): How long to keep retrying the target, in seconds.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error (Optional[Callable[Exception]]): If given, the on_error
+ callback will be called with each retryable exception raised by the
+ target. Any error raised by this function will *not* be caught.
+ exception_factory: A function that is called when the retryable reaches
+ a terminal failure state, used to construct an exception to be raised.
+ It takes a list of all exceptions encountered, a retry.RetryFailureReason
+ enum indicating the failure cause, and the original timeout value
+ as arguments. It should return a tuple of the exception to be raised,
+ along with the cause exception if any. The default implementation will raise
+ a RetryError on timeout, or the last exception encountered otherwise.
+ deadline (float): DEPRECATED use ``timeout`` instead. For backward
+ compatibility, if set it will override the ``timeout`` parameter.
+
+ Returns:
+ Any: the return value of the target function.
+
+ Raises:
+ ValueError: If the sleep generator stops yielding values.
+ Exception: a custom exception specified by the exception_factory if provided.
+ If no exception_factory is provided:
+ google.api_core.RetryError: If the timeout is exceeded while retrying.
+ Exception: If the target raises an error that isn't retryable.
+ """
+
+ timeout = kwargs.get("deadline", timeout)
+
+ deadline = time.monotonic() + timeout if timeout is not None else None
+ error_list: list[Exception] = []
+ sleep_iter = iter(sleep_generator)
+
+ # continue trying until an attempt completes, or a terminal exception is raised in _retry_error_helper
+ # TODO: support max_attempts argument: https://github.com/googleapis/python-api-core/issues/535
+ while True:
+ try:
+ return await target()
+ # pylint: disable=broad-except
+ # This function explicitly must deal with broad exceptions.
+ except Exception as exc:
+ # defer to shared logic for handling errors
+ next_sleep = _retry_error_helper(
+ exc,
+ deadline,
+ sleep_iter,
+ error_list,
+ predicate,
+ on_error,
+ exception_factory,
+ timeout,
+ )
+ # if exception not raised, sleep before next attempt
+ await asyncio.sleep(next_sleep)
+
+
+class AsyncRetry(_BaseRetry):
+ """Exponential retry decorator for async coroutines.
+
+ This class is a decorator used to add exponential back-off retry behavior
+ to an RPC call.
+
+ Although the default behavior is to retry transient API errors, a
+ different predicate can be provided to retry other exceptions.
+
+ Args:
+ predicate (Callable[Exception]): A callable that should return ``True``
+ if the given exception is retryable.
+ initial (float): The minimum amount of time to delay in seconds. This
+ must be greater than 0.
+ maximum (float): The maximum amount of time to delay in seconds.
+ multiplier (float): The multiplier applied to the delay.
+ timeout (Optional[float]): How long to keep retrying in seconds.
+ Note: timeout is only checked before initiating a retry, so the target may
+ run past the timeout value as long as it is healthy.
+ on_error (Optional[Callable[Exception]]): A function to call while processing
+ a retryable exception. Any error raised by this function will
+ *not* be caught.
+ deadline (float): DEPRECATED use ``timeout`` instead. If set it will
+ override ``timeout`` parameter.
+ """
+
+ def __call__(
+ self,
+ func: Callable[..., Awaitable[_R]],
+ on_error: Callable[[Exception], Any] | None = None,
+ ) -> Callable[_P, Awaitable[_R]]:
+ """Wrap a callable with retry behavior.
+
+ Args:
+ func (Callable): The callable or stream to add retry behavior to.
+ on_error (Optional[Callable[Exception]]): If given, the
+ on_error callback will be called with each retryable exception
+ raised by the wrapped function. Any error raised by this
+ function will *not* be caught. If on_error was specified in the
+ constructor, this value will be ignored.
+
+ Returns:
+ Callable: A callable that will invoke ``func`` with retry
+ behavior.
+ """
+ if self._on_error is not None:
+ on_error = self._on_error
+
+ @functools.wraps(func)
+ async def retry_wrapped_func(*args: _P.args, **kwargs: _P.kwargs) -> _R:
+ """A wrapper that calls target function with retry."""
+ sleep_generator = exponential_sleep_generator(
+ self._initial, self._maximum, multiplier=self._multiplier
+ )
+ return await retry_target(
+ functools.partial(func, *args, **kwargs),
+ predicate=self._predicate,
+ sleep_generator=sleep_generator,
+ timeout=self._timeout,
+ on_error=on_error,
+ )
+
+ return retry_wrapped_func
diff --git a/google/api_core/retry_async.py b/google/api_core/retry_async.py
index 739e88d9..90a2d5ad 100644
--- a/google/api_core/retry_async.py
+++ b/google/api_core/retry_async.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,301 +11,24 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-"""Helpers for retrying coroutine functions with exponential back-off.
-
-The :class:`AsyncRetry` decorator shares most functionality and behavior with
-:class:`Retry`, but supports coroutine functions. Please refer to description
-of :class:`Retry` for more details.
-
-By default, this decorator will retry transient
-API errors (see :func:`if_transient_error`). For example:
-
-.. code-block:: python
-
- @retry_async.AsyncRetry()
- async def call_flaky_rpc():
- return await client.flaky_rpc()
-
- # Will retry flaky_rpc() if it raises transient API errors.
- result = await call_flaky_rpc()
-
-You can pass a custom predicate to retry on different exceptions, such as
-waiting for an eventually consistent item to be available:
-
-.. code-block:: python
-
- @retry_async.AsyncRetry(predicate=retry_async.if_exception_type(exceptions.NotFound))
- async def check_if_exists():
- return await client.does_thing_exist()
-
- is_available = await check_if_exists()
-
-Some client library methods apply retry automatically. These methods can accept
-a ``retry`` parameter that allows you to configure the behavior:
-
-.. code-block:: python
-
- my_retry = retry_async.AsyncRetry(deadline=60)
- result = await client.some_method(retry=my_retry)
-
-"""
-
-import asyncio
-import datetime
-import functools
-import logging
-
-from google.api_core import datetime_helpers
-from google.api_core import exceptions
-from google.api_core.retry import exponential_sleep_generator
+#
+# The following imports are for backwards compatibility with https://github.com/googleapis/python-api-core/blob/4d7d2edee2c108d43deb151e6e0fdceb56b73275/google/api_core/retry_async.py
+#
+# TODO: Revert these imports on the next major version release (https://github.com/googleapis/python-api-core/issues/576)
+from google.api_core import datetime_helpers # noqa: F401
+from google.api_core import exceptions # noqa: F401
+from google.api_core.retry import exponential_sleep_generator # noqa: F401
from google.api_core.retry import if_exception_type # noqa: F401
-from google.api_core.retry import if_transient_error
-
-
-_LOGGER = logging.getLogger(__name__)
-_DEFAULT_INITIAL_DELAY = 1.0 # seconds
-_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
-_DEFAULT_DELAY_MULTIPLIER = 2.0
-_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
-_DEFAULT_TIMEOUT = 60.0 * 2.0 # seconds
-
-
-async def retry_target(
- target, predicate, sleep_generator, timeout=None, on_error=None, **kwargs
-):
- """Call a function and retry if it fails.
-
- This is the lowest-level retry helper. Generally, you'll use the
- higher-level retry helper :class:`Retry`.
-
- Args:
- target(Callable): The function to call and retry. This must be a
- nullary function - apply arguments with `functools.partial`.
- predicate (Callable[Exception]): A callable used to determine if an
- exception raised by the target should be considered retryable.
- It should return True to retry or False otherwise.
- sleep_generator (Iterable[float]): An infinite iterator that determines
- how long to sleep between retries.
- timeout (float): How long to keep retrying the target, in seconds.
- on_error (Callable[Exception]): A function to call while processing a
- retryable exception. Any error raised by this function will *not*
- be caught.
- deadline (float): DEPRECATED use ``timeout`` instead. For backward
- compatibility, if set it will override the ``timeout`` parameter.
-
- Returns:
- Any: the return value of the target function.
-
- Raises:
- google.api_core.RetryError: If the deadline is exceeded while retrying.
- ValueError: If the sleep generator stops yielding values.
- Exception: If the target raises a method that isn't retryable.
- """
-
- timeout = kwargs.get("deadline", timeout)
-
- deadline_dt = (
- (datetime_helpers.utcnow() + datetime.timedelta(seconds=timeout))
- if timeout
- else None
- )
-
- last_exc = None
-
- for sleep in sleep_generator:
- try:
- if not deadline_dt:
- return await target()
- else:
- return await asyncio.wait_for(
- target(),
- timeout=(deadline_dt - datetime_helpers.utcnow()).total_seconds(),
- )
- # pylint: disable=broad-except
- # This function explicitly must deal with broad exceptions.
- except Exception as exc:
- if not predicate(exc) and not isinstance(exc, asyncio.TimeoutError):
- raise
- last_exc = exc
- if on_error is not None:
- on_error(exc)
-
- now = datetime_helpers.utcnow()
-
- if deadline_dt:
- if deadline_dt <= now:
- # Chains the raising RetryError with the root cause error,
- # which helps observability and debugability.
- raise exceptions.RetryError(
- "Timeout of {:.1f}s exceeded while calling target function".format(
- timeout
- ),
- last_exc,
- ) from last_exc
- else:
- time_to_deadline = (deadline_dt - now).total_seconds()
- sleep = min(time_to_deadline, sleep)
-
- _LOGGER.debug(
- "Retrying due to {}, sleeping {:.1f}s ...".format(last_exc, sleep)
- )
- await asyncio.sleep(sleep)
-
- raise ValueError("Sleep generator stopped yielding sleep values.")
-
-
-class AsyncRetry:
- """Exponential retry decorator for async functions.
-
- This class is a decorator used to add exponential back-off retry behavior
- to an RPC call.
-
- Although the default behavior is to retry transient API errors, a
- different predicate can be provided to retry other exceptions.
-
- Args:
- predicate (Callable[Exception]): A callable that should return ``True``
- if the given exception is retryable.
- initial (float): The minimum a,out of time to delay in seconds. This
- must be greater than 0.
- maximum (float): The maximum amount of time to delay in seconds.
- multiplier (float): The multiplier applied to the delay.
- timeout (float): How long to keep retrying in seconds.
- on_error (Callable[Exception]): A function to call while processing
- a retryable exception. Any error raised by this function will
- *not* be caught.
- deadline (float): DEPRECATED use ``timeout`` instead. If set it will
- override ``timeout`` parameter.
- """
-
- def __init__(
- self,
- predicate=if_transient_error,
- initial=_DEFAULT_INITIAL_DELAY,
- maximum=_DEFAULT_MAXIMUM_DELAY,
- multiplier=_DEFAULT_DELAY_MULTIPLIER,
- timeout=_DEFAULT_TIMEOUT,
- on_error=None,
- **kwargs
- ):
- self._predicate = predicate
- self._initial = initial
- self._multiplier = multiplier
- self._maximum = maximum
- self._timeout = kwargs.get("deadline", timeout)
- self._deadline = self._timeout
- self._on_error = on_error
-
- def __call__(self, func, on_error=None):
- """Wrap a callable with retry behavior.
-
- Args:
- func (Callable): The callable to add retry behavior to.
- on_error (Callable[Exception]): A function to call while processing
- a retryable exception. Any error raised by this function will
- *not* be caught.
-
- Returns:
- Callable: A callable that will invoke ``func`` with retry
- behavior.
- """
- if self._on_error is not None:
- on_error = self._on_error
-
- @functools.wraps(func)
- async def retry_wrapped_func(*args, **kwargs):
- """A wrapper that calls target function with retry."""
- target = functools.partial(func, *args, **kwargs)
- sleep_generator = exponential_sleep_generator(
- self._initial, self._maximum, multiplier=self._multiplier
- )
- return await retry_target(
- target,
- self._predicate,
- sleep_generator,
- self._timeout,
- on_error=on_error,
- )
-
- return retry_wrapped_func
-
- def _replace(
- self,
- predicate=None,
- initial=None,
- maximum=None,
- multiplier=None,
- timeout=None,
- on_error=None,
- ):
- return AsyncRetry(
- predicate=predicate or self._predicate,
- initial=initial or self._initial,
- maximum=maximum or self._maximum,
- multiplier=multiplier or self._multiplier,
- timeout=timeout or self._timeout,
- on_error=on_error or self._on_error,
- )
-
- def with_deadline(self, deadline):
- """Return a copy of this retry with the given deadline.
- DEPRECATED: use :meth:`with_timeout` instead.
-
- Args:
- deadline (float): How long to keep retrying.
-
- Returns:
- AsyncRetry: A new retry instance with the given deadline.
- """
- return self._replace(timeout=deadline)
-
- def with_timeout(self, timeout):
- """Return a copy of this retry with the given timeout.
-
- Args:
- timeout (float): How long to keep retrying, in seconds.
-
- Returns:
- AsyncRetry: A new retry instance with the given timeout.
- """
- return self._replace(timeout=timeout)
-
- def with_predicate(self, predicate):
- """Return a copy of this retry with the given predicate.
-
- Args:
- predicate (Callable[Exception]): A callable that should return
- ``True`` if the given exception is retryable.
-
- Returns:
- AsyncRetry: A new retry instance with the given predicate.
- """
- return self._replace(predicate=predicate)
-
- def with_delay(self, initial=None, maximum=None, multiplier=None):
- """Return a copy of this retry with the given delay options.
-
- Args:
- initial (float): The minimum amount of time to delay. This must
- be greater than 0.
- maximum (float): The maximum amount of time to delay.
- multiplier (float): The multiplier applied to the delay.
-
- Returns:
- AsyncRetry: A new retry instance with the given predicate.
- """
- return self._replace(initial=initial, maximum=maximum, multiplier=multiplier)
-
- def __str__(self):
- return (
- "".format(
- self._predicate,
- self._initial,
- self._maximum,
- self._multiplier,
- self._timeout,
- self._on_error,
- )
- )
+from google.api_core.retry import if_transient_error # noqa: F401
+from google.api_core.retry.retry_unary_async import AsyncRetry
+from google.api_core.retry.retry_unary_async import retry_target
+
+__all__ = (
+ "AsyncRetry",
+ "datetime_helpers",
+ "exceptions",
+ "exponential_sleep_generator",
+ "if_exception_type",
+ "if_transient_error",
+ "retry_target",
+)
diff --git a/google/api_core/timeout.py b/google/api_core/timeout.py
index 868e3e9f..55b195e9 100644
--- a/google/api_core/timeout.py
+++ b/google/api_core/timeout.py
@@ -102,8 +102,7 @@ def __call__(self, func):
def func_with_timeout(*args, **kwargs):
"""Wrapped function that adds timeout."""
- remaining_timeout = self._timeout
- if remaining_timeout is not None:
+ if self._timeout is not None:
# All calculations are in seconds
now_timestamp = self._clock().timestamp()
@@ -114,8 +113,19 @@ def func_with_timeout(*args, **kwargs):
now_timestamp = first_attempt_timestamp
time_since_first_attempt = now_timestamp - first_attempt_timestamp
- # Avoid setting negative timeout
- kwargs["timeout"] = max(0, self._timeout - time_since_first_attempt)
+ remaining_timeout = self._timeout - time_since_first_attempt
+
+ # Although the `deadline` parameter in `google.api_core.retry.Retry`
+ # is deprecated, and should be treated the same as the `timeout`,
+ # it is still possible for the `deadline` argument in
+ # `google.api_core.retry.Retry` to be larger than the `timeout`.
+ # See https://github.com/googleapis/python-api-core/issues/654
+ # Only positive non-zero timeouts are supported.
+ # Revert back to the initial timeout for negative or 0 timeout values.
+ if remaining_timeout < 1:
+ remaining_timeout = self._timeout
+
+ kwargs["timeout"] = remaining_timeout
return func(*args, **kwargs)
diff --git a/google/api_core/universe.py b/google/api_core/universe.py
new file mode 100644
index 00000000..35669642
--- /dev/null
+++ b/google/api_core/universe.py
@@ -0,0 +1,82 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for universe domain."""
+
+from typing import Any, Optional
+
+DEFAULT_UNIVERSE = "googleapis.com"
+
+
+class EmptyUniverseError(ValueError):
+ def __init__(self):
+ message = "Universe Domain cannot be an empty string."
+ super().__init__(message)
+
+
+class UniverseMismatchError(ValueError):
+ def __init__(self, client_universe, credentials_universe):
+ message = (
+ f"The configured universe domain ({client_universe}) does not match the universe domain "
+ f"found in the credentials ({credentials_universe}). "
+ "If you haven't configured the universe domain explicitly, "
+ f"`{DEFAULT_UNIVERSE}` is the default."
+ )
+ super().__init__(message)
+
+
+def determine_domain(
+ client_universe_domain: Optional[str], universe_domain_env: Optional[str]
+) -> str:
+ """Return the universe domain used by the client.
+
+ Args:
+ client_universe_domain (Optional[str]): The universe domain configured via the client options.
+ universe_domain_env (Optional[str]): The universe domain configured via the
+ "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable.
+
+ Returns:
+ str: The universe domain to be used by the client.
+
+ Raises:
+ ValueError: If the universe domain is an empty string.
+ """
+ universe_domain = DEFAULT_UNIVERSE
+ if client_universe_domain is not None:
+ universe_domain = client_universe_domain
+ elif universe_domain_env is not None:
+ universe_domain = universe_domain_env
+ if len(universe_domain.strip()) == 0:
+ raise EmptyUniverseError
+ return universe_domain
+
+
+def compare_domains(client_universe: str, credentials: Any) -> bool:
+ """Returns True iff the universe domains used by the client and credentials match.
+
+ Args:
+ client_universe (str): The universe domain configured via the client options.
+ credentials Any: The credentials being used in the client.
+
+ Returns:
+ bool: True iff client_universe matches the universe in credentials.
+
+ Raises:
+ ValueError: when client_universe does not match the universe in credentials.
+ """
+ credentials_universe = getattr(credentials, "universe_domain", DEFAULT_UNIVERSE)
+
+ if client_universe != credentials_universe:
+ raise UniverseMismatchError(client_universe, credentials_universe)
+ return True
diff --git a/google/api_core/version.py b/google/api_core/version.py
index e6e35743..f882cac3 100644
--- a/google/api_core/version.py
+++ b/google/api_core/version.py
@@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-__version__ = "2.11.0"
+__version__ = "2.25.0"
diff --git a/google/api_core/version_header.py b/google/api_core/version_header.py
new file mode 100644
index 00000000..cf1972ac
--- /dev/null
+++ b/google/api_core/version_header.py
@@ -0,0 +1,29 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+API_VERSION_METADATA_KEY = "x-goog-api-version"
+
+
+def to_api_version_header(version_identifier):
+ """Returns data for the API Version header for the given `version_identifier`.
+
+ Args:
+ version_identifier (str): The version identifier to be used in the
+ tuple returned.
+
+ Returns:
+ Tuple(str, str): A tuple containing the API Version metadata key and
+ value.
+ """
+ return (API_VERSION_METADATA_KEY, version_identifier)
diff --git a/mypy.ini b/mypy.ini
deleted file mode 100644
index ce33582a..00000000
--- a/mypy.ini
+++ /dev/null
@@ -1,4 +0,0 @@
-[mypy]
-python_version = 3.7
-namespace_packages = True
-ignore_missing_imports = True
diff --git a/noxfile.py b/noxfile.py
index 748485a7..ac21330e 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -15,7 +15,9 @@
from __future__ import absolute_import
import os
import pathlib
+import re
import shutil
+import unittest
# https://github.com/google/importlab/issues/25
import nox # pytype: disable=import-error
@@ -26,6 +28,8 @@
# Black and flake8 clash on the syntax for ignoring flake8's F401 in this file.
BLACK_EXCLUDES = ["--exclude", "^/google/api_core/operations_v1/__init__.py"]
+PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+
DEFAULT_PYTHON_VERSION = "3.10"
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
@@ -34,6 +38,8 @@
"unit",
"unit_grpc_gcp",
"unit_wo_grpc",
+ "unit_w_prerelease_deps",
+ "unit_w_async_rest_extra",
"cover",
"pytype",
"mypy",
@@ -43,15 +49,8 @@
"docs",
]
-
-def _greater_or_equal_than_37(version_string):
- tokens = version_string.split(".")
- for i, token in enumerate(tokens):
- try:
- tokens[i] = int(token)
- except ValueError:
- pass
- return tokens >= [3, 7]
+# Error if a python version is missing
+nox.options.error_on_missing_interpreters = True
@nox.session(python=DEFAULT_PYTHON_VERSION)
@@ -82,7 +81,37 @@ def blacken(session):
session.run("black", *BLACK_EXCLUDES, *BLACK_PATHS)
-def default(session, install_grpc=True):
+def install_prerelease_dependencies(session, constraints_path):
+ with open(constraints_path, encoding="utf-8") as constraints_file:
+ constraints_text = constraints_file.read()
+ # Ignore leading whitespace and comment lines.
+ constraints_deps = [
+ match.group(1)
+ for match in re.finditer(
+ r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
+ )
+ ]
+ session.install(*constraints_deps)
+ prerel_deps = [
+ "google-auth",
+ "googleapis-common-protos",
+ "grpcio",
+ "grpcio-status",
+ "proto-plus",
+ "protobuf",
+ ]
+
+ for dep in prerel_deps:
+ session.install("--pre", "--no-deps", "--upgrade", dep)
+
+ # Remaining dependencies
+ other_deps = [
+ "requests",
+ ]
+ session.install(*other_deps)
+
+
+def default(session, install_grpc=True, prerelease=False, install_async_rest=False):
"""Default unit test session.
This is intended to be run **without** an interpreter set, so
@@ -90,24 +119,66 @@ def default(session, install_grpc=True):
Python corresponding to the ``nox`` binary the ``PATH`` can
run the tests.
"""
- constraints_path = str(
- CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
- )
+ if prerelease and not install_grpc:
+ unittest.skip("The pre-release session cannot be run without grpc")
session.install(
"dataclasses",
- "mock",
- # Revert to just "pytest" once
- # https://github.com/pytest-dev/pytest/issues/10451 is fixed
- "pytest<7.2.0",
+ "mock; python_version=='3.7'",
+ "pytest",
"pytest-cov",
"pytest-xdist",
)
+ install_extras = []
if install_grpc:
- session.install("-e", ".[grpc]", "-c", constraints_path)
+ # Note: The extra is called `grpc` and not `grpcio`.
+ install_extras.append("grpc")
+
+ constraints_dir = str(CURRENT_DIRECTORY / "testing")
+ if install_async_rest:
+ install_extras.append("async_rest")
+ constraints_type = "async-rest-"
+ else:
+ constraints_type = ""
+
+ lib_with_extras = f".[{','.join(install_extras)}]" if len(install_extras) else "."
+ if prerelease:
+ install_prerelease_dependencies(
+ session,
+ f"{constraints_dir}/constraints-{constraints_type}{PYTHON_VERSIONS[0]}.txt",
+ )
+ # This *must* be the last install command to get the package from source.
+ session.install("-e", lib_with_extras, "--no-deps")
else:
- session.install("-e", ".", "-c", constraints_path)
+ constraints_file = (
+ f"{constraints_dir}/constraints-{constraints_type}{session.python}.txt"
+ )
+ # fall back to standard constraints file
+ if not pathlib.Path(constraints_file).exists():
+ constraints_file = f"{constraints_dir}/constraints-{session.python}.txt"
+
+ session.install(
+ "-e",
+ lib_with_extras,
+ "-c",
+ constraints_file,
+ )
+
+ # Print out package versions of dependencies
+ session.run(
+ "python", "-c", "import google.protobuf; print(google.protobuf.__version__)"
+ )
+ # Support for proto.version was added in v1.23.0
+ # https://github.com/googleapis/proto-plus-python/releases/tag/v1.23.0
+ session.run(
+ "python",
+ "-c",
+ """import proto; hasattr(proto, "version") and print(proto.version.__version__)""",
+ )
+ if install_grpc:
+ session.run("python", "-c", "import grpc; print(grpc.__version__)")
+ session.run("python", "-c", "import google.auth; print(google.auth.__version__)")
pytest_args = [
"python",
@@ -131,29 +202,37 @@ def default(session, install_grpc=True):
),
]
- # Inject AsyncIO content and proto-plus, if version >= 3.7.
- # proto-plus is needed for a field mask test in test_protobuf_helpers.py
- if _greater_or_equal_than_37(session.python):
- session.install("asyncmock", "pytest-asyncio", "proto-plus")
+ session.install("asyncmock", "pytest-asyncio")
- # Having positional arguments means the user wants to run specific tests.
- # Best not to add additional tests to that list.
- if not session.posargs:
- pytest_args.append("--cov=tests.asyncio")
- pytest_args.append(os.path.join("tests", "asyncio"))
+ # Having positional arguments means the user wants to run specific tests.
+ # Best not to add additional tests to that list.
+ if not session.posargs:
+ pytest_args.append("--cov=tests.asyncio")
+ pytest_args.append(os.path.join("tests", "asyncio"))
session.run(*pytest_args)
-@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11"])
+@nox.session(python=PYTHON_VERSIONS)
def unit(session):
"""Run the unit test suite."""
default(session)
-@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11"])
+@nox.session(python=PYTHON_VERSIONS)
+def unit_w_prerelease_deps(session):
+ """Run the unit test suite."""
+ default(session, prerelease=True)
+
+
+@nox.session(python=PYTHON_VERSIONS)
def unit_grpc_gcp(session):
- """Run the unit test suite with grpcio-gcp installed."""
+ """
+ Run the unit test suite with grpcio-gcp installed.
+ `grpcio-gcp` doesn't support protobuf 4+.
+ Remove extra `grpcgcp` when protobuf 3.x is dropped.
+ https://github.com/googleapis/python-api-core/issues/594
+ """
constraints_path = str(
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
)
@@ -165,12 +244,18 @@ def unit_grpc_gcp(session):
default(session)
-@nox.session(python=["3.8", "3.10", "3.11"])
+@nox.session(python=PYTHON_VERSIONS)
def unit_wo_grpc(session):
"""Run the unit test suite w/o grpcio installed"""
default(session, install_grpc=False)
+@nox.session(python=PYTHON_VERSIONS)
+def unit_w_async_rest_extra(session):
+ """Run the unit test suite with the `async_rest` extra"""
+ default(session, install_async_rest=True)
+
+
@nox.session(python=DEFAULT_PYTHON_VERSION)
def lint_setup_py(session):
"""Verify that setup.py is valid (including RST check)."""
@@ -179,23 +264,23 @@ def lint_setup_py(session):
session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
-@nox.session(python="3.8")
+@nox.session(python=DEFAULT_PYTHON_VERSION)
def pytype(session):
"""Run type-checking."""
- session.install(".[grpc]", "pytype >= 2019.3.21")
+ session.install(".[grpc]", "pytype")
session.run("pytype")
@nox.session(python=DEFAULT_PYTHON_VERSION)
def mypy(session):
"""Run type-checking."""
- session.install(".[grpc]", "mypy")
+ session.install(".[grpc,async_rest]", "mypy")
session.install(
"types-setuptools",
"types-requests",
"types-protobuf",
- "types-mock",
"types-dataclasses",
+ "types-mock; python_version=='3.7'",
)
session.run("mypy", "google", "tests")
@@ -212,12 +297,25 @@ def cover(session):
session.run("coverage", "erase")
-@nox.session(python="3.9")
+@nox.session(python="3.10")
def docs(session):
"""Build the docs for this library."""
session.install("-e", ".[grpc]")
- session.install("sphinx==4.2.0", "alabaster", "recommonmark")
+ session.install(
+ # We need to pin to specific versions of the `sphinxcontrib-*` packages
+ # which still support sphinx 4.x.
+ # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+ # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+ "sphinxcontrib-applehelp==1.0.4",
+ "sphinxcontrib-devhelp==1.0.2",
+ "sphinxcontrib-htmlhelp==2.0.1",
+ "sphinxcontrib-qthelp==1.0.3",
+ "sphinxcontrib-serializinghtml==1.1.5",
+ "sphinx==4.5.0",
+ "alabaster",
+ "recommonmark",
+ )
shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
session.run(
@@ -234,13 +332,24 @@ def docs(session):
)
-@nox.session(python="3.9")
+@nox.session(python="3.10")
def docfx(session):
"""Build the docfx yaml files for this library."""
session.install("-e", ".")
session.install(
- "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml"
+ # We need to pin to specific versions of the `sphinxcontrib-*` packages
+ # which still support sphinx 4.x.
+ # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+ # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+ "sphinxcontrib-applehelp==1.0.4",
+ "sphinxcontrib-devhelp==1.0.2",
+ "sphinxcontrib-htmlhelp==2.0.1",
+ "sphinxcontrib-qthelp==1.0.3",
+ "sphinxcontrib-serializinghtml==1.1.5",
+ "gcp-sphinx-docfx-yaml",
+ "alabaster",
+ "recommonmark",
)
shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
diff --git a/owlbot.py b/owlbot.py
index 5a83032e..58bc7517 100644
--- a/owlbot.py
+++ b/owlbot.py
@@ -26,28 +26,15 @@
excludes = [
"noxfile.py", # pytype
"setup.cfg", # pytype
- ".flake8", # flake8-import-order, layout
".coveragerc", # layout
"CONTRIBUTING.rst", # no systests
- ".github/workflows/unittest.yml", # exclude unittest gh action
- ".github/workflows/lint.yml", # exclude lint gh action
+ ".github/workflows/unittest.yml", # exclude unittest gh action
+ ".github/workflows/lint.yml", # exclude lint gh action
"README.rst",
]
templated_files = common.py_library(microgenerator=True, cov_level=100)
s.move(templated_files, excludes=excludes)
-# Add pytype support
-s.replace(
- ".gitignore",
- """\
-.pytest_cache
-""",
- """\
-.pytest_cache
-.pytype
-""",
-)
-
python.configure_previous_major_version_branches()
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..da404ab3
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,107 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "google-api-core"
+authors = [{ name = "Google LLC", email = "googleapis-packages@google.com" }]
+license = { text = "Apache 2.0" }
+requires-python = ">=3.7"
+readme = "README.rst"
+description = "Google API client core library"
+classifiers = [
+ # Should be one of:
+ # "Development Status :: 3 - Alpha"
+ # "Development Status :: 4 - Beta"
+ # "Development Status :: 5 - Production/Stable"
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: Apache Software License",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
+ "Operating System :: OS Independent",
+ "Topic :: Internet",
+]
+dependencies = [
+ "googleapis-common-protos >= 1.56.2, < 2.0.0",
+ "protobuf >= 3.19.5, < 7.0.0, != 3.20.0, != 3.20.1, != 4.21.0, != 4.21.1, != 4.21.2, != 4.21.3, != 4.21.4, != 4.21.5",
+ "proto-plus >= 1.22.3, < 2.0.0",
+ "proto-plus >= 1.25.0, < 2.0.0; python_version >= '3.13'",
+ "google-auth >= 2.14.1, < 3.0.0",
+ "requests >= 2.18.0, < 3.0.0",
+]
+dynamic = ["version"]
+
+[project.urls]
+Documentation = "https://googleapis.dev/python/google-api-core/latest/"
+Repository = "https://github.com/googleapis/python-api-core"
+
+[project.optional-dependencies]
+async_rest = ["google-auth[aiohttp] >= 2.35.0, < 3.0.0"]
+grpc = [
+ "grpcio >= 1.33.2, < 2.0.0",
+ "grpcio >= 1.49.1, < 2.0.0; python_version >= '3.11'",
+ "grpcio-status >= 1.33.2, < 2.0.0",
+ "grpcio-status >= 1.49.1, < 2.0.0; python_version >= '3.11'",
+]
+grpcgcp = ["grpcio-gcp >= 0.2.2, < 1.0.0"]
+grpcio-gcp = ["grpcio-gcp >= 0.2.2, < 1.0.0"]
+
+[tool.setuptools.dynamic]
+version = { attr = "google.api_core.version.__version__" }
+
+[tool.setuptools.packages.find]
+# Only include packages under the 'google' namespace. Do not include tests,
+# benchmarks, etc.
+include = ["google*"]
+
+[tool.mypy]
+python_version = "3.7"
+namespace_packages = true
+ignore_missing_imports = true
+
+[tool.pytest]
+filterwarnings = [
+ # treat all warnings as errors
+ "error",
+ # Remove once https://github.com/pytest-dev/pytest-cov/issues/621 is fixed
+ "ignore:.*The --rsyncdir command line argument and rsyncdirs config variable are deprecated:DeprecationWarning",
+ # Remove once https://github.com/protocolbuffers/protobuf/issues/12186 is fixed
+ "ignore:.*custom tp_new.*in Python 3.14:DeprecationWarning",
+ # Remove once support for python 3.7 is dropped
+ # This warning only appears when using python 3.7
+ "ignore:.*Using or importing the ABCs from.*collections:DeprecationWarning",
+ # Remove once support for grpcio-gcp is deprecated
+ # See https://github.com/googleapis/python-api-core/blob/42e8b6e6f426cab749b34906529e8aaf3f133d75/google/api_core/grpc_helpers.py#L39-L45
+ "ignore:.*Support for grpcio-gcp is deprecated:DeprecationWarning",
+ "ignore: The `compression` argument is ignored for grpc_gcp.secure_channel creation:DeprecationWarning",
+ "ignore:The `attempt_direct_path` argument is ignored for grpc_gcp.secure_channel creation:DeprecationWarning",
+ # Remove once the minimum supported version of googleapis-common-protos is 1.62.0
+ "ignore:.*pkg_resources.declare_namespace:DeprecationWarning",
+ "ignore:.*pkg_resources is deprecated as an API:DeprecationWarning",
+ # Remove once https://github.com/grpc/grpc/issues/35086 is fixed (and version newer than 1.60.0 is published)
+ "ignore:There is no current event loop:DeprecationWarning",
+ # Remove after support for Python 3.7 is dropped
+ "ignore:After January 1, 2024, new releases of this library will drop support for Python 3.7:DeprecationWarning",
+]
diff --git a/renovate.json b/renovate.json
index 39b2a0ec..c7875c46 100644
--- a/renovate.json
+++ b/renovate.json
@@ -5,7 +5,7 @@
":preserveSemverRanges",
":disableDependencyDashboard"
],
- "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"],
+ "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml"],
"pip_requirements": {
"fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"]
}
diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh
index 21f6d2a2..120b0ddc 100755
--- a/scripts/decrypt-secrets.sh
+++ b/scripts/decrypt-secrets.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# Copyright 2015 Google Inc. All rights reserved.
+# Copyright 2024 Google LLC All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py
index 91b59676..8f5e248a 100644
--- a/scripts/readme-gen/readme_gen.py
+++ b/scripts/readme-gen/readme_gen.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-# Copyright 2016 Google Inc
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -33,17 +33,17 @@
autoescape=True,
)
-README_TMPL = jinja_env.get_template('README.tmpl.rst')
+README_TMPL = jinja_env.get_template("README.tmpl.rst")
def get_help(file):
- return subprocess.check_output(['python', file, '--help']).decode()
+ return subprocess.check_output(["python", file, "--help"]).decode()
def main():
parser = argparse.ArgumentParser()
- parser.add_argument('source')
- parser.add_argument('--destination', default='README.rst')
+ parser.add_argument("source")
+ parser.add_argument("--destination", default="README.rst")
args = parser.parse_args()
@@ -51,9 +51,9 @@ def main():
root = os.path.dirname(source)
destination = os.path.join(root, args.destination)
- jinja_env.globals['get_help'] = get_help
+ jinja_env.globals["get_help"] = get_help
- with io.open(source, 'r') as f:
+ with io.open(source, "r") as f:
config = yaml.load(f)
# This allows get_help to execute in the right directory.
@@ -61,9 +61,9 @@ def main():
output = README_TMPL.render(config)
- with io.open(destination, 'w') as f:
+ with io.open(destination, "w") as f:
f.write(output)
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/setup.py b/setup.py
index 6050e5c5..168877fa 100644
--- a/setup.py
+++ b/setup.py
@@ -12,95 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import io
-import os
-
import setuptools
-# Package metadata.
-
-name = "google-api-core"
-description = "Google API client core library"
-
-# Should be one of:
-# 'Development Status :: 3 - Alpha'
-# 'Development Status :: 4 - Beta'
-# 'Development Status :: 5 - Production/Stable'
-release_status = "Development Status :: 5 - Production/Stable"
-dependencies = [
- "googleapis-common-protos >= 1.56.2, < 2.0dev",
- "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
- "google-auth >= 2.14.1, < 3.0dev",
- "requests >= 2.18.0, < 3.0.0dev",
-]
-extras = {
- "grpc": [
- "grpcio >= 1.33.2, < 2.0dev",
- "grpcio >= 1.49.1, < 2.0dev; python_version>='3.11'",
- "grpcio-status >= 1.33.2, < 2.0dev",
- "grpcio-status >= 1.49.1, < 2.0dev; python_version>='3.11'",
- ],
- "grpcgcp": "grpcio-gcp >= 0.2.2, < 1.0dev",
- "grpcio-gcp": "grpcio-gcp >= 0.2.2, < 1.0dev",
-}
-
-
-# Setup boilerplate below this line.
-
-package_root = os.path.abspath(os.path.dirname(__file__))
-
-
-version = {}
-with open(os.path.join(package_root, "google/api_core/version.py")) as fp:
- exec(fp.read(), version)
-version = version["__version__"]
-
-readme_filename = os.path.join(package_root, "README.rst")
-with io.open(readme_filename, encoding="utf-8") as readme_file:
- readme = readme_file.read()
-
-# Only include packages under the 'google' namespace. Do not include tests,
-# benchmarks, etc.
-packages = [
- package for package in setuptools.find_packages() if package.startswith("google")
-]
-
-# Determine which namespaces are needed.
-namespaces = ["google"]
-if "google.cloud" in packages:
- namespaces.append("google.cloud")
-
-
-setuptools.setup(
- name=name,
- version=version,
- description=description,
- long_description=readme,
- author="Google LLC",
- author_email="googleapis-packages@google.com",
- license="Apache 2.0",
- url="https://github.com/googleapis/python-api-core",
- classifiers=[
- release_status,
- "Intended Audience :: Developers",
- "License :: OSI Approved :: Apache Software License",
- "Programming Language :: Python",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Operating System :: OS Independent",
- "Topic :: Internet",
- ],
- platforms="Posix; MacOS X; Windows",
- packages=packages,
- namespace_packages=namespaces,
- install_requires=dependencies,
- extras_require=extras,
- python_requires=">=3.7",
- include_package_data=True,
- zip_safe=False,
-)
+setuptools.setup()
diff --git a/testing/constraints-3.12.txt b/testing/constraints-3.12.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/testing/constraints-3.13.txt b/testing/constraints-3.13.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/testing/constraints-3.14.txt b/testing/constraints-3.14.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt
index ec041297..4ce1c899 100644
--- a/testing/constraints-3.7.txt
+++ b/testing/constraints-3.7.txt
@@ -9,7 +9,7 @@ googleapis-common-protos==1.56.2
protobuf==3.19.5
google-auth==2.14.1
requests==2.18.0
-packaging==14.3
grpcio==1.33.2
grpcio-status==1.33.2
grpcio-gcp==0.2.2
+proto-plus==1.22.3
diff --git a/testing/constraints-async-rest-3.7.txt b/testing/constraints-async-rest-3.7.txt
new file mode 100644
index 00000000..7aedeb1c
--- /dev/null
+++ b/testing/constraints-async-rest-3.7.txt
@@ -0,0 +1,17 @@
+# This constraints file is used to check that lower bounds
+# are correct in setup.py
+# List *all* library dependencies and extras in this file.
+# Pin the version to the lower bound.
+#
+# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev",
+# Then this file should have foo==1.14.0
+googleapis-common-protos==1.56.2
+protobuf==3.19.5
+google-auth==2.35.0
+# from google-auth[aiohttp]
+aiohttp==3.6.2
+requests==2.20.0
+grpcio==1.33.2
+grpcio-status==1.33.2
+grpcio-gcp==0.2.2
+proto-plus==1.22.3
diff --git a/tests/asyncio/future/test_async_future.py b/tests/asyncio/future/test_async_future.py
index 1e9ae334..659f41cf 100644
--- a/tests/asyncio/future/test_async_future.py
+++ b/tests/asyncio/future/test_async_future.py
@@ -13,8 +13,8 @@
# limitations under the License.
import asyncio
+from unittest import mock
-import mock
import pytest
from google.api_core import exceptions
@@ -47,7 +47,6 @@ async def test_polling_future_constructor():
@pytest.mark.asyncio
async def test_set_result():
future = AsyncFuture()
- callback = mock.Mock()
future.set_result(1)
diff --git a/tests/asyncio/gapic/test_method_async.py b/tests/asyncio/gapic/test_method_async.py
index 02d883f6..cc4e7de8 100644
--- a/tests/asyncio/gapic/test_method_async.py
+++ b/tests/asyncio/gapic/test_method_async.py
@@ -14,11 +14,15 @@
import datetime
-import mock
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
import pytest
try:
- from grpc import aio
+ from grpc import aio, Compression
except ImportError:
pytest.skip("No GRPC", allow_module_level=True)
@@ -77,6 +81,7 @@ async def test_wrap_method_with_custom_client_info():
api_core_version=3,
gapic_version=4,
client_library_version=5,
+ protobuf_runtime_version=6,
)
fake_call = grpc_helpers_async.FakeUnaryUnaryCall()
method = mock.Mock(spec=aio.UnaryUnaryMultiCallable, return_value=fake_call)
@@ -92,6 +97,35 @@ async def test_wrap_method_with_custom_client_info():
assert client_info.to_grpc_metadata() in metadata
+@pytest.mark.asyncio
+async def test_wrap_method_with_no_compression():
+ fake_call = grpc_helpers_async.FakeUnaryUnaryCall()
+ method = mock.Mock(spec=aio.UnaryUnaryMultiCallable, return_value=fake_call)
+
+ wrapped_method = gapic_v1.method_async.wrap_method(method)
+
+ await wrapped_method(1, 2, meep="moop", compression=None)
+
+ method.assert_called_once_with(1, 2, meep="moop", metadata=mock.ANY)
+
+
+@pytest.mark.asyncio
+async def test_wrap_method_with_custom_compression():
+ compression = Compression.Gzip
+ fake_call = grpc_helpers_async.FakeUnaryUnaryCall()
+ method = mock.Mock(spec=aio.UnaryUnaryMultiCallable, return_value=fake_call)
+
+ wrapped_method = gapic_v1.method_async.wrap_method(
+ method, default_compression=compression
+ )
+
+ await wrapped_method(1, 2, meep="moop", compression=Compression.Deflate)
+
+ method.assert_called_once_with(
+ 1, 2, meep="moop", metadata=mock.ANY, compression=Compression.Deflate
+ )
+
+
@pytest.mark.asyncio
async def test_invoke_wrapped_method_with_metadata():
fake_call = grpc_helpers_async.FakeUnaryUnaryCall()
@@ -126,7 +160,7 @@ async def test_invoke_wrapped_method_with_metadata_as_none():
@mock.patch("asyncio.sleep")
@pytest.mark.asyncio
-async def test_wrap_method_with_default_retry_and_timeout(unused_sleep):
+async def test_wrap_method_with_default_retry_timeout_and_compression(unused_sleep):
fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42)
method = mock.Mock(
spec=aio.UnaryUnaryMultiCallable,
@@ -135,15 +169,18 @@ async def test_wrap_method_with_default_retry_and_timeout(unused_sleep):
default_retry = retry_async.AsyncRetry()
default_timeout = timeout.ConstantTimeout(60)
+ default_compression = Compression.Gzip
wrapped_method = gapic_v1.method_async.wrap_method(
- method, default_retry, default_timeout
+ method, default_retry, default_timeout, default_compression
)
result = await wrapped_method()
assert result == 42
assert method.call_count == 2
- method.assert_called_with(timeout=60, metadata=mock.ANY)
+ method.assert_called_with(
+ timeout=60, compression=default_compression, metadata=mock.ANY
+ )
@mock.patch("asyncio.sleep")
@@ -157,23 +194,27 @@ async def test_wrap_method_with_default_retry_and_timeout_using_sentinel(unused_
default_retry = retry_async.AsyncRetry()
default_timeout = timeout.ConstantTimeout(60)
+ default_compression = Compression.Gzip
wrapped_method = gapic_v1.method_async.wrap_method(
- method, default_retry, default_timeout
+ method, default_retry, default_timeout, default_compression
)
result = await wrapped_method(
retry=gapic_v1.method_async.DEFAULT,
timeout=gapic_v1.method_async.DEFAULT,
+ compression=gapic_v1.method_async.DEFAULT,
)
assert result == 42
assert method.call_count == 2
- method.assert_called_with(timeout=60, metadata=mock.ANY)
+ method.assert_called_with(
+ timeout=60, compression=Compression.Gzip, metadata=mock.ANY
+ )
@mock.patch("asyncio.sleep")
@pytest.mark.asyncio
-async def test_wrap_method_with_overriding_retry_and_timeout(unused_sleep):
+async def test_wrap_method_with_overriding_retry_timeout_and_compression(unused_sleep):
fake_call = grpc_helpers_async.FakeUnaryUnaryCall(42)
method = mock.Mock(
spec=aio.UnaryUnaryMultiCallable,
@@ -182,8 +223,9 @@ async def test_wrap_method_with_overriding_retry_and_timeout(unused_sleep):
default_retry = retry_async.AsyncRetry()
default_timeout = timeout.ConstantTimeout(60)
+ default_compression = Compression.Gzip
wrapped_method = gapic_v1.method_async.wrap_method(
- method, default_retry, default_timeout
+ method, default_retry, default_timeout, default_compression
)
result = await wrapped_method(
@@ -191,11 +233,14 @@ async def test_wrap_method_with_overriding_retry_and_timeout(unused_sleep):
retry_async.if_exception_type(exceptions.NotFound)
),
timeout=timeout.ConstantTimeout(22),
+ compression=Compression.Deflate,
)
assert result == 42
assert method.call_count == 2
- method.assert_called_with(timeout=22, metadata=mock.ANY)
+ method.assert_called_with(
+ timeout=22, compression=Compression.Deflate, metadata=mock.ANY
+ )
@pytest.mark.asyncio
@@ -212,3 +257,14 @@ async def test_wrap_method_with_overriding_timeout_as_a_number():
assert result == 42
method.assert_called_once_with(timeout=22, metadata=mock.ANY)
+
+
+@pytest.mark.asyncio
+async def test_wrap_method_without_wrap_errors():
+ fake_call = mock.AsyncMock()
+
+ wrapped_method = gapic_v1.method_async.wrap_method(fake_call, kind="rest")
+ with mock.patch("google.api_core.grpc_helpers_async.wrap_errors") as method:
+ await wrapped_method()
+
+ method.assert_not_called()
diff --git a/tests/asyncio/operations_v1/test_operations_async_client.py b/tests/asyncio/operations_v1/test_operations_async_client.py
index 34236da7..e5b20dcd 100644
--- a/tests/asyncio/operations_v1/test_operations_async_client.py
+++ b/tests/asyncio/operations_v1/test_operations_async_client.py
@@ -12,11 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import mock
+from unittest import mock
+
import pytest
try:
- from grpc import aio
+ from grpc import aio, Compression
except ImportError: # pragma: NO COVER
pytest.skip("No GRPC", allow_module_level=True)
@@ -42,10 +43,13 @@ async def test_get_operation():
)
client = operations_v1.OperationsAsyncClient(mocked_channel)
- response = await client.get_operation("name", metadata=[("header", "foo")])
+ response = await client.get_operation(
+ "name", metadata=[("header", "foo")], compression=Compression.Gzip
+ )
assert method.call_count == 1
assert tuple(method.call_args_list[0])[0][0].name == "name"
assert ("header", "foo") in tuple(method.call_args_list[0])[1]["metadata"]
+ assert tuple(method.call_args_list[0])[1]["compression"] == Compression.Gzip
assert ("x-goog-request-params", "name=name") in tuple(method.call_args_list[0])[1][
"metadata"
]
@@ -63,7 +67,9 @@ async def test_list_operations():
mocked_channel, method, fake_call = _mock_grpc_objects(list_response)
client = operations_v1.OperationsAsyncClient(mocked_channel)
- pager = await client.list_operations("name", "filter", metadata=[("header", "foo")])
+ pager = await client.list_operations(
+ "name", "filter", metadata=[("header", "foo")], compression=Compression.Gzip
+ )
assert isinstance(pager, page_iterator_async.AsyncIterator)
responses = []
@@ -74,6 +80,7 @@ async def test_list_operations():
assert method.call_count == 1
assert ("header", "foo") in tuple(method.call_args_list[0])[1]["metadata"]
+ assert tuple(method.call_args_list[0])[1]["compression"] == Compression.Gzip
assert ("x-goog-request-params", "name=name") in tuple(method.call_args_list[0])[1][
"metadata"
]
@@ -88,11 +95,14 @@ async def test_delete_operation():
mocked_channel, method, fake_call = _mock_grpc_objects(empty_pb2.Empty())
client = operations_v1.OperationsAsyncClient(mocked_channel)
- await client.delete_operation("name", metadata=[("header", "foo")])
+ await client.delete_operation(
+ "name", metadata=[("header", "foo")], compression=Compression.Gzip
+ )
assert method.call_count == 1
assert tuple(method.call_args_list[0])[0][0].name == "name"
assert ("header", "foo") in tuple(method.call_args_list[0])[1]["metadata"]
+ assert tuple(method.call_args_list[0])[1]["compression"] == Compression.Gzip
assert ("x-goog-request-params", "name=name") in tuple(method.call_args_list[0])[1][
"metadata"
]
@@ -103,11 +113,14 @@ async def test_cancel_operation():
mocked_channel, method, fake_call = _mock_grpc_objects(empty_pb2.Empty())
client = operations_v1.OperationsAsyncClient(mocked_channel)
- await client.cancel_operation("name", metadata=[("header", "foo")])
+ await client.cancel_operation(
+ "name", metadata=[("header", "foo")], compression=Compression.Gzip
+ )
assert method.call_count == 1
assert tuple(method.call_args_list[0])[0][0].name == "name"
assert ("header", "foo") in tuple(method.call_args_list[0])[1]["metadata"]
+ assert tuple(method.call_args_list[0])[1]["compression"] == Compression.Gzip
assert ("x-goog-request-params", "name=name") in tuple(method.call_args_list[0])[1][
"metadata"
]
diff --git a/tests/asyncio/retry/__init__.py b/tests/asyncio/retry/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/asyncio/retry/test_retry_streaming_async.py b/tests/asyncio/retry/test_retry_streaming_async.py
new file mode 100644
index 00000000..e44f5361
--- /dev/null
+++ b/tests/asyncio/retry/test_retry_streaming_async.py
@@ -0,0 +1,601 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import datetime
+import re
+
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
+
+import pytest
+
+from google.api_core import exceptions
+from google.api_core import retry_async
+from google.api_core.retry import retry_streaming_async
+
+from ...unit.retry.test_retry_base import Test_BaseRetry
+
+
+@pytest.mark.asyncio
+async def test_retry_streaming_target_bad_sleep_generator():
+ from google.api_core.retry.retry_streaming_async import retry_target_stream
+
+ with pytest.raises(ValueError, match="Sleep generator"):
+ await retry_target_stream(None, lambda x: True, [], None).__anext__()
+
+
+@mock.patch("asyncio.sleep", autospec=True)
+@pytest.mark.asyncio
+async def test_retry_streaming_target_dynamic_backoff(sleep):
+ """
+ sleep_generator should be iterated after on_error, to support dynamic backoff
+ """
+ from functools import partial
+ from google.api_core.retry.retry_streaming_async import retry_target_stream
+
+ sleep.side_effect = RuntimeError("stop after sleep")
+ # start with empty sleep generator; values are added after exception in push_sleep_value
+ sleep_values = []
+ error_target = partial(TestAsyncStreamingRetry._generator_mock, error_on=0)
+ inserted_sleep = 99
+
+ def push_sleep_value(err):
+ sleep_values.append(inserted_sleep)
+
+ with pytest.raises(RuntimeError):
+ await retry_target_stream(
+ error_target,
+ predicate=lambda x: True,
+ sleep_generator=sleep_values,
+ on_error=push_sleep_value,
+ ).__anext__()
+ assert sleep.call_count == 1
+ sleep.assert_called_once_with(inserted_sleep)
+
+
+class TestAsyncStreamingRetry(Test_BaseRetry):
+ def _make_one(self, *args, **kwargs):
+ return retry_streaming_async.AsyncStreamingRetry(*args, **kwargs)
+
+ def test___str__(self):
+ def if_exception_type(exc):
+ return bool(exc) # pragma: NO COVER
+
+ # Explicitly set all attributes as changed Retry defaults should not
+ # cause this test to start failing.
+ retry_ = retry_streaming_async.AsyncStreamingRetry(
+ predicate=if_exception_type,
+ initial=1.0,
+ maximum=60.0,
+ multiplier=2.0,
+ timeout=120.0,
+ on_error=None,
+ )
+ assert re.match(
+ (
+ r", "
+ r"initial=1.0, maximum=60.0, multiplier=2.0, timeout=120.0, "
+ r"on_error=None>"
+ ),
+ str(retry_),
+ )
+
+ @staticmethod
+ async def _generator_mock(
+ num=5,
+ error_on=None,
+ exceptions_seen=None,
+ sleep_time=0,
+ ):
+ """
+ Helper to create a mock generator that yields a number of values
+ Generator can optionally raise an exception on a specific iteration
+
+ Args:
+ - num (int): the number of values to yield
+ - error_on (int): if given, the generator will raise a ValueError on the specified iteration
+ - exceptions_seen (list): if given, the generator will append any exceptions to this list before raising
+ - sleep_time (int): if given, the generator will asyncio.sleep for this many seconds before yielding each value
+ """
+ try:
+ for i in range(num):
+ if sleep_time:
+ await asyncio.sleep(sleep_time)
+ if error_on is not None and i == error_on:
+ raise ValueError("generator mock error")
+ yield i
+ except (Exception, BaseException, GeneratorExit) as e:
+ # keep track of exceptions seen by generator
+ if exceptions_seen is not None:
+ exceptions_seen.append(e)
+ raise
+
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___generator_success(self, sleep):
+ """
+ Test that a retry-decorated generator yields values as expected
+ This test checks a generator with no issues
+ """
+ from collections.abc import AsyncGenerator
+
+ retry_ = retry_streaming_async.AsyncStreamingRetry()
+ decorated = retry_(self._generator_mock)
+
+ num = 10
+ generator = await decorated(num)
+ # check types
+ assert isinstance(generator, AsyncGenerator)
+ assert isinstance(self._generator_mock(num), AsyncGenerator)
+ # check yield contents
+ unpacked = [i async for i in generator]
+ assert len(unpacked) == num
+ expected = [i async for i in self._generator_mock(num)]
+ for a, b in zip(unpacked, expected):
+ assert a == b
+ sleep.assert_not_called()
+
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___generator_retry(self, sleep):
+ """
+ Tests that a retry-decorated generator will retry on errors
+ """
+ on_error = mock.Mock(return_value=None)
+ retry_ = retry_streaming_async.AsyncStreamingRetry(
+ on_error=on_error,
+ predicate=retry_async.if_exception_type(ValueError),
+ timeout=None,
+ )
+ generator = await retry_(self._generator_mock)(error_on=3)
+ # error thrown on 3
+ # generator should contain 0, 1, 2 looping
+ unpacked = [await generator.__anext__() for i in range(10)]
+ assert unpacked == [0, 1, 2, 0, 1, 2, 0, 1, 2, 0]
+ assert on_error.call_count == 3
+ await generator.aclose()
+
+ @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n)
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.parametrize("use_deadline_arg", [True, False])
+ @pytest.mark.asyncio
+ async def test___call___generator_retry_hitting_timeout(
+ self, sleep, uniform, use_deadline_arg
+ ):
+ """
+ Tests that a retry-decorated generator will throw a RetryError
+ after using the time budget
+ """
+ import time
+
+ timeout_val = 9.9
+ # support "deadline" as an alias for "timeout"
+ timeout_kwarg = (
+ {"timeout": timeout_val}
+ if not use_deadline_arg
+ else {"deadline": timeout_val}
+ )
+
+ on_error = mock.Mock()
+ retry_ = retry_streaming_async.AsyncStreamingRetry(
+ predicate=retry_async.if_exception_type(ValueError),
+ initial=1.0,
+ maximum=1024.0,
+ multiplier=2.0,
+ **timeout_kwarg,
+ )
+
+ time_now = time.monotonic()
+ now_patcher = mock.patch(
+ "time.monotonic",
+ return_value=time_now,
+ )
+
+ decorated = retry_(self._generator_mock, on_error=on_error)
+ generator = await decorated(error_on=1)
+
+ with now_patcher as patched_now:
+ # Make sure that calls to fake asyncio.sleep() also advance the mocked
+ # time clock.
+ def increase_time(sleep_delay):
+ patched_now.return_value += sleep_delay
+
+ sleep.side_effect = increase_time
+
+ with pytest.raises(exceptions.RetryError):
+ [i async for i in generator]
+
+ assert on_error.call_count == 4
+ # check the delays
+ assert sleep.call_count == 3 # once between each successive target calls
+ last_wait = sleep.call_args.args[0]
+ total_wait = sum(call_args.args[0] for call_args in sleep.call_args_list)
+ # next wait would have put us over, so ended early
+ assert last_wait == 4
+ assert total_wait == 7
+
+ @pytest.mark.asyncio
+ async def test___call___generator_cancellations(self):
+ """
+ cancel calls should propagate to the generator
+ """
+ # test without cancel as retryable
+ retry_ = retry_streaming_async.AsyncStreamingRetry()
+ utcnow = datetime.datetime.now(datetime.timezone.utc)
+ mock.patch("google.api_core.datetime_helpers.utcnow", return_value=utcnow)
+ generator = await retry_(self._generator_mock)(sleep_time=0.2)
+ assert await generator.__anext__() == 0
+ task = asyncio.create_task(generator.__anext__())
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ with pytest.raises(StopAsyncIteration):
+ await generator.__anext__()
+
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___with_generator_send(self, sleep):
+ """
+ Send should be passed through retry into target generator
+ """
+
+ async def _mock_send_gen():
+ """
+ always yield whatever was sent in
+ """
+ in_ = yield
+ while True:
+ in_ = yield in_
+
+ retry_ = retry_streaming_async.AsyncStreamingRetry()
+
+ decorated = retry_(_mock_send_gen)
+
+ generator = await decorated()
+ result = await generator.__anext__()
+ # first yield should be None
+ assert result is None
+ in_messages = ["test_1", "hello", "world"]
+ out_messages = []
+ for msg in in_messages:
+ recv = await generator.asend(msg)
+ out_messages.append(recv)
+ assert in_messages == out_messages
+ await generator.aclose()
+
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___generator_send_retry(self, sleep):
+ """
+ Send should be retried if target generator raises an error
+ """
+ on_error = mock.Mock(return_value=None)
+ retry_ = retry_streaming_async.AsyncStreamingRetry(
+ on_error=on_error,
+ predicate=retry_async.if_exception_type(ValueError),
+ timeout=None,
+ )
+ generator = await retry_(self._generator_mock)(error_on=3)
+ with pytest.raises(TypeError) as exc_info:
+ await generator.asend("cannot send to fresh generator")
+ assert exc_info.match("can't send non-None value")
+ await generator.aclose()
+
+ # error thrown on 3
+ # generator should contain 0, 1, 2 looping
+ generator = await retry_(self._generator_mock)(error_on=3)
+ assert await generator.__anext__() == 0
+ unpacked = [await generator.asend(i) for i in range(10)]
+ assert unpacked == [1, 2, 0, 1, 2, 0, 1, 2, 0, 1]
+ assert on_error.call_count == 3
+ await generator.aclose()
+
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___with_generator_close(self, sleep):
+ """
+ Close should be passed through retry into target generator
+ """
+ retry_ = retry_streaming_async.AsyncStreamingRetry()
+ decorated = retry_(self._generator_mock)
+ exception_list = []
+ generator = await decorated(10, exceptions_seen=exception_list)
+ for i in range(2):
+ await generator.__anext__()
+ await generator.aclose()
+
+ assert isinstance(exception_list[0], GeneratorExit)
+ with pytest.raises(StopAsyncIteration):
+ # calling next on closed generator should raise error
+ await generator.__anext__()
+
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___with_new_generator_close(self, sleep):
+ """
+ Close should be passed through retry into target generator,
+ even when it hasn't been iterated yet
+ """
+ retry_ = retry_streaming_async.AsyncStreamingRetry()
+ decorated = retry_(self._generator_mock)
+ exception_list = []
+ generator = await decorated(10, exceptions_seen=exception_list)
+ await generator.aclose()
+
+ with pytest.raises(StopAsyncIteration):
+ # calling next on closed generator should raise error
+ await generator.__anext__()
+
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___with_generator_throw(self, sleep):
+ """
+ Throw should be passed through retry into target generator
+ """
+
+ # The generator should not retry when it encounters a non-retryable error
+ retry_ = retry_streaming_async.AsyncStreamingRetry(
+ predicate=retry_async.if_exception_type(ValueError),
+ )
+ decorated = retry_(self._generator_mock)
+ exception_list = []
+ generator = await decorated(10, exceptions_seen=exception_list)
+ for i in range(2):
+ await generator.__anext__()
+ with pytest.raises(BufferError):
+ await generator.athrow(BufferError("test"))
+ assert isinstance(exception_list[0], BufferError)
+ with pytest.raises(StopAsyncIteration):
+ # calling next on closed generator should raise error
+ await generator.__anext__()
+
+ # In contrast, the generator should retry if we throw a retryable exception
+ exception_list = []
+ generator = await decorated(10, exceptions_seen=exception_list)
+ for i in range(2):
+ await generator.__anext__()
+ throw_val = await generator.athrow(ValueError("test"))
+ assert throw_val == 0
+ assert isinstance(exception_list[0], ValueError)
+ # calling next on generator should not raise error, because it was retried
+ assert await generator.__anext__() == 1
+
+ @pytest.mark.parametrize("awaitable_wrapped", [True, False])
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___with_iterable_send(self, sleep, awaitable_wrapped):
+ """
+ Send should work like next if the wrapped iterable does not support it
+ """
+ retry_ = retry_streaming_async.AsyncStreamingRetry()
+
+ def iterable_fn():
+ class CustomIterable:
+ def __init__(self):
+ self.i = -1
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ self.i += 1
+ return self.i
+
+ return CustomIterable()
+
+ if awaitable_wrapped:
+
+ async def wrapper():
+ return iterable_fn()
+
+ decorated = retry_(wrapper)
+ else:
+ decorated = retry_(iterable_fn)
+
+ retryable = await decorated()
+ # initiate the generator by calling next
+ result = await retryable.__anext__()
+ assert result == 0
+ # test sending values
+ assert await retryable.asend("test") == 1
+ assert await retryable.asend("test2") == 2
+ assert await retryable.asend("test3") == 3
+ await retryable.aclose()
+
+ @pytest.mark.parametrize("awaitable_wrapped", [True, False])
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___with_iterable_close(self, sleep, awaitable_wrapped):
+ """
+ close should be handled by wrapper if wrapped iterable does not support it
+ """
+ retry_ = retry_streaming_async.AsyncStreamingRetry()
+
+ def iterable_fn():
+ class CustomIterable:
+ def __init__(self):
+ self.i = -1
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ self.i += 1
+ return self.i
+
+ return CustomIterable()
+
+ if awaitable_wrapped:
+
+ async def wrapper():
+ return iterable_fn()
+
+ decorated = retry_(wrapper)
+ else:
+ decorated = retry_(iterable_fn)
+
+ # try closing active generator
+ retryable = await decorated()
+ assert await retryable.__anext__() == 0
+ await retryable.aclose()
+ with pytest.raises(StopAsyncIteration):
+ await retryable.__anext__()
+ # try closing new generator
+ new_retryable = await decorated()
+ await new_retryable.aclose()
+ with pytest.raises(StopAsyncIteration):
+ await new_retryable.__anext__()
+
+ @pytest.mark.parametrize("awaitable_wrapped", [True, False])
+ @mock.patch("asyncio.sleep", autospec=True)
+ @pytest.mark.asyncio
+ async def test___call___with_iterable_throw(self, sleep, awaitable_wrapped):
+ """
+ Throw should work even if the wrapped iterable does not support it
+ """
+
+ predicate = retry_async.if_exception_type(ValueError)
+ retry_ = retry_streaming_async.AsyncStreamingRetry(predicate=predicate)
+
+ def iterable_fn():
+ class CustomIterable:
+ def __init__(self):
+ self.i = -1
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ self.i += 1
+ return self.i
+
+ return CustomIterable()
+
+ if awaitable_wrapped:
+
+ async def wrapper():
+ return iterable_fn()
+
+ decorated = retry_(wrapper)
+ else:
+ decorated = retry_(iterable_fn)
+
+ # try throwing with active generator
+ retryable = await decorated()
+ assert await retryable.__anext__() == 0
+ # should swallow errors in predicate
+ await retryable.athrow(ValueError("test"))
+ # should raise errors not in predicate
+ with pytest.raises(BufferError):
+ await retryable.athrow(BufferError("test"))
+ with pytest.raises(StopAsyncIteration):
+ await retryable.__anext__()
+ # try throwing with new generator
+ new_retryable = await decorated()
+ with pytest.raises(BufferError):
+ await new_retryable.athrow(BufferError("test"))
+ with pytest.raises(StopAsyncIteration):
+ await new_retryable.__anext__()
+
+ @pytest.mark.asyncio
+ async def test_exc_factory_non_retryable_error(self):
+ """
+ generator should give the option to override exception creation logic
+ test when non-retryable error is thrown
+ """
+ from google.api_core.retry import RetryFailureReason
+ from google.api_core.retry.retry_streaming_async import retry_target_stream
+
+ timeout = 6
+ sent_errors = [ValueError("test"), ValueError("test2"), BufferError("test3")]
+ expected_final_err = RuntimeError("done")
+ expected_source_err = ZeroDivisionError("test4")
+
+ def factory(*args, **kwargs):
+ assert len(kwargs) == 0
+ assert args[0] == sent_errors
+ assert args[1] == RetryFailureReason.NON_RETRYABLE_ERROR
+ assert args[2] == timeout
+ return expected_final_err, expected_source_err
+
+ generator = retry_target_stream(
+ self._generator_mock,
+ retry_async.if_exception_type(ValueError),
+ [0] * 3,
+ timeout=timeout,
+ exception_factory=factory,
+ )
+ # initialize the generator
+ await generator.__anext__()
+ # trigger some retryable errors
+ await generator.athrow(sent_errors[0])
+ await generator.athrow(sent_errors[1])
+ # trigger a non-retryable error
+ with pytest.raises(expected_final_err.__class__) as exc_info:
+ await generator.athrow(sent_errors[2])
+ assert exc_info.value == expected_final_err
+ assert exc_info.value.__cause__ == expected_source_err
+
+ @pytest.mark.asyncio
+ async def test_exc_factory_timeout(self):
+ """
+ generator should give the option to override exception creation logic
+ test when timeout is exceeded
+ """
+ import time
+ from google.api_core.retry import RetryFailureReason
+ from google.api_core.retry.retry_streaming_async import retry_target_stream
+
+ timeout = 2
+ time_now = time.monotonic()
+ now_patcher = mock.patch(
+ "time.monotonic",
+ return_value=time_now,
+ )
+
+ with now_patcher as patched_now:
+ timeout = 2
+ sent_errors = [ValueError("test"), ValueError("test2"), ValueError("test3")]
+ expected_final_err = RuntimeError("done")
+ expected_source_err = ZeroDivisionError("test4")
+
+ def factory(*args, **kwargs):
+ assert len(kwargs) == 0
+ assert args[0] == sent_errors
+ assert args[1] == RetryFailureReason.TIMEOUT
+ assert args[2] == timeout
+ return expected_final_err, expected_source_err
+
+ generator = retry_target_stream(
+ self._generator_mock,
+ retry_async.if_exception_type(ValueError),
+ [0] * 3,
+ timeout=timeout,
+ exception_factory=factory,
+ )
+ # initialize the generator
+ await generator.__anext__()
+ # trigger some retryable errors
+ await generator.athrow(sent_errors[0])
+ await generator.athrow(sent_errors[1])
+ # trigger a timeout
+ patched_now.return_value += timeout + 1
+ with pytest.raises(expected_final_err.__class__) as exc_info:
+ await generator.athrow(sent_errors[2])
+ assert exc_info.value == expected_final_err
+ assert exc_info.value.__cause__ == expected_source_err
diff --git a/tests/asyncio/test_retry_async.py b/tests/asyncio/retry/test_retry_unary_async.py
similarity index 65%
rename from tests/asyncio/test_retry_async.py
rename to tests/asyncio/retry/test_retry_unary_async.py
index 14807eb5..e7fdc963 100644
--- a/tests/asyncio/test_retry_async.py
+++ b/tests/asyncio/retry/test_retry_unary_async.py
@@ -15,12 +15,18 @@
import datetime
import re
-import mock
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
import pytest
from google.api_core import exceptions
from google.api_core import retry_async
+from ...unit.retry.test_retry_base import Test_BaseRetry
+
@mock.patch("asyncio.sleep", autospec=True)
@mock.patch(
@@ -97,23 +103,25 @@ async def test_retry_target_non_retryable_error(utcnow, sleep):
@mock.patch("asyncio.sleep", autospec=True)
-@mock.patch("google.api_core.datetime_helpers.utcnow", autospec=True)
+@mock.patch("time.monotonic", autospec=True)
+@pytest.mark.parametrize("use_deadline_arg", [True, False])
@pytest.mark.asyncio
-async def test_retry_target_deadline_exceeded(utcnow, sleep):
+async def test_retry_target_timeout_exceeded(monotonic, sleep, use_deadline_arg):
predicate = retry_async.if_exception_type(ValueError)
exception = ValueError("meep")
target = mock.Mock(side_effect=exception)
# Setup the timeline so that the first call takes 5 seconds but the second
- # call takes 6, which puts the retry over the deadline.
- utcnow.side_effect = [
- # The first call to utcnow establishes the start of the timeline.
- datetime.datetime.min,
- datetime.datetime.min + datetime.timedelta(seconds=5),
- datetime.datetime.min + datetime.timedelta(seconds=11),
- ]
+ # call takes 6, which puts the retry over the timeout.
+ monotonic.side_effect = [0, 5, 11]
+
+ timeout_val = 10
+ # support "deadline" as an alias for "timeout"
+ timeout_kwarg = (
+ {"timeout": timeout_val} if not use_deadline_arg else {"deadline": timeout_val}
+ )
with pytest.raises(exceptions.RetryError) as exc_info:
- await retry_async.retry_target(target, predicate, range(10), deadline=10)
+ await retry_async.retry_target(target, predicate, range(10), **timeout_kwarg)
assert exc_info.value.cause == exception
assert exc_info.match("Timeout of 10.0s exceeded")
@@ -128,113 +136,39 @@ async def test_retry_target_deadline_exceeded(utcnow, sleep):
@pytest.mark.asyncio
async def test_retry_target_bad_sleep_generator():
with pytest.raises(ValueError, match="Sleep generator"):
- await retry_async.retry_target(
- mock.sentinel.target, mock.sentinel.predicate, [], None
- )
+ await retry_async.retry_target(mock.sentinel.target, lambda x: True, [], None)
-class TestAsyncRetry:
- def test_constructor_defaults(self):
- retry_ = retry_async.AsyncRetry()
- assert retry_._predicate == retry_async.if_transient_error
- assert retry_._initial == 1
- assert retry_._maximum == 60
- assert retry_._multiplier == 2
- assert retry_._deadline == 120
- assert retry_._on_error is None
-
- def test_constructor_options(self):
- _some_function = mock.Mock()
-
- retry_ = retry_async.AsyncRetry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=_some_function,
- )
- assert retry_._predicate == mock.sentinel.predicate
- assert retry_._initial == 1
- assert retry_._maximum == 2
- assert retry_._multiplier == 3
- assert retry_._deadline == 4
- assert retry_._on_error is _some_function
-
- def test_with_deadline(self):
- retry_ = retry_async.AsyncRetry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_deadline(42)
- assert retry_ is not new_retry
- assert new_retry._deadline == 42
-
- # the rest of the attributes should remain the same
- assert new_retry._predicate is retry_._predicate
- assert new_retry._initial == retry_._initial
- assert new_retry._maximum == retry_._maximum
- assert new_retry._multiplier == retry_._multiplier
- assert new_retry._on_error is retry_._on_error
-
- def test_with_predicate(self):
- retry_ = retry_async.AsyncRetry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_predicate(mock.sentinel.predicate)
- assert retry_ is not new_retry
- assert new_retry._predicate == mock.sentinel.predicate
-
- # the rest of the attributes should remain the same
- assert new_retry._deadline == retry_._deadline
- assert new_retry._initial == retry_._initial
- assert new_retry._maximum == retry_._maximum
- assert new_retry._multiplier == retry_._multiplier
- assert new_retry._on_error is retry_._on_error
-
- def test_with_delay_noop(self):
- retry_ = retry_async.AsyncRetry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
+@mock.patch("asyncio.sleep", autospec=True)
+@pytest.mark.asyncio
+async def test_retry_target_dynamic_backoff(sleep):
+ """
+ sleep_generator should be iterated after on_error, to support dynamic backoff
+ """
+ sleep.side_effect = RuntimeError("stop after sleep")
+ # start with empty sleep generator; values are added after exception in push_sleep_value
+ sleep_values = []
+ exception = ValueError("trigger retry")
+ error_target = mock.Mock(side_effect=exception)
+ inserted_sleep = 99
+
+ def push_sleep_value(err):
+ sleep_values.append(inserted_sleep)
+
+ with pytest.raises(RuntimeError):
+ await retry_async.retry_target(
+ error_target,
+ predicate=lambda x: True,
+ sleep_generator=sleep_values,
+ on_error=push_sleep_value,
)
- new_retry = retry_.with_delay()
- assert retry_ is not new_retry
- assert new_retry._initial == retry_._initial
- assert new_retry._maximum == retry_._maximum
- assert new_retry._multiplier == retry_._multiplier
+ assert sleep.call_count == 1
+ sleep.assert_called_once_with(inserted_sleep)
- def test_with_delay(self):
- retry_ = retry_async.AsyncRetry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_delay(initial=1, maximum=2, multiplier=3)
- assert retry_ is not new_retry
- assert new_retry._initial == 1
- assert new_retry._maximum == 2
- assert new_retry._multiplier == 3
- # the rest of the attributes should remain the same
- assert new_retry._deadline == retry_._deadline
- assert new_retry._predicate is retry_._predicate
- assert new_retry._on_error is retry_._on_error
+class TestAsyncRetry(Test_BaseRetry):
+ def _make_one(self, *args, **kwargs):
+ return retry_async.AsyncRetry(*args, **kwargs)
def test___str__(self):
def if_exception_type(exc):
@@ -247,7 +181,7 @@ def if_exception_type(exc):
initial=1.0,
maximum=60.0,
multiplier=2.0,
- deadline=120.0,
+ timeout=120.0,
on_error=None,
)
assert re.match(
@@ -280,7 +214,6 @@ async def test___call___and_execute_success(self, sleep):
@mock.patch("asyncio.sleep", autospec=True)
@pytest.mark.asyncio
async def test___call___and_execute_retry(self, sleep, uniform):
-
on_error = mock.Mock(spec=["__call__"], side_effect=[None])
retry_ = retry_async.AsyncRetry(
predicate=retry_async.if_exception_type(ValueError)
@@ -304,21 +237,17 @@ async def test___call___and_execute_retry(self, sleep, uniform):
@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n)
@mock.patch("asyncio.sleep", autospec=True)
@pytest.mark.asyncio
- async def test___call___and_execute_retry_hitting_deadline(self, sleep, uniform):
-
+ async def test___call___and_execute_retry_hitting_timeout(self, sleep, uniform):
on_error = mock.Mock(spec=["__call__"], side_effect=[None] * 10)
retry_ = retry_async.AsyncRetry(
predicate=retry_async.if_exception_type(ValueError),
initial=1.0,
maximum=1024.0,
multiplier=2.0,
- deadline=9.9,
+ timeout=30.9,
)
- utcnow = datetime.datetime.utcnow()
- utcnow_patcher = mock.patch(
- "google.api_core.datetime_helpers.utcnow", return_value=utcnow
- )
+ monotonic_patcher = mock.patch("time.monotonic", return_value=0)
target = mock.AsyncMock(spec=["__call__"], side_effect=[ValueError()] * 10)
# __name__ is needed by functools.partial.
@@ -327,11 +256,11 @@ async def test___call___and_execute_retry_hitting_deadline(self, sleep, uniform)
decorated = retry_(target, on_error=on_error)
target.assert_not_called()
- with utcnow_patcher as patched_utcnow:
+ with monotonic_patcher as patched_monotonic:
# Make sure that calls to fake asyncio.sleep() also advance the mocked
# time clock.
def increase_time(sleep_delay):
- patched_utcnow.return_value += datetime.timedelta(seconds=sleep_delay)
+ patched_monotonic.return_value += sleep_delay
sleep.side_effect = increase_time
@@ -347,8 +276,17 @@ def increase_time(sleep_delay):
last_wait = sleep.call_args.args[0]
total_wait = sum(call_args.args[0] for call_args in sleep.call_args_list)
- assert last_wait == 2.9 # and not 8.0, because the last delay was shortened
- assert total_wait == 9.9 # the same as the deadline
+ assert last_wait == 8.0
+ # Next attempt would be scheduled in 16 secs, 15 + 16 = 31 > 30.9, thus
+ # we do not even wait for it to be scheduled (30.9 is configured timeout).
+ # This changes the previous logic of shortening the last attempt to fit
+ # in the timeout. The previous logic was removed to make Python retry
+ # logic consistent with the other languages and to not disrupt the
+ # randomized retry delays distribution by artificially increasing a
+ # probability of scheduling two (instead of one) last attempts with very
+ # short delay between them, while the second retry having very low chance
+ # of succeeding anyways.
+ assert total_wait == 15.0
@mock.patch("asyncio.sleep", autospec=True)
@pytest.mark.asyncio
diff --git a/tests/asyncio/test_grpc_helpers_async.py b/tests/asyncio/test_grpc_helpers_async.py
index 33ce8212..aa8d5d10 100644
--- a/tests/asyncio/test_grpc_helpers_async.py
+++ b/tests/asyncio/test_grpc_helpers_async.py
@@ -12,7 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import mock
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
import pytest # noqa: I202
try:
@@ -97,12 +101,40 @@ async def test_common_methods_in_wrapped_call():
assert mock_call.wait_for_connection.call_count == 1
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "callable_type,expected_wrapper_type",
+ [
+ (grpc.aio.UnaryStreamMultiCallable, grpc_helpers_async._WrappedUnaryStreamCall),
+ (grpc.aio.StreamUnaryMultiCallable, grpc_helpers_async._WrappedStreamUnaryCall),
+ (
+ grpc.aio.StreamStreamMultiCallable,
+ grpc_helpers_async._WrappedStreamStreamCall,
+ ),
+ ],
+)
+async def test_wrap_errors_w_stream_type(callable_type, expected_wrapper_type):
+ class ConcreteMulticallable(callable_type):
+ def __call__(self, *args, **kwargs):
+ raise NotImplementedError("Should not be called")
+
+ with mock.patch.object(
+ grpc_helpers_async, "_wrap_stream_errors"
+ ) as wrap_stream_errors:
+ callable_ = ConcreteMulticallable()
+ grpc_helpers_async.wrap_errors(callable_)
+ assert wrap_stream_errors.call_count == 1
+ wrap_stream_errors.assert_called_once_with(callable_, expected_wrapper_type)
+
+
@pytest.mark.asyncio
async def test_wrap_stream_errors_unary_stream():
mock_call = mock.Mock(aio.UnaryStreamCall, autospec=True)
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedUnaryStreamCall
+ )
await wrapped_callable(1, 2, three="four")
multicallable.assert_called_once_with(1, 2, three="four")
@@ -114,7 +146,9 @@ async def test_wrap_stream_errors_stream_unary():
mock_call = mock.Mock(aio.StreamUnaryCall, autospec=True)
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamUnaryCall
+ )
await wrapped_callable(1, 2, three="four")
multicallable.assert_called_once_with(1, 2, three="four")
@@ -126,24 +160,15 @@ async def test_wrap_stream_errors_stream_stream():
mock_call = mock.Mock(aio.StreamStreamCall, autospec=True)
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamStreamCall
+ )
await wrapped_callable(1, 2, three="four")
multicallable.assert_called_once_with(1, 2, three="four")
assert mock_call.wait_for_connection.call_count == 1
-@pytest.mark.asyncio
-async def test_wrap_stream_errors_type_error():
- mock_call = mock.Mock()
- multicallable = mock.Mock(return_value=mock_call)
-
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
-
- with pytest.raises(TypeError):
- await wrapped_callable()
-
-
@pytest.mark.asyncio
async def test_wrap_stream_errors_raised():
grpc_error = RpcErrorImpl(grpc.StatusCode.INVALID_ARGUMENT)
@@ -151,7 +176,9 @@ async def test_wrap_stream_errors_raised():
mock_call.wait_for_connection = mock.AsyncMock(side_effect=[grpc_error])
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamStreamCall
+ )
with pytest.raises(exceptions.InvalidArgument):
await wrapped_callable()
@@ -166,7 +193,9 @@ async def test_wrap_stream_errors_read():
mock_call.read = mock.AsyncMock(side_effect=grpc_error)
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamStreamCall
+ )
wrapped_call = await wrapped_callable(1, 2, three="four")
multicallable.assert_called_once_with(1, 2, three="four")
@@ -189,7 +218,9 @@ async def test_wrap_stream_errors_aiter():
mock_call.__aiter__ = mock.Mock(return_value=mocked_aiter)
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamStreamCall
+ )
wrapped_call = await wrapped_callable()
with pytest.raises(exceptions.InvalidArgument) as exc_info:
@@ -210,7 +241,9 @@ async def test_wrap_stream_errors_aiter_non_rpc_error():
mock_call.__aiter__ = mock.Mock(return_value=mocked_aiter)
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamStreamCall
+ )
wrapped_call = await wrapped_callable()
with pytest.raises(TypeError) as exc_info:
@@ -224,7 +257,9 @@ async def test_wrap_stream_errors_aiter_called_multiple_times():
mock_call = mock.Mock(aio.StreamStreamCall, autospec=True)
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamStreamCall
+ )
wrapped_call = await wrapped_callable()
assert wrapped_call.__aiter__() == wrapped_call.__aiter__()
@@ -239,7 +274,9 @@ async def test_wrap_stream_errors_write():
mock_call.done_writing = mock.AsyncMock(side_effect=[None, grpc_error])
multicallable = mock.Mock(return_value=mock_call)
- wrapped_callable = grpc_helpers_async._wrap_stream_errors(multicallable)
+ wrapped_callable = grpc_helpers_async._wrap_stream_errors(
+ multicallable, grpc_helpers_async._WrappedStreamStreamCall
+ )
wrapped_call = await wrapped_callable()
@@ -266,6 +303,28 @@ def test_wrap_errors_non_streaming(wrap_unary_errors):
wrap_unary_errors.assert_called_once_with(callable_)
+def test_grpc_async_stream():
+ """
+ GrpcAsyncStream type should be both an AsyncIterator and a grpc.aio.Call.
+ """
+ instance = grpc_helpers_async.GrpcAsyncStream[int]()
+ assert isinstance(instance, grpc.aio.Call)
+ # should implement __aiter__ and __anext__
+ assert hasattr(instance, "__aiter__")
+ it = instance.__aiter__()
+ assert hasattr(it, "__anext__")
+
+
+def test_awaitable_grpc_call():
+ """
+ AwaitableGrpcCall type should be an Awaitable and a grpc.aio.Call.
+ """
+ instance = grpc_helpers_async.AwaitableGrpcCall()
+ assert isinstance(instance, grpc.aio.Call)
+ # should implement __await__
+ assert hasattr(instance, "__await__")
+
+
@mock.patch("google.api_core.grpc_helpers_async._wrap_stream_errors")
def test_wrap_errors_streaming(wrap_stream_errors):
callable_ = mock.create_autospec(aio.UnaryStreamMultiCallable)
@@ -273,35 +332,67 @@ def test_wrap_errors_streaming(wrap_stream_errors):
result = grpc_helpers_async.wrap_errors(callable_)
assert result == wrap_stream_errors.return_value
- wrap_stream_errors.assert_called_once_with(callable_)
+ wrap_stream_errors.assert_called_once_with(
+ callable_, grpc_helpers_async._WrappedUnaryStreamCall
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@pytest.mark.parametrize(
+ "attempt_direct_path,target,expected_target",
+ [
+ (None, "example.com:443", "example.com:443"),
+ (False, "example.com:443", "example.com:443"),
+ (True, "example.com:443", "google-c2p:///example.com"),
+ (True, "dns:///example.com", "google-c2p:///example.com"),
+ (True, "another-c2p:///example.com", "another-c2p:///example.com"),
+ ],
+)
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
return_value=(mock.sentinel.credentials, mock.sentinel.project),
)
@mock.patch("grpc.aio.secure_channel")
-def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_call):
- target = "example.com:443"
+def test_create_channel_implicit(
+ grpc_secure_channel,
+ google_auth_default,
+ composite_creds_call,
+ attempt_direct_path,
+ target,
+ expected_target,
+):
composite_creds = composite_creds_call.return_value
- channel = grpc_helpers_async.create_channel(target)
+ channel = grpc_helpers_async.create_channel(
+ target, attempt_direct_path=attempt_direct_path
+ )
assert channel is grpc_secure_channel.return_value
- default.assert_called_once_with(scopes=None, default_scopes=None)
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ google_auth_default.assert_called_once_with(scopes=None, default_scopes=None)
+ grpc_secure_channel.assert_called_once_with(
+ expected_target, composite_creds, compression=None
+ )
+@pytest.mark.parametrize(
+ "attempt_direct_path,target, expected_target",
+ [
+ (None, "example.com:443", "example.com:443"),
+ (False, "example.com:443", "example.com:443"),
+ (True, "example.com:443", "google-c2p:///example.com"),
+ (True, "dns:///example.com", "google-c2p:///example.com"),
+ (True, "another-c2p:///example.com", "another-c2p:///example.com"),
+ ],
+)
@mock.patch("google.auth.transport.grpc.AuthMetadataPlugin", autospec=True)
@mock.patch(
"google.auth.transport.requests.Request",
autospec=True,
return_value=mock.sentinel.Request,
)
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
@@ -309,23 +400,40 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c
)
@mock.patch("grpc.aio.secure_channel")
def test_create_channel_implicit_with_default_host(
- grpc_secure_channel, default, composite_creds_call, request, auth_metadata_plugin
+ grpc_secure_channel,
+ google_auth_default,
+ composite_creds_call,
+ request,
+ auth_metadata_plugin,
+ attempt_direct_path,
+ target,
+ expected_target,
):
- target = "example.com:443"
default_host = "example.com"
composite_creds = composite_creds_call.return_value
- channel = grpc_helpers_async.create_channel(target, default_host=default_host)
+ channel = grpc_helpers_async.create_channel(
+ target, default_host=default_host, attempt_direct_path=attempt_direct_path
+ )
assert channel is grpc_secure_channel.return_value
- default.assert_called_once_with(scopes=None, default_scopes=None)
+ google_auth_default.assert_called_once_with(scopes=None, default_scopes=None)
auth_metadata_plugin.assert_called_once_with(
mock.sentinel.credentials, mock.sentinel.Request, default_host=default_host
)
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ expected_target, composite_creds, compression=None
+ )
+@pytest.mark.parametrize(
+ "attempt_direct_path",
+ [
+ None,
+ False,
+ ],
+)
@mock.patch("grpc.composite_channel_credentials")
@mock.patch(
"google.auth.default",
@@ -333,21 +441,36 @@ def test_create_channel_implicit_with_default_host(
)
@mock.patch("grpc.aio.secure_channel")
def test_create_channel_implicit_with_ssl_creds(
- grpc_secure_channel, default, composite_creds_call
+ grpc_secure_channel, default, composite_creds_call, attempt_direct_path
):
target = "example.com:443"
ssl_creds = grpc.ssl_channel_credentials()
- grpc_helpers_async.create_channel(target, ssl_credentials=ssl_creds)
+ grpc_helpers_async.create_channel(
+ target, ssl_credentials=ssl_creds, attempt_direct_path=attempt_direct_path
+ )
default.assert_called_once_with(scopes=None, default_scopes=None)
composite_creds_call.assert_called_once_with(ssl_creds, mock.ANY)
composite_creds = composite_creds_call.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+def test_create_channel_implicit_with_ssl_creds_attempt_direct_path_true():
+ target = "example.com:443"
+ ssl_creds = grpc.ssl_channel_credentials()
+ with pytest.raises(
+ ValueError, match="Using ssl_credentials with Direct Path is not supported"
+ ):
+ grpc_helpers_async.create_channel(
+ target, ssl_credentials=ssl_creds, attempt_direct_path=True
+ )
+
+
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
@@ -365,10 +488,12 @@ def test_create_channel_implicit_with_scopes(
assert channel is grpc_secure_channel.return_value
default.assert_called_once_with(scopes=["one", "two"], default_scopes=None)
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
@@ -382,13 +507,15 @@ def test_create_channel_implicit_with_default_scopes(
composite_creds = composite_creds_call.return_value
channel = grpc_helpers_async.create_channel(
- target, default_scopes=["three", "four"]
+ target, default_scopes=["three", "four"], compression=grpc.Compression.Gzip
)
assert channel is grpc_secure_channel.return_value
default.assert_called_once_with(scopes=None, default_scopes=["three", "four"])
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=grpc.Compression.Gzip
+ )
def test_create_channel_explicit_with_duplicate_credentials():
@@ -404,7 +531,7 @@ def test_create_channel_explicit_with_duplicate_credentials():
assert "mutually exclusive" in str(excinfo.value)
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("google.auth.credentials.with_scopes_if_required", autospec=True)
@mock.patch("grpc.aio.secure_channel")
def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_creds_call):
@@ -412,17 +539,19 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred
composite_creds = composite_creds_call.return_value
channel = grpc_helpers_async.create_channel(
- target, credentials=mock.sentinel.credentials
+ target, credentials=mock.sentinel.credentials, compression=grpc.Compression.Gzip
)
auth_creds.assert_called_once_with(
mock.sentinel.credentials, scopes=None, default_scopes=None
)
assert channel is grpc_secure_channel.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=grpc.Compression.Gzip
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.aio.secure_channel")
def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_call):
target = "example.com:443"
@@ -433,15 +562,20 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal
credentials.requires_scopes = True
channel = grpc_helpers_async.create_channel(
- target, credentials=credentials, scopes=scopes
+ target,
+ credentials=credentials,
+ scopes=scopes,
+ compression=grpc.Compression.Gzip,
)
credentials.with_scopes.assert_called_once_with(scopes, default_scopes=None)
assert channel is grpc_secure_channel.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=grpc.Compression.Gzip
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.aio.secure_channel")
def test_create_channel_explicit_default_scopes(
grpc_secure_channel, composite_creds_call
@@ -454,17 +588,22 @@ def test_create_channel_explicit_default_scopes(
credentials.requires_scopes = True
channel = grpc_helpers_async.create_channel(
- target, credentials=credentials, default_scopes=default_scopes
+ target,
+ credentials=credentials,
+ default_scopes=default_scopes,
+ compression=grpc.Compression.Gzip,
)
credentials.with_scopes.assert_called_once_with(
scopes=None, default_scopes=default_scopes
)
assert channel is grpc_secure_channel.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=grpc.Compression.Gzip
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.aio.secure_channel")
def test_create_channel_explicit_with_quota_project(
grpc_secure_channel, composite_creds_call
@@ -482,10 +621,12 @@ def test_create_channel_explicit_with_quota_project(
credentials.with_quota_project.assert_called_once_with("project-foo")
assert channel is grpc_secure_channel.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.aio.secure_channel")
@mock.patch(
"google.auth.load_credentials_from_file",
@@ -508,10 +649,12 @@ def test_create_channel_with_credentials_file(
credentials_file, scopes=None, default_scopes=None
)
assert channel is grpc_secure_channel.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.aio.secure_channel")
@mock.patch(
"google.auth.load_credentials_from_file",
@@ -535,10 +678,12 @@ def test_create_channel_with_credentials_file_and_scopes(
credentials_file, scopes=scopes, default_scopes=None
)
assert channel is grpc_secure_channel.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.aio.secure_channel")
@mock.patch(
"google.auth.load_credentials_from_file",
@@ -562,7 +707,9 @@ def test_create_channel_with_credentials_file_and_default_scopes(
credentials_file, scopes=None, default_scopes=default_scopes
)
assert channel is grpc_secure_channel.return_value
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
@mock.patch("grpc.aio.secure_channel")
diff --git a/tests/asyncio/test_operation_async.py b/tests/asyncio/test_operation_async.py
index 127ba634..9d9fb5d2 100644
--- a/tests/asyncio/test_operation_async.py
+++ b/tests/asyncio/test_operation_async.py
@@ -13,9 +13,14 @@
# limitations under the License.
-import mock
import pytest
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
+
try:
import grpc # noqa: F401
except ImportError: # pragma: NO COVER
@@ -79,6 +84,7 @@ async def test_constructor():
assert await future.running()
+@pytest.mark.asyncio
def test_metadata():
expected_metadata = struct_pb2.Struct()
future, _, _ = make_operation_future(
@@ -171,6 +177,7 @@ async def test_unexpected_result(unused_sleep):
assert "Unexpected state" in "{!r}".format(exception)
+@pytest.mark.asyncio
def test_from_gapic():
operation_proto = make_operation_proto(done=True)
operations_client = mock.create_autospec(
diff --git a/tests/asyncio/test_page_iterator_async.py b/tests/asyncio/test_page_iterator_async.py
index 75f9e1cf..63e26d02 100644
--- a/tests/asyncio/test_page_iterator_async.py
+++ b/tests/asyncio/test_page_iterator_async.py
@@ -14,7 +14,11 @@
import inspect
-import mock
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
import pytest
from google.api_core import page_iterator_async
@@ -106,6 +110,7 @@ async def test__page_aiter_increment(self):
await page_aiter.__anext__()
assert iterator.num_results == 1
+ await page_aiter.aclose()
@pytest.mark.asyncio
async def test__page_aiter_no_increment(self):
@@ -118,6 +123,7 @@ async def test__page_aiter_no_increment(self):
# results should still be 0 after fetching a page.
assert iterator.num_results == 0
+ await page_aiter.aclose()
@pytest.mark.asyncio
async def test__items_aiter(self):
diff --git a/tests/asyncio/test_rest_streaming_async.py b/tests/asyncio/test_rest_streaming_async.py
new file mode 100644
index 00000000..c9caa2b1
--- /dev/null
+++ b/tests/asyncio/test_rest_streaming_async.py
@@ -0,0 +1,378 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO: set random.seed explicitly in each test function.
+# See related issue: https://github.com/googleapis/python-api-core/issues/689.
+
+import datetime
+import logging
+import random
+import time
+from typing import List, AsyncIterator
+
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
+
+import pytest # noqa: I202
+
+import proto
+
+try:
+ from google.auth.aio.transport import Response
+except ImportError:
+ pytest.skip(
+ "google-api-core[async_rest] is required to test asynchronous rest streaming.",
+ allow_module_level=True,
+ )
+
+from google.api_core import rest_streaming_async
+from google.api import http_pb2
+from google.api import httpbody_pb2
+
+
+from ..helpers import Composer, Song, EchoResponse, parse_responses
+
+
+__protobuf__ = proto.module(package=__name__)
+SEED = int(time.time())
+logging.info(f"Starting async rest streaming tests with random seed: {SEED}")
+random.seed(SEED)
+
+
+async def mock_async_gen(data, chunk_size=1):
+ for i in range(0, len(data)): # pragma: NO COVER
+ chunk = data[i : i + chunk_size]
+ yield chunk.encode("utf-8")
+
+
+class ResponseMock(Response):
+ class _ResponseItr(AsyncIterator[bytes]):
+ def __init__(self, _response_bytes: bytes, random_split=False):
+ self._responses_bytes = _response_bytes
+ self._idx = 0
+ self._random_split = random_split
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ if self._idx >= len(self._responses_bytes):
+ raise StopAsyncIteration
+ if self._random_split:
+ n = random.randint(1, len(self._responses_bytes[self._idx :]))
+ else:
+ n = 1
+ x = self._responses_bytes[self._idx : self._idx + n]
+ self._idx += n
+ return x
+
+ def __init__(
+ self,
+ responses: List[proto.Message],
+ response_cls,
+ random_split=False,
+ ):
+ self._responses = responses
+ self._random_split = random_split
+ self._response_message_cls = response_cls
+
+ def _parse_responses(self):
+ return parse_responses(self._response_message_cls, self._responses)
+
+ @property
+ async def headers(self):
+ raise NotImplementedError()
+
+ @property
+ async def status_code(self):
+ raise NotImplementedError()
+
+ async def close(self):
+ raise NotImplementedError()
+
+ async def content(self, chunk_size=None):
+ itr = self._ResponseItr(
+ self._parse_responses(), random_split=self._random_split
+ )
+ async for chunk in itr:
+ yield chunk
+
+ async def read(self):
+ raise NotImplementedError()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [(False, True), (False, False)],
+)
+async def test_next_simple(random_split, resp_message_is_proto_plus):
+ if resp_message_is_proto_plus:
+ response_type = EchoResponse
+ responses = [EchoResponse(content="hello world"), EchoResponse(content="yes")]
+ else:
+ response_type = httpbody_pb2.HttpBody
+ responses = [
+ httpbody_pb2.HttpBody(content_type="hello world"),
+ httpbody_pb2.HttpBody(content_type="yes"),
+ ]
+
+ resp = ResponseMock(
+ responses=responses, random_split=random_split, response_cls=response_type
+ )
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ idx = 0
+ async for response in itr:
+ assert response == responses[idx]
+ idx += 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [
+ (True, True),
+ (False, True),
+ (True, False),
+ (False, False),
+ ],
+)
+async def test_next_nested(random_split, resp_message_is_proto_plus):
+ if resp_message_is_proto_plus:
+ response_type = Song
+ responses = [
+ Song(title="some song", composer=Composer(given_name="some name")),
+ Song(title="another song", date_added=datetime.datetime(2021, 12, 17)),
+ ]
+ else:
+ # Although `http_pb2.HttpRule`` is used in the response, any response message
+ # can be used which meets this criteria for the test of having a nested field.
+ response_type = http_pb2.HttpRule
+ responses = [
+ http_pb2.HttpRule(
+ selector="some selector",
+ custom=http_pb2.CustomHttpPattern(kind="some kind"),
+ ),
+ http_pb2.HttpRule(
+ selector="another selector",
+ custom=http_pb2.CustomHttpPattern(path="some path"),
+ ),
+ ]
+ resp = ResponseMock(
+ responses=responses, random_split=random_split, response_cls=response_type
+ )
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ idx = 0
+ async for response in itr:
+ assert response == responses[idx]
+ idx += 1
+ assert idx == len(responses)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [
+ (True, True),
+ (False, True),
+ (True, False),
+ (False, False),
+ ],
+)
+async def test_next_stress(random_split, resp_message_is_proto_plus):
+ n = 50
+ if resp_message_is_proto_plus:
+ response_type = Song
+ responses = [
+ Song(title="title_%d" % i, composer=Composer(given_name="name_%d" % i))
+ for i in range(n)
+ ]
+ else:
+ response_type = http_pb2.HttpRule
+ responses = [
+ http_pb2.HttpRule(
+ selector="selector_%d" % i,
+ custom=http_pb2.CustomHttpPattern(path="path_%d" % i),
+ )
+ for i in range(n)
+ ]
+ resp = ResponseMock(
+ responses=responses, random_split=random_split, response_cls=response_type
+ )
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ idx = 0
+ async for response in itr:
+ assert response == responses[idx]
+ idx += 1
+ assert idx == n
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [
+ (True, True),
+ (False, True),
+ (True, False),
+ (False, False),
+ ],
+)
+async def test_next_escaped_characters_in_string(
+ random_split, resp_message_is_proto_plus
+):
+ if resp_message_is_proto_plus:
+ response_type = Song
+ composer_with_relateds = Composer()
+ relateds = ["Artist A", "Artist B"]
+ composer_with_relateds.relateds = relateds
+
+ responses = [
+ Song(
+ title='ti"tle\nfoo\tbar{}', composer=Composer(given_name="name\n\n\n")
+ ),
+ Song(
+ title='{"this is weird": "totally"}',
+ composer=Composer(given_name="\\{}\\"),
+ ),
+ Song(title='\\{"key": ["value",]}\\', composer=composer_with_relateds),
+ ]
+ else:
+ response_type = http_pb2.Http
+ responses = [
+ http_pb2.Http(
+ rules=[
+ http_pb2.HttpRule(
+ selector='ti"tle\nfoo\tbar{}',
+ custom=http_pb2.CustomHttpPattern(kind="name\n\n\n"),
+ )
+ ]
+ ),
+ http_pb2.Http(
+ rules=[
+ http_pb2.HttpRule(
+ selector='{"this is weird": "totally"}',
+ custom=http_pb2.CustomHttpPattern(kind="\\{}\\"),
+ )
+ ]
+ ),
+ http_pb2.Http(
+ rules=[
+ http_pb2.HttpRule(
+ selector='\\{"key": ["value",]}\\',
+ custom=http_pb2.CustomHttpPattern(kind="\\{}\\"),
+ )
+ ]
+ ),
+ ]
+ resp = ResponseMock(
+ responses=responses, random_split=random_split, response_cls=response_type
+ )
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ idx = 0
+ async for response in itr:
+ assert response == responses[idx]
+ idx += 1
+ assert idx == len(responses)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response_type", [EchoResponse, httpbody_pb2.HttpBody])
+async def test_next_not_array(response_type):
+
+ data = '{"hello": 0}'
+ with mock.patch.object(
+ ResponseMock, "content", return_value=mock_async_gen(data)
+ ) as mock_method:
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ with pytest.raises(ValueError):
+ await itr.__anext__()
+ mock_method.assert_called_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response_type", [EchoResponse, httpbody_pb2.HttpBody])
+async def test_cancel(response_type):
+ with mock.patch.object(
+ ResponseMock, "close", new_callable=mock.AsyncMock
+ ) as mock_method:
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ await itr.cancel()
+ mock_method.assert_called_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response_type", [EchoResponse, httpbody_pb2.HttpBody])
+async def test_iterator_as_context_manager(response_type):
+ with mock.patch.object(
+ ResponseMock, "close", new_callable=mock.AsyncMock
+ ) as mock_method:
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ async with rest_streaming_async.AsyncResponseIterator(resp, response_type):
+ pass
+ mock_method.assert_called_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "response_type,return_value",
+ [
+ (EchoResponse, bytes('[{"content": "hello"}, {', "utf-8")),
+ (httpbody_pb2.HttpBody, bytes('[{"content_type": "hello"}, {', "utf-8")),
+ ],
+)
+async def test_check_buffer(response_type, return_value):
+ with mock.patch.object(
+ ResponseMock,
+ "_parse_responses",
+ return_value=return_value,
+ ):
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ with pytest.raises(ValueError):
+ await itr.__anext__()
+ await itr.__anext__()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("response_type", [EchoResponse, httpbody_pb2.HttpBody])
+async def test_next_html(response_type):
+
+ data = ""
+ with mock.patch.object(
+ ResponseMock, "content", return_value=mock_async_gen(data)
+ ) as mock_method:
+ resp = ResponseMock(responses=[], response_cls=response_type)
+
+ itr = rest_streaming_async.AsyncResponseIterator(resp, response_type)
+ with pytest.raises(ValueError):
+ await itr.__anext__()
+ mock_method.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_invalid_response_class():
+ class SomeClass:
+ pass
+
+ resp = ResponseMock(responses=[], response_cls=SomeClass)
+ with pytest.raises(
+ ValueError,
+ match="Response message class must be a subclass of proto.Message or google.protobuf.message.Message",
+ ):
+ rest_streaming_async.AsyncResponseIterator(resp, SomeClass)
diff --git a/tests/helpers.py b/tests/helpers.py
new file mode 100644
index 00000000..3429d511
--- /dev/null
+++ b/tests/helpers.py
@@ -0,0 +1,71 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for tests"""
+
+import logging
+from typing import List
+
+import proto
+
+from google.protobuf import duration_pb2
+from google.protobuf import timestamp_pb2
+from google.protobuf.json_format import MessageToJson
+
+
+class Genre(proto.Enum):
+ GENRE_UNSPECIFIED = 0
+ CLASSICAL = 1
+ JAZZ = 2
+ ROCK = 3
+
+
+class Composer(proto.Message):
+ given_name = proto.Field(proto.STRING, number=1)
+ family_name = proto.Field(proto.STRING, number=2)
+ relateds = proto.RepeatedField(proto.STRING, number=3)
+ indices = proto.MapField(proto.STRING, proto.STRING, number=4)
+
+
+class Song(proto.Message):
+ composer = proto.Field(Composer, number=1)
+ title = proto.Field(proto.STRING, number=2)
+ lyrics = proto.Field(proto.STRING, number=3)
+ year = proto.Field(proto.INT32, number=4)
+ genre = proto.Field(Genre, number=5)
+ is_five_mins_longer = proto.Field(proto.BOOL, number=6)
+ score = proto.Field(proto.DOUBLE, number=7)
+ likes = proto.Field(proto.INT64, number=8)
+ duration = proto.Field(duration_pb2.Duration, number=9)
+ date_added = proto.Field(timestamp_pb2.Timestamp, number=10)
+
+
+class EchoResponse(proto.Message):
+ content = proto.Field(proto.STRING, number=1)
+
+
+def parse_responses(response_message_cls, all_responses: List[proto.Message]) -> bytes:
+ # json.dumps returns a string surrounded with quotes that need to be stripped
+ # in order to be an actual JSON.
+ json_responses = [
+ (
+ response_message_cls.to_json(response).strip('"')
+ if issubclass(response_message_cls, proto.Message)
+ else MessageToJson(response).strip('"')
+ )
+ for response in all_responses
+ ]
+ logging.info(f"Sending JSON stream: {json_responses}")
+ ret_val = "[{}]".format(",".join(json_responses))
+ return bytes(ret_val, "utf-8")
diff --git a/tests/unit/future/test__helpers.py b/tests/unit/future/test__helpers.py
index 98afc599..a37efdd4 100644
--- a/tests/unit/future/test__helpers.py
+++ b/tests/unit/future/test__helpers.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import mock
+from unittest import mock
from google.api_core.future import _helpers
diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py
index f5d9b4f1..2f66f230 100644
--- a/tests/unit/future/test_polling.py
+++ b/tests/unit/future/test_polling.py
@@ -15,8 +15,8 @@
import concurrent.futures
import threading
import time
+from unittest import mock
-import mock
import pytest
from google.api_core import exceptions, retry
diff --git a/tests/unit/gapic/test_method.py b/tests/unit/gapic/test_method.py
index 03c2260d..8896429c 100644
--- a/tests/unit/gapic/test_method.py
+++ b/tests/unit/gapic/test_method.py
@@ -13,8 +13,8 @@
# limitations under the License.
import datetime
+from unittest import mock
-import mock
import pytest
try:
@@ -76,6 +76,7 @@ def test_wrap_method_with_custom_client_info():
api_core_version=3,
gapic_version=4,
client_library_version=5,
+ protobuf_runtime_version=6,
)
method = mock.Mock(spec=["__call__"])
@@ -121,21 +122,24 @@ def test_invoke_wrapped_method_with_metadata_as_none():
@mock.patch("time.sleep")
-def test_wrap_method_with_default_retry_and_timeout(unused_sleep):
+def test_wrap_method_with_default_retry_and_timeout_and_compression(unused_sleep):
method = mock.Mock(
spec=["__call__"], side_effect=[exceptions.InternalServerError(None), 42]
)
default_retry = retry.Retry()
default_timeout = timeout.ConstantTimeout(60)
+ default_compression = grpc.Compression.Gzip
wrapped_method = google.api_core.gapic_v1.method.wrap_method(
- method, default_retry, default_timeout
+ method, default_retry, default_timeout, default_compression
)
result = wrapped_method()
assert result == 42
assert method.call_count == 2
- method.assert_called_with(timeout=60, metadata=mock.ANY)
+ method.assert_called_with(
+ timeout=60, compression=default_compression, metadata=mock.ANY
+ )
@mock.patch("time.sleep")
@@ -145,37 +149,45 @@ def test_wrap_method_with_default_retry_and_timeout_using_sentinel(unused_sleep)
)
default_retry = retry.Retry()
default_timeout = timeout.ConstantTimeout(60)
+ default_compression = grpc.Compression.Gzip
wrapped_method = google.api_core.gapic_v1.method.wrap_method(
- method, default_retry, default_timeout
+ method, default_retry, default_timeout, default_compression
)
result = wrapped_method(
retry=google.api_core.gapic_v1.method.DEFAULT,
timeout=google.api_core.gapic_v1.method.DEFAULT,
+ compression=google.api_core.gapic_v1.method.DEFAULT,
)
assert result == 42
assert method.call_count == 2
- method.assert_called_with(timeout=60, metadata=mock.ANY)
+ method.assert_called_with(
+ timeout=60, compression=default_compression, metadata=mock.ANY
+ )
@mock.patch("time.sleep")
-def test_wrap_method_with_overriding_retry_and_timeout(unused_sleep):
+def test_wrap_method_with_overriding_retry_timeout_compression(unused_sleep):
method = mock.Mock(spec=["__call__"], side_effect=[exceptions.NotFound(None), 42])
default_retry = retry.Retry()
default_timeout = timeout.ConstantTimeout(60)
+ default_compression = grpc.Compression.Gzip
wrapped_method = google.api_core.gapic_v1.method.wrap_method(
- method, default_retry, default_timeout
+ method, default_retry, default_timeout, default_compression
)
result = wrapped_method(
retry=retry.Retry(retry.if_exception_type(exceptions.NotFound)),
timeout=timeout.ConstantTimeout(22),
+ compression=grpc.Compression.Deflate,
)
assert result == 42
assert method.call_count == 2
- method.assert_called_with(timeout=22, metadata=mock.ANY)
+ method.assert_called_with(
+ timeout=22, compression=grpc.Compression.Deflate, metadata=mock.ANY
+ )
def test_wrap_method_with_overriding_timeout_as_a_number():
@@ -190,3 +202,24 @@ def test_wrap_method_with_overriding_timeout_as_a_number():
assert result == 42
method.assert_called_once_with(timeout=22, metadata=mock.ANY)
+
+
+def test_wrap_method_with_call():
+ method = mock.Mock()
+ mock_call = mock.Mock()
+ method.with_call.return_value = 42, mock_call
+
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(method, with_call=True)
+ result = wrapped_method()
+ assert len(result) == 2
+ assert result[0] == 42
+ assert result[1] == mock_call
+
+
+def test_wrap_method_with_call_not_supported():
+ """Raises an error if wrapped callable doesn't have with_call method."""
+ method = lambda: None # noqa: E731
+
+ with pytest.raises(ValueError) as exc_info:
+ google.api_core.gapic_v1.method.wrap_method(method, with_call=True)
+ assert "with_call=True is only supported for unary calls" in str(exc_info.value)
diff --git a/tests/unit/gapic/test_routing_header.py b/tests/unit/gapic/test_routing_header.py
index 9d31eb39..2c8c7546 100644
--- a/tests/unit/gapic/test_routing_header.py
+++ b/tests/unit/gapic/test_routing_header.py
@@ -70,3 +70,34 @@ def test_to_grpc_metadata():
params = [("name", "meep"), ("book.read", "1")]
metadata = routing_header.to_grpc_metadata(params)
assert metadata == (routing_header.ROUTING_METADATA_KEY, "name=meep&book.read=1")
+
+
+@pytest.mark.parametrize(
+ "key,value,expected",
+ [
+ ("book.read", "1", "book.read=1"),
+ ("name", "me/ep", "name=me/ep"),
+ ("\\", "=", "%5C=%3D"),
+ (b"hello", "world", "hello=world"),
+ ("✔️", "✌️", "%E2%9C%94%EF%B8%8F=%E2%9C%8C%EF%B8%8F"),
+ ],
+)
+def test__urlencode_param(key, value, expected):
+ result = routing_header._urlencode_param(key, value)
+ assert result == expected
+
+
+def test__urlencode_param_caching_performance():
+ import time
+
+ key = "key" * 100
+ value = "value" * 100
+ # time with empty cache
+ start_time = time.perf_counter()
+ routing_header._urlencode_param(key, value)
+ duration = time.perf_counter() - start_time
+ second_start_time = time.perf_counter()
+ routing_header._urlencode_param(key, value)
+ second_duration = time.perf_counter() - second_start_time
+ # second call should be approximately 10 times faster
+ assert second_duration < duration / 10
diff --git a/tests/unit/operations_v1/test_operations_rest_client.py b/tests/unit/operations_v1/test_operations_rest_client.py
index 4ab4f1f7..d1f6e0eb 100644
--- a/tests/unit/operations_v1/test_operations_rest_client.py
+++ b/tests/unit/operations_v1/test_operations_rest_client.py
@@ -15,23 +15,31 @@
#
import os
-import mock
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
+
import pytest
+from typing import Any, List
try:
import grpc # noqa: F401
except ImportError: # pragma: NO COVER
pytest.skip("No GRPC", allow_module_level=True)
from requests import Response # noqa I201
-from requests.sessions import Session
+from google.auth.transport.requests import AuthorizedSession
from google.api_core import client_options
from google.api_core import exceptions as core_exceptions
from google.api_core import gapic_v1
from google.api_core.operations_v1 import AbstractOperationsClient
+
+import google.auth
from google.api_core.operations_v1 import pagers
+from google.api_core.operations_v1 import pagers_async
from google.api_core.operations_v1 import transports
-import google.auth
from google.auth import credentials as ga_credentials
from google.auth.exceptions import MutualTLSChannelError
from google.longrunning import operations_pb2
@@ -39,6 +47,16 @@
from google.protobuf import json_format # type: ignore
from google.rpc import status_pb2 # type: ignore
+try:
+ import aiohttp # noqa: F401
+ import google.auth.aio.transport
+ from google.auth.aio.transport.sessions import AsyncAuthorizedSession
+ from google.api_core.operations_v1 import AsyncOperationsRestClient
+ from google.auth.aio import credentials as ga_credentials_async
+
+ GOOGLE_AUTH_AIO_INSTALLED = True
+except ImportError:
+ GOOGLE_AUTH_AIO_INSTALLED = False
HTTP_OPTIONS = {
"google.longrunning.Operations.CancelOperation": [
@@ -55,17 +73,62 @@
],
}
+PYPARAM_CLIENT: List[Any] = [
+ AbstractOperationsClient,
+]
+PYPARAM_CLIENT_TRANSPORT_NAME = [
+ [AbstractOperationsClient, transports.OperationsRestTransport, "rest"],
+]
+PYPARAM_CLIENT_TRANSPORT_CREDENTIALS = [
+ [
+ AbstractOperationsClient,
+ transports.OperationsRestTransport,
+ ga_credentials.AnonymousCredentials(),
+ ],
+]
+
+if GOOGLE_AUTH_AIO_INSTALLED:
+ PYPARAM_CLIENT.append(AsyncOperationsRestClient)
+ PYPARAM_CLIENT_TRANSPORT_NAME.append(
+ [
+ AsyncOperationsRestClient,
+ transports.AsyncOperationsRestTransport,
+ "rest_asyncio",
+ ]
+ )
+ PYPARAM_CLIENT_TRANSPORT_CREDENTIALS.append(
+ [
+ AsyncOperationsRestClient,
+ transports.AsyncOperationsRestTransport,
+ ga_credentials_async.AnonymousCredentials(),
+ ]
+ )
+
def client_cert_source_callback():
return b"cert bytes", b"key bytes"
-def _get_operations_client(http_options=HTTP_OPTIONS):
- transport = transports.rest.OperationsRestTransport(
- credentials=ga_credentials.AnonymousCredentials(), http_options=http_options
+def _get_session_type(is_async: bool):
+ return (
+ AsyncAuthorizedSession
+ if is_async and GOOGLE_AUTH_AIO_INSTALLED
+ else AuthorizedSession
)
- return AbstractOperationsClient(transport=transport)
+
+def _get_operations_client(is_async: bool, http_options=HTTP_OPTIONS):
+ if is_async and GOOGLE_AUTH_AIO_INSTALLED:
+ async_transport = transports.rest_asyncio.AsyncOperationsRestTransport(
+ credentials=ga_credentials_async.AnonymousCredentials(),
+ http_options=http_options,
+ )
+ return AsyncOperationsRestClient(transport=async_transport)
+ else:
+ sync_transport = transports.rest.OperationsRestTransport(
+ credentials=ga_credentials.AnonymousCredentials(), http_options=http_options
+ )
+ return AbstractOperationsClient(transport=sync_transport)
# If default endpoint is localhost, then default mtls endpoint will be the same.
@@ -79,57 +142,69 @@ def modify_default_endpoint(client):
)
-def test__get_default_mtls_endpoint():
+# TODO: Add support for mtls in async rest
+@pytest.mark.parametrize(
+ "client_class",
+ [
+ AbstractOperationsClient,
+ ],
+)
+def test__get_default_mtls_endpoint(client_class):
api_endpoint = "example.googleapis.com"
api_mtls_endpoint = "example.mtls.googleapis.com"
sandbox_endpoint = "example.sandbox.googleapis.com"
sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com"
non_googleapi = "api.example.com"
- assert AbstractOperationsClient._get_default_mtls_endpoint(None) is None
+ assert client_class._get_default_mtls_endpoint(None) is None
+ assert client_class._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint
assert (
- AbstractOperationsClient._get_default_mtls_endpoint(api_endpoint)
- == api_mtls_endpoint
+ client_class._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint
)
assert (
- AbstractOperationsClient._get_default_mtls_endpoint(api_mtls_endpoint)
- == api_mtls_endpoint
- )
- assert (
- AbstractOperationsClient._get_default_mtls_endpoint(sandbox_endpoint)
+ client_class._get_default_mtls_endpoint(sandbox_endpoint)
== sandbox_mtls_endpoint
)
assert (
- AbstractOperationsClient._get_default_mtls_endpoint(sandbox_mtls_endpoint)
+ client_class._get_default_mtls_endpoint(sandbox_mtls_endpoint)
== sandbox_mtls_endpoint
)
- assert (
- AbstractOperationsClient._get_default_mtls_endpoint(non_googleapi)
- == non_googleapi
- )
+ assert client_class._get_default_mtls_endpoint(non_googleapi) == non_googleapi
-@pytest.mark.parametrize("client_class", [AbstractOperationsClient])
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
def test_operations_client_from_service_account_info(client_class):
creds = ga_credentials.AnonymousCredentials()
- with mock.patch.object(
- service_account.Credentials, "from_service_account_info"
- ) as factory:
- factory.return_value = creds
- info = {"valid": True}
- client = client_class.from_service_account_info(info)
- assert client.transport._credentials == creds
- assert isinstance(client, client_class)
+ if "async" in str(client_class):
+ # TODO(): Add support for service account info to async REST transport.
+ with pytest.raises(NotImplementedError):
+ info = {"valid": True}
+ client_class.from_service_account_info(info)
+ else:
+ with mock.patch.object(
+ service_account.Credentials, "from_service_account_info"
+ ) as factory:
+ factory.return_value = creds
+ info = {"valid": True}
+ client = client_class.from_service_account_info(info)
+ assert client.transport._credentials == creds
+ assert isinstance(client, client_class)
- assert client.transport._host == "https://longrunning.googleapis.com"
+ assert client.transport._host == "https://longrunning.googleapis.com"
@pytest.mark.parametrize(
- "transport_class,transport_name", [(transports.OperationsRestTransport, "rest")]
+ "transport_class",
+ [
+ transports.OperationsRestTransport,
+ # TODO(https://github.com/googleapis/python-api-core/issues/706): Add support for
+ # service account credentials in transports.AsyncOperationsRestTransport
+ ],
)
-def test_operations_client_service_account_always_use_jwt(
- transport_class, transport_name
-):
+def test_operations_client_service_account_always_use_jwt(transport_class):
with mock.patch.object(
service_account.Credentials, "with_always_use_jwt_access", create=True
) as use_jwt:
@@ -145,35 +220,53 @@ def test_operations_client_service_account_always_use_jwt(
use_jwt.assert_not_called()
-@pytest.mark.parametrize("client_class", [AbstractOperationsClient])
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
def test_operations_client_from_service_account_file(client_class):
- creds = ga_credentials.AnonymousCredentials()
- with mock.patch.object(
- service_account.Credentials, "from_service_account_file"
- ) as factory:
- factory.return_value = creds
- client = client_class.from_service_account_file("dummy/file/path.json")
- assert client.transport._credentials == creds
- assert isinstance(client, client_class)
- client = client_class.from_service_account_json("dummy/file/path.json")
- assert client.transport._credentials == creds
- assert isinstance(client, client_class)
+ if "async" in str(client_class):
+ # TODO(): Add support for service account creds to async REST transport.
+ with pytest.raises(NotImplementedError):
+ client_class.from_service_account_file("dummy/file/path.json")
+ else:
+ creds = ga_credentials.AnonymousCredentials()
+ with mock.patch.object(
+ service_account.Credentials, "from_service_account_file"
+ ) as factory:
+ factory.return_value = creds
+ client = client_class.from_service_account_file("dummy/file/path.json")
+ assert client.transport._credentials == creds
+ assert isinstance(client, client_class)
+
+ client = client_class.from_service_account_json("dummy/file/path.json")
+ assert client.transport._credentials == creds
+ assert isinstance(client, client_class)
- assert client.transport._host == "https://longrunning.googleapis.com"
+ assert client.transport._host == "https://longrunning.googleapis.com"
-def test_operations_client_get_transport_class():
- transport = AbstractOperationsClient.get_transport_class()
+@pytest.mark.parametrize(
+ "client_class,transport_class,transport_name",
+ PYPARAM_CLIENT_TRANSPORT_NAME,
+)
+def test_operations_client_get_transport_class(
+ client_class, transport_class, transport_name
+):
+ transport = client_class.get_transport_class()
available_transports = [
transports.OperationsRestTransport,
]
+ if GOOGLE_AUTH_AIO_INSTALLED:
+ available_transports.append(transports.AsyncOperationsRestTransport)
assert transport in available_transports
- transport = AbstractOperationsClient.get_transport_class("rest")
- assert transport == transports.OperationsRestTransport
+ transport = client_class.get_transport_class(transport_name)
+ assert transport == transport_class
+# TODO(): Update this test case to include async REST once we have support for MTLS.
@pytest.mark.parametrize(
"client_class,transport_class,transport_name",
[(AbstractOperationsClient, transports.OperationsRestTransport, "rest")],
@@ -186,22 +279,21 @@ def test_operations_client_get_transport_class():
def test_operations_client_client_options(
client_class, transport_class, transport_name
):
- # Check that if channel is provided we won't create a new one.
- with mock.patch.object(AbstractOperationsClient, "get_transport_class") as gtc:
- transport = transport_class(credentials=ga_credentials.AnonymousCredentials())
- client = client_class(transport=transport)
- gtc.assert_not_called()
+ # # Check that if channel is provided we won't create a new one.
+ # with mock.patch.object(AbstractOperationsBaseClient, "get_transport_class") as gtc:
+ # client = client_class(transport=transport_class())
+ # gtc.assert_not_called()
- # Check that if channel is provided via str we will create a new one.
- with mock.patch.object(AbstractOperationsClient, "get_transport_class") as gtc:
- client = client_class(transport=transport_name)
- gtc.assert_called()
+ # # Check that if channel is provided via str we will create a new one.
+ # with mock.patch.object(AbstractOperationsBaseClient, "get_transport_class") as gtc:
+ # client = client_class(transport=transport_name)
+ # gtc.assert_called()
# Check the case api_endpoint is provided.
options = client_options.ClientOptions(api_endpoint="squid.clam.whelk")
with mock.patch.object(transport_class, "__init__") as patched:
patched.return_value = None
- client = client_class(client_options=options)
+ client = client_class(client_options=options, transport=transport_name)
patched.assert_called_once_with(
credentials=None,
credentials_file=None,
@@ -218,7 +310,7 @@ def test_operations_client_client_options(
with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}):
with mock.patch.object(transport_class, "__init__") as patched:
patched.return_value = None
- client = client_class()
+ client = client_class(transport=transport_name)
patched.assert_called_once_with(
credentials=None,
credentials_file=None,
@@ -235,7 +327,7 @@ def test_operations_client_client_options(
with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}):
with mock.patch.object(transport_class, "__init__") as patched:
patched.return_value = None
- client = client_class()
+ client = client_class(transport=transport_name)
patched.assert_called_once_with(
credentials=None,
credentials_file=None,
@@ -264,7 +356,7 @@ def test_operations_client_client_options(
options = client_options.ClientOptions(quota_project_id="octopus")
with mock.patch.object(transport_class, "__init__") as patched:
patched.return_value = None
- client = client_class(client_options=options)
+ client = client_class(client_options=options, transport=transport_name)
patched.assert_called_once_with(
credentials=None,
credentials_file=None,
@@ -277,6 +369,7 @@ def test_operations_client_client_options(
)
+# TODO: Add support for mtls in async REST
@pytest.mark.parametrize(
"client_class,transport_class,transport_name,use_client_cert_env",
[
@@ -393,7 +486,7 @@ def fake_init(client_cert_source_for_mtls=None, **kwargs):
@pytest.mark.parametrize(
"client_class,transport_class,transport_name",
- [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")],
+ PYPARAM_CLIENT_TRANSPORT_NAME,
)
def test_operations_client_client_options_scopes(
client_class, transport_class, transport_name
@@ -402,52 +495,59 @@ def test_operations_client_client_options_scopes(
options = client_options.ClientOptions(
scopes=["1", "2"],
)
- with mock.patch.object(transport_class, "__init__") as patched:
- patched.return_value = None
- client = client_class(client_options=options)
- patched.assert_called_once_with(
- credentials=None,
- credentials_file=None,
- host=client.DEFAULT_ENDPOINT,
- scopes=["1", "2"],
- client_cert_source_for_mtls=None,
- quota_project_id=None,
- client_info=transports.base.DEFAULT_CLIENT_INFO,
- always_use_jwt_access=True,
- )
+ if "async" in str(client_class):
+ # TODO(): Add support for scopes to async REST transport.
+ with pytest.raises(core_exceptions.AsyncRestUnsupportedParameterError):
+ client_class(client_options=options, transport=transport_name)
+ else:
+ with mock.patch.object(transport_class, "__init__") as patched:
+ patched.return_value = None
+ client = client_class(client_options=options, transport=transport_name)
+ patched.assert_called_once_with(
+ credentials=None,
+ credentials_file=None,
+ host=client.DEFAULT_ENDPOINT,
+ scopes=["1", "2"],
+ client_cert_source_for_mtls=None,
+ quota_project_id=None,
+ client_info=transports.base.DEFAULT_CLIENT_INFO,
+ always_use_jwt_access=True,
+ )
@pytest.mark.parametrize(
"client_class,transport_class,transport_name",
- [(AbstractOperationsClient, transports.OperationsRestTransport, "rest")],
+ PYPARAM_CLIENT_TRANSPORT_NAME,
)
def test_operations_client_client_options_credentials_file(
client_class, transport_class, transport_name
):
# Check the case credentials file is provided.
options = client_options.ClientOptions(credentials_file="credentials.json")
- with mock.patch.object(transport_class, "__init__") as patched:
- patched.return_value = None
- client = client_class(client_options=options)
- patched.assert_called_once_with(
- credentials=None,
- credentials_file="credentials.json",
- host=client.DEFAULT_ENDPOINT,
- scopes=None,
- client_cert_source_for_mtls=None,
- quota_project_id=None,
- client_info=transports.base.DEFAULT_CLIENT_INFO,
- always_use_jwt_access=True,
- )
-
+ if "async" in str(client_class):
+ # TODO(): Add support for credentials file to async REST transport.
+ with pytest.raises(core_exceptions.AsyncRestUnsupportedParameterError):
+ client_class(client_options=options, transport=transport_name)
+ else:
+ with mock.patch.object(transport_class, "__init__") as patched:
+ patched.return_value = None
+ client = client_class(client_options=options, transport=transport_name)
+ patched.assert_called_once_with(
+ credentials=None,
+ credentials_file="credentials.json",
+ host=client.DEFAULT_ENDPOINT,
+ scopes=None,
+ client_cert_source_for_mtls=None,
+ quota_project_id=None,
+ client_info=transports.base.DEFAULT_CLIENT_INFO,
+ always_use_jwt_access=True,
+ )
-def test_list_operations_rest(
- transport: str = "rest", request_type=operations_pb2.ListOperationsRequest
-):
- client = _get_operations_client()
+def test_list_operations_rest():
+ client = _get_operations_client(is_async=False)
# Mock the http request call within the method and fake a response.
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
# Designate an appropriate value for the returned response.
return_value = operations_pb2.ListOperationsResponse(
next_page_token="next_page_token_value",
@@ -477,10 +577,49 @@ def test_list_operations_rest(
assert response.next_page_token == "next_page_token_value"
+@pytest.mark.asyncio
+async def test_list_operations_rest_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+
+ client = _get_operations_client(is_async=True)
+ # Mock the http request call within the method and fake a response.
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ # Designate an appropriate value for the returned response.
+ return_value = operations_pb2.ListOperationsResponse(
+ next_page_token="next_page_token_value",
+ )
+
+ # Wrap the value into a proper Response obj
+ response_value = mock.Mock()
+ response_value.status_code = 200
+ json_return_value = json_format.MessageToJson(return_value)
+ response_value.read = mock.AsyncMock(
+ return_value=json_return_value.encode("UTF-8")
+ )
+ req.return_value = response_value
+ response = await client.list_operations(
+ name="operations", filter_="my_filter", page_size=10, page_token="abc"
+ )
+
+ actual_args = req.call_args
+ assert actual_args.args[0] == "GET"
+ assert actual_args.args[1] == "https://longrunning.googleapis.com/v3/operations"
+ assert actual_args.kwargs["params"] == [
+ ("filter", "my_filter"),
+ ("pageSize", 10),
+ ("pageToken", "abc"),
+ ]
+
+ # Establish that the response is the type that we expect.
+ assert isinstance(response, pagers_async.ListOperationsAsyncPager)
+ assert response.next_page_token == "next_page_token_value"
+
+
def test_list_operations_rest_failure():
- client = _get_operations_client(http_options=None)
+ client = _get_operations_client(is_async=False, http_options=None)
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
response_value = Response()
response_value.status_code = 400
mock_request = mock.MagicMock()
@@ -492,13 +631,31 @@ def test_list_operations_rest_failure():
client.list_operations(name="operations")
+@pytest.mark.asyncio
+async def test_list_operations_rest_failure_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+
+ client = _get_operations_client(is_async=True, http_options=None)
+
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ response_value = mock.Mock()
+ response_value.status_code = 400
+ response_value.read = mock.AsyncMock(return_value=b"{}")
+ mock_request = mock.MagicMock()
+ mock_request.method = "GET"
+ mock_request.url = "https://longrunning.googleapis.com:443/v1/operations"
+ response_value.request = mock_request
+ req.return_value = response_value
+ with pytest.raises(core_exceptions.GoogleAPIError):
+ await client.list_operations(name="operations")
+
+
def test_list_operations_rest_pager():
- client = AbstractOperationsClient(
- credentials=ga_credentials.AnonymousCredentials(),
- )
+ client = _get_operations_client(is_async=False, http_options=None)
# Mock the http request call within the method and fake a response.
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
# TODO(kbandes): remove this mock unless there's a good reason for it.
# with mock.patch.object(path_template, 'transcode') as transcode:
# Set the response as a series of pages
@@ -545,13 +702,80 @@ def test_list_operations_rest_pager():
assert page_.next_page_token == token
-def test_get_operation_rest(
- transport: str = "rest", request_type=operations_pb2.GetOperationRequest
-):
- client = _get_operations_client()
+@pytest.mark.asyncio
+async def test_list_operations_rest_pager_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+ client = _get_operations_client(is_async=True, http_options=None)
# Mock the http request call within the method and fake a response.
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ # TODO(kbandes): remove this mock unless there's a good reason for it.
+ # with mock.patch.object(path_template, 'transcode') as transcode:
+ # Set the response as a series of pages
+ response = (
+ operations_pb2.ListOperationsResponse(
+ operations=[
+ operations_pb2.Operation(),
+ operations_pb2.Operation(),
+ operations_pb2.Operation(),
+ ],
+ next_page_token="abc",
+ ),
+ operations_pb2.ListOperationsResponse(
+ operations=[],
+ next_page_token="def",
+ ),
+ operations_pb2.ListOperationsResponse(
+ operations=[operations_pb2.Operation()],
+ next_page_token="ghi",
+ ),
+ operations_pb2.ListOperationsResponse(
+ operations=[operations_pb2.Operation(), operations_pb2.Operation()],
+ ),
+ )
+ # Two responses for two calls
+ response = response + response
+
+ # Wrap the values into proper Response objs
+ response = tuple(json_format.MessageToJson(x) for x in response)
+ return_values = tuple(mock.Mock() for i in response)
+ for return_val, response_val in zip(return_values, response):
+ return_val.read = mock.AsyncMock(return_value=response_val.encode("UTF-8"))
+ return_val.status_code = 200
+ req.side_effect = return_values
+
+ pager = await client.list_operations(name="operations")
+
+ responses = []
+ async for response in pager:
+ responses.append(response)
+
+ results = list(responses)
+ assert len(results) == 6
+ assert all(isinstance(i, operations_pb2.Operation) for i in results)
+ pager = await client.list_operations(name="operations")
+
+ responses = []
+ async for response in pager:
+ responses.append(response)
+
+ assert len(responses) == 6
+ assert all(isinstance(i, operations_pb2.Operation) for i in results)
+
+ pages = []
+
+ async for page in pager.pages:
+ pages.append(page)
+ for page_, token in zip(pages, ["", "", "", "abc", "def", "ghi", ""]):
+ assert page_.next_page_token == token
+
+
+def test_get_operation_rest():
+ client = _get_operations_client(is_async=False)
+
+ # Mock the http request call within the method and fake a response.
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
# Designate an appropriate value for the returned response.
return_value = operations_pb2.Operation(
name="operations/sample1",
@@ -580,10 +804,46 @@ def test_get_operation_rest(
assert response.done is True
+@pytest.mark.asyncio
+async def test_get_operation_rest_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+ client = _get_operations_client(is_async=True)
+
+ # Mock the http request call within the method and fake a response.
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ # Designate an appropriate value for the returned response.
+ return_value = operations_pb2.Operation(
+ name="operations/sample1",
+ done=True,
+ error=status_pb2.Status(code=411),
+ )
+
+ # Wrap the value into a proper Response obj
+ response_value = mock.Mock()
+ response_value.status_code = 200
+ json_return_value = json_format.MessageToJson(return_value)
+ response_value.read = mock.AsyncMock(return_value=json_return_value)
+ req.return_value = response_value
+ response = await client.get_operation("operations/sample1")
+
+ actual_args = req.call_args
+ assert actual_args.args[0] == "GET"
+ assert (
+ actual_args.args[1]
+ == "https://longrunning.googleapis.com/v3/operations/sample1"
+ )
+
+ # Establish that the response is the type that we expect.
+ assert isinstance(response, operations_pb2.Operation)
+ assert response.name == "operations/sample1"
+ assert response.done is True
+
+
def test_get_operation_rest_failure():
- client = _get_operations_client(http_options=None)
+ client = _get_operations_client(is_async=False, http_options=None)
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
response_value = Response()
response_value.status_code = 400
mock_request = mock.MagicMock()
@@ -595,13 +855,30 @@ def test_get_operation_rest_failure():
client.get_operation("sample0/operations/sample1")
-def test_delete_operation_rest(
- transport: str = "rest", request_type=operations_pb2.DeleteOperationRequest
-):
- client = _get_operations_client()
+@pytest.mark.asyncio
+async def test_get_operation_rest_failure_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+ client = _get_operations_client(is_async=True, http_options=None)
+
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ response_value = mock.Mock()
+ response_value.status_code = 400
+ response_value.read = mock.AsyncMock(return_value=b"{}")
+ mock_request = mock.MagicMock()
+ mock_request.method = "GET"
+ mock_request.url = "https://longrunning.googleapis.com/v1/operations/sample1"
+ response_value.request = mock_request
+ req.return_value = response_value
+ with pytest.raises(core_exceptions.GoogleAPIError):
+ await client.get_operation("sample0/operations/sample1")
+
+
+def test_delete_operation_rest():
+ client = _get_operations_client(is_async=False)
# Mock the http request call within the method and fake a response.
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
# Wrap the value into a proper Response obj
response_value = Response()
response_value.status_code = 200
@@ -618,10 +895,36 @@ def test_delete_operation_rest(
)
+@pytest.mark.asyncio
+async def test_delete_operation_rest_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+ client = _get_operations_client(is_async=True)
+
+ # Mock the http request call within the method and fake a response.
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ # Wrap the value into a proper Response obj
+ response_value = mock.Mock()
+ response_value.status_code = 200
+ json_return_value = ""
+ response_value.read = mock.AsyncMock(
+ return_value=json_return_value.encode("UTF-8")
+ )
+ req.return_value = response_value
+ await client.delete_operation(name="operations/sample1")
+ assert req.call_count == 1
+ actual_args = req.call_args
+ assert actual_args.args[0] == "DELETE"
+ assert (
+ actual_args.args[1]
+ == "https://longrunning.googleapis.com/v3/operations/sample1"
+ )
+
+
def test_delete_operation_rest_failure():
- client = _get_operations_client(http_options=None)
+ client = _get_operations_client(is_async=False, http_options=None)
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
response_value = Response()
response_value.status_code = 400
mock_request = mock.MagicMock()
@@ -633,11 +936,30 @@ def test_delete_operation_rest_failure():
client.delete_operation(name="sample0/operations/sample1")
-def test_cancel_operation_rest(transport: str = "rest"):
- client = _get_operations_client()
+@pytest.mark.asyncio
+async def test_delete_operation_rest_failure_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+ client = _get_operations_client(is_async=True, http_options=None)
+
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ response_value = mock.Mock()
+ response_value.status_code = 400
+ response_value.read = mock.AsyncMock(return_value=b"{}")
+ mock_request = mock.MagicMock()
+ mock_request.method = "DELETE"
+ mock_request.url = "https://longrunning.googleapis.com/v1/operations/sample1"
+ response_value.request = mock_request
+ req.return_value = response_value
+ with pytest.raises(core_exceptions.GoogleAPIError):
+ await client.delete_operation(name="sample0/operations/sample1")
+
+
+def test_cancel_operation_rest():
+ client = _get_operations_client(is_async=False)
# Mock the http request call within the method and fake a response.
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
# Wrap the value into a proper Response obj
response_value = Response()
response_value.status_code = 200
@@ -654,10 +976,36 @@ def test_cancel_operation_rest(transport: str = "rest"):
)
+@pytest.mark.asyncio
+async def test_cancel_operation_rest_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+ client = _get_operations_client(is_async=True)
+
+ # Mock the http request call within the method and fake a response.
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ # Wrap the value into a proper Response obj
+ response_value = mock.Mock()
+ response_value.status_code = 200
+ json_return_value = ""
+ response_value.read = mock.AsyncMock(
+ return_value=json_return_value.encode("UTF-8")
+ )
+ req.return_value = response_value
+ await client.cancel_operation(name="operations/sample1")
+ assert req.call_count == 1
+ actual_args = req.call_args
+ assert actual_args.args[0] == "POST"
+ assert (
+ actual_args.args[1]
+ == "https://longrunning.googleapis.com/v3/operations/sample1:cancel"
+ )
+
+
def test_cancel_operation_rest_failure():
- client = _get_operations_client(http_options=None)
+ client = _get_operations_client(is_async=False, http_options=None)
- with mock.patch.object(Session, "request") as req:
+ with mock.patch.object(_get_session_type(is_async=False), "request") as req:
response_value = Response()
response_value.status_code = 400
mock_request = mock.MagicMock()
@@ -671,52 +1019,79 @@ def test_cancel_operation_rest_failure():
client.cancel_operation(name="sample0/operations/sample1")
-def test_credentials_transport_error():
+@pytest.mark.asyncio
+async def test_cancel_operation_rest_failure_async():
+ if not GOOGLE_AUTH_AIO_INSTALLED:
+ pytest.skip("Skipped because google-api-core[async_rest] is not installed")
+ client = _get_operations_client(is_async=True, http_options=None)
+
+ with mock.patch.object(_get_session_type(is_async=True), "request") as req:
+ response_value = mock.Mock()
+ response_value.status_code = 400
+ response_value.read = mock.AsyncMock(return_value=b"{}")
+ mock_request = mock.MagicMock()
+ mock_request.method = "POST"
+ mock_request.url = (
+ "https://longrunning.googleapis.com/v1/operations/sample1:cancel"
+ )
+ response_value.request = mock_request
+ req.return_value = response_value
+ with pytest.raises(core_exceptions.GoogleAPIError):
+ await client.cancel_operation(name="sample0/operations/sample1")
+
+
+@pytest.mark.parametrize(
+ "client_class,transport_class,credentials",
+ PYPARAM_CLIENT_TRANSPORT_CREDENTIALS,
+)
+def test_credentials_transport_error(client_class, transport_class, credentials):
+
# It is an error to provide credentials and a transport instance.
- transport = transports.OperationsRestTransport(
- credentials=ga_credentials.AnonymousCredentials(),
- )
+ transport = transport_class(credentials=credentials)
with pytest.raises(ValueError):
- AbstractOperationsClient(
+ client_class(
credentials=ga_credentials.AnonymousCredentials(),
transport=transport,
)
# It is an error to provide a credentials file and a transport instance.
- transport = transports.OperationsRestTransport(
- credentials=ga_credentials.AnonymousCredentials(),
- )
+ transport = transport_class(credentials=credentials)
with pytest.raises(ValueError):
- AbstractOperationsClient(
+ client_class(
client_options={"credentials_file": "credentials.json"},
transport=transport,
)
# It is an error to provide scopes and a transport instance.
- transport = transports.OperationsRestTransport(
- credentials=ga_credentials.AnonymousCredentials(),
- )
+ transport = transport_class(credentials=credentials)
with pytest.raises(ValueError):
- AbstractOperationsClient(
+ client_class(
client_options={"scopes": ["1", "2"]},
transport=transport,
)
-def test_transport_instance():
+@pytest.mark.parametrize(
+ "client_class,transport_class,credentials",
+ PYPARAM_CLIENT_TRANSPORT_CREDENTIALS,
+)
+def test_transport_instance(client_class, transport_class, credentials):
# A client may be instantiated with a custom transport instance.
- transport = transports.OperationsRestTransport(
- credentials=ga_credentials.AnonymousCredentials(),
+ transport = transport_class(
+ credentials=credentials,
)
- client = AbstractOperationsClient(transport=transport)
+ client = client_class(transport=transport)
assert client.transport is transport
-@pytest.mark.parametrize("transport_class", [transports.OperationsRestTransport])
-def test_transport_adc(transport_class):
+@pytest.mark.parametrize(
+ "client_class,transport_class,credentials",
+ PYPARAM_CLIENT_TRANSPORT_CREDENTIALS,
+)
+def test_transport_adc(client_class, transport_class, credentials):
# Test default credentials are used if not provided.
with mock.patch.object(google.auth, "default") as adc:
- adc.return_value = (ga_credentials.AnonymousCredentials(), None)
+ adc.return_value = (credentials, None)
transport_class()
adc.assert_called_once()
@@ -788,32 +1163,59 @@ def test_operations_base_transport_with_adc():
adc.assert_called_once()
-def test_operations_auth_adc():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_operations_auth_adc(client_class):
# If no credentials are provided, we should use ADC credentials.
with mock.patch.object(google.auth, "default", autospec=True) as adc:
adc.return_value = (ga_credentials.AnonymousCredentials(), None)
- AbstractOperationsClient()
- adc.assert_called_once_with(
- scopes=None,
- default_scopes=(),
- quota_project_id=None,
- )
+
+ if "async" in str(client_class).lower():
+ # TODO(): Add support for adc to async REST transport.
+ # NOTE: Ideally, the logic for adc shouldn't be called if transport
+ # is set to async REST. If the user does not configure credentials
+ # of type `google.auth.aio.credentials.Credentials`,
+ # we should raise an exception to avoid the adc workflow.
+ with pytest.raises(google.auth.exceptions.InvalidType):
+ client_class()
+ else:
+ client_class()
+ adc.assert_called_once_with(
+ scopes=None,
+ default_scopes=(),
+ quota_project_id=None,
+ )
-def test_operations_http_transport_client_cert_source_for_mtls():
+# TODO(https://github.com/googleapis/python-api-core/issues/705): Add
+# testing for `transports.AsyncOperationsRestTransport` once MTLS is supported
+# in `google.auth.aio.transport`.
+@pytest.mark.parametrize(
+ "transport_class",
+ [
+ transports.OperationsRestTransport,
+ ],
+)
+def test_operations_http_transport_client_cert_source_for_mtls(transport_class):
cred = ga_credentials.AnonymousCredentials()
with mock.patch(
"google.auth.transport.requests.AuthorizedSession.configure_mtls_channel"
) as mock_configure_mtls_channel:
- transports.OperationsRestTransport(
+ transport_class(
credentials=cred, client_cert_source_for_mtls=client_cert_source_callback
)
mock_configure_mtls_channel.assert_called_once_with(client_cert_source_callback)
-def test_operations_host_no_port():
- client = AbstractOperationsClient(
- credentials=ga_credentials.AnonymousCredentials(),
+@pytest.mark.parametrize(
+ "client_class,transport_class,credentials",
+ PYPARAM_CLIENT_TRANSPORT_CREDENTIALS,
+)
+def test_operations_host_no_port(client_class, transport_class, credentials):
+ client = client_class(
+ credentials=credentials,
client_options=client_options.ClientOptions(
api_endpoint="longrunning.googleapis.com"
),
@@ -821,9 +1223,13 @@ def test_operations_host_no_port():
assert client.transport._host == "https://longrunning.googleapis.com"
-def test_operations_host_with_port():
- client = AbstractOperationsClient(
- credentials=ga_credentials.AnonymousCredentials(),
+@pytest.mark.parametrize(
+ "client_class,transport_class,credentials",
+ PYPARAM_CLIENT_TRANSPORT_CREDENTIALS,
+)
+def test_operations_host_with_port(client_class, transport_class, credentials):
+ client = client_class(
+ credentials=credentials,
client_options=client_options.ClientOptions(
api_endpoint="longrunning.googleapis.com:8000"
),
@@ -831,127 +1237,165 @@ def test_operations_host_with_port():
assert client.transport._host == "https://longrunning.googleapis.com:8000"
-def test_common_billing_account_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_common_billing_account_path(client_class):
billing_account = "squid"
expected = "billingAccounts/{billing_account}".format(
billing_account=billing_account,
)
- actual = AbstractOperationsClient.common_billing_account_path(billing_account)
+ actual = client_class.common_billing_account_path(billing_account)
assert expected == actual
-def test_parse_common_billing_account_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_parse_common_billing_account_path(client_class):
expected = {
"billing_account": "clam",
}
- path = AbstractOperationsClient.common_billing_account_path(**expected)
+ path = client_class.common_billing_account_path(**expected)
# Check that the path construction is reversible.
- actual = AbstractOperationsClient.parse_common_billing_account_path(path)
+ actual = client_class.parse_common_billing_account_path(path)
assert expected == actual
-def test_common_folder_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_common_folder_path(client_class):
folder = "whelk"
expected = "folders/{folder}".format(
folder=folder,
)
- actual = AbstractOperationsClient.common_folder_path(folder)
+ actual = client_class.common_folder_path(folder)
assert expected == actual
-def test_parse_common_folder_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_parse_common_folder_path(client_class):
expected = {
"folder": "octopus",
}
- path = AbstractOperationsClient.common_folder_path(**expected)
+ path = client_class.common_folder_path(**expected)
# Check that the path construction is reversible.
- actual = AbstractOperationsClient.parse_common_folder_path(path)
+ actual = client_class.parse_common_folder_path(path)
assert expected == actual
-def test_common_organization_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_common_organization_path(client_class):
organization = "oyster"
expected = "organizations/{organization}".format(
organization=organization,
)
- actual = AbstractOperationsClient.common_organization_path(organization)
+ actual = client_class.common_organization_path(organization)
assert expected == actual
-def test_parse_common_organization_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_parse_common_organization_path(client_class):
expected = {
"organization": "nudibranch",
}
- path = AbstractOperationsClient.common_organization_path(**expected)
+ path = client_class.common_organization_path(**expected)
# Check that the path construction is reversible.
- actual = AbstractOperationsClient.parse_common_organization_path(path)
+ actual = client_class.parse_common_organization_path(path)
assert expected == actual
-def test_common_project_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_common_project_path(client_class):
project = "cuttlefish"
expected = "projects/{project}".format(
project=project,
)
- actual = AbstractOperationsClient.common_project_path(project)
+ actual = client_class.common_project_path(project)
assert expected == actual
-def test_parse_common_project_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_parse_common_project_path(client_class):
expected = {
"project": "mussel",
}
- path = AbstractOperationsClient.common_project_path(**expected)
+ path = client_class.common_project_path(**expected)
# Check that the path construction is reversible.
- actual = AbstractOperationsClient.parse_common_project_path(path)
+ actual = client_class.parse_common_project_path(path)
assert expected == actual
-def test_common_location_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_common_location_path(client_class):
project = "winkle"
location = "nautilus"
expected = "projects/{project}/locations/{location}".format(
project=project,
location=location,
)
- actual = AbstractOperationsClient.common_location_path(project, location)
+ actual = client_class.common_location_path(project, location)
assert expected == actual
-def test_parse_common_location_path():
+@pytest.mark.parametrize(
+ "client_class",
+ PYPARAM_CLIENT,
+)
+def test_parse_common_location_path(client_class):
expected = {
"project": "scallop",
"location": "abalone",
}
- path = AbstractOperationsClient.common_location_path(**expected)
+ path = client_class.common_location_path(**expected)
# Check that the path construction is reversible.
- actual = AbstractOperationsClient.parse_common_location_path(path)
+ actual = client_class.parse_common_location_path(path)
assert expected == actual
-def test_client_withDEFAULT_CLIENT_INFO():
+@pytest.mark.parametrize(
+ "client_class,transport_class,credentials",
+ PYPARAM_CLIENT_TRANSPORT_CREDENTIALS,
+)
+def test_client_withDEFAULT_CLIENT_INFO(client_class, transport_class, credentials):
client_info = gapic_v1.client_info.ClientInfo()
-
- with mock.patch.object(
- transports.OperationsTransport, "_prep_wrapped_messages"
- ) as prep:
- AbstractOperationsClient(
- credentials=ga_credentials.AnonymousCredentials(),
+ with mock.patch.object(transport_class, "_prep_wrapped_messages") as prep:
+ client_class(
+ credentials=credentials,
client_info=client_info,
)
prep.assert_called_once_with(client_info)
- with mock.patch.object(
- transports.OperationsTransport, "_prep_wrapped_messages"
- ) as prep:
- transport_class = AbstractOperationsClient.get_transport_class()
+ with mock.patch.object(transport_class, "_prep_wrapped_messages") as prep:
transport_class(
- credentials=ga_credentials.AnonymousCredentials(),
+ credentials=credentials,
client_info=client_info,
)
prep.assert_called_once_with(client_info)
diff --git a/tests/unit/retry/__init__.py b/tests/unit/retry/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/unit/retry/test_retry_base.py b/tests/unit/retry/test_retry_base.py
new file mode 100644
index 00000000..212c4293
--- /dev/null
+++ b/tests/unit/retry/test_retry_base.py
@@ -0,0 +1,293 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import itertools
+import re
+from unittest import mock
+
+import pytest
+import requests.exceptions
+
+from google.api_core import exceptions
+from google.api_core import retry
+from google.auth import exceptions as auth_exceptions
+
+
+def test_if_exception_type():
+ predicate = retry.if_exception_type(ValueError)
+
+ assert predicate(ValueError())
+ assert not predicate(TypeError())
+
+
+def test_if_exception_type_multiple():
+ predicate = retry.if_exception_type(ValueError, TypeError)
+
+ assert predicate(ValueError())
+ assert predicate(TypeError())
+ assert not predicate(RuntimeError())
+
+
+def test_if_transient_error():
+ assert retry.if_transient_error(exceptions.InternalServerError(""))
+ assert retry.if_transient_error(exceptions.TooManyRequests(""))
+ assert retry.if_transient_error(exceptions.ServiceUnavailable(""))
+ assert retry.if_transient_error(requests.exceptions.ConnectionError(""))
+ assert retry.if_transient_error(requests.exceptions.ChunkedEncodingError(""))
+ assert retry.if_transient_error(auth_exceptions.TransportError(""))
+ assert not retry.if_transient_error(exceptions.InvalidArgument(""))
+
+
+# Make uniform return half of its maximum, which will be the calculated
+# sleep time.
+@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n)
+def test_exponential_sleep_generator_base_2(uniform):
+ gen = retry.exponential_sleep_generator(1, 60, multiplier=2)
+
+ result = list(itertools.islice(gen, 8))
+ assert result == [1, 2, 4, 8, 16, 32, 60, 60]
+
+
+def test_build_retry_error_empty_list():
+ """
+ attempt to build a retry error with no errors encountered
+ should return a generic RetryError
+ """
+ from google.api_core.retry import build_retry_error
+ from google.api_core.retry import RetryFailureReason
+
+ reason = RetryFailureReason.NON_RETRYABLE_ERROR
+ src, cause = build_retry_error([], reason, 10)
+ assert isinstance(src, exceptions.RetryError)
+ assert cause is None
+ assert src.message == "Unknown error"
+
+
+def test_build_retry_error_timeout_message():
+ """
+ should provide helpful error message when timeout is reached
+ """
+ from google.api_core.retry import build_retry_error
+ from google.api_core.retry import RetryFailureReason
+
+ reason = RetryFailureReason.TIMEOUT
+ cause = RuntimeError("timeout")
+ src, found_cause = build_retry_error([ValueError(), cause], reason, 10)
+ assert isinstance(src, exceptions.RetryError)
+ assert src.message == "Timeout of 10.0s exceeded"
+ # should attach appropriate cause
+ assert found_cause is cause
+
+
+def test_build_retry_error_empty_timeout():
+ """
+ attempt to build a retry error when timeout is None
+ should return a generic timeout error message
+ """
+ from google.api_core.retry import build_retry_error
+ from google.api_core.retry import RetryFailureReason
+
+ reason = RetryFailureReason.TIMEOUT
+ src, _ = build_retry_error([], reason, None)
+ assert isinstance(src, exceptions.RetryError)
+ assert src.message == "Timeout exceeded"
+
+
+class Test_BaseRetry(object):
+ def _make_one(self, *args, **kwargs):
+ return retry.retry_base._BaseRetry(*args, **kwargs)
+
+ def test_constructor_defaults(self):
+ retry_ = self._make_one()
+ assert retry_._predicate == retry.if_transient_error
+ assert retry_._initial == 1
+ assert retry_._maximum == 60
+ assert retry_._multiplier == 2
+ assert retry_._timeout == 120
+ assert retry_._on_error is None
+ assert retry_.timeout == 120
+ assert retry_.timeout == 120
+
+ def test_constructor_options(self):
+ _some_function = mock.Mock()
+
+ retry_ = self._make_one(
+ predicate=mock.sentinel.predicate,
+ initial=1,
+ maximum=2,
+ multiplier=3,
+ timeout=4,
+ on_error=_some_function,
+ )
+ assert retry_._predicate == mock.sentinel.predicate
+ assert retry_._initial == 1
+ assert retry_._maximum == 2
+ assert retry_._multiplier == 3
+ assert retry_._timeout == 4
+ assert retry_._on_error is _some_function
+
+ @pytest.mark.parametrize("use_deadline", [True, False])
+ @pytest.mark.parametrize("value", [None, 0, 1, 4, 42, 5.5])
+ def test_with_timeout(self, use_deadline, value):
+ retry_ = self._make_one(
+ predicate=mock.sentinel.predicate,
+ initial=1,
+ maximum=2,
+ multiplier=3,
+ timeout=4,
+ on_error=mock.sentinel.on_error,
+ )
+ new_retry = (
+ retry_.with_timeout(value)
+ if not use_deadline
+ else retry_.with_deadline(value)
+ )
+ assert retry_ is not new_retry
+ assert new_retry._timeout == value
+ assert (
+ new_retry.timeout == value
+ if not use_deadline
+ else new_retry.deadline == value
+ )
+
+ # the rest of the attributes should remain the same
+ assert new_retry._predicate is retry_._predicate
+ assert new_retry._initial == retry_._initial
+ assert new_retry._maximum == retry_._maximum
+ assert new_retry._multiplier == retry_._multiplier
+ assert new_retry._on_error is retry_._on_error
+
+ def test_with_predicate(self):
+ retry_ = self._make_one(
+ predicate=mock.sentinel.predicate,
+ initial=1,
+ maximum=2,
+ multiplier=3,
+ timeout=4,
+ on_error=mock.sentinel.on_error,
+ )
+ new_retry = retry_.with_predicate(mock.sentinel.predicate)
+ assert retry_ is not new_retry
+ assert new_retry._predicate == mock.sentinel.predicate
+
+ # the rest of the attributes should remain the same
+ assert new_retry._timeout == retry_._timeout
+ assert new_retry._initial == retry_._initial
+ assert new_retry._maximum == retry_._maximum
+ assert new_retry._multiplier == retry_._multiplier
+ assert new_retry._on_error is retry_._on_error
+
+ def test_with_delay_noop(self):
+ retry_ = self._make_one(
+ predicate=mock.sentinel.predicate,
+ initial=1,
+ maximum=2,
+ multiplier=3,
+ timeout=4,
+ on_error=mock.sentinel.on_error,
+ )
+ new_retry = retry_.with_delay()
+ assert retry_ is not new_retry
+ assert new_retry._initial == retry_._initial
+ assert new_retry._maximum == retry_._maximum
+ assert new_retry._multiplier == retry_._multiplier
+
+ @pytest.mark.parametrize(
+ "originals,updated,expected",
+ [
+ [(1, 2, 3), (4, 5, 6), (4, 5, 6)],
+ [(1, 2, 3), (0, 0, 0), (0, 0, 0)],
+ [(1, 2, 3), (None, None, None), (1, 2, 3)],
+ [(0, 0, 0), (None, None, None), (0, 0, 0)],
+ [(1, 2, 3), (None, 0.5, None), (1, 0.5, 3)],
+ [(1, 2, 3), (None, 0.5, 4), (1, 0.5, 4)],
+ [(1, 2, 3), (9, None, None), (9, 2, 3)],
+ ],
+ )
+ def test_with_delay(self, originals, updated, expected):
+ retry_ = self._make_one(
+ predicate=mock.sentinel.predicate,
+ initial=originals[0],
+ maximum=originals[1],
+ multiplier=originals[2],
+ timeout=14,
+ on_error=mock.sentinel.on_error,
+ )
+ new_retry = retry_.with_delay(
+ initial=updated[0], maximum=updated[1], multiplier=updated[2]
+ )
+ assert retry_ is not new_retry
+ assert new_retry._initial == expected[0]
+ assert new_retry._maximum == expected[1]
+ assert new_retry._multiplier == expected[2]
+
+ # the rest of the attributes should remain the same
+ assert new_retry._timeout == retry_._timeout
+ assert new_retry._predicate is retry_._predicate
+ assert new_retry._on_error is retry_._on_error
+
+ def test_with_delay_partial_options(self):
+ retry_ = self._make_one(
+ predicate=mock.sentinel.predicate,
+ initial=1,
+ maximum=2,
+ multiplier=3,
+ timeout=4,
+ on_error=mock.sentinel.on_error,
+ )
+ new_retry = retry_.with_delay(initial=4)
+ assert retry_ is not new_retry
+ assert new_retry._initial == 4
+ assert new_retry._maximum == 2
+ assert new_retry._multiplier == 3
+
+ new_retry = retry_.with_delay(maximum=4)
+ assert retry_ is not new_retry
+ assert new_retry._initial == 1
+ assert new_retry._maximum == 4
+ assert new_retry._multiplier == 3
+
+ new_retry = retry_.with_delay(multiplier=4)
+ assert retry_ is not new_retry
+ assert new_retry._initial == 1
+ assert new_retry._maximum == 2
+ assert new_retry._multiplier == 4
+
+ # the rest of the attributes should remain the same
+ assert new_retry._timeout == retry_._timeout
+ assert new_retry._predicate is retry_._predicate
+ assert new_retry._on_error is retry_._on_error
+
+ def test___str__(self):
+ def if_exception_type(exc):
+ return bool(exc) # pragma: NO COVER
+
+ # Explicitly set all attributes as changed Retry defaults should not
+ # cause this test to start failing.
+ retry_ = self._make_one(
+ predicate=if_exception_type,
+ initial=1.0,
+ maximum=60.0,
+ multiplier=2.0,
+ timeout=120.0,
+ on_error=None,
+ )
+ assert re.match(
+ (
+ r"<_BaseRetry predicate=, "
+ r"initial=1.0, maximum=60.0, multiplier=2.0, timeout=120.0, "
+ r"on_error=None>"
+ ),
+ str(retry_),
+ )
diff --git a/tests/unit/retry/test_retry_imports.py b/tests/unit/retry/test_retry_imports.py
new file mode 100644
index 00000000..597909fc
--- /dev/null
+++ b/tests/unit/retry/test_retry_imports.py
@@ -0,0 +1,33 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def test_legacy_imports_retry_unary_sync():
+ # TODO: Delete this test when when we revert these imports on the
+ # next major version release
+ # (https://github.com/googleapis/python-api-core/issues/576)
+ from google.api_core.retry import datetime_helpers # noqa: F401
+ from google.api_core.retry import exceptions # noqa: F401
+ from google.api_core.retry import auth_exceptions # noqa: F401
+
+
+def test_legacy_imports_retry_unary_async():
+ # TODO: Delete this test when when we revert these imports on the
+ # next major version release
+ # (https://github.com/googleapis/python-api-core/issues/576)
+ from google.api_core import retry_async # noqa: F401
+
+ # See https://github.com/googleapis/python-api-core/issues/586
+ # for context on why we need to test this import this explicitly.
+ from google.api_core.retry_async import AsyncRetry # noqa: F401
diff --git a/tests/unit/retry/test_retry_streaming.py b/tests/unit/retry/test_retry_streaming.py
new file mode 100644
index 00000000..2499b2ae
--- /dev/null
+++ b/tests/unit/retry/test_retry_streaming.py
@@ -0,0 +1,505 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
+
+import pytest
+
+from google.api_core import exceptions
+from google.api_core import retry
+from google.api_core.retry import retry_streaming
+
+from .test_retry_base import Test_BaseRetry
+
+
+def test_retry_streaming_target_bad_sleep_generator():
+ with pytest.raises(
+ ValueError, match="Sleep generator stopped yielding sleep values"
+ ):
+ next(retry_streaming.retry_target_stream(None, lambda x: True, [], None))
+
+
+@mock.patch("time.sleep", autospec=True)
+def test_retry_streaming_target_dynamic_backoff(sleep):
+ """
+ sleep_generator should be iterated after on_error, to support dynamic backoff
+ """
+ from functools import partial
+
+ sleep.side_effect = RuntimeError("stop after sleep")
+ # start with empty sleep generator; values are added after exception in push_sleep_value
+ sleep_values = []
+ error_target = partial(TestStreamingRetry._generator_mock, error_on=0)
+ inserted_sleep = 99
+
+ def push_sleep_value(err):
+ sleep_values.append(inserted_sleep)
+
+ with pytest.raises(RuntimeError):
+ next(
+ retry_streaming.retry_target_stream(
+ error_target,
+ predicate=lambda x: True,
+ sleep_generator=sleep_values,
+ on_error=push_sleep_value,
+ )
+ )
+ assert sleep.call_count == 1
+ sleep.assert_called_once_with(inserted_sleep)
+
+
+class TestStreamingRetry(Test_BaseRetry):
+ def _make_one(self, *args, **kwargs):
+ return retry_streaming.StreamingRetry(*args, **kwargs)
+
+ def test___str__(self):
+ def if_exception_type(exc):
+ return bool(exc) # pragma: NO COVER
+
+ # Explicitly set all attributes as changed Retry defaults should not
+ # cause this test to start failing.
+ retry_ = retry_streaming.StreamingRetry(
+ predicate=if_exception_type,
+ initial=1.0,
+ maximum=60.0,
+ multiplier=2.0,
+ timeout=120.0,
+ on_error=None,
+ )
+ assert re.match(
+ (
+ r", "
+ r"initial=1.0, maximum=60.0, multiplier=2.0, timeout=120.0, "
+ r"on_error=None>"
+ ),
+ str(retry_),
+ )
+
+ @staticmethod
+ def _generator_mock(
+ num=5,
+ error_on=None,
+ return_val=None,
+ exceptions_seen=None,
+ ):
+ """
+ Helper to create a mock generator that yields a number of values
+ Generator can optionally raise an exception on a specific iteration
+
+ Args:
+ - num (int): the number of values to yield. After this, the generator will return `return_val`
+ - error_on (int): if given, the generator will raise a ValueError on the specified iteration
+ - return_val (any): if given, the generator will return this value after yielding num values
+ - exceptions_seen (list): if given, the generator will append any exceptions to this list before raising
+ """
+ try:
+ for i in range(num):
+ if error_on is not None and i == error_on:
+ raise ValueError("generator mock error")
+ yield i
+ return return_val
+ except (Exception, BaseException, GeneratorExit) as e:
+ # keep track of exceptions seen by generator
+ if exceptions_seen is not None:
+ exceptions_seen.append(e)
+ raise
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___success(self, sleep):
+ """
+ Test that a retry-decorated generator yields values as expected
+ This test checks a generator with no issues
+ """
+ import types
+ import collections
+
+ retry_ = retry_streaming.StreamingRetry()
+
+ decorated = retry_(self._generator_mock)
+
+ num = 10
+ result = decorated(num)
+ # check types
+ assert isinstance(decorated(num), collections.abc.Iterable)
+ assert isinstance(decorated(num), types.GeneratorType)
+ assert isinstance(self._generator_mock(num), collections.abc.Iterable)
+ assert isinstance(self._generator_mock(num), types.GeneratorType)
+ # check yield contents
+ unpacked = [i for i in result]
+ assert len(unpacked) == num
+ for a, b in zip(unpacked, self._generator_mock(num)):
+ assert a == b
+ sleep.assert_not_called()
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___retry(self, sleep):
+ """
+ Tests that a retry-decorated generator will retry on errors
+ """
+ on_error = mock.Mock(return_value=None)
+ retry_ = retry_streaming.StreamingRetry(
+ on_error=on_error,
+ predicate=retry.if_exception_type(ValueError),
+ timeout=None,
+ )
+ result = retry_(self._generator_mock)(error_on=3)
+ # error thrown on 3
+ # generator should contain 0, 1, 2 looping
+ unpacked = [next(result) for i in range(10)]
+ assert unpacked == [0, 1, 2, 0, 1, 2, 0, 1, 2, 0]
+ assert on_error.call_count == 3
+
+ @mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n)
+ @mock.patch("time.sleep", autospec=True)
+ @pytest.mark.parametrize("use_deadline_arg", [True, False])
+ def test___call___retry_hitting_timeout(self, sleep, uniform, use_deadline_arg):
+ """
+ Tests that a retry-decorated generator will throw a RetryError
+ after using the time budget
+ """
+ import time
+
+ timeout_val = 30.9
+ # support "deadline" as an alias for "timeout"
+ timeout_kwarg = (
+ {"timeout": timeout_val}
+ if not use_deadline_arg
+ else {"deadline": timeout_val}
+ )
+
+ on_error = mock.Mock(return_value=None)
+ retry_ = retry_streaming.StreamingRetry(
+ predicate=retry.if_exception_type(ValueError),
+ initial=1.0,
+ maximum=1024.0,
+ multiplier=2.0,
+ **timeout_kwarg,
+ )
+
+ timenow = time.monotonic()
+ now_patcher = mock.patch(
+ "time.monotonic",
+ return_value=timenow,
+ )
+
+ decorated = retry_(self._generator_mock, on_error=on_error)
+ generator = decorated(error_on=1)
+ with now_patcher as patched_now:
+ # Make sure that calls to fake time.sleep() also advance the mocked
+ # time clock.
+ def increase_time(sleep_delay):
+ patched_now.return_value += sleep_delay
+
+ sleep.side_effect = increase_time
+ with pytest.raises(exceptions.RetryError):
+ [i for i in generator]
+
+ assert on_error.call_count == 5
+ # check the delays
+ assert sleep.call_count == 4 # once between each successive target calls
+ last_wait = sleep.call_args.args[0]
+ total_wait = sum(call_args.args[0] for call_args in sleep.call_args_list)
+ assert last_wait == 8.0
+ assert total_wait == 15.0
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_generator_send(self, sleep):
+ """
+ Send should be passed through retry into target generator
+ """
+
+ def _mock_send_gen():
+ """
+ always yield whatever was sent in
+ """
+ in_ = yield
+ while True:
+ in_ = yield in_
+
+ retry_ = retry_streaming.StreamingRetry()
+
+ decorated = retry_(_mock_send_gen)
+
+ generator = decorated()
+ result = next(generator)
+ # first yield should be None
+ assert result is None
+ in_messages = ["test_1", "hello", "world"]
+ out_messages = []
+ for msg in in_messages:
+ recv = generator.send(msg)
+ out_messages.append(recv)
+ assert in_messages == out_messages
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_generator_send_retry(self, sleep):
+ """
+ Send should support retries like next
+ """
+ on_error = mock.Mock(return_value=None)
+ retry_ = retry_streaming.StreamingRetry(
+ on_error=on_error,
+ predicate=retry.if_exception_type(ValueError),
+ timeout=None,
+ )
+ result = retry_(self._generator_mock)(error_on=3)
+ with pytest.raises(TypeError) as exc_info:
+ # calling first send with non-None input should raise a TypeError
+ result.send("can not send to fresh generator")
+ assert exc_info.match("can't send non-None value")
+ # initiate iteration with None
+ result = retry_(self._generator_mock)(error_on=3)
+ assert result.send(None) == 0
+ # error thrown on 3
+ # generator should contain 0, 1, 2 looping
+ unpacked = [result.send(i) for i in range(10)]
+ assert unpacked == [1, 2, 0, 1, 2, 0, 1, 2, 0, 1]
+ assert on_error.call_count == 3
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_iterable_send(self, sleep):
+ """
+ send should raise attribute error if wrapped iterator does not support it
+ """
+ retry_ = retry_streaming.StreamingRetry()
+
+ def iterable_fn(n):
+ return iter(range(n))
+
+ decorated = retry_(iterable_fn)
+ generator = decorated(5)
+ # initialize
+ next(generator)
+ # call send
+ with pytest.raises(AttributeError):
+ generator.send("test")
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_iterable_close(self, sleep):
+ """
+ close should be handled by wrapper if wrapped iterable does not support it
+ """
+ retry_ = retry_streaming.StreamingRetry()
+
+ def iterable_fn(n):
+ return iter(range(n))
+
+ decorated = retry_(iterable_fn)
+
+ # try closing active generator
+ retryable = decorated(10)
+ assert next(retryable) == 0
+ retryable.close()
+ with pytest.raises(StopIteration):
+ next(retryable)
+
+ # try closing a new generator
+ retryable = decorated(10)
+ retryable.close()
+ with pytest.raises(StopIteration):
+ next(retryable)
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_iterable_throw(self, sleep):
+ """
+ Throw should work even if the wrapped iterable does not support it
+ """
+ predicate = retry.if_exception_type(ValueError)
+ retry_ = retry_streaming.StreamingRetry(predicate=predicate)
+
+ def iterable_fn(n):
+ return iter(range(n))
+
+ decorated = retry_(iterable_fn)
+
+ # try throwing with active generator
+ retryable = decorated(10)
+ assert next(retryable) == 0
+ # should swallow errors in predicate
+ retryable.throw(ValueError)
+ assert next(retryable) == 1
+ # should raise on other errors
+ with pytest.raises(TypeError):
+ retryable.throw(TypeError)
+ with pytest.raises(StopIteration):
+ next(retryable)
+
+ # try throwing with a new generator
+ retryable = decorated(10)
+ with pytest.raises(ValueError):
+ retryable.throw(ValueError)
+ with pytest.raises(StopIteration):
+ next(retryable)
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_generator_return(self, sleep):
+ """
+ Generator return value should be passed through retry decorator
+ """
+ retry_ = retry_streaming.StreamingRetry()
+
+ decorated = retry_(self._generator_mock)
+
+ expected_value = "done"
+ generator = decorated(5, return_val=expected_value)
+ found_value = None
+ try:
+ while True:
+ next(generator)
+ except StopIteration as e:
+ found_value = e.value
+ assert found_value == expected_value
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_generator_close(self, sleep):
+ """
+ Close should be passed through retry into target generator
+ """
+ retry_ = retry_streaming.StreamingRetry()
+
+ decorated = retry_(self._generator_mock)
+
+ exception_list = []
+ generator = decorated(10, exceptions_seen=exception_list)
+ for i in range(2):
+ next(generator)
+ generator.close()
+ assert isinstance(exception_list[0], GeneratorExit)
+ with pytest.raises(StopIteration):
+ # calling next on closed generator should raise error
+ next(generator)
+
+ @mock.patch("time.sleep", autospec=True)
+ def test___call___with_generator_throw(self, sleep):
+ """
+ Throw should be passed through retry into target generator
+ """
+ retry_ = retry_streaming.StreamingRetry(
+ predicate=retry.if_exception_type(ValueError),
+ )
+ decorated = retry_(self._generator_mock)
+
+ exception_list = []
+ generator = decorated(10, exceptions_seen=exception_list)
+ for i in range(2):
+ next(generator)
+ with pytest.raises(BufferError):
+ generator.throw(BufferError("test"))
+ assert isinstance(exception_list[0], BufferError)
+ with pytest.raises(StopIteration):
+ # calling next on closed generator should raise error
+ next(generator)
+ # should retry if throw retryable exception
+ exception_list = []
+ generator = decorated(10, exceptions_seen=exception_list)
+ for i in range(2):
+ next(generator)
+ val = generator.throw(ValueError("test"))
+ assert val == 0
+ assert isinstance(exception_list[0], ValueError)
+ # calling next on closed generator should not raise error
+ assert next(generator) == 1
+
+ def test_exc_factory_non_retryable_error(self):
+ """
+ generator should give the option to override exception creation logic
+ test when non-retryable error is thrown
+ """
+ from google.api_core.retry import RetryFailureReason
+ from google.api_core.retry.retry_streaming import retry_target_stream
+
+ timeout = None
+ sent_errors = [ValueError("test"), ValueError("test2"), BufferError("test3")]
+ expected_final_err = RuntimeError("done")
+ expected_source_err = ZeroDivisionError("test4")
+
+ def factory(*args, **kwargs):
+ assert len(kwargs) == 0
+ assert args[0] == sent_errors
+ assert args[1] == RetryFailureReason.NON_RETRYABLE_ERROR
+ assert args[2] == timeout
+ return expected_final_err, expected_source_err
+
+ generator = retry_target_stream(
+ self._generator_mock,
+ retry.if_exception_type(ValueError),
+ [0] * 3,
+ timeout=timeout,
+ exception_factory=factory,
+ )
+ # initialize generator
+ next(generator)
+ # trigger some retryable errors
+ generator.throw(sent_errors[0])
+ generator.throw(sent_errors[1])
+ # trigger a non-retryable error
+ with pytest.raises(expected_final_err.__class__) as exc_info:
+ generator.throw(sent_errors[2])
+ assert exc_info.value == expected_final_err
+ assert exc_info.value.__cause__ == expected_source_err
+
+ def test_exc_factory_timeout(self):
+ """
+ generator should give the option to override exception creation logic
+ test when timeout is exceeded
+ """
+ import time
+ from google.api_core.retry import RetryFailureReason
+ from google.api_core.retry.retry_streaming import retry_target_stream
+
+ timeout = 2
+ time_now = time.monotonic()
+ now_patcher = mock.patch(
+ "time.monotonic",
+ return_value=time_now,
+ )
+
+ with now_patcher as patched_now:
+ timeout = 2
+ sent_errors = [ValueError("test"), ValueError("test2"), ValueError("test3")]
+ expected_final_err = RuntimeError("done")
+ expected_source_err = ZeroDivisionError("test4")
+
+ def factory(*args, **kwargs):
+ assert len(kwargs) == 0
+ assert args[0] == sent_errors
+ assert args[1] == RetryFailureReason.TIMEOUT
+ assert args[2] == timeout
+ return expected_final_err, expected_source_err
+
+ generator = retry_target_stream(
+ self._generator_mock,
+ retry.if_exception_type(ValueError),
+ [0] * 3,
+ timeout=timeout,
+ exception_factory=factory,
+ check_timeout_on_yield=True,
+ )
+ # initialize generator
+ next(generator)
+ # trigger some retryable errors
+ generator.throw(sent_errors[0])
+ generator.throw(sent_errors[1])
+ # trigger a timeout
+ patched_now.return_value += timeout + 1
+ with pytest.raises(expected_final_err.__class__) as exc_info:
+ generator.throw(sent_errors[2])
+ assert exc_info.value == expected_final_err
+ assert exc_info.value.__cause__ == expected_source_err
diff --git a/tests/unit/test_retry.py b/tests/unit/retry/test_retry_unary.py
similarity index 55%
rename from tests/unit/test_retry.py
rename to tests/unit/retry/test_retry_unary.py
index ec27056d..f5bbcff7 100644
--- a/tests/unit/test_retry.py
+++ b/tests/unit/retry/test_retry_unary.py
@@ -13,51 +13,19 @@
# limitations under the License.
import datetime
-import itertools
+import pytest
import re
-import mock
-import pytest
-import requests.exceptions
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
from google.api_core import exceptions
from google.api_core import retry
-from google.auth import exceptions as auth_exceptions
-
-
-def test_if_exception_type():
- predicate = retry.if_exception_type(ValueError)
-
- assert predicate(ValueError())
- assert not predicate(TypeError())
-
-
-def test_if_exception_type_multiple():
- predicate = retry.if_exception_type(ValueError, TypeError)
-
- assert predicate(ValueError())
- assert predicate(TypeError())
- assert not predicate(RuntimeError())
-
-
-def test_if_transient_error():
- assert retry.if_transient_error(exceptions.InternalServerError(""))
- assert retry.if_transient_error(exceptions.TooManyRequests(""))
- assert retry.if_transient_error(exceptions.ServiceUnavailable(""))
- assert retry.if_transient_error(requests.exceptions.ConnectionError(""))
- assert retry.if_transient_error(requests.exceptions.ChunkedEncodingError(""))
- assert retry.if_transient_error(auth_exceptions.TransportError(""))
- assert not retry.if_transient_error(exceptions.InvalidArgument(""))
-
-# Make uniform return half of its maximum, which will be the calculated
-# sleep time.
-@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n)
-def test_exponential_sleep_generator_base_2(uniform):
- gen = retry.exponential_sleep_generator(1, 60, multiplier=2)
-
- result = list(itertools.islice(gen, 8))
- assert result == [1, 2, 4, 8, 16, 32, 60, 60]
+from .test_retry_base import Test_BaseRetry
@mock.patch("time.sleep", autospec=True)
@@ -129,26 +97,45 @@ def test_retry_target_non_retryable_error(utcnow, sleep):
sleep.assert_not_called()
+@mock.patch("asyncio.sleep", autospec=True)
+@mock.patch(
+ "google.api_core.datetime_helpers.utcnow",
+ return_value=datetime.datetime.min,
+ autospec=True,
+)
+@pytest.mark.asyncio
+async def test_retry_target_warning_for_retry(utcnow, sleep):
+ predicate = retry.if_exception_type(ValueError)
+ target = mock.AsyncMock(spec=["__call__"])
+
+ with pytest.warns(Warning) as exc_info:
+ # Note: predicate is just a filler and doesn't affect the test
+ retry.retry_target(target, predicate, range(10), None)
+
+ assert len(exc_info) == 2
+ assert str(exc_info[0].message) == retry.retry_unary._ASYNC_RETRY_WARNING
+ sleep.assert_not_called()
+
+
@mock.patch("time.sleep", autospec=True)
-@mock.patch("google.api_core.datetime_helpers.utcnow", autospec=True)
-def test_retry_target_deadline_exceeded(utcnow, sleep):
+@mock.patch("time.monotonic", autospec=True)
+@pytest.mark.parametrize("use_deadline_arg", [True, False])
+def test_retry_target_timeout_exceeded(monotonic, sleep, use_deadline_arg):
predicate = retry.if_exception_type(ValueError)
exception = ValueError("meep")
target = mock.Mock(side_effect=exception)
# Setup the timeline so that the first call takes 5 seconds but the second
- # call takes 6, which puts the retry over the deadline.
- utcnow.side_effect = [
- # The first call to utcnow establishes the start of the timeline.
- datetime.datetime.min,
- datetime.datetime.min + datetime.timedelta(seconds=5),
- datetime.datetime.min + datetime.timedelta(seconds=11),
- ]
+ # call takes 6, which puts the retry over the timeout.
+ monotonic.side_effect = [0, 5, 11]
+
+ # support "deadline" as an alias for "timeout"
+ kwargs = {"timeout": 10} if not use_deadline_arg else {"deadline": 10}
with pytest.raises(exceptions.RetryError) as exc_info:
- retry.retry_target(target, predicate, range(10), deadline=10)
+ retry.retry_target(target, predicate, range(10), **kwargs)
assert exc_info.value.cause == exception
- assert exc_info.match("Deadline of 10.0s exceeded")
+ assert exc_info.match("Timeout of 10.0s exceeded")
assert exc_info.match("last exception: meep")
assert target.call_count == 2
@@ -159,145 +146,38 @@ def test_retry_target_deadline_exceeded(utcnow, sleep):
def test_retry_target_bad_sleep_generator():
with pytest.raises(ValueError, match="Sleep generator"):
- retry.retry_target(mock.sentinel.target, mock.sentinel.predicate, [], None)
+ retry.retry_target(mock.sentinel.target, lambda x: True, [], None)
-class TestRetry(object):
- def test_constructor_defaults(self):
- retry_ = retry.Retry()
- assert retry_._predicate == retry.if_transient_error
- assert retry_._initial == 1
- assert retry_._maximum == 60
- assert retry_._multiplier == 2
- assert retry_._deadline == 120
- assert retry_._on_error is None
- assert retry_.deadline == 120
- assert retry_.timeout == 120
-
- def test_constructor_options(self):
- _some_function = mock.Mock()
-
- retry_ = retry.Retry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=_some_function,
+@mock.patch("time.sleep", autospec=True)
+def test_retry_target_dynamic_backoff(sleep):
+ """
+ sleep_generator should be iterated after on_error, to support dynamic backoff
+ """
+ sleep.side_effect = RuntimeError("stop after sleep")
+ # start with empty sleep generator; values are added after exception in push_sleep_value
+ sleep_values = []
+ exception = ValueError("trigger retry")
+ error_target = mock.Mock(side_effect=exception)
+ inserted_sleep = 99
+
+ def push_sleep_value(err):
+ sleep_values.append(inserted_sleep)
+
+ with pytest.raises(RuntimeError):
+ retry.retry_target(
+ error_target,
+ predicate=lambda x: True,
+ sleep_generator=sleep_values,
+ on_error=push_sleep_value,
)
- assert retry_._predicate == mock.sentinel.predicate
- assert retry_._initial == 1
- assert retry_._maximum == 2
- assert retry_._multiplier == 3
- assert retry_._deadline == 4
- assert retry_._on_error is _some_function
+ assert sleep.call_count == 1
+ sleep.assert_called_once_with(inserted_sleep)
- def test_with_deadline(self):
- retry_ = retry.Retry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_deadline(42)
- assert retry_ is not new_retry
- assert new_retry._deadline == 42
-
- # the rest of the attributes should remain the same
- assert new_retry._predicate is retry_._predicate
- assert new_retry._initial == retry_._initial
- assert new_retry._maximum == retry_._maximum
- assert new_retry._multiplier == retry_._multiplier
- assert new_retry._on_error is retry_._on_error
-
- def test_with_predicate(self):
- retry_ = retry.Retry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_predicate(mock.sentinel.predicate)
- assert retry_ is not new_retry
- assert new_retry._predicate == mock.sentinel.predicate
-
- # the rest of the attributes should remain the same
- assert new_retry._deadline == retry_._deadline
- assert new_retry._initial == retry_._initial
- assert new_retry._maximum == retry_._maximum
- assert new_retry._multiplier == retry_._multiplier
- assert new_retry._on_error is retry_._on_error
-
- def test_with_delay_noop(self):
- retry_ = retry.Retry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_delay()
- assert retry_ is not new_retry
- assert new_retry._initial == retry_._initial
- assert new_retry._maximum == retry_._maximum
- assert new_retry._multiplier == retry_._multiplier
- def test_with_delay(self):
- retry_ = retry.Retry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_delay(initial=5, maximum=6, multiplier=7)
- assert retry_ is not new_retry
- assert new_retry._initial == 5
- assert new_retry._maximum == 6
- assert new_retry._multiplier == 7
-
- # the rest of the attributes should remain the same
- assert new_retry._deadline == retry_._deadline
- assert new_retry._predicate is retry_._predicate
- assert new_retry._on_error is retry_._on_error
-
- def test_with_delay_partial_options(self):
- retry_ = retry.Retry(
- predicate=mock.sentinel.predicate,
- initial=1,
- maximum=2,
- multiplier=3,
- deadline=4,
- on_error=mock.sentinel.on_error,
- )
- new_retry = retry_.with_delay(initial=4)
- assert retry_ is not new_retry
- assert new_retry._initial == 4
- assert new_retry._maximum == 2
- assert new_retry._multiplier == 3
-
- new_retry = retry_.with_delay(maximum=4)
- assert retry_ is not new_retry
- assert new_retry._initial == 1
- assert new_retry._maximum == 4
- assert new_retry._multiplier == 3
-
- new_retry = retry_.with_delay(multiplier=4)
- assert retry_ is not new_retry
- assert new_retry._initial == 1
- assert new_retry._maximum == 2
- assert new_retry._multiplier == 4
-
- # the rest of the attributes should remain the same
- assert new_retry._deadline == retry_._deadline
- assert new_retry._predicate is retry_._predicate
- assert new_retry._on_error is retry_._on_error
+class TestRetry(Test_BaseRetry):
+ def _make_one(self, *args, **kwargs):
+ return retry.Retry(*args, **kwargs)
def test___str__(self):
def if_exception_type(exc):
@@ -310,7 +190,7 @@ def if_exception_type(exc):
initial=1.0,
maximum=60.0,
multiplier=2.0,
- deadline=120.0,
+ timeout=120.0,
on_error=None,
)
assert re.match(
@@ -341,7 +221,6 @@ def test___call___and_execute_success(self, sleep):
@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n)
@mock.patch("time.sleep", autospec=True)
def test___call___and_execute_retry(self, sleep, uniform):
-
on_error = mock.Mock(spec=["__call__"], side_effect=[None])
retry_ = retry.Retry(predicate=retry.if_exception_type(ValueError))
@@ -362,21 +241,17 @@ def test___call___and_execute_retry(self, sleep, uniform):
@mock.patch("random.uniform", autospec=True, side_effect=lambda m, n: n)
@mock.patch("time.sleep", autospec=True)
- def test___call___and_execute_retry_hitting_deadline(self, sleep, uniform):
-
+ def test___call___and_execute_retry_hitting_timeout(self, sleep, uniform):
on_error = mock.Mock(spec=["__call__"], side_effect=[None] * 10)
retry_ = retry.Retry(
predicate=retry.if_exception_type(ValueError),
initial=1.0,
maximum=1024.0,
multiplier=2.0,
- deadline=30.9,
+ timeout=30.9,
)
- utcnow = datetime.datetime.utcnow()
- utcnow_patcher = mock.patch(
- "google.api_core.datetime_helpers.utcnow", return_value=utcnow
- )
+ monotonic_patcher = mock.patch("time.monotonic", return_value=0)
target = mock.Mock(spec=["__call__"], side_effect=[ValueError()] * 10)
# __name__ is needed by functools.partial.
@@ -385,11 +260,11 @@ def test___call___and_execute_retry_hitting_deadline(self, sleep, uniform):
decorated = retry_(target, on_error=on_error)
target.assert_not_called()
- with utcnow_patcher as patched_utcnow:
+ with monotonic_patcher as patched_monotonic:
# Make sure that calls to fake time.sleep() also advance the mocked
# time clock.
def increase_time(sleep_delay):
- patched_utcnow.return_value += datetime.timedelta(seconds=sleep_delay)
+ patched_monotonic.return_value += sleep_delay
sleep.side_effect = increase_time
@@ -409,7 +284,7 @@ def increase_time(sleep_delay):
# Next attempt would be scheduled in 16 secs, 15 + 16 = 31 > 30.9, thus
# we do not even wait for it to be scheduled (30.9 is configured timeout).
# This changes the previous logic of shortening the last attempt to fit
- # in the deadline. The previous logic was removed to make Python retry
+ # in the timeout. The previous logic was removed to make Python retry
# logic consistent with the other languages and to not disrupt the
# randomized retry delays distribution by artificially increasing a
# probability of scheduling two (instead of one) last attempts with very
diff --git a/tests/unit/test_bidi.py b/tests/unit/test_bidi.py
index f5e2b72b..0e7b018c 100644
--- a/tests/unit/test_bidi.py
+++ b/tests/unit/test_bidi.py
@@ -16,8 +16,14 @@
import logging
import queue
import threading
+import time
+
+try:
+ from unittest import mock
+ from unittest.mock import AsyncMock # pragma: NO COVER # noqa: F401
+except ImportError: # pragma: NO COVER
+ import mock # type: ignore
-import mock
import pytest
try:
@@ -291,6 +297,9 @@ def test_close(self):
# ensure the request queue was signaled to stop.
assert bidi_rpc.pending_requests == 1
assert bidi_rpc._request_queue.get() is None
+ # ensure request and callbacks are cleaned up
+ assert bidi_rpc._initial_request is None
+ assert not bidi_rpc._callbacks
def test_close_no_rpc(self):
bidi_rpc = bidi.BidiRpc(None)
@@ -618,6 +627,8 @@ def cancel_side_effect():
assert bidi_rpc.pending_requests == 1
assert bidi_rpc._request_queue.get() is None
assert bidi_rpc._finalized
+ assert bidi_rpc._initial_request is None
+ assert not bidi_rpc._callbacks
def test_reopen_failure_on_rpc_restart(self):
error1 = ValueError("1")
@@ -772,6 +783,7 @@ def on_response(response):
consumer.stop()
assert consumer.is_active is False
+ assert consumer._on_response is None
def test_wake_on_error(self):
should_continue = threading.Event()
@@ -804,6 +816,21 @@ def test_wake_on_error(self):
while consumer.is_active:
pass
+ def test_rpc_callback_fires_when_consumer_start_fails(self):
+ expected_exception = exceptions.InvalidArgument(
+ "test", response=grpc.StatusCode.INVALID_ARGUMENT
+ )
+ callback = mock.Mock(spec=["__call__"])
+
+ rpc, _ = make_rpc()
+ bidi_rpc = bidi.BidiRpc(rpc)
+ bidi_rpc.add_done_callback(callback)
+ bidi_rpc._start_rpc.side_effect = expected_exception
+
+ consumer = bidi.BackgroundConsumer(bidi_rpc, on_response=None)
+ consumer.start()
+ assert callback.call_args.args[0] == grpc.StatusCode.INVALID_ARGUMENT
+
def test_consumer_expected_error(self, caplog):
caplog.set_level(logging.DEBUG)
@@ -864,6 +891,30 @@ def close_side_effect():
consumer.stop()
assert consumer.is_active is False
+ assert consumer._on_response is None
# calling stop twice should not result in an error.
consumer.stop()
+
+ def test_stop_error_logs(self, caplog):
+ """
+ Closing the client should result in no internal error logs
+
+ https://github.com/googleapis/python-api-core/issues/788
+ """
+ caplog.set_level(logging.DEBUG)
+ bidi_rpc = mock.create_autospec(bidi.BidiRpc, instance=True)
+ bidi_rpc.is_active = True
+ on_response = mock.Mock(spec=["__call__"])
+
+ consumer = bidi.BackgroundConsumer(bidi_rpc, on_response)
+
+ consumer.start()
+ consumer.stop()
+ # let the background thread run for a while before exiting
+ time.sleep(0.1)
+ bidi_rpc.is_active = False
+ # running thread should not result in error logs
+ error_logs = [r.message for r in caplog.records if r.levelname == "ERROR"]
+ assert not error_logs, f"Found unexpected ERROR logs: {error_logs}"
+ bidi_rpc.is_active = False
diff --git a/tests/unit/test_client_info.py b/tests/unit/test_client_info.py
index 3361fef6..3eacabca 100644
--- a/tests/unit/test_client_info.py
+++ b/tests/unit/test_client_info.py
@@ -46,6 +46,7 @@ def test_constructor_options():
client_library_version="5",
user_agent="6",
rest_version="7",
+ protobuf_runtime_version="8",
)
assert info.python_version == "1"
@@ -55,11 +56,15 @@ def test_constructor_options():
assert info.client_library_version == "5"
assert info.user_agent == "6"
assert info.rest_version == "7"
+ assert info.protobuf_runtime_version == "8"
def test_to_user_agent_minimal():
info = client_info.ClientInfo(
- python_version="1", api_core_version="2", grpc_version=None
+ python_version="1",
+ api_core_version="2",
+ grpc_version=None,
+ protobuf_runtime_version=None,
)
user_agent = info.to_user_agent()
@@ -75,11 +80,12 @@ def test_to_user_agent_full():
gapic_version="4",
client_library_version="5",
user_agent="app-name/1.0",
+ protobuf_runtime_version="6",
)
user_agent = info.to_user_agent()
- assert user_agent == "app-name/1.0 gl-python/1 grpc/2 gax/3 gapic/4 gccl/5"
+ assert user_agent == "app-name/1.0 gl-python/1 grpc/2 gax/3 gapic/4 gccl/5 pb/6"
def test_to_user_agent_rest():
diff --git a/tests/unit/test_client_logging.py b/tests/unit/test_client_logging.py
new file mode 100644
index 00000000..b3b0b5c8
--- /dev/null
+++ b/tests/unit/test_client_logging.py
@@ -0,0 +1,140 @@
+import json
+import logging
+from unittest import mock
+
+from google.api_core.client_logging import (
+ setup_logging,
+ initialize_logging,
+ StructuredLogFormatter,
+)
+
+
+def reset_logger(scope):
+ logger = logging.getLogger(scope)
+ logger.handlers = []
+ logger.setLevel(logging.NOTSET)
+ logger.propagate = True
+
+
+def test_setup_logging_w_no_scopes():
+ with mock.patch("google.api_core.client_logging._BASE_LOGGER_NAME", "foogle"):
+ setup_logging()
+ base_logger = logging.getLogger("foogle")
+ assert base_logger.handlers == []
+ assert not base_logger.propagate
+ assert base_logger.level == logging.NOTSET
+
+ reset_logger("foogle")
+
+
+def test_setup_logging_w_base_scope():
+ with mock.patch("google.api_core.client_logging._BASE_LOGGER_NAME", "foogle"):
+ setup_logging("foogle")
+ base_logger = logging.getLogger("foogle")
+ assert isinstance(base_logger.handlers[0], logging.StreamHandler)
+ assert not base_logger.propagate
+ assert base_logger.level == logging.DEBUG
+
+ reset_logger("foogle")
+
+
+def test_setup_logging_w_configured_scope():
+ with mock.patch("google.api_core.client_logging._BASE_LOGGER_NAME", "foogle"):
+ base_logger = logging.getLogger("foogle")
+ base_logger.propagate = False
+ setup_logging("foogle")
+ assert base_logger.handlers == []
+ assert not base_logger.propagate
+ assert base_logger.level == logging.NOTSET
+
+ reset_logger("foogle")
+
+
+def test_setup_logging_w_module_scope():
+ with mock.patch("google.api_core.client_logging._BASE_LOGGER_NAME", "foogle"):
+ setup_logging("foogle.bar")
+
+ base_logger = logging.getLogger("foogle")
+ assert base_logger.handlers == []
+ assert not base_logger.propagate
+ assert base_logger.level == logging.NOTSET
+
+ module_logger = logging.getLogger("foogle.bar")
+ assert isinstance(module_logger.handlers[0], logging.StreamHandler)
+ assert not module_logger.propagate
+ assert module_logger.level == logging.DEBUG
+
+ reset_logger("foogle")
+ reset_logger("foogle.bar")
+
+
+def test_setup_logging_w_incorrect_scope():
+ with mock.patch("google.api_core.client_logging._BASE_LOGGER_NAME", "foogle"):
+ setup_logging("abc")
+
+ base_logger = logging.getLogger("foogle")
+ assert base_logger.handlers == []
+ assert not base_logger.propagate
+ assert base_logger.level == logging.NOTSET
+
+ # TODO(https://github.com/googleapis/python-api-core/issues/759): update test once we add logic to ignore an incorrect scope.
+ logger = logging.getLogger("abc")
+ assert isinstance(logger.handlers[0], logging.StreamHandler)
+ assert not logger.propagate
+ assert logger.level == logging.DEBUG
+
+ reset_logger("foogle")
+ reset_logger("abc")
+
+
+def test_initialize_logging():
+
+ with mock.patch("os.getenv", return_value="foogle.bar"):
+ with mock.patch("google.api_core.client_logging._BASE_LOGGER_NAME", "foogle"):
+ initialize_logging()
+
+ base_logger = logging.getLogger("foogle")
+ assert base_logger.handlers == []
+ assert not base_logger.propagate
+ assert base_logger.level == logging.NOTSET
+
+ module_logger = logging.getLogger("foogle.bar")
+ assert isinstance(module_logger.handlers[0], logging.StreamHandler)
+ assert not module_logger.propagate
+ assert module_logger.level == logging.DEBUG
+
+ # Check that `initialize_logging()` is a no-op after the first time by verifying that user-set configs are not modified:
+ base_logger.propagate = True
+ module_logger.propagate = True
+
+ initialize_logging()
+
+ assert base_logger.propagate
+ assert module_logger.propagate
+
+ reset_logger("foogle")
+ reset_logger("foogle.bar")
+
+
+def test_structured_log_formatter():
+ # TODO(https://github.com/googleapis/python-api-core/issues/761): Test additional fields when implemented.
+ record = logging.LogRecord(
+ name="Appelation",
+ level=logging.DEBUG,
+ msg="This is a test message.",
+ pathname="some/path",
+ lineno=25,
+ args=None,
+ exc_info=None,
+ )
+
+ # Extra fields:
+ record.rpcName = "bar"
+
+ formatted_msg = StructuredLogFormatter().format(record)
+ parsed_msg = json.loads(formatted_msg)
+
+ assert parsed_msg["name"] == "Appelation"
+ assert parsed_msg["severity"] == "DEBUG"
+ assert parsed_msg["message"] == "This is a test message."
+ assert parsed_msg["rpcName"] == "bar"
diff --git a/tests/unit/test_client_options.py b/tests/unit/test_client_options.py
index d56a1b3a..396d6627 100644
--- a/tests/unit/test_client_options.py
+++ b/tests/unit/test_client_options.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from re import match
import pytest
from google.api_core import client_options
@@ -37,6 +38,7 @@ def test_constructor():
"https://www.googleapis.com/auth/cloud-platform.read-only",
],
api_audience="foo2.googleapis.com",
+ universe_domain="googleapis.com",
)
assert options.api_endpoint == "foo.googleapis.com"
@@ -48,6 +50,7 @@ def test_constructor():
"https://www.googleapis.com/auth/cloud-platform.read-only",
]
assert options.api_audience == "foo2.googleapis.com"
+ assert options.universe_domain == "googleapis.com"
def test_constructor_with_encrypted_cert_source():
@@ -109,6 +112,7 @@ def test_from_dict():
options = client_options.from_dict(
{
"api_endpoint": "foo.googleapis.com",
+ "universe_domain": "googleapis.com",
"client_cert_source": get_client_cert,
"quota_project_id": "quote-proj",
"credentials_file": "path/to/credentials.json",
@@ -121,6 +125,7 @@ def test_from_dict():
)
assert options.api_endpoint == "foo.googleapis.com"
+ assert options.universe_domain == "googleapis.com"
assert options.client_cert_source() == (b"cert", b"key")
assert options.quota_project_id == "quote-proj"
assert options.credentials_file == "path/to/credentials.json"
@@ -144,10 +149,22 @@ def test_from_dict_bad_argument():
def test_repr():
- options = client_options.ClientOptions(api_endpoint="foo.googleapis.com")
-
- assert (
- repr(options)
- == "ClientOptions: {'api_endpoint': 'foo.googleapis.com', 'client_cert_source': None, 'client_encrypted_cert_source': None, 'api_key': None}"
- or "ClientOptions: {'client_encrypted_cert_source': None, 'client_cert_source': None, 'api_endpoint': 'foo.googleapis.com', 'api_key': None}"
+ expected_keys = set(
+ [
+ "api_endpoint",
+ "universe_domain",
+ "client_cert_source",
+ "client_encrypted_cert_source",
+ "quota_project_id",
+ "credentials_file",
+ "scopes",
+ "api_key",
+ "api_audience",
+ ]
)
+ options = client_options.ClientOptions(api_endpoint="foo.googleapis.com")
+ options_repr = repr(options)
+ options_keys = vars(options).keys()
+ assert match(r"ClientOptions:", options_repr)
+ assert match(r".*'api_endpoint': 'foo.googleapis.com'.*", options_repr)
+ assert options_keys == expected_keys
diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py
index 07a36817..e3f8f909 100644
--- a/tests/unit/test_exceptions.py
+++ b/tests/unit/test_exceptions.py
@@ -14,8 +14,8 @@
import http.client
import json
+from unittest import mock
-import mock
import pytest
import requests
diff --git a/tests/unit/test_extended_operation.py b/tests/unit/test_extended_operation.py
index c551bfa8..ab550662 100644
--- a/tests/unit/test_extended_operation.py
+++ b/tests/unit/test_extended_operation.py
@@ -15,8 +15,8 @@
import dataclasses
import enum
import typing
+from unittest import mock
-import mock
import pytest
from google.api_core import exceptions
@@ -33,11 +33,23 @@ class StatusCode(enum.Enum):
DONE = 1
PENDING = 2
+ class LROCustomErrors:
+ class LROCustomError:
+ def __init__(self, code: str = "", message: str = ""):
+ self.code = code
+ self.message = message
+
+ def __init__(self, errors: typing.List[LROCustomError] = []):
+ self.errors = errors
+
name: str
status: StatusCode
error_code: typing.Optional[int] = None
error_message: typing.Optional[str] = None
armor_class: typing.Optional[int] = None
+ # Note: `error` can be removed once proposal A from
+ # b/284179390 is implemented.
+ error: typing.Optional[LROCustomErrors] = None
# Note: in generated clients, this property must be generated for each
# extended operation message type.
@@ -170,6 +182,35 @@ def test_error():
with pytest.raises(exceptions.BadRequest):
ex_op.result()
+ # Test GCE custom LRO Error. See b/284179390
+ # Note: This test case can be removed once proposal A from
+ # b/284179390 is implemented.
+ _EXCEPTION_CODE = "INCOMPATIBLE_BACKEND_SERVICES"
+ _EXCEPTION_MESSAGE = "Validation failed for instance group"
+ responses = [
+ CustomOperation(
+ name=TEST_OPERATION_NAME,
+ status=CustomOperation.StatusCode.DONE,
+ error_code=400,
+ error_message="Bad request",
+ error=CustomOperation.LROCustomErrors(
+ errors=[
+ CustomOperation.LROCustomErrors.LROCustomError(
+ code=_EXCEPTION_CODE, message=_EXCEPTION_MESSAGE
+ )
+ ]
+ ),
+ ),
+ ]
+
+ ex_op, _, _ = make_extended_operation(responses)
+
+ # Defaults to CallError when grpc is not installed
+ with pytest.raises(
+ exceptions.BadRequest, match=f"{_EXCEPTION_CODE}: {_EXCEPTION_MESSAGE}"
+ ):
+ ex_op.result()
+
# Inconsistent result
responses = [
CustomOperation(
diff --git a/tests/unit/test_general_helpers.py b/tests/unit/test_general_helpers.py
deleted file mode 100644
index 82d6d4b6..00000000
--- a/tests/unit/test_general_helpers.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2022, Google LLC All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py
index a99dc280..8de9d8c0 100644
--- a/tests/unit/test_grpc_helpers.py
+++ b/tests/unit/test_grpc_helpers.py
@@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import mock
+from unittest import mock
+
import pytest
try:
@@ -195,6 +196,23 @@ def test_trailing_metadata(self):
wrapped.trailing_metadata.assert_called_once_with()
+class TestGrpcStream(Test_StreamingResponseIterator):
+ @staticmethod
+ def _make_one(wrapped, **kw):
+ return grpc_helpers.GrpcStream(wrapped, **kw)
+
+ def test_grpc_stream_attributes(self):
+ """
+ Should be both a grpc.Call and an iterable
+ """
+ call = self._make_one(None)
+ assert isinstance(call, grpc.Call)
+ # should implement __iter__
+ assert hasattr(call, "__iter__")
+ it = call.__iter__()
+ assert hasattr(it, "__next__")
+
+
def test_wrap_stream_okay():
expected_responses = [1, 2, 3]
callable_ = mock.Mock(spec=["__call__"], return_value=iter(expected_responses))
@@ -348,36 +366,72 @@ def test_wrap_errors_streaming(wrap_stream_errors):
wrap_stream_errors.assert_called_once_with(callable_)
-@mock.patch("grpc.composite_channel_credentials")
+@pytest.mark.parametrize(
+ "attempt_direct_path,target,expected_target",
+ [
+ (None, "example.com:443", "example.com:443"),
+ (False, "example.com:443", "example.com:443"),
+ (True, "example.com:443", "google-c2p:///example.com"),
+ (True, "dns:///example.com", "google-c2p:///example.com"),
+ (True, "another-c2p:///example.com", "another-c2p:///example.com"),
+ ],
+)
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
return_value=(mock.sentinel.credentials, mock.sentinel.project),
)
@mock.patch("grpc.secure_channel")
-def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_call):
- target = "example.com:443"
+def test_create_channel_implicit(
+ grpc_secure_channel,
+ google_auth_default,
+ composite_creds_call,
+ attempt_direct_path,
+ target,
+ expected_target,
+):
composite_creds = composite_creds_call.return_value
- channel = grpc_helpers.create_channel(target)
+ channel = grpc_helpers.create_channel(
+ target,
+ compression=grpc.Compression.Gzip,
+ attempt_direct_path=attempt_direct_path,
+ )
assert channel is grpc_secure_channel.return_value
- default.assert_called_once_with(scopes=None, default_scopes=None)
+ google_auth_default.assert_called_once_with(scopes=None, default_scopes=None)
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
- grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
+ # The original target is the expected target
+ expected_target = target
+ grpc_secure_channel.assert_called_once_with(
+ expected_target, composite_creds, None
+ )
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ expected_target, composite_creds, compression=grpc.Compression.Gzip
+ )
+@pytest.mark.parametrize(
+ "attempt_direct_path,target, expected_target",
+ [
+ (None, "example.com:443", "example.com:443"),
+ (False, "example.com:443", "example.com:443"),
+ (True, "example.com:443", "google-c2p:///example.com"),
+ (True, "dns:///example.com", "google-c2p:///example.com"),
+ (True, "another-c2p:///example.com", "another-c2p:///example.com"),
+ ],
+)
@mock.patch("google.auth.transport.grpc.AuthMetadataPlugin", autospec=True)
@mock.patch(
"google.auth.transport.requests.Request",
autospec=True,
return_value=mock.sentinel.Request,
)
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
@@ -385,27 +439,48 @@ def test_create_channel_implicit(grpc_secure_channel, default, composite_creds_c
)
@mock.patch("grpc.secure_channel")
def test_create_channel_implicit_with_default_host(
- grpc_secure_channel, default, composite_creds_call, request, auth_metadata_plugin
+ grpc_secure_channel,
+ google_auth_default,
+ composite_creds_call,
+ request,
+ auth_metadata_plugin,
+ attempt_direct_path,
+ target,
+ expected_target,
):
- target = "example.com:443"
default_host = "example.com"
composite_creds = composite_creds_call.return_value
- channel = grpc_helpers.create_channel(target, default_host=default_host)
+ channel = grpc_helpers.create_channel(
+ target, default_host=default_host, attempt_direct_path=attempt_direct_path
+ )
assert channel is grpc_secure_channel.return_value
- default.assert_called_once_with(scopes=None, default_scopes=None)
+ google_auth_default.assert_called_once_with(scopes=None, default_scopes=None)
auth_metadata_plugin.assert_called_once_with(
mock.sentinel.credentials, mock.sentinel.Request, default_host=default_host
)
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
- grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
+ # The original target is the expected target
+ expected_target = target
+ grpc_secure_channel.assert_called_once_with(
+ expected_target, composite_creds, None
+ )
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ expected_target, composite_creds, compression=None
+ )
+@pytest.mark.parametrize(
+ "attempt_direct_path",
+ [
+ None,
+ False,
+ ],
+)
@mock.patch("grpc.composite_channel_credentials")
@mock.patch(
"google.auth.default",
@@ -414,13 +489,15 @@ def test_create_channel_implicit_with_default_host(
)
@mock.patch("grpc.secure_channel")
def test_create_channel_implicit_with_ssl_creds(
- grpc_secure_channel, default, composite_creds_call
+ grpc_secure_channel, default, composite_creds_call, attempt_direct_path
):
target = "example.com:443"
ssl_creds = grpc.ssl_channel_credentials()
- grpc_helpers.create_channel(target, ssl_credentials=ssl_creds)
+ grpc_helpers.create_channel(
+ target, ssl_credentials=ssl_creds, attempt_direct_path=attempt_direct_path
+ )
default.assert_called_once_with(scopes=None, default_scopes=None)
@@ -430,10 +507,23 @@ def test_create_channel_implicit_with_ssl_creds(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+def test_create_channel_implicit_with_ssl_creds_attempt_direct_path_true():
+ target = "example.com:443"
+ ssl_creds = grpc.ssl_channel_credentials()
+ with pytest.raises(
+ ValueError, match="Using ssl_credentials with Direct Path is not supported"
+ ):
+ grpc_helpers.create_channel(
+ target, ssl_credentials=ssl_creds, attempt_direct_path=True
+ )
+
+
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
@@ -455,10 +545,12 @@ def test_create_channel_implicit_with_scopes(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch(
"google.auth.default",
autospec=True,
@@ -480,7 +572,9 @@ def test_create_channel_implicit_with_default_scopes(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
def test_create_channel_explicit_with_duplicate_credentials():
@@ -494,7 +588,7 @@ def test_create_channel_explicit_with_duplicate_credentials():
)
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("google.auth.credentials.with_scopes_if_required", autospec=True)
@mock.patch("grpc.secure_channel")
def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_creds_call):
@@ -512,10 +606,12 @@ def test_create_channel_explicit(grpc_secure_channel, auth_creds, composite_cred
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.secure_channel")
def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_call):
target = "example.com:443"
@@ -536,10 +632,12 @@ def test_create_channel_explicit_scoped(grpc_secure_channel, composite_creds_cal
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.secure_channel")
def test_create_channel_explicit_default_scopes(
grpc_secure_channel, composite_creds_call
@@ -564,10 +662,12 @@ def test_create_channel_explicit_default_scopes(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.secure_channel")
def test_create_channel_explicit_with_quota_project(
grpc_secure_channel, composite_creds_call
@@ -590,10 +690,12 @@ def test_create_channel_explicit_with_quota_project(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.secure_channel")
@mock.patch(
"google.auth.load_credentials_from_file",
@@ -619,10 +721,12 @@ def test_create_channel_with_credentials_file(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.secure_channel")
@mock.patch(
"google.auth.load_credentials_from_file",
@@ -651,10 +755,12 @@ def test_create_channel_with_credentials_file_and_scopes(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
-@mock.patch("grpc.composite_channel_credentials")
+@mock.patch("grpc.compute_engine_channel_credentials")
@mock.patch("grpc.secure_channel")
@mock.patch(
"google.auth.load_credentials_from_file",
@@ -683,7 +789,9 @@ def test_create_channel_with_credentials_file_and_default_scopes(
if grpc_helpers.HAS_GRPC_GCP: # pragma: NO COVER
grpc_secure_channel.assert_called_once_with(target, composite_creds, None)
else:
- grpc_secure_channel.assert_called_once_with(target, composite_creds)
+ grpc_secure_channel.assert_called_once_with(
+ target, composite_creds, compression=None
+ )
@pytest.mark.skipif(
@@ -813,6 +921,7 @@ def test_call_info(self):
stub = operations_pb2.OperationsStub(channel)
expected_request = operations_pb2.GetOperationRequest(name="meep")
expected_response = operations_pb2.Operation(name="moop")
+ expected_compression = grpc.Compression.NoCompression
expected_metadata = [("red", "blue"), ("two", "shoe")]
expected_credentials = mock.sentinel.credentials
channel.GetOperation.response = expected_response
@@ -820,6 +929,7 @@ def test_call_info(self):
response = stub.GetOperation(
expected_request,
timeout=42,
+ compression=expected_compression,
metadata=expected_metadata,
credentials=expected_credentials,
)
@@ -827,7 +937,13 @@ def test_call_info(self):
assert response == expected_response
assert channel.requests == [("GetOperation", expected_request)]
assert channel.GetOperation.calls == [
- (expected_request, 42, expected_metadata, expected_credentials)
+ (
+ expected_request,
+ 42,
+ expected_metadata,
+ expected_credentials,
+ expected_compression,
+ )
]
def test_unary_unary(self):
diff --git a/tests/unit/test_iam.py b/tests/unit/test_iam.py
index fbd242e5..3de15288 100644
--- a/tests/unit/test_iam.py
+++ b/tests/unit/test_iam.py
@@ -167,14 +167,15 @@ def test_owners_getter(self):
assert policy.owners == expected
def test_owners_setter(self):
- import warnings
from google.api_core.iam import OWNER_ROLE
MEMBER = "user:phred@example.com"
expected = set([MEMBER])
policy = self._make_one()
- with warnings.catch_warnings(record=True) as warned:
+ with pytest.warns(
+ DeprecationWarning, match="Assigning to 'owners' is deprecated."
+ ) as warned:
policy.owners = [MEMBER]
(warning,) = warned
@@ -191,14 +192,15 @@ def test_editors_getter(self):
assert policy.editors == expected
def test_editors_setter(self):
- import warnings
from google.api_core.iam import EDITOR_ROLE
MEMBER = "user:phred@example.com"
expected = set([MEMBER])
policy = self._make_one()
- with warnings.catch_warnings(record=True) as warned:
+ with pytest.warns(
+ DeprecationWarning, match="Assigning to 'editors' is deprecated."
+ ) as warned:
policy.editors = [MEMBER]
(warning,) = warned
@@ -215,14 +217,15 @@ def test_viewers_getter(self):
assert policy.viewers == expected
def test_viewers_setter(self):
- import warnings
from google.api_core.iam import VIEWER_ROLE
MEMBER = "user:phred@example.com"
expected = set([MEMBER])
policy = self._make_one()
- with warnings.catch_warnings(record=True) as warned:
+ with pytest.warns(
+ DeprecationWarning, match="Assigning to 'viewers' is deprecated."
+ ) as warned:
policy.viewers = [MEMBER]
(warning,) = warned
@@ -337,12 +340,13 @@ def test_to_api_repr_binding_wo_members(self):
assert policy.to_api_repr() == {}
def test_to_api_repr_binding_w_duplicates(self):
- import warnings
from google.api_core.iam import OWNER_ROLE
OWNER = "group:cloud-logs@google.com"
policy = self._make_one()
- with warnings.catch_warnings(record=True):
+ with pytest.warns(
+ DeprecationWarning, match="Assigning to 'owners' is deprecated."
+ ):
policy.owners = [OWNER, OWNER]
assert policy.to_api_repr() == {
"bindings": [{"role": OWNER_ROLE, "members": [OWNER]}]
diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py
index f029866c..80680720 100644
--- a/tests/unit/test_operation.py
+++ b/tests/unit/test_operation.py
@@ -13,7 +13,8 @@
# limitations under the License.
-import mock
+from unittest import mock
+
import pytest
try:
diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py
new file mode 100644
index 00000000..8100a496
--- /dev/null
+++ b/tests/unit/test_packaging.py
@@ -0,0 +1,28 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+
+
+def test_namespace_package_compat(tmp_path):
+ # The ``google`` namespace package should not be masked
+ # by the presence of ``google-api-core``.
+ google = tmp_path / "google"
+ google.mkdir()
+ google.joinpath("othermod.py").write_text("")
+ env = dict(os.environ, PYTHONPATH=str(tmp_path))
+ cmd = [sys.executable, "-m", "google.othermod"]
+ subprocess.check_call(cmd, env=env)
diff --git a/tests/unit/test_page_iterator.py b/tests/unit/test_page_iterator.py
index cf43aedf..560722c5 100644
--- a/tests/unit/test_page_iterator.py
+++ b/tests/unit/test_page_iterator.py
@@ -14,8 +14,8 @@
import math
import types
+from unittest import mock
-import mock
import pytest
from google.api_core import page_iterator
diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py
index 808b36f3..c34dd0f3 100644
--- a/tests/unit/test_path_template.py
+++ b/tests/unit/test_path_template.py
@@ -13,8 +13,8 @@
# limitations under the License.
from __future__ import unicode_literals
+from unittest import mock
-import mock
import pytest
from google.api import auth_pb2
diff --git a/tests/unit/test_protobuf_helpers.py b/tests/unit/test_protobuf_helpers.py
index 3df45df5..5678d3bc 100644
--- a/tests/unit/test_protobuf_helpers.py
+++ b/tests/unit/test_protobuf_helpers.py
@@ -12,9 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import sys
-
import pytest
+import re
from google.api import http_pb2
from google.api_core import protobuf_helpers
@@ -67,7 +66,12 @@ def test_from_any_pb_failure():
in_message = any_pb2.Any()
in_message.Pack(date_pb2.Date(year=1990))
- with pytest.raises(TypeError):
+ with pytest.raises(
+ TypeError,
+ match=re.escape(
+ "Could not convert `google.type.Date` with underlying type `google.protobuf.any_pb2.Any` to `google.type.TimeOfDay`"
+ ),
+ ):
protobuf_helpers.from_any_pb(timeofday_pb2.TimeOfDay, in_message)
@@ -476,11 +480,6 @@ def test_field_mask_different_level_diffs():
]
-@pytest.mark.skipif(
- sys.version_info.major == 2,
- reason="Field names with trailing underscores can only be created"
- "through proto-plus, which is Python 3 only.",
-)
def test_field_mask_ignore_trailing_underscore():
import proto
@@ -496,11 +495,6 @@ class Foo(proto.Message):
]
-@pytest.mark.skipif(
- sys.version_info.major == 2,
- reason="Field names with trailing underscores can only be created"
- "through proto-plus, which is Python 3 only.",
-)
def test_field_mask_ignore_trailing_underscore_with_nesting():
import proto
diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py
index a44c83c0..0f998dfe 100644
--- a/tests/unit/test_rest_streaming.py
+++ b/tests/unit/test_rest_streaming.py
@@ -24,47 +24,18 @@
import requests
from google.api_core import rest_streaming
-from google.protobuf import duration_pb2
-from google.protobuf import timestamp_pb2
+from google.api import http_pb2
+from google.api import httpbody_pb2
+
+from ..helpers import Composer, Song, EchoResponse, parse_responses
__protobuf__ = proto.module(package=__name__)
SEED = int(time.time())
-logging.info(f"Starting rest streaming tests with random seed: {SEED}")
+logging.info(f"Starting sync rest streaming tests with random seed: {SEED}")
random.seed(SEED)
-class Genre(proto.Enum):
- GENRE_UNSPECIFIED = 0
- CLASSICAL = 1
- JAZZ = 2
- ROCK = 3
-
-
-class Composer(proto.Message):
- given_name = proto.Field(proto.STRING, number=1)
- family_name = proto.Field(proto.STRING, number=2)
- relateds = proto.RepeatedField(proto.STRING, number=3)
- indices = proto.MapField(proto.STRING, proto.STRING, number=4)
-
-
-class Song(proto.Message):
- composer = proto.Field(Composer, number=1)
- title = proto.Field(proto.STRING, number=2)
- lyrics = proto.Field(proto.STRING, number=3)
- year = proto.Field(proto.INT32, number=4)
- genre = proto.Field(Genre, number=5)
- is_five_mins_longer = proto.Field(proto.BOOL, number=6)
- score = proto.Field(proto.DOUBLE, number=7)
- likes = proto.Field(proto.INT64, number=8)
- duration = proto.Field(duration_pb2.Duration, number=9)
- date_added = proto.Field(timestamp_pb2.Timestamp, number=10)
-
-
-class EchoResponse(proto.Message):
- content = proto.Field(proto.STRING, number=1)
-
-
class ResponseMock(requests.Response):
class _ResponseItr:
def __init__(self, _response_bytes: bytes, random_split=False):
@@ -94,123 +65,232 @@ def __init__(
self._random_split = random_split
self._response_message_cls = response_cls
- def _parse_responses(self, responses: List[proto.Message]) -> bytes:
- # json.dumps returns a string surrounded with quotes that need to be stripped
- # in order to be an actual JSON.
- json_responses = [
- self._response_message_cls.to_json(r).strip('"') for r in responses
- ]
- logging.info(f"Sending JSON stream: {json_responses}")
- ret_val = "[{}]".format(",".join(json_responses))
- return bytes(ret_val, "utf-8")
+ def _parse_responses(self):
+ return parse_responses(self._response_message_cls, self._responses)
def close(self):
raise NotImplementedError()
def iter_content(self, *args, **kwargs):
return self._ResponseItr(
- self._parse_responses(self._responses),
+ self._parse_responses(),
random_split=self._random_split,
)
-@pytest.mark.parametrize("random_split", [False])
-def test_next_simple(random_split):
- responses = [EchoResponse(content="hello world"), EchoResponse(content="yes")]
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [(False, True), (False, False)],
+)
+def test_next_simple(random_split, resp_message_is_proto_plus):
+ if resp_message_is_proto_plus:
+ response_type = EchoResponse
+ responses = [EchoResponse(content="hello world"), EchoResponse(content="yes")]
+ else:
+ response_type = httpbody_pb2.HttpBody
+ responses = [
+ httpbody_pb2.HttpBody(content_type="hello world"),
+ httpbody_pb2.HttpBody(content_type="yes"),
+ ]
+
resp = ResponseMock(
- responses=responses, random_split=random_split, response_cls=EchoResponse
+ responses=responses, random_split=random_split, response_cls=response_type
)
- itr = rest_streaming.ResponseIterator(resp, EchoResponse)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
assert list(itr) == responses
-@pytest.mark.parametrize("random_split", [True, False])
-def test_next_nested(random_split):
- responses = [
- Song(title="some song", composer=Composer(given_name="some name")),
- Song(title="another song", date_added=datetime.datetime(2021, 12, 17)),
- ]
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [
+ (True, True),
+ (False, True),
+ (True, False),
+ (False, False),
+ ],
+)
+def test_next_nested(random_split, resp_message_is_proto_plus):
+ if resp_message_is_proto_plus:
+ response_type = Song
+ responses = [
+ Song(title="some song", composer=Composer(given_name="some name")),
+ Song(title="another song", date_added=datetime.datetime(2021, 12, 17)),
+ ]
+ else:
+ # Although `http_pb2.HttpRule`` is used in the response, any response message
+ # can be used which meets this criteria for the test of having a nested field.
+ response_type = http_pb2.HttpRule
+ responses = [
+ http_pb2.HttpRule(
+ selector="some selector",
+ custom=http_pb2.CustomHttpPattern(kind="some kind"),
+ ),
+ http_pb2.HttpRule(
+ selector="another selector",
+ custom=http_pb2.CustomHttpPattern(path="some path"),
+ ),
+ ]
resp = ResponseMock(
- responses=responses, random_split=random_split, response_cls=Song
+ responses=responses, random_split=random_split, response_cls=response_type
)
- itr = rest_streaming.ResponseIterator(resp, Song)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
assert list(itr) == responses
-@pytest.mark.parametrize("random_split", [True, False])
-def test_next_stress(random_split):
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [
+ (True, True),
+ (False, True),
+ (True, False),
+ (False, False),
+ ],
+)
+def test_next_stress(random_split, resp_message_is_proto_plus):
n = 50
- responses = [
- Song(title="title_%d" % i, composer=Composer(given_name="name_%d" % i))
- for i in range(n)
- ]
+ if resp_message_is_proto_plus:
+ response_type = Song
+ responses = [
+ Song(title="title_%d" % i, composer=Composer(given_name="name_%d" % i))
+ for i in range(n)
+ ]
+ else:
+ response_type = http_pb2.HttpRule
+ responses = [
+ http_pb2.HttpRule(
+ selector="selector_%d" % i,
+ custom=http_pb2.CustomHttpPattern(path="path_%d" % i),
+ )
+ for i in range(n)
+ ]
resp = ResponseMock(
- responses=responses, random_split=random_split, response_cls=Song
+ responses=responses, random_split=random_split, response_cls=response_type
)
- itr = rest_streaming.ResponseIterator(resp, Song)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
assert list(itr) == responses
-@pytest.mark.parametrize("random_split", [True, False])
-def test_next_escaped_characters_in_string(random_split):
- composer_with_relateds = Composer()
- relateds = ["Artist A", "Artist B"]
- composer_with_relateds.relateds = relateds
-
- responses = [
- Song(title='ti"tle\nfoo\tbar{}', composer=Composer(given_name="name\n\n\n")),
- Song(
- title='{"this is weird": "totally"}', composer=Composer(given_name="\\{}\\")
- ),
- Song(title='\\{"key": ["value",]}\\', composer=composer_with_relateds),
- ]
+@pytest.mark.parametrize(
+ "random_split,resp_message_is_proto_plus",
+ [
+ (True, True),
+ (False, True),
+ (True, False),
+ (False, False),
+ ],
+)
+def test_next_escaped_characters_in_string(random_split, resp_message_is_proto_plus):
+ if resp_message_is_proto_plus:
+ response_type = Song
+ composer_with_relateds = Composer()
+ relateds = ["Artist A", "Artist B"]
+ composer_with_relateds.relateds = relateds
+
+ responses = [
+ Song(
+ title='ti"tle\nfoo\tbar{}', composer=Composer(given_name="name\n\n\n")
+ ),
+ Song(
+ title='{"this is weird": "totally"}',
+ composer=Composer(given_name="\\{}\\"),
+ ),
+ Song(title='\\{"key": ["value",]}\\', composer=composer_with_relateds),
+ ]
+ else:
+ response_type = http_pb2.Http
+ responses = [
+ http_pb2.Http(
+ rules=[
+ http_pb2.HttpRule(
+ selector='ti"tle\nfoo\tbar{}',
+ custom=http_pb2.CustomHttpPattern(kind="name\n\n\n"),
+ )
+ ]
+ ),
+ http_pb2.Http(
+ rules=[
+ http_pb2.HttpRule(
+ selector='{"this is weird": "totally"}',
+ custom=http_pb2.CustomHttpPattern(kind="\\{}\\"),
+ )
+ ]
+ ),
+ http_pb2.Http(
+ rules=[
+ http_pb2.HttpRule(
+ selector='\\{"key": ["value",]}\\',
+ custom=http_pb2.CustomHttpPattern(kind="\\{}\\"),
+ )
+ ]
+ ),
+ ]
resp = ResponseMock(
- responses=responses, random_split=random_split, response_cls=Song
+ responses=responses, random_split=random_split, response_cls=response_type
)
- itr = rest_streaming.ResponseIterator(resp, Song)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
assert list(itr) == responses
-def test_next_not_array():
+@pytest.mark.parametrize("response_type", [EchoResponse, httpbody_pb2.HttpBody])
+def test_next_not_array(response_type):
with patch.object(
ResponseMock, "iter_content", return_value=iter('{"hello": 0}')
) as mock_method:
-
- resp = ResponseMock(responses=[], response_cls=EchoResponse)
- itr = rest_streaming.ResponseIterator(resp, EchoResponse)
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
with pytest.raises(ValueError):
next(itr)
mock_method.assert_called_once()
-def test_cancel():
+@pytest.mark.parametrize("response_type", [EchoResponse, httpbody_pb2.HttpBody])
+def test_cancel(response_type):
with patch.object(ResponseMock, "close", return_value=None) as mock_method:
- resp = ResponseMock(responses=[], response_cls=EchoResponse)
- itr = rest_streaming.ResponseIterator(resp, EchoResponse)
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
itr.cancel()
mock_method.assert_called_once()
-def test_check_buffer():
+@pytest.mark.parametrize(
+ "response_type,return_value",
+ [
+ (EchoResponse, bytes('[{"content": "hello"}, {', "utf-8")),
+ (httpbody_pb2.HttpBody, bytes('[{"content_type": "hello"}, {', "utf-8")),
+ ],
+)
+def test_check_buffer(response_type, return_value):
with patch.object(
ResponseMock,
"_parse_responses",
- return_value=bytes('[{"content": "hello"}, {', "utf-8"),
+ return_value=return_value,
):
- resp = ResponseMock(responses=[], response_cls=EchoResponse)
- itr = rest_streaming.ResponseIterator(resp, EchoResponse)
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
with pytest.raises(ValueError):
next(itr)
next(itr)
-def test_next_html():
+@pytest.mark.parametrize("response_type", [EchoResponse, httpbody_pb2.HttpBody])
+def test_next_html(response_type):
with patch.object(
ResponseMock, "iter_content", return_value=iter("")
) as mock_method:
-
- resp = ResponseMock(responses=[], response_cls=EchoResponse)
- itr = rest_streaming.ResponseIterator(resp, EchoResponse)
+ resp = ResponseMock(responses=[], response_cls=response_type)
+ itr = rest_streaming.ResponseIterator(resp, response_type)
with pytest.raises(ValueError):
next(itr)
mock_method.assert_called_once()
+
+
+def test_invalid_response_class():
+ class SomeClass:
+ pass
+
+ resp = ResponseMock(responses=[], response_cls=SomeClass)
+ with pytest.raises(
+ ValueError,
+ match="Response message class must be a subclass of proto.Message or google.protobuf.message.Message",
+ ):
+ rest_streaming.ResponseIterator(resp, SomeClass)
diff --git a/tests/unit/test_timeout.py b/tests/unit/test_timeout.py
index a83a2ecb..2c20202b 100644
--- a/tests/unit/test_timeout.py
+++ b/tests/unit/test_timeout.py
@@ -14,8 +14,7 @@
import datetime
import itertools
-
-import mock
+from unittest import mock
from google.api_core import timeout as timeouts
@@ -58,10 +57,10 @@ def test___str__(self):
def test_apply(self):
target = mock.Mock(spec=["__call__", "__name__"], __name__="target")
- datetime.datetime.utcnow()
+ datetime.datetime.now(tz=datetime.timezone.utc)
datetime.timedelta(seconds=1)
- now = datetime.datetime.utcnow()
+ now = datetime.datetime.now(tz=datetime.timezone.utc)
times = [
now,
@@ -85,17 +84,17 @@ def _clock():
wrapped()
target.assert_called_with(timeout=3.0)
wrapped()
- target.assert_called_with(timeout=0.0)
+ target.assert_called_with(timeout=42.0)
wrapped()
- target.assert_called_with(timeout=0.0)
+ target.assert_called_with(timeout=42.0)
def test_apply_no_timeout(self):
target = mock.Mock(spec=["__call__", "__name__"], __name__="target")
- datetime.datetime.utcnow()
+ datetime.datetime.now(tz=datetime.timezone.utc)
datetime.timedelta(seconds=1)
- now = datetime.datetime.utcnow()
+ now = datetime.datetime.now(tz=datetime.timezone.utc)
times = [
now,
diff --git a/tests/unit/test_universe.py b/tests/unit/test_universe.py
new file mode 100644
index 00000000..214e00ac
--- /dev/null
+++ b/tests/unit/test_universe.py
@@ -0,0 +1,63 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from google.api_core import universe
+
+
+class _Fake_Credentials:
+ def __init__(self, universe_domain=None):
+ if universe_domain:
+ self.universe_domain = universe_domain
+
+
+def test_determine_domain():
+ domain_client = "foo.com"
+ domain_env = "bar.com"
+
+ assert universe.determine_domain(domain_client, domain_env) == domain_client
+ assert universe.determine_domain(None, domain_env) == domain_env
+ assert universe.determine_domain(domain_client, None) == domain_client
+ assert universe.determine_domain(None, None) == universe.DEFAULT_UNIVERSE
+
+ with pytest.raises(universe.EmptyUniverseError):
+ universe.determine_domain("", None)
+
+ with pytest.raises(universe.EmptyUniverseError):
+ universe.determine_domain(None, "")
+
+
+def test_compare_domains():
+ fake_domain = "foo.com"
+ another_fake_domain = "bar.com"
+
+ assert universe.compare_domains(universe.DEFAULT_UNIVERSE, _Fake_Credentials())
+ assert universe.compare_domains(fake_domain, _Fake_Credentials(fake_domain))
+
+ with pytest.raises(universe.UniverseMismatchError) as excinfo:
+ universe.compare_domains(
+ universe.DEFAULT_UNIVERSE, _Fake_Credentials(fake_domain)
+ )
+ assert str(excinfo.value).find(universe.DEFAULT_UNIVERSE) >= 0
+ assert str(excinfo.value).find(fake_domain) >= 0
+
+ with pytest.raises(universe.UniverseMismatchError) as excinfo:
+ universe.compare_domains(fake_domain, _Fake_Credentials())
+ assert str(excinfo.value).find(fake_domain) >= 0
+ assert str(excinfo.value).find(universe.DEFAULT_UNIVERSE) >= 0
+
+ with pytest.raises(universe.UniverseMismatchError) as excinfo:
+ universe.compare_domains(fake_domain, _Fake_Credentials(another_fake_domain))
+ assert str(excinfo.value).find(fake_domain) >= 0
+ assert str(excinfo.value).find(another_fake_domain) >= 0
diff --git a/google/__init__.py b/tests/unit/test_version_header.py
similarity index 55%
rename from google/__init__.py
rename to tests/unit/test_version_header.py
index 9f1d5491..ea7028e2 100644
--- a/google/__init__.py
+++ b/tests/unit/test_version_header.py
@@ -1,10 +1,10 @@
-# Copyright 2016 Google LLC
+# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,14 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Google namespace package."""
+import pytest
-try:
- import pkg_resources
+from google.api_core import version_header
- pkg_resources.declare_namespace(__name__)
-except ImportError:
- import pkgutil
- # See: https://github.com/python/mypy/issues/1422
- __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore
+@pytest.mark.parametrize("version_identifier", ["some_value", ""])
+def test_to_api_version_header(version_identifier):
+ value = version_header.to_api_version_header(version_identifier)
+ assert value == (version_header.API_VERSION_METADATA_KEY, version_identifier)