diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 7519fa3a..f33299dd 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:0e18b9475fbeb12d9ad4302283171edebb6baf2dfca1bd215ee3b34ed79d95d7 + digest: sha256:899d5d7cc340fa8ef9d8ae1a8cfba362c6898584f779e156f25ee828ba824610 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ee818917..1b023b72 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,5 +8,5 @@ # @googleapis/yoshi-python @googleapis/actools-python are the default owners for changes in this repo * @googleapis/yoshi-python @googleapis/actools-python -# @googleapis/python-samples-owners @googleapis/actools-python are the default owners for samples changes -/samples/ @googleapis/python-samples-owners @googleapis/actools-python +# @googleapis/python-samples-reviewers @googleapis/actools-python are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/actools-python diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index e621885d..67fcd59f 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,3 +1,28 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings +# Rules for main branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `main` +- pattern: main + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: true + requiredStatusCheckContexts: + - 'cla/google' + # No Kokoro: the following are Github actions + - 'lint-mypy' + - 'unit-3.6' + - 'unit-3.7' + - 'unit-3.8' + - 'unit-3.9' + - 'unit-3.10' + - 'unit_grpc_gcp-3.6' + - 'unit_grpc_gcp-3.7' + - 'unit_grpc_gcp-3.8' + - 'unit_grpc_gcp-3.9' + - 'unit_grpc_gcp-3.10' + - 'unit_wo_grpc-3.6' + - 'unit_wo_grpc-3.10' + - 'cover' permissionRules: - team: actools-python permission: admin diff --git a/.github/workflows/unittest_lint.yml b/.github/workflows/unittest_lint.yml new file mode 100644 index 00000000..302be970 --- /dev/null +++ b/.github/workflows/unittest_lint.yml @@ -0,0 +1,119 @@ +name: "Lint / Unit tests / Cover / Mypy" + +on: + pull_request: + branches: + - main + + +jobs: + + run-lint-mypy: + name: lint-mypy + runs-on: ubuntu-latest + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + + - name: Run lint + run: | + nox -s lint + + - name: Run lint_setup_py + run: | + nox -s lint_setup_py + + - name: Run mypy + run: | + nox -s mypy + + run-unittests: + name: unit${{ matrix.option }}-${{ matrix.python }} + runs-on: ubuntu-latest + strategy: + matrix: + option: ["", "_grpc_gcp", "_wo_grpc"] + python: + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10" + exclude: + - option: "_wo_grpc" + python: 3.7 + - option: "_wo_grpc" + python: 3.8 + - option: "_wo_grpc" + python: 3.9 + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + + - name: Run unit tests + env: + COVERAGE_FILE: .coverage${{ matrix.option }}-${{matrix.python }} + run: | + nox -s unit${{ matrix.option }}-${{ matrix.python }} + + - name: Upload coverage results + uses: actions/upload-artifact@v2 + with: + name: coverage-artifacts + path: .coverage${{ matrix.option }}-${{ matrix.python }} + + report-coverage: + name: cover + runs-on: ubuntu-latest + needs: + - run-unittests + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Install coverage + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install coverage + + - name: Download coverage results + uses: actions/download-artifact@v2 + with: + name: coverage-artifacts + path: .coverage-results/ + + - name: Report coverage results + run: | + coverage combine .coverage-results/.coverage* + coverage report --show-missing --fail-under=100 diff --git a/.repo-metadata.json b/.repo-metadata.json index e16c9d27..0f0abd93 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,7 +2,7 @@ "name": "google-api-core", "name_pretty": "Google API client core library", "client_documentation": "https://googleapis.dev/python/google-api-core/latest", - "release_level": "ga", + "release_level": "stable", "language": "python", "library_type": "CORE", "repo": "googleapis/python-api-core", diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d47daa..e6177e0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ [1]: https://pypi.org/project/google-api-core/#history +## [2.4.0](https://www.github.com/googleapis/python-api-core/compare/v2.3.2...v2.4.0) (2022-01-11) + + +### Features + +* add support for 'error_info' ([#315](https://www.github.com/googleapis/python-api-core/issues/315)) ([cc46aa6](https://www.github.com/googleapis/python-api-core/commit/cc46aa68ec184871330d16a6c767f57a4f0eb633)) +* iterator for processing JSON responses in REST streaming. ([#317](https://www.github.com/googleapis/python-api-core/issues/317)) ([f9f2696](https://www.github.com/googleapis/python-api-core/commit/f9f26969842b456ea372bed941d712b7a9ab7239)) + +## [2.3.2](https://www.github.com/googleapis/python-api-core/compare/v2.3.1...v2.3.2) (2021-12-16) + + +### Bug Fixes + +* address broken wheels in version 2.3.1 + +## [2.3.1](https://www.github.com/googleapis/python-api-core/compare/v2.3.0...v2.3.1) (2021-12-15) + + +### Bug Fixes +* exclude function target from retry deadline exceeded exception message ([#318](https://www.github.com/googleapis/python-api-core/issues/318)) ([34ebdcc](https://www.github.com/googleapis/python-api-core/commit/34ebdcc251d4f3d7d496e8e0b78847645a06650b)) + ## [2.3.0](https://www.github.com/googleapis/python-api-core/compare/v2.2.2...v2.3.0) (2021-11-25) diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py index 6b1b6f7e..24b65ee0 100644 --- a/google/api_core/exceptions.py +++ b/google/api_core/exceptions.py @@ -104,6 +104,8 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): details (Sequence[Any]): An optional list of objects defined in google.rpc.error_details. response (Union[requests.Request, grpc.Call]): The response or gRPC call metadata. + error_info (Union[error_details_pb2.ErrorInfo, None]): An optional object containing error info + (google.rpc.error_details.ErrorInfo). """ code: Union[int, None] = None @@ -122,13 +124,14 @@ class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): This may be ``None`` if the exception does not match up to a gRPC error. """ - def __init__(self, message, errors=(), details=(), response=None): + def __init__(self, message, errors=(), details=(), response=None, error_info=None): super(GoogleAPICallError, self).__init__(message) self.message = message """str: The exception message.""" self._errors = errors self._details = details self._response = response + self._error_info = error_info def __str__(self): if self.details: @@ -136,6 +139,42 @@ def __str__(self): else: return "{} {}".format(self.code, self.message) + @property + def reason(self): + """The reason of the error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing reason of the error. + """ + return self._error_info.reason if self._error_info else None + + @property + def domain(self): + """The logical grouping to which the "reason" belongs. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing a logical grouping to which the "reason" belongs. + """ + return self._error_info.domain if self._error_info else None + + @property + def metadata(self): + """Additional structured details about this error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[Dict[str, str], None]: An optional object containing structured details about the error. + """ + return self._error_info.metadata if self._error_info else None + @property def errors(self): """Detailed error information. @@ -433,13 +472,26 @@ def from_http_response(response): errors = payload.get("error", {}).get("errors", ()) # In JSON, details are already formatted in developer-friendly way. details = payload.get("error", {}).get("details", ()) + error_info = list( + filter( + lambda detail: detail.get("@type", "") + == "type.googleapis.com/google.rpc.ErrorInfo", + details, + ) + ) + error_info = error_info[0] if error_info else None message = "{method} {url}: {error}".format( - method=response.request.method, url=response.request.url, error=error_message + method=response.request.method, url=response.request.url, error=error_message, ) exception = from_http_status( - response.status_code, message, errors=errors, details=details, response=response + response.status_code, + message, + errors=errors, + details=details, + response=response, + error_info=error_info, ) return exception @@ -490,10 +542,10 @@ def _parse_grpc_error_details(rpc_exc): try: status = rpc_status.from_call(rpc_exc) except NotImplementedError: # workaround - return [] + return [], None if not status: - return [] + return [], None possible_errors = [ error_details_pb2.BadRequest, @@ -507,6 +559,7 @@ def _parse_grpc_error_details(rpc_exc): error_details_pb2.Help, error_details_pb2.LocalizedMessage, ] + error_info = None error_details = [] for detail in status.details: matched_detail_cls = list( @@ -519,7 +572,9 @@ def _parse_grpc_error_details(rpc_exc): info = matched_detail_cls[0]() detail.Unpack(info) error_details.append(info) - return error_details + if isinstance(info, error_details_pb2.ErrorInfo): + error_info = info + return error_details, error_info def from_grpc_error(rpc_exc): @@ -535,12 +590,14 @@ def from_grpc_error(rpc_exc): # NOTE(lidiz) All gRPC error shares the parent class grpc.RpcError. # However, check for grpc.RpcError breaks backward compatibility. if isinstance(rpc_exc, grpc.Call) or _is_informative_grpc_error(rpc_exc): + details, err_info = _parse_grpc_error_details(rpc_exc) return from_grpc_status( rpc_exc.code(), rpc_exc.details(), errors=(rpc_exc,), - details=_parse_grpc_error_details(rpc_exc), + details=details, response=rpc_exc, + error_info=err_info, ) else: return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/google/api_core/rest_streaming.py b/google/api_core/rest_streaming.py new file mode 100644 index 00000000..69f5b41b --- /dev/null +++ b/google/api_core/rest_streaming.py @@ -0,0 +1,114 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for server-side streaming in REST.""" + +from collections import deque +import string +from typing import Deque + +import requests + + +class ResponseIterator: + """Iterator over REST API responses. + + Args: + response (requests.Response): An API response object. + response_message_cls (Callable[proto.Message]): A proto + class expected to be returned from an API. + """ + + def __init__(self, response: requests.Response, response_message_cls): + self._response = response + self._response_message_cls = response_message_cls + # Inner iterator over HTTP response's content. + self._response_itr = self._response.iter_content(decode_unicode=True) + # Contains a list of JSON responses ready to be sent to user. + self._ready_objs: Deque[str] = deque() + # Current JSON response being built. + self._obj = "" + # Keeps track of the nesting level within a JSON object. + self._level = 0 + # Keeps track whether HTTP response is currently sending values + # inside of a string value. + self._in_string = False + # Whether an escape symbol "\" was encountered. + self._escape_next = False + + def cancel(self): + """Cancel existing streaming operation. + """ + self._response.close() + + def _process_chunk(self, chunk: str): + if self._level == 0: + if chunk[0] != "[": + raise ValueError( + "Can only parse array of JSON objects, instead got %s" % chunk + ) + for char in chunk: + if char == "{": + if self._level == 1: + # Level 1 corresponds to the outermost JSON object + # (i.e. the one we care about). + self._obj = "" + if not self._in_string: + self._level += 1 + self._obj += char + elif char == "}": + self._obj += char + if not self._in_string: + self._level -= 1 + if not self._in_string and self._level == 1: + self._ready_objs.append(self._obj) + elif char == '"': + # Helps to deal with an escaped quotes inside of a string. + if not self._escape_next: + self._in_string = not self._in_string + self._obj += char + elif char in string.whitespace: + if self._in_string: + self._obj += char + elif char == "[": + if self._level == 0: + self._level += 1 + else: + self._obj += char + elif char == "]": + if self._level == 1: + self._level -= 1 + else: + self._obj += char + else: + self._obj += char + self._escape_next = not self._escape_next if char == "\\" else False + + def __next__(self): + while not self._ready_objs: + try: + chunk = next(self._response_itr) + self._process_chunk(chunk) + except StopIteration as e: + if self._level > 0: + raise ValueError("Unfinished stream: %s" % self._obj) + raise e + return self._grab() + + def _grab(self): + # Add extra quotes to make json.loads happy. + return self._response_message_cls.from_json(self._ready_objs.popleft()) + + def __iter__(self): + return self diff --git a/google/api_core/retry.py b/google/api_core/retry.py index bd3a4a65..ce496937 100644 --- a/google/api_core/retry.py +++ b/google/api_core/retry.py @@ -203,8 +203,8 @@ def retry_target(target, predicate, sleep_generator, deadline, on_error=None): if deadline_datetime is not None: if deadline_datetime <= now: raise exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target + "Deadline of {:.1f}s exceeded while calling target function".format( + deadline ), last_exc, ) from last_exc diff --git a/google/api_core/retry_async.py b/google/api_core/retry_async.py index 2dfa2f6e..68a25597 100644 --- a/google/api_core/retry_async.py +++ b/google/api_core/retry_async.py @@ -132,8 +132,8 @@ async def retry_target(target, predicate, sleep_generator, deadline, on_error=No # Chains the raising RetryError with the root cause error, # which helps observability and debugability. raise exceptions.RetryError( - "Deadline of {:.1f}s exceeded while calling {}".format( - deadline, target + "Deadline of {:.1f}s exceeded while calling target function".format( + deadline ), last_exc, ) from last_exc diff --git a/google/api_core/version.py b/google/api_core/version.py index 999199f5..fe11624d 100644 --- a/google/api_core/version.py +++ b/google/api_core/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.3.0" +__version__ = "2.4.0" diff --git a/noxfile.py b/noxfile.py index db37c561..003a276d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -132,7 +132,7 @@ def unit(session): default(session) -@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) def unit_grpc_gcp(session): """Run the unit test suite with grpcio-gcp installed.""" constraints_path = str( diff --git a/tests/asyncio/test_retry_async.py b/tests/asyncio/test_retry_async.py index 9e51044b..873caaf1 100644 --- a/tests/asyncio/test_retry_async.py +++ b/tests/asyncio/test_retry_async.py @@ -120,6 +120,10 @@ async def test_retry_target_deadline_exceeded(utcnow, sleep): assert exc_info.match("last exception: meep") assert target.call_count == 2 + # Ensure the exception message does not include the target fn: + # it may be a partial with user data embedded + assert str(target) not in exc_info.exconly() + @pytest.mark.asyncio async def test_retry_target_bad_sleep_generator(): diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py index 622f58ab..4169ad44 100644 --- a/tests/unit/test_exceptions.py +++ b/tests/unit/test_exceptions.py @@ -275,31 +275,56 @@ def create_bad_request_details(): return status_detail +def create_error_info_details(): + info = error_details_pb2.ErrorInfo( + reason="SERVICE_DISABLED", + domain="googleapis.com", + metadata={ + "consumer": "projects/455411330361", + "service": "translate.googleapis.com", + }, + ) + status_detail = any_pb2.Any() + status_detail.Pack(info) + return status_detail + + def test_error_details_from_rest_response(): bad_request_detail = create_bad_request_details() + error_info_detail = create_error_info_details() status = status_pb2.Status() status.code = 3 status.message = ( "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." ) status.details.append(bad_request_detail) + status.details.append(error_info_detail) # See JSON schema in https://cloud.google.com/apis/design/errors#http_mapping http_response = make_response( - json.dumps({"error": json.loads(json_format.MessageToJson(status))}).encode( - "utf-8" - ) + json.dumps( + {"error": json.loads(json_format.MessageToJson(status, sort_keys=True))} + ).encode("utf-8") ) exception = exceptions.from_http_response(http_response) - want_error_details = [json.loads(json_format.MessageToJson(bad_request_detail))] + want_error_details = [ + json.loads(json_format.MessageToJson(bad_request_detail)), + json.loads(json_format.MessageToJson(error_info_detail)), + ] assert want_error_details == exception.details + # 404 POST comes from make_response. assert str(exception) == ( "404 POST https://example.com/: 3 INVALID_ARGUMENT:" " One of content, or gcs_content_uri must be set." " [{'@type': 'type.googleapis.com/google.rpc.BadRequest'," - " 'fieldViolations': [{'field': 'document.content'," - " 'description': 'Must have some text content to annotate.'}]}]" + " 'fieldViolations': [{'description': 'Must have some text content to annotate.'," + " 'field': 'document.content'}]}," + " {'@type': 'type.googleapis.com/google.rpc.ErrorInfo'," + " 'domain': 'googleapis.com'," + " 'metadata': {'consumer': 'projects/455411330361'," + " 'service': 'translate.googleapis.com'}," + " 'reason': 'SERVICE_DISABLED'}]" ) @@ -311,6 +336,11 @@ def test_error_details_from_v1_rest_response(): ) exception = exceptions.from_http_response(response) assert exception.details == [] + assert ( + exception.reason is None + and exception.domain is None + and exception.metadata is None + ) @pytest.mark.skipif(grpc is None, reason="gRPC not importable") @@ -320,8 +350,10 @@ def test_error_details_from_grpc_response(): status.message = ( "3 INVALID_ARGUMENT: One of content, or gcs_content_uri must be set." ) - status_detail = create_bad_request_details() - status.details.append(status_detail) + status_br_detail = create_bad_request_details() + status_ei_detail = create_error_info_details() + status.details.append(status_br_detail) + status.details.append(status_ei_detail) # Actualy error doesn't matter as long as its grpc.Call, # because from_call is mocked. @@ -331,8 +363,13 @@ def test_error_details_from_grpc_response(): exception = exceptions.from_grpc_error(error) bad_request_detail = error_details_pb2.BadRequest() - status_detail.Unpack(bad_request_detail) - assert exception.details == [bad_request_detail] + error_info_detail = error_details_pb2.ErrorInfo() + status_br_detail.Unpack(bad_request_detail) + status_ei_detail.Unpack(error_info_detail) + assert exception.details == [bad_request_detail, error_info_detail] + assert exception.reason == error_info_detail.reason + assert exception.domain == error_info_detail.domain + assert exception.metadata == error_info_detail.metadata @pytest.mark.skipif(grpc is None, reason="gRPC not importable") @@ -351,3 +388,8 @@ def test_error_details_from_grpc_response_unknown_error(): m.return_value = status exception = exceptions.from_grpc_error(error) assert exception.details == [status_detail] + assert ( + exception.reason is None + and exception.domain is None + and exception.metadata is None + ) diff --git a/tests/unit/test_rest_streaming.py b/tests/unit/test_rest_streaming.py new file mode 100644 index 00000000..4be59580 --- /dev/null +++ b/tests/unit/test_rest_streaming.py @@ -0,0 +1,211 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import random +import time +from typing import List +from unittest.mock import patch + +import proto +import pytest +import requests + +from google.api_core import rest_streaming +from google.protobuf import duration_pb2 +from google.protobuf import timestamp_pb2 + + +SEED = int(time.time()) +logging.info(f"Starting rest streaming tests with random seed: {SEED}") +random.seed(SEED) + + +class Genre(proto.Enum): + GENRE_UNSPECIFIED = 0 + CLASSICAL = 1 + JAZZ = 2 + ROCK = 3 + + +class Composer(proto.Message): + given_name = proto.Field(proto.STRING, number=1) + family_name = proto.Field(proto.STRING, number=2) + relateds = proto.RepeatedField(proto.STRING, number=3) + indices = proto.MapField(proto.STRING, proto.STRING, number=4) + + +class Song(proto.Message): + composer = proto.Field(Composer, number=1) + title = proto.Field(proto.STRING, number=2) + lyrics = proto.Field(proto.STRING, number=3) + year = proto.Field(proto.INT32, number=4) + genre = proto.Field(Genre, number=5) + is_five_mins_longer = proto.Field(proto.BOOL, number=6) + score = proto.Field(proto.DOUBLE, number=7) + likes = proto.Field(proto.INT64, number=8) + duration = proto.Field(duration_pb2.Duration, number=9) + date_added = proto.Field(timestamp_pb2.Timestamp, number=10) + + +class EchoResponse(proto.Message): + content = proto.Field(proto.STRING, number=1) + + +class ResponseMock(requests.Response): + class _ResponseItr: + def __init__(self, _response_bytes: bytes, random_split=False): + self._responses_bytes = _response_bytes + self._i = 0 + self._random_split = random_split + + def __next__(self): + if self._i == len(self._responses_bytes): + raise StopIteration + if self._random_split: + n = random.randint(1, len(self._responses_bytes[self._i :])) + else: + n = 1 + x = self._responses_bytes[self._i : self._i + n] + self._i += n + return x.decode("utf-8") + + def __init__( + self, responses: List[proto.Message], response_cls, random_split=False, + ): + super().__init__() + self._responses = responses + self._random_split = random_split + self._response_message_cls = response_cls + + def _parse_responses(self, responses: List[proto.Message]) -> bytes: + # json.dumps returns a string surrounded with quotes that need to be stripped + # in order to be an actual JSON. + json_responses = [ + self._response_message_cls.to_json(r).strip('"') for r in responses + ] + logging.info(f"Sending JSON stream: {json_responses}") + ret_val = "[{}]".format(",".join(json_responses)) + return bytes(ret_val, "utf-8") + + def close(self): + raise NotImplementedError() + + def iter_content(self, *args, **kwargs): + return self._ResponseItr( + self._parse_responses(self._responses), random_split=self._random_split, + ) + + +@pytest.mark.parametrize("random_split", [False]) +def test_next_simple(random_split): + responses = [EchoResponse(content="hello world"), EchoResponse(content="yes")] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=EchoResponse + ) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_nested(random_split): + responses = [ + Song(title="some song", composer=Composer(given_name="some name")), + Song(title="another song", date_added=datetime.datetime(2021, 12, 17)), + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_stress(random_split): + n = 50 + responses = [ + Song(title="title_%d" % i, composer=Composer(given_name="name_%d" % i)) + for i in range(n) + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +@pytest.mark.parametrize("random_split", [True, False]) +def test_next_escaped_characters_in_string(random_split): + composer_with_relateds = Composer() + relateds = ["Artist A", "Artist B"] + composer_with_relateds.relateds = relateds + + responses = [ + Song(title='ti"tle\nfoo\tbar{}', composer=Composer(given_name="name\n\n\n")), + Song( + title='{"this is weird": "totally"}', composer=Composer(given_name="\\{}\\") + ), + Song(title='\\{"key": ["value",]}\\', composer=composer_with_relateds), + ] + resp = ResponseMock( + responses=responses, random_split=random_split, response_cls=Song + ) + itr = rest_streaming.ResponseIterator(resp, Song) + assert list(itr) == responses + + +def test_next_not_array(): + with patch.object( + ResponseMock, "iter_content", return_value=iter('{"hello": 0}') + ) as mock_method: + + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + mock_method.assert_called_once() + + +def test_cancel(): + with patch.object(ResponseMock, "close", return_value=None) as mock_method: + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + itr.cancel() + mock_method.assert_called_once() + + +def test_check_buffer(): + with patch.object( + ResponseMock, + "_parse_responses", + return_value=bytes('[{"content": "hello"}, {', "utf-8"), + ): + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + next(itr) + + +def test_next_html(): + with patch.object( + ResponseMock, "iter_content", return_value=iter("") + ) as mock_method: + + resp = ResponseMock(responses=[], response_cls=EchoResponse) + itr = rest_streaming.ResponseIterator(resp, EchoResponse) + with pytest.raises(ValueError): + next(itr) + mock_method.assert_called_once() diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 199ca559..74c5d77c 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -152,6 +152,10 @@ def test_retry_target_deadline_exceeded(utcnow, sleep): assert exc_info.match("last exception: meep") assert target.call_count == 2 + # Ensure the exception message does not include the target fn: + # it may be a partial with user data embedded + assert str(target) not in exc_info.exconly() + def test_retry_target_bad_sleep_generator(): with pytest.raises(ValueError, match="Sleep generator"):