From 7eb354f6c80557cba572138f0bd76f08e2ed3d90 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Mon, 7 Jun 2021 15:56:57 -0400 Subject: [PATCH 01/10] chore: migrate to owl bot (#170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: migrate to owl bot * chore: use the latest post processor image * run the post processor * remove synth.py and synth.metadata * attempt to fix coverage failure * chore: raise coverage requirements to 100 * 🦉 Updates from OwlBot Co-authored-by: Bu Sun Kim Co-authored-by: Owl Bot --- .flake8 | 1 + .github/.OwlBot.lock.yaml | 4 + .github/.OwlBot.yaml | 28 ++++ .github/header-checker-lint.yml | 15 ++ .gitignore | 4 +- .kokoro/build.sh | 26 +++- .kokoro/docs/docs-presubmit.cfg | 11 ++ .kokoro/release.sh | 4 +- .kokoro/release/common.cfg | 14 +- .kokoro/samples/python3.6/periodic-head.cfg | 11 ++ .kokoro/samples/python3.7/periodic-head.cfg | 11 ++ .kokoro/samples/python3.8/periodic-head.cfg | 11 ++ .kokoro/test-samples-against-head.sh | 28 ++++ .kokoro/test-samples-impl.sh | 102 ++++++++++++++ .kokoro/test-samples.sh | 96 +++---------- .kokoro/trampoline_v2.sh | 2 +- .pre-commit-config.yaml | 31 ++++ .trampolinerc | 1 + CONTRIBUTING.rst | 59 ++++---- LICENSE | 7 +- MANIFEST.in | 4 +- docs/_static/custom.css | 18 ++- docs/conf.py | 13 ++ noxfile.py | 85 +++++++---- synth.py => owlbot.py | 125 ++++++++-------- renovate.json | 6 +- synth.metadata | 149 -------------------- testing/constraints-3.6.txt | 3 +- 28 files changed, 491 insertions(+), 378 deletions(-) create mode 100644 .github/.OwlBot.lock.yaml create mode 100644 .github/.OwlBot.yaml create mode 100644 .github/header-checker-lint.yml create mode 100644 .kokoro/samples/python3.6/periodic-head.cfg create mode 100644 .kokoro/samples/python3.7/periodic-head.cfg create mode 100644 .kokoro/samples/python3.8/periodic-head.cfg create mode 100755 .kokoro/test-samples-against-head.sh create mode 100755 .kokoro/test-samples-impl.sh create mode 100644 .pre-commit-config.yaml rename synth.py => owlbot.py (60%) delete mode 100644 synth.metadata diff --git a/.flake8 b/.flake8 index ed931638..29227d4c 100644 --- a/.flake8 +++ b/.flake8 @@ -26,6 +26,7 @@ exclude = *_pb2.py # Standard linting exemptions. + **/.nox/** __pycache__, .git, *.pyc, diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml new file mode 100644 index 00000000..8051d189 --- /dev/null +++ b/.github/.OwlBot.lock.yaml @@ -0,0 +1,4 @@ +docker: + digest: sha256:0856ca711da1fd5ec9d6d7da6c50aa0bbf550fb94acb47b55159a640791987bf + image: gcr.io/repo-automation-bots/owlbot-python:latest + diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml new file mode 100644 index 00000000..83c15486 --- /dev/null +++ b/.github/.OwlBot.yaml @@ -0,0 +1,28 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + image: gcr.io/repo-automation-bots/owlbot-python:latest + +deep-remove-regex: + - /owl-bot-staging + +deep-copy-regex: + - source: /google/datastore/(v.*)/.*-py/(.*) + dest: /owl-bot-staging/datastore/$1/$2 + - source: /google/datastore/admin/(v.*)/.*-py/(.*) + dest: /owl-bot-staging/datastore_admin/$1/$2 + +begin-after-commit-hash: de97bb0aeade880aba2cd71a55c06dbc4cd2b337 + diff --git a/.github/header-checker-lint.yml b/.github/header-checker-lint.yml new file mode 100644 index 00000000..6fe78aa7 --- /dev/null +++ b/.github/header-checker-lint.yml @@ -0,0 +1,15 @@ +{"allowedCopyrightHolders": ["Google LLC"], + "allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"], + "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt", "**/__init__.py", "samples/**/constraints.txt", "samples/**/constraints-test.txt"], + "sourceFileExtensions": [ + "ts", + "js", + "java", + "sh", + "Dockerfile", + "yaml", + "py", + "html", + "txt" + ] +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index b9daa52f..b4243ced 100644 --- a/.gitignore +++ b/.gitignore @@ -50,8 +50,10 @@ docs.metadata # Virtual environment env/ + +# Test logs coverage.xml -sponge_log.xml +*sponge_log.xml # System test environment variables. system_tests/local_test_setup diff --git a/.kokoro/build.sh b/.kokoro/build.sh index df77f979..59bd450e 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -15,7 +15,11 @@ set -eo pipefail -cd github/python-datastore +if [[ -z "${PROJECT_ROOT:-}" ]]; then + PROJECT_ROOT="github/python-datastore" +fi + +cd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 @@ -30,16 +34,26 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") # Remove old nox -python3.6 -m pip uninstall --yes --quiet nox-automation +python3 -m pip uninstall --yes --quiet nox-automation # Install nox -python3.6 -m pip install --upgrade --quiet nox -python3.6 -m nox --version +python3 -m pip install --upgrade --quiet nox +python3 -m nox --version + +# If this is a continuous build, send the test log to the FlakyBot. +# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. +if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then + cleanup() { + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + } + trap cleanup EXIT HUP +fi # If NOX_SESSION is set, it only runs the specified session, # otherwise run all the sessions. if [[ -n "${NOX_SESSION:-}" ]]; then - python3.6 -m nox -s "${NOX_SESSION:-}" + python3 -m nox -s ${NOX_SESSION:-} else - python3.6 -m nox + python3 -m nox fi diff --git a/.kokoro/docs/docs-presubmit.cfg b/.kokoro/docs/docs-presubmit.cfg index 11181078..049a9863 100644 --- a/.kokoro/docs/docs-presubmit.cfg +++ b/.kokoro/docs/docs-presubmit.cfg @@ -15,3 +15,14 @@ env_vars: { key: "TRAMPOLINE_IMAGE_UPLOAD" value: "false" } + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-datastore/.kokoro/build.sh" +} + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "docs docfx" +} diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 95282f08..c593dd9d 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google_cloud_pypi_password") +TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") cd github/python-datastore python3 setup.py sdist bdist_wheel -twine upload --username gcloudpypi --password "${TWINE_PASSWORD}" dist/* +twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index b4f0c6ad..8571f251 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,18 +23,8 @@ env_vars: { value: "github/python-datastore/.kokoro/release.sh" } -# Fetch PyPI password -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "google_cloud_pypi_password" - } - } -} - # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" -} \ No newline at end of file + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" +} diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg new file mode 100644 index 00000000..f9cfcd33 --- /dev/null +++ b/.kokoro/samples/python3.6/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.7/periodic-head.cfg b/.kokoro/samples/python3.7/periodic-head.cfg new file mode 100644 index 00000000..f9cfcd33 --- /dev/null +++ b/.kokoro/samples/python3.7/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.8/periodic-head.cfg b/.kokoro/samples/python3.8/periodic-head.cfg new file mode 100644 index 00000000..f9cfcd33 --- /dev/null +++ b/.kokoro/samples/python3.8/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh new file mode 100755 index 00000000..7503e762 --- /dev/null +++ b/.kokoro/test-samples-against-head.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A customized test runner for samples. +# +# For periodic builds, you can specify this file for testing against head. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-datastore + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh new file mode 100755 index 00000000..cf5de74c --- /dev/null +++ b/.kokoro/test-samples-impl.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Exit early if samples directory doesn't exist +if [ ! -d "./samples" ]; then + echo "No tests run. `./samples` not found" + exit 0 +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.6 -m pip install --upgrade --quiet nox + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.6 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the FlakyBot. + # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index c4163d79..b3014bb9 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The default test runner for samples. +# +# For periodic builds, we rewinds the repo to the latest release, and +# run test-samples-impl.sh. # `-e` enables the script to automatically fail when a command fails # `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero @@ -24,87 +28,19 @@ cd github/python-datastore # Run periodic samples tests at latest release if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + # preserving the test runner implementation. + cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + echo "Now we rewind the repo back to the latest release..." LATEST_RELEASE=$(git describe --abbrev=0 --tags) git checkout $LATEST_RELEASE -fi - -# Exit early if samples directory doesn't exist -if [ ! -d "./samples" ]; then - echo "No tests run. `./samples` not found" - exit 0 -fi - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Install nox -python3.6 -m pip install --upgrade --quiet nox - -# Use secrets acessor service account to get secrets -if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then - gcloud auth activate-service-account \ - --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ - --project="cloud-devrel-kokoro-resources" -fi - -# This script will create 3 files: -# - testing/test-env.sh -# - testing/service-account.json -# - testing/client-secrets.json -./scripts/decrypt-secrets.sh - -source ./testing/test-env.sh -export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json - -# For cloud-run session, we activate the service account for gcloud sdk. -gcloud auth activate-service-account \ - --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" - -export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json - -echo -e "\n******************** TESTING PROJECTS ********************" - -# Switch to 'fail at end' to allow all tests to complete before exiting. -set +e -# Use RTN to return a non-zero value if the test fails. -RTN=0 -ROOT=$(pwd) -# Find all requirements.txt in the samples directory (may break on whitespace). -for file in samples/**/requirements.txt; do - cd "$ROOT" - # Navigate to the project folder. - file=$(dirname "$file") - cd "$file" - - echo "------------------------------------------------------------" - echo "- testing $file" - echo "------------------------------------------------------------" - - # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" - EXIT=$? - - # If this is a periodic build, send the test log to the Build Cop Bot. - # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/buildcop. - if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop - $KOKORO_GFILE_DIR/linux_amd64/buildcop + echo "The current head is: " + echo $(git rev-parse --verify HEAD) + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + # move back the test runner implementation if there's no file. + if [ ! -f .kokoro/test-samples-impl.sh ]; then + cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh fi +fi - if [[ $EXIT -ne 0 ]]; then - RTN=1 - echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" - else - echo -e "\n Testing completed.\n" - fi - -done -cd "$ROOT" - -# Workaround for Kokoro permissions issue: delete secrets -rm testing/{test-env.sh,client-secrets.json,service-account.json} - -exit "$RTN" +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 719bcd5b..4af6cdc2 100755 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -159,7 +159,7 @@ if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then "KOKORO_GITHUB_COMMIT" "KOKORO_GITHUB_PULL_REQUEST_NUMBER" "KOKORO_GITHUB_PULL_REQUEST_COMMIT" - # For Build Cop Bot + # For FlakyBot "KOKORO_GITHUB_COMMIT_URL" "KOKORO_GITHUB_PULL_REQUEST_URL" ) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..4f00c7cf --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml +- repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 diff --git a/.trampolinerc b/.trampolinerc index 995ee291..383b6ec8 100644 --- a/.trampolinerc +++ b/.trampolinerc @@ -24,6 +24,7 @@ required_envvars+=( pass_down_envvars+=( "STAGING_BUCKET" "V2_STAGING_BUCKET" + "NOX_SESSION" ) # Prevent unintentional override on the default image. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2002603b..0f183f3c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,8 +21,8 @@ In order to add a feature: - The feature must be documented in both the API and narrative documentation. -- The feature must work fully on the following CPython versions: 2.7, - 3.5, 3.6, 3.7 and 3.8 on both UNIX and Windows. +- The feature must work fully on the following CPython versions: + 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -70,9 +70,14 @@ We use `nox `__ to instrument our tests. - To test your changes, run unit tests with ``nox``:: $ nox -s unit-2.7 - $ nox -s unit-3.7 + $ nox -s unit-3.8 $ ... +- Args to pytest can be passed through the nox command separated by a `--`. For + example, to run a single test:: + + $ nox -s unit-3.8 -- -k + .. note:: The unit tests and system tests are described in the @@ -93,8 +98,12 @@ On Debian/Ubuntu:: ************ Coding Style ************ +- We use the automatic code formatter ``black``. You can run it using + the nox session ``blacken``. This will eliminate many lint errors. Run via:: + + $ nox -s blacken -- PEP8 compliance, with exceptions defined in the linter configuration. +- PEP8 compliance is required, with exceptions defined in the linter configuration. If you have ``nox`` installed, you can test that you have not introduced any non-compliant code via:: @@ -111,6 +120,16 @@ Coding Style should point to the official ``googleapis`` checkout and the the branch should be the main branch on that remote (``master``). +- This repository contains configuration for the + `pre-commit `__ tool, which automates checking + our linters during a commit. If you have it installed on your ``$PATH``, + you can enable enforcing those checks via: + +.. code-block:: bash + + $ pre-commit install + pre-commit installed at .git/hooks/pre-commit + Exceptions to PEP8: - Many unit tests use a helper method, ``_call_fut`` ("FUT" is short for @@ -123,33 +142,25 @@ Running System Tests - To run system tests, you can execute:: - $ nox -s system-3.7 + # Run all system tests + $ nox -s system-3.8 $ nox -s system-2.7 + # Run a single system test + $ nox -s system-3.8 -- -k + + .. note:: System tests are only configured to run under Python 2.7 and - Python 3.7. For expediency, we do not run them in older versions + Python 3.8. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local auth settings and change some configuration in your project to run all the tests. -- System tests will be run against an actual project and - so you'll need to provide some environment variables to facilitate - authentication to your project: - - - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file; - Such a file can be downloaded directly from the developer's console by clicking - "Generate new JSON key". See private key - `docs `__ - for more details. - -- Once you have downloaded your json keys, set the environment variable - ``GOOGLE_APPLICATION_CREDENTIALS`` to the absolute path of the json file:: - - $ export GOOGLE_APPLICATION_CREDENTIALS="/Users//path/to/app_credentials.json" +- System tests will be run against an actual project. You should use local credentials from gcloud when possible. See `Best practices for application authentication `__. Some tests require a service account. For those tests see `Authenticating as a service account `__. - You'll need to create composite `indexes `__ @@ -176,7 +187,6 @@ Running System Tests $ python tests/system/utils/clear_datastore.py - ************* Test Coverage ************* @@ -217,25 +227,24 @@ Supported Python Versions We support: -- `Python 3.5`_ - `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ +- `Python 3.9`_ -.. _Python 3.5: https://docs.python.org/3.5/ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ +.. _Python 3.9: https://docs.python.org/3.9/ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-datastore/blob/master/noxfile.py -Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020. We also explicitly decided to support Python 3 beginning with version -3.5. Reasons for this include: +3.6. Reasons for this include: - Encouraging use of newest versions of Python 3 - Taking the lead of `prominent`_ open-source `projects`_ diff --git a/LICENSE b/LICENSE index a8ee855d..d6456956 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ - Apache License + + Apache License Version 2.0, January 2004 - https://www.apache.org/licenses/ + http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -192,7 +193,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/MANIFEST.in b/MANIFEST.in index e9e29d12..e783f4c6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -16,10 +16,10 @@ # Generated by synthtool. DO NOT EDIT! include README.rst LICENSE -recursive-include google *.json *.proto +recursive-include google *.json *.proto py.typed recursive-include tests * global-exclude *.py[co] global-exclude __pycache__ # Exclude scripts for samples readmegen -prune scripts/readme-gen \ No newline at end of file +prune scripts/readme-gen diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 0abaf229..b0a29546 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,4 +1,20 @@ div#python2-eol { border-color: red; border-width: medium; -} \ No newline at end of file +} + +/* Ensure minimum width for 'Parameters' / 'Returns' column */ +dl.field-list > dt { + min-width: 100px +} + +/* Insert space between methods for readability */ +dl.method { + padding-top: 10px; + padding-bottom: 10px +} + +/* Insert empty space between classes */ +dl.class { + padding-bottom: 50px +} diff --git a/docs/conf.py b/docs/conf.py index 46cba6ca..ee4a5a8f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # google-cloud-datastore documentation build configuration file # diff --git a/noxfile.py b/noxfile.py index a6c9bf64..4dd9e06c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,6 +18,7 @@ from __future__ import absolute_import import os +import pathlib import shutil import nox @@ -30,6 +31,22 @@ SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +# 'docfx' is excluded since it only needs to run in 'docs-presubmit' +nox.options.sessions = [ + "unit", + "system", + "cover", + "lint", + "lint_setup_py", + "blacken", + "docs", +] + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + @nox.session(python=DEFAULT_PYTHON_VERSION) def lint(session): @@ -45,16 +62,9 @@ def lint(session): session.run("flake8", "google", "tests") -@nox.session(python="3.6") +@nox.session(python=DEFAULT_PYTHON_VERSION) def blacken(session): - """Run black. - - Format code to uniform standard. - - This currently uses Python 3.6 due to the automated Kokoro run of synthtool. - That run uses an image that doesn't have 3.6 installed. Before updating this - check the state of the `gcp_ubuntu_config` we use for that Kokoro run. - """ + """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) session.run( "black", *BLACK_PATHS, @@ -70,18 +80,23 @@ def lint_setup_py(session): def default(session): # Install all test dependencies, then install this package in-place. - session.install( - "mock", "pytest", "pytest-cov", + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) - session.install("-e", ".") + session.install("asyncmock", "pytest-asyncio", "-c", constraints_path) + + session.install("mock", "pytest", "pytest-cov", "-c", constraints_path) + + session.install("-e", ".", "-c", constraints_path) # Run py.test against the unit tests. session.run( "py.test", "--quiet", - "--cov=google.cloud.datastore", - "--cov=google.cloud", - "--cov=tests.unit", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=google/cloud", + "--cov=tests/unit", "--cov-append", "--cov-config=.coveragerc", "--cov-report=", @@ -101,15 +116,18 @@ def unit(session): @nox.parametrize("disable_grpc", [False, True]) def system(session, disable_grpc): """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) system_test_path = os.path.join("tests", "system.py") system_test_folder_path = os.path.join("tests", "system") # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": session.skip("RUN_SYSTEM_TESTS is set to false, skipping") - # Sanity check: Only run tests if the environment variable is set. - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") system_test_exists = os.path.exists(system_test_path) system_test_folder_exists = os.path.exists(system_test_folder_path) @@ -122,10 +140,8 @@ def system(session, disable_grpc): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. - session.install( - "mock", "pytest", "google-cloud-testutils", - ) - session.install("-e", ".") + session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) + session.install("-e", ".", "-c", constraints_path) env = {} if disable_grpc: @@ -133,10 +149,23 @@ def system(session, disable_grpc): # Run py.test against the system tests. if system_test_exists: - session.run("py.test", "--quiet", system_test_path, env=env, *session.posargs) + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + env=env, + *session.posargs, + ) + if system_test_folder_exists: session.run( - "py.test", "--quiet", system_test_folder_path, env=env, *session.posargs + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + env=env, + *session.posargs, ) @@ -148,7 +177,7 @@ def cover(session): test runs (not system test runs), and then erases coverage data. """ session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under=97") + session.run("coverage", "report", "--show-missing", "--fail-under=100") session.run("coverage", "erase") @@ -158,7 +187,7 @@ def docs(session): """Build the docs for this library.""" session.install("-e", ".") - session.install("sphinx<3.0.0", "alabaster", "recommonmark") + session.install("sphinx==4.0.1", "alabaster", "recommonmark") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( @@ -180,7 +209,9 @@ def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml") + session.install( + "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" + ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/synth.py b/owlbot.py similarity index 60% rename from synth.py rename to owlbot.py index 6a59fc5b..8017fb4a 100644 --- a/synth.py +++ b/owlbot.py @@ -16,75 +16,48 @@ import synthtool as s from synthtool import gcp -gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() -# ---------------------------------------------------------------------------- -# Generate datastore GAPIC layer -# ---------------------------------------------------------------------------- -library = gapic.py_library( - service="datastore", - version="v1", - bazel_target="//google/datastore/v1:datastore-v1-py", - include_protos=True, -) - -s.move(library / "google/cloud/datastore_v1") - -s.move( - library / f"tests/", - f"tests", -) -s.move(library / "scripts") - -# ---------------------------------------------------------------------------- -# Generate datastore admin GAPIC layer -# ---------------------------------------------------------------------------- -library = gapic.py_library( - service="datastore_admin", - version="v1", - bazel_target="//google/datastore/admin/v1:datastore-admin-v1-py", - include_protos=True, -) - -s.move( - library / "google/cloud/datastore_admin_v1", - "google/cloud/datastore_admin_v1" -) - +# This library ships clients for two different APIs, +# Datastore and Datastore Admin +datastore_default_version = "v1" +datastore_admin_default_version = "v1" + +for library in s.get_staging_dirs(datastore_default_version): + if library.parent.absolute() == 'datastore': + s.move(library / f"google/cloud/datastore_{library.name}") + s.move(library / f"tests/") + s.move(library / "scripts") + +for library in s.get_staging_dirs(datastore_admin_default_version): + if library.parent.absolute() == 'datastore_admin': + s.replace( + library / "google/**/datastore_admin_client.py", + "google-cloud-datastore-admin", + "google-cloud-datstore" + ) -s.move( - library / f"tests/", - f"tests", -) + # Remove spurious markup + s.replace( + "google/**/datastore_admin/client.py", + "\s+---------------------------------(-)+", + "" + ) -s.move(library / "scripts") -s.replace( - "google/**/datastore_admin_client.py", - "google-cloud-datastore-admin", - "google-cloud-datstore" -) + s.move(library / f"google/cloud/datastore_admin_{library.name}") + s.move(library / f"tests") + s.move(library / "scripts") -# Remove spurious markup -s.replace( - "google/**/datastore_admin/client.py", - "\s+---------------------------------(-)+", - "" -) +s.remove_staging_dirs() # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- -# TODO: cov_level should be 99%, reduced due to regression in test coverage. templated_files = common.py_library( - unit_cov_level=97, - cov_level=97, - unit_test_python_versions=["3.6", "3.7", "3.8", "3.9"], - system_test_python_versions=["3.8"], + microgenerator=True, ) s.move(templated_files, excludes=["docs/multiprocessing.rst", ".coveragerc"]) -s.replace("noxfile.py", """["']sphinx['"]""", '''"sphinx<3.0.0"''') # Preserve system tests w/ GOOGLE_DISABLE_GRPC set (#133, PR #136) s.replace( @@ -116,22 +89,40 @@ def system(session, disable_grpc): s.replace( "noxfile.py", - """\ - session.run\("py.test", "--quiet", system_test_path, \*session.posargs\) -""", - """\ - session.run("py.test", "--quiet", system_test_path, env=env, *session.posargs) + """session\.run\( + "py\.test", + "--quiet", + f"--junitxml=system_\{session\.python\}_sponge_log\.xml", + system_test_path, + \*session\.posargs + \)""", + """session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + env=env, + *session.posargs + ) """, ) s.replace( "noxfile.py", - """\ - session.run\("py.test", "--quiet", system_test_folder_path, \*session.posargs\) -""", - """\ - session.run( - "py.test", "--quiet", system_test_folder_path, env=env, *session.posargs + """session\.run\( + "py\.test", + "--quiet", + f"--junitxml=system_\{session\.python\}_sponge_log\.xml", + system_test_folder_path, + \*session\.posargs + \)""", + """session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + env=env, + *session.posargs ) """, ) diff --git a/renovate.json b/renovate.json index 4fa94931..c0489556 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,9 @@ { "extends": [ "config:base", ":preserveSemverRanges" - ] + ], + "ignorePaths": [".pre-commit-config.yaml"], + "pip_requirements": { + "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] + } } diff --git a/synth.metadata b/synth.metadata deleted file mode 100644 index c8b752ae..00000000 --- a/synth.metadata +++ /dev/null @@ -1,149 +0,0 @@ -{ - "sources": [ - { - "git": { - "name": ".", - "remote": "https://github.com/googleapis/python-datastore.git", - "sha": "3c8da54de9a6837d6d96b6befd3b0ef084d8ff7f" - } - }, - { - "git": { - "name": "googleapis", - "remote": "https://github.com/googleapis/googleapis.git", - "sha": "7f31f40209008ad24058579e7112e45fc9d5715e", - "internalRef": "339939234" - } - }, - { - "git": { - "name": "synthtool", - "remote": "https://github.com/googleapis/synthtool.git", - "sha": "a073c873f3928c561bdf87fdfbf1d081d1998984" - } - } - ], - "destinations": [ - { - "client": { - "source": "googleapis", - "apiName": "datastore", - "apiVersion": "v1", - "language": "python", - "generator": "bazel" - } - }, - { - "client": { - "source": "googleapis", - "apiName": "datastore_admin", - "apiVersion": "v1", - "language": "python", - "generator": "bazel" - } - } - ], - "generatedFiles": [ - ".flake8", - ".github/CONTRIBUTING.md", - ".github/ISSUE_TEMPLATE/bug_report.md", - ".github/ISSUE_TEMPLATE/feature_request.md", - ".github/ISSUE_TEMPLATE/support_request.md", - ".github/PULL_REQUEST_TEMPLATE.md", - ".github/release-please.yml", - ".github/snippet-bot.yml", - ".gitignore", - ".kokoro/build.sh", - ".kokoro/continuous/common.cfg", - ".kokoro/continuous/continuous.cfg", - ".kokoro/docker/docs/Dockerfile", - ".kokoro/docker/docs/fetch_gpg_keys.sh", - ".kokoro/docs/common.cfg", - ".kokoro/docs/docs-presubmit.cfg", - ".kokoro/docs/docs.cfg", - ".kokoro/populate-secrets.sh", - ".kokoro/presubmit/common.cfg", - ".kokoro/presubmit/presubmit.cfg", - ".kokoro/publish-docs.sh", - ".kokoro/release.sh", - ".kokoro/release/common.cfg", - ".kokoro/release/release.cfg", - ".kokoro/samples/lint/common.cfg", - ".kokoro/samples/lint/continuous.cfg", - ".kokoro/samples/lint/periodic.cfg", - ".kokoro/samples/lint/presubmit.cfg", - ".kokoro/samples/python3.6/common.cfg", - ".kokoro/samples/python3.6/continuous.cfg", - ".kokoro/samples/python3.6/periodic.cfg", - ".kokoro/samples/python3.6/presubmit.cfg", - ".kokoro/samples/python3.7/common.cfg", - ".kokoro/samples/python3.7/continuous.cfg", - ".kokoro/samples/python3.7/periodic.cfg", - ".kokoro/samples/python3.7/presubmit.cfg", - ".kokoro/samples/python3.8/common.cfg", - ".kokoro/samples/python3.8/continuous.cfg", - ".kokoro/samples/python3.8/periodic.cfg", - ".kokoro/samples/python3.8/presubmit.cfg", - ".kokoro/test-samples.sh", - ".kokoro/trampoline.sh", - ".kokoro/trampoline_v2.sh", - ".trampolinerc", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.rst", - "LICENSE", - "MANIFEST.in", - "docs/_static/custom.css", - "docs/_templates/layout.html", - "docs/conf.py", - "google/cloud/datastore_admin_v1/__init__.py", - "google/cloud/datastore_admin_v1/proto/datastore_admin.proto", - "google/cloud/datastore_admin_v1/proto/index.proto", - "google/cloud/datastore_admin_v1/py.typed", - "google/cloud/datastore_admin_v1/services/__init__.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/__init__.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/client.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/pagers.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/transports/__init__.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py", - "google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py", - "google/cloud/datastore_admin_v1/types/__init__.py", - "google/cloud/datastore_admin_v1/types/datastore_admin.py", - "google/cloud/datastore_admin_v1/types/index.py", - "google/cloud/datastore_v1/__init__.py", - "google/cloud/datastore_v1/proto/datastore.proto", - "google/cloud/datastore_v1/proto/entity.proto", - "google/cloud/datastore_v1/proto/query.proto", - "google/cloud/datastore_v1/py.typed", - "google/cloud/datastore_v1/services/__init__.py", - "google/cloud/datastore_v1/services/datastore/__init__.py", - "google/cloud/datastore_v1/services/datastore/async_client.py", - "google/cloud/datastore_v1/services/datastore/client.py", - "google/cloud/datastore_v1/services/datastore/transports/__init__.py", - "google/cloud/datastore_v1/services/datastore/transports/base.py", - "google/cloud/datastore_v1/services/datastore/transports/grpc.py", - "google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py", - "google/cloud/datastore_v1/types/__init__.py", - "google/cloud/datastore_v1/types/datastore.py", - "google/cloud/datastore_v1/types/entity.py", - "google/cloud/datastore_v1/types/query.py", - "noxfile.py", - "renovate.json", - "scripts/decrypt-secrets.sh", - "scripts/fixup_datastore_admin_v1_keywords.py", - "scripts/fixup_datastore_v1_keywords.py", - "scripts/readme-gen/readme_gen.py", - "scripts/readme-gen/templates/README.tmpl.rst", - "scripts/readme-gen/templates/auth.tmpl.rst", - "scripts/readme-gen/templates/auth_api_key.tmpl.rst", - "scripts/readme-gen/templates/install_deps.tmpl.rst", - "scripts/readme-gen/templates/install_portaudio.tmpl.rst", - "setup.cfg", - "testing/.gitignore", - "tests/unit/gapic/datastore_admin_v1/__init__.py", - "tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py", - "tests/unit/gapic/datastore_v1/__init__.py", - "tests/unit/gapic/datastore_v1/test_datastore.py" - ] -} \ No newline at end of file diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 20a821c2..01fc45a4 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -8,4 +8,5 @@ google-api-core==1.22.2 google-cloud-core==1.4.0 proto-plus==1.4.0 -libcst==0.2.5 \ No newline at end of file +libcst==0.2.5 +google-auth==1.24.0 # TODO: remove when google-auth>=1.25.0 is required through google-api-core \ No newline at end of file From 885456252e9d0a1858122a0182a9f9e15fc5bcbf Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 15:39:56 -0400 Subject: [PATCH 02/10] chore: new owl bot post processor docker image (#177) Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:9d6a2d613e2c04c07ecdb6c287e3931890f6d30266ab5ee4ee412f748dc98341 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 5 ++--- docs/conf.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 8051d189..c1ef6e60 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,4 +1,3 @@ docker: - digest: sha256:0856ca711da1fd5ec9d6d7da6c50aa0bbf550fb94acb47b55159a640791987bf - image: gcr.io/repo-automation-bots/owlbot-python:latest - + image: gcr.io/repo-automation-bots/owlbot-python:latest + digest: sha256:9d6a2d613e2c04c07ecdb6c287e3931890f6d30266ab5ee4ee412f748dc98341 diff --git a/docs/conf.py b/docs/conf.py index ee4a5a8f..2c48bd48 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -363,6 +363,7 @@ "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), } From 2d1446fec9e1b13503d4e109ec6133d14c281f85 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Tue, 15 Jun 2021 14:38:55 -0700 Subject: [PATCH 03/10] chore: create flakybot.yaml (#178) --- .github/flakybot.yaml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/flakybot.yaml diff --git a/.github/flakybot.yaml b/.github/flakybot.yaml new file mode 100644 index 00000000..cb83375f --- /dev/null +++ b/.github/flakybot.yaml @@ -0,0 +1 @@ +issuePriority: p2 From 1717b8a956cc86e48ccad96512cf079d024f1563 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 17 Jun 2021 10:58:08 +0000 Subject: [PATCH 04/10] chore: new owl bot post processor docker image (#179) Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:58c7342b0bccf85028100adaa3d856cb4a871c22ca9c01960d996e66c40548ce --- .github/.OwlBot.lock.yaml | 2 +- docs/conf.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index c1ef6e60..ea06d395 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:9d6a2d613e2c04c07ecdb6c287e3931890f6d30266ab5ee4ee412f748dc98341 + digest: sha256:58c7342b0bccf85028100adaa3d856cb4a871c22ca9c01960d996e66c40548ce diff --git a/docs/conf.py b/docs/conf.py index 2c48bd48..f1b50787 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -80,9 +80,9 @@ master_doc = "index" # General information about the project. -project = u"google-cloud-datastore" -copyright = u"2019, Google" -author = u"Google APIs" +project = "google-cloud-datastore" +copyright = "2019, Google" +author = "Google APIs" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -281,7 +281,7 @@ ( master_doc, "google-cloud-datastore.tex", - u"google-cloud-datastore Documentation", + "google-cloud-datastore Documentation", author, "manual", ) @@ -316,7 +316,7 @@ ( master_doc, "google-cloud-datastore", - u"google-cloud-datastore Documentation", + "google-cloud-datastore Documentation", [author], 1, ) @@ -335,7 +335,7 @@ ( master_doc, "google-cloud-datastore", - u"google-cloud-datastore Documentation", + "google-cloud-datastore Documentation", author, "google-cloud-datastore", "google-cloud-datastore Library", From 6efde70db751bf708091b24a932ab8571bd981a6 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sat, 19 Jun 2021 02:02:03 +0000 Subject: [PATCH 05/10] docs: omit mention of Python 2.7 in 'CONTRIBUTING.rst' (#1127) (#181) Source-Link: https://github.com/googleapis/synthtool/commit/b91f129527853d5b756146a0b5044481fb4e09a8 Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:b6169fc6a5207b11800a7c002d0c5c2bc6d82697185ca12e666f44031468cfcd --- .github/.OwlBot.lock.yaml | 2 +- CONTRIBUTING.rst | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ea06d395..cc49c6a3 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:58c7342b0bccf85028100adaa3d856cb4a871c22ca9c01960d996e66c40548ce + digest: sha256:b6169fc6a5207b11800a7c002d0c5c2bc6d82697185ca12e666f44031468cfcd diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0f183f3c..d70e876a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -69,7 +69,6 @@ We use `nox `__ to instrument our tests. - To test your changes, run unit tests with ``nox``:: - $ nox -s unit-2.7 $ nox -s unit-3.8 $ ... @@ -144,7 +143,6 @@ Running System Tests # Run all system tests $ nox -s system-3.8 - $ nox -s system-2.7 # Run a single system test $ nox -s system-3.8 -- -k @@ -152,9 +150,8 @@ Running System Tests .. note:: - System tests are only configured to run under Python 2.7 and - Python 3.8. For expediency, we do not run them in older versions - of Python 3. + System tests are only configured to run under Python 3.8. + For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local auth settings and change some configuration in your project to From 53c2927d22020e5d7c303fcca7f69825af96318a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Sun, 20 Jun 2021 01:34:03 +0000 Subject: [PATCH 06/10] chore: update precommit hook pre-commit/pre-commit-hooks to v4 (#1083) (#183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [pre-commit/pre-commit-hooks](https://togithub.com/pre-commit/pre-commit-hooks) | repository | major | `v3.4.0` -> `v4.0.1` | --- ### Release Notes
pre-commit/pre-commit-hooks ### [`v4.0.1`](https://togithub.com/pre-commit/pre-commit-hooks/releases/v4.0.1) [Compare Source](https://togithub.com/pre-commit/pre-commit-hooks/compare/v4.0.0...v4.0.1) ##### Fixes - `check-shebang-scripts-are-executable` fix entry point. - [#​602](https://togithub.com/pre-commit/pre-commit-hooks/issues/602) issue by [@​Person-93](https://togithub.com/Person-93). - [#​603](https://togithub.com/pre-commit/pre-commit-hooks/issues/603) PR by [@​scop](https://togithub.com/scop). ### [`v4.0.0`](https://togithub.com/pre-commit/pre-commit-hooks/releases/v4.0.0) [Compare Source](https://togithub.com/pre-commit/pre-commit-hooks/compare/v3.4.0...v4.0.0) ##### Features - `check-json`: report duplicate keys. - [#​558](https://togithub.com/pre-commit/pre-commit-hooks/issues/558) PR by [@​AdityaKhursale](https://togithub.com/AdityaKhursale). - [#​554](https://togithub.com/pre-commit/pre-commit-hooks/issues/554) issue by [@​adamchainz](https://togithub.com/adamchainz). - `no-commit-to-branch`: add `main` to default blocked branches. - [#​565](https://togithub.com/pre-commit/pre-commit-hooks/issues/565) PR by [@​ndevenish](https://togithub.com/ndevenish). - `check-case-conflict`: check conflicts in directory names as well. - [#​575](https://togithub.com/pre-commit/pre-commit-hooks/issues/575) PR by [@​slsyy](https://togithub.com/slsyy). - [#​70](https://togithub.com/pre-commit/pre-commit-hooks/issues/70) issue by [@​andyjack](https://togithub.com/andyjack). - `check-vcs-permalinks`: forbid other branch names. - [#​582](https://togithub.com/pre-commit/pre-commit-hooks/issues/582) PR by [@​jack1142](https://togithub.com/jack1142). - [#​581](https://togithub.com/pre-commit/pre-commit-hooks/issues/581) issue by [@​jack1142](https://togithub.com/jack1142). - `check-shebang-scripts-are-executable`: new hook which ensures shebang'd scripts are executable. - [#​545](https://togithub.com/pre-commit/pre-commit-hooks/issues/545) PR by [@​scop](https://togithub.com/scop). ##### Fixes - `check-executables-have-shebangs`: Short circuit shebang lookup on windows. - [#​544](https://togithub.com/pre-commit/pre-commit-hooks/issues/544) PR by [@​scop](https://togithub.com/scop). - `requirements-txt-fixer`: Fix comments which have indentation - [#​549](https://togithub.com/pre-commit/pre-commit-hooks/issues/549) PR by [@​greshilov](https://togithub.com/greshilov). - [#​548](https://togithub.com/pre-commit/pre-commit-hooks/issues/548) issue by [@​greshilov](https://togithub.com/greshilov). - `pretty-format-json`: write to stdout using UTF-8 encoding. - [#​571](https://togithub.com/pre-commit/pre-commit-hooks/issues/571) PR by [@​jack1142](https://togithub.com/jack1142). - [#​570](https://togithub.com/pre-commit/pre-commit-hooks/issues/570) issue by [@​jack1142](https://togithub.com/jack1142). - Use more inclusive language. - [#​599](https://togithub.com/pre-commit/pre-commit-hooks/issues/599) PR by [@​asottile](https://togithub.com/asottile). ##### Breaking changes - Remove deprecated hooks: `flake8`, `pyflakes`, `autopep8-wrapper`. - [#​597](https://togithub.com/pre-commit/pre-commit-hooks/issues/597) PR by [@​asottile](https://togithub.com/asottile).
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻️ **Rebasing**: Renovate will not automatically rebase this PR, because other commits have been found. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/synthtool). Source-Link: https://github.com/googleapis/synthtool/commit/333fd90856f1454380514bc59fc0936cdaf1c202 Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:b8c131c558606d3cea6e18f8e87befbd448c1482319b0db3c5d5388fa6ea72e3 --- .github/.OwlBot.lock.yaml | 2 +- .pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index cc49c6a3..9602d540 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:b6169fc6a5207b11800a7c002d0c5c2bc6d82697185ca12e666f44031468cfcd + digest: sha256:b8c131c558606d3cea6e18f8e87befbd448c1482319b0db3c5d5388fa6ea72e3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f00c7cf..62eb5a77 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: trailing-whitespace - id: end-of-file-fixer From b9079151b154ffaa65425ba82e3e8c13d8d641db Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 22 Jun 2021 20:08:32 +0000 Subject: [PATCH 07/10] chore: add kokoro 3.9 config templates (#1128) (#185) Source-Link: https://github.com/googleapis/synthtool/commit/b0eb8a8b30b46a3c98d23c23107acb748c6601a1 Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:df50e8d462f86d6bcb42f27ecad55bb12c404f1c65de9c6fe4c4d25120080bd6 --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/samples/python3.9/common.cfg | 40 +++++++++++++++++++++ .kokoro/samples/python3.9/continuous.cfg | 6 ++++ .kokoro/samples/python3.9/periodic-head.cfg | 11 ++++++ .kokoro/samples/python3.9/periodic.cfg | 6 ++++ .kokoro/samples/python3.9/presubmit.cfg | 6 ++++ 6 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 .kokoro/samples/python3.9/common.cfg create mode 100644 .kokoro/samples/python3.9/continuous.cfg create mode 100644 .kokoro/samples/python3.9/periodic-head.cfg create mode 100644 .kokoro/samples/python3.9/periodic.cfg create mode 100644 .kokoro/samples/python3.9/presubmit.cfg diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 9602d540..0954585f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:b8c131c558606d3cea6e18f8e87befbd448c1482319b0db3c5d5388fa6ea72e3 + digest: sha256:df50e8d462f86d6bcb42f27ecad55bb12c404f1c65de9c6fe4c4d25120080bd6 diff --git a/.kokoro/samples/python3.9/common.cfg b/.kokoro/samples/python3.9/common.cfg new file mode 100644 index 00000000..6e6deb84 --- /dev/null +++ b/.kokoro/samples/python3.9/common.cfg @@ -0,0 +1,40 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Specify which tests to run +env_vars: { + key: "RUN_TESTS_SESSION" + value: "py-3.9" +} + +# Declare build specific Cloud project. +env_vars: { + key: "BUILD_SPECIFIC_GCLOUD_PROJECT" + value: "python-docs-samples-tests-py39" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-datastore/.kokoro/test-samples.sh" +} + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" +} + +# Download secrets for samples +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-datastore/.kokoro/trampoline.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.9/continuous.cfg b/.kokoro/samples/python3.9/continuous.cfg new file mode 100644 index 00000000..a1c8d975 --- /dev/null +++ b/.kokoro/samples/python3.9/continuous.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.9/periodic-head.cfg b/.kokoro/samples/python3.9/periodic-head.cfg new file mode 100644 index 00000000..f9cfcd33 --- /dev/null +++ b/.kokoro/samples/python3.9/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.9/periodic.cfg b/.kokoro/samples/python3.9/periodic.cfg new file mode 100644 index 00000000..50fec964 --- /dev/null +++ b/.kokoro/samples/python3.9/periodic.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "False" +} \ No newline at end of file diff --git a/.kokoro/samples/python3.9/presubmit.cfg b/.kokoro/samples/python3.9/presubmit.cfg new file mode 100644 index 00000000..a1c8d975 --- /dev/null +++ b/.kokoro/samples/python3.9/presubmit.cfg @@ -0,0 +1,6 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} \ No newline at end of file From e24333fa1bb78246ef633a2c4315b8e4ed78c04f Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 25 Jun 2021 17:15:01 -0400 Subject: [PATCH 08/10] chore(python): simplify nox steps in CONTRIBUTING.rst (#187) Source-Link: https://github.com/googleapis/synthtool/commit/26558bae8976a985d73c2d98c31d8612273f907d Post-Processor: gcr.io/repo-automation-bots/owlbot-python:latest@sha256:99d90d097e4a4710cc8658ee0b5b963f4426d0e424819787c3ac1405c9a26719 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- CONTRIBUTING.rst | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0954585f..e2b39f94 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:df50e8d462f86d6bcb42f27ecad55bb12c404f1c65de9c6fe4c4d25120080bd6 + digest: sha256:99d90d097e4a4710cc8658ee0b5b963f4426d0e424819787c3ac1405c9a26719 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d70e876a..63d4d9de 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -68,14 +68,12 @@ Using ``nox`` We use `nox `__ to instrument our tests. - To test your changes, run unit tests with ``nox``:: + $ nox -s unit - $ nox -s unit-3.8 - $ ... +- To run a single unit test:: -- Args to pytest can be passed through the nox command separated by a `--`. For - example, to run a single test:: + $ nox -s unit-3.9 -- -k - $ nox -s unit-3.8 -- -k .. note:: @@ -142,7 +140,7 @@ Running System Tests - To run system tests, you can execute:: # Run all system tests - $ nox -s system-3.8 + $ nox -s system # Run a single system test $ nox -s system-3.8 -- -k @@ -240,8 +238,8 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-datastore/blob/master/noxfile.py -We also explicitly decided to support Python 3 beginning with version -3.6. Reasons for this include: +We also explicitly decided to support Python 3 beginning with version 3.6. +Reasons for this include: - Encouraging use of newest versions of Python 3 - Taking the lead of `prominent`_ open-source `projects`_ From d0481bf8caa84a829808e7f512fda8709f38d0cc Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 9 Jul 2021 14:36:04 -0400 Subject: [PATCH 09/10] perf: further avoid using proto-plus wrapper when unmarshalling entities (#190) Always unwrap to get the raw protobuf message, rather than the proto-plus wrapper. We are back to within a few percent of the older/faster version on my comparison test. Closes #150. --- google/cloud/datastore/helpers.py | 93 +++++++++------------------ tests/unit/test_batch.py | 4 +- tests/unit/test_client.py | 7 +-- tests/unit/test_helpers.py | 100 +++++++++++------------------- tests/unit/test_query.py | 2 + 5 files changed, 73 insertions(+), 133 deletions(-) diff --git a/google/cloud/datastore/helpers.py b/google/cloud/datastore/helpers.py index 5627d8a3..7222fbdf 100644 --- a/google/cloud/datastore/helpers.py +++ b/google/cloud/datastore/helpers.py @@ -22,6 +22,7 @@ from google.protobuf import struct_pb2 from google.type import latlng_pb2 +from proto.datetime_helpers import DatetimeWithNanoseconds from google.cloud._helpers import _datetime_to_pb_timestamp from google.cloud.datastore_v1.types import datastore as datastore_pb2 @@ -33,8 +34,8 @@ def _get_meaning(value_pb, is_list=False): """Get the meaning from a protobuf value. - :type value_pb: :class:`.entity_pb2.Value` - :param value_pb: The protobuf value to be checked for an + :type value_pb: :class:`.entity_pb2.Value._pb` + :param value_pb: The *raw* protobuf value to be checked for an associated meaning. :type is_list: bool @@ -47,14 +48,9 @@ def _get_meaning(value_pb, is_list=False): means it just returns a list of meanings. If all the list meanings agree, it just condenses them. """ - meaning = None if is_list: - values = ( - value_pb._pb.array_value.values - if hasattr(value_pb, "_pb") - else value_pb.array_value.values - ) + values = value_pb.array_value.values # An empty list will have no values, hence no shared meaning # set among them. @@ -65,16 +61,18 @@ def _get_meaning(value_pb, is_list=False): # the rest which may be enum/int values. all_meanings = [_get_meaning(sub_value_pb) for sub_value_pb in values] unique_meanings = set(all_meanings) + if len(unique_meanings) == 1: # If there is a unique meaning, we preserve it. - meaning = unique_meanings.pop() + return unique_meanings.pop() else: # We know len(value_pb.array_value.values) > 0. # If the meaning is not unique, just return all of them. - meaning = all_meanings + return all_meanings + elif value_pb.meaning: # Simple field (int32). - meaning = value_pb.meaning + return value_pb.meaning - return meaning + return None def _new_value_pb(entity_pb, name): @@ -89,29 +87,12 @@ def _new_value_pb(entity_pb, name): :rtype: :class:`.entity_pb2.Value` :returns: The new ``Value`` protobuf that was added to the entity. """ - properties = entity_pb.properties - try: - properties = properties._pb - except AttributeError: - # TODO(microgenerator): shouldn't need this. the issue is that - # we have wrapped and non-wrapped protos coming here. - pass + # TODO(microgenerator): shouldn't need this. the issue is that + # we have wrapped and non-wrapped protos coming here. + properties = getattr(entity_pb.properties, "_pb", entity_pb.properties) return properties.get_or_create(name) -def _property_tuples(entity_pb): - """Iterator of name, ``Value`` tuples from entity properties. - - :type entity_pb: :class:`.entity_pb2.Entity` - :param entity_pb: An entity protobuf to add a new property to. - - :rtype: :class:`generator` - :returns: An iterator that yields tuples of a name and ``Value`` - corresponding to properties on the entity. - """ - return iter(entity_pb.properties.items()) - - def entity_from_protobuf(pb): """Factory method for creating an entity based on a protobuf. @@ -124,21 +105,18 @@ def entity_from_protobuf(pb): :rtype: :class:`google.cloud.datastore.entity.Entity` :returns: The entity derived from the protobuf. """ - if not isinstance(pb, entity_pb2.Entity): - proto_pb = entity_pb2.Entity.wrap(pb) - else: - proto_pb = pb + if isinstance(pb, entity_pb2.Entity): pb = pb._pb key = None - if "key" in proto_pb: # Message field (Key) - key = key_from_protobuf(proto_pb.key) + if pb.HasField("key"): # Message field (Key) + key = key_from_protobuf(pb.key) entity_props = {} entity_meanings = {} exclude_from_indexes = [] - for prop_name, value_pb in _property_tuples(proto_pb._pb): + for prop_name, value_pb in pb.properties.items(): value = _get_value_from_value_pb(value_pb) entity_props[prop_name] = value @@ -384,7 +362,7 @@ def _pb_attr_value(val): return name + "_value", value -def _get_value_from_value_pb(value): +def _get_value_from_value_pb(pb): """Given a protobuf for a Value, get the correct value. The Cloud Datastore Protobuf API returns a Property Protobuf which @@ -394,56 +372,47 @@ def _get_value_from_value_pb(value): Some work is done to coerce the return value into a more useful type (particularly in the case of a timestamp value, or a key value). - :type value_pb: :class:`.entity_pb2.Value` - :param value_pb: The Value Protobuf. + :type pb: :class:`.entity_pb2.Value._pb` + :param pb: The *raw* Value Protobuf. :rtype: object :returns: The value provided by the Protobuf. :raises: :class:`ValueError ` if no value type has been set. """ - if not getattr(value, "_pb", False): - # Coerce raw pb type into proto-plus pythonic type. - value = entity_pb2.Value.wrap(value) - - value_type = value._pb.WhichOneof("value_type") + value_type = pb.WhichOneof("value_type") if value_type == "timestamp_value": - # Do not access `._pb` here, as that returns a Timestamp proto, - # but this should return a Pythonic `DatetimeWithNanoseconds` value, - # which is found at `value.timestamp_value` - result = value.timestamp_value + result = DatetimeWithNanoseconds.from_timestamp_pb(pb.timestamp_value) elif value_type == "key_value": - result = key_from_protobuf(value._pb.key_value) + result = key_from_protobuf(pb.key_value) elif value_type == "boolean_value": - result = value._pb.boolean_value + result = pb.boolean_value elif value_type == "double_value": - result = value._pb.double_value + result = pb.double_value elif value_type == "integer_value": - result = value._pb.integer_value + result = pb.integer_value elif value_type == "string_value": - result = value._pb.string_value + result = pb.string_value elif value_type == "blob_value": - result = value._pb.blob_value + result = pb.blob_value elif value_type == "entity_value": - result = entity_from_protobuf(value._pb.entity_value) + result = entity_from_protobuf(pb.entity_value) elif value_type == "array_value": result = [ - _get_value_from_value_pb(value) for value in value._pb.array_value.values + _get_value_from_value_pb(item_value) for item_value in pb.array_value.values ] elif value_type == "geo_point_value": - result = GeoPoint( - value._pb.geo_point_value.latitude, value._pb.geo_point_value.longitude, - ) + result = GeoPoint(pb.geo_point_value.latitude, pb.geo_point_value.longitude,) elif value_type == "null_value": result = None diff --git a/tests/unit/test_batch.py b/tests/unit/test_batch.py index 78c1db20..ead00623 100644 --- a/tests/unit/test_batch.py +++ b/tests/unit/test_batch.py @@ -118,8 +118,6 @@ def test_put_entity_w_partial_key(self): self.assertEqual(batch._partial_key_entities, [entity]) def test_put_entity_w_completed_key(self): - from google.cloud.datastore.helpers import _property_tuples - project = "PROJECT" properties = {"foo": "bar", "baz": "qux", "spam": [1, 2, 3], "frotz": []} client = _Client(project) @@ -134,7 +132,7 @@ def test_put_entity_w_completed_key(self): mutated_entity = _mutated_pb(self, batch.mutations, "upsert") self.assertEqual(mutated_entity.key, key._key) - prop_dict = dict(_property_tuples(mutated_entity)) + prop_dict = dict(mutated_entity.properties.items()) self.assertEqual(len(prop_dict), 4) self.assertFalse(prop_dict["foo"].exclude_from_indexes) self.assertTrue(prop_dict["baz"].exclude_from_indexes) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 3c75a5fb..5127fd60 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -802,7 +802,6 @@ def test_put_multi_w_single_empty_entity(self): def test_put_multi_no_batch_w_partial_key_w_retry_w_timeout(self): from google.cloud.datastore_v1.types import datastore as datastore_pb2 - from google.cloud.datastore.helpers import _property_tuples entity = _Entity(foo=u"bar") key = entity.key = _Key(_Key.kind, None) @@ -838,15 +837,13 @@ def test_put_multi_no_batch_w_partial_key_w_retry_w_timeout(self): mutated_entity = _mutated_pb(self, mutations, "insert") self.assertEqual(mutated_entity.key, key.to_protobuf()) - prop_list = list(_property_tuples(mutated_entity)) + prop_list = list(mutated_entity.properties.items()) self.assertTrue(len(prop_list), 1) name, value_pb = prop_list[0] self.assertEqual(name, "foo") self.assertEqual(value_pb.string_value, u"bar") def test_put_multi_existing_batch_w_completed_key(self): - from google.cloud.datastore.helpers import _property_tuples - creds = _make_credentials() client = self._make_one(credentials=creds) entity = _Entity(foo=u"bar") @@ -859,7 +856,7 @@ def test_put_multi_existing_batch_w_completed_key(self): mutated_entity = _mutated_pb(self, CURR_BATCH.mutations, "upsert") self.assertEqual(mutated_entity.key, key.to_protobuf()) - prop_list = list(_property_tuples(mutated_entity)) + prop_list = list(mutated_entity.properties.items()) self.assertTrue(len(prop_list), 1) name, value_pb = prop_list[0] self.assertEqual(name, "foo") diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py index 5b602cff..c37499ca 100644 --- a/tests/unit/test_helpers.py +++ b/tests/unit/test_helpers.py @@ -33,28 +33,6 @@ def test_it(self): self.assertEqual(entity_pb._pb.properties[name], result) -class Test__property_tuples(unittest.TestCase): - def _call_fut(self, entity_pb): - from google.cloud.datastore.helpers import _property_tuples - - return _property_tuples(entity_pb) - - def test_it(self): - import types - from google.cloud.datastore_v1.types import entity as entity_pb2 - from google.cloud.datastore.helpers import _new_value_pb - - entity_pb = entity_pb2.Entity() - name1 = "foo" - name2 = "bar" - val_pb1 = _new_value_pb(entity_pb, name1) - val_pb2 = _new_value_pb(entity_pb, name2) - - result = self._call_fut(entity_pb) - self.assertIsInstance(result, types.GeneratorType) - self.assertEqual(sorted(result), sorted([(name1, val_pb1), (name2, val_pb2)])) - - class Test_entity_from_protobuf(unittest.TestCase): def _call_fut(self, val): from google.cloud.datastore.helpers import entity_from_protobuf @@ -221,11 +199,9 @@ def _call_fut(self, entity): return entity_to_protobuf(entity) def _compare_entity_proto(self, entity_pb1, entity_pb2): - from google.cloud.datastore.helpers import _property_tuples - self.assertEqual(entity_pb1.key, entity_pb2.key) - value_list1 = sorted(_property_tuples(entity_pb1)) - value_list2 = sorted(_property_tuples(entity_pb2)) + value_list1 = sorted(entity_pb1.properties.items()) + value_list2 = sorted(entity_pb2.properties.items()) self.assertEqual(len(value_list1), len(value_list2)) for pair1, pair2 in zip(value_list1, value_list2): name1, val1 = pair1 @@ -668,12 +644,12 @@ def _call_fut(self, pb): return _get_value_from_value_pb(pb) - def _makePB(self, attr_name, value): + def _makePB(self, attr_name, attr_value): from google.cloud.datastore_v1.types import entity as entity_pb2 - pb = entity_pb2.Value() - setattr(pb, attr_name, value) - return pb + value = entity_pb2.Value() + setattr(value._pb, attr_name, attr_value) + return value def test_datetime(self): import calendar @@ -683,67 +659,67 @@ def test_datetime(self): micros = 4375 utc = datetime.datetime(2014, 9, 16, 10, 19, 32, micros, UTC) - pb = entity_pb2.Value() - pb._pb.timestamp_value.seconds = calendar.timegm(utc.timetuple()) - pb._pb.timestamp_value.nanos = 1000 * micros - self.assertEqual(self._call_fut(pb), utc) + value = entity_pb2.Value() + value._pb.timestamp_value.seconds = calendar.timegm(utc.timetuple()) + value._pb.timestamp_value.nanos = 1000 * micros + self.assertEqual(self._call_fut(value._pb), utc) def test_key(self): from google.cloud.datastore_v1.types import entity as entity_pb2 from google.cloud.datastore.key import Key - pb = entity_pb2.Value() + value = entity_pb2.Value() expected = Key("KIND", 1234, project="PROJECT").to_protobuf() - pb.key_value._pb.CopyFrom(expected._pb) - found = self._call_fut(pb) + value.key_value._pb.CopyFrom(expected._pb) + found = self._call_fut(value._pb) self.assertEqual(found.to_protobuf(), expected) def test_bool(self): - pb = self._makePB("boolean_value", False) - self.assertEqual(self._call_fut(pb), False) + value = self._makePB("boolean_value", False) + self.assertEqual(self._call_fut(value._pb), False) def test_float(self): - pb = self._makePB("double_value", 3.1415926) - self.assertEqual(self._call_fut(pb), 3.1415926) + value = self._makePB("double_value", 3.1415926) + self.assertEqual(self._call_fut(value._pb), 3.1415926) def test_int(self): - pb = self._makePB("integer_value", 42) - self.assertEqual(self._call_fut(pb), 42) + value = self._makePB("integer_value", 42) + self.assertEqual(self._call_fut(value._pb), 42) def test_bytes(self): - pb = self._makePB("blob_value", b"str") - self.assertEqual(self._call_fut(pb), b"str") + value = self._makePB("blob_value", b"str") + self.assertEqual(self._call_fut(value._pb), b"str") def test_unicode(self): - pb = self._makePB("string_value", u"str") - self.assertEqual(self._call_fut(pb), u"str") + value = self._makePB("string_value", u"str") + self.assertEqual(self._call_fut(value._pb), u"str") def test_entity(self): from google.cloud.datastore_v1.types import entity as entity_pb2 from google.cloud.datastore.entity import Entity from google.cloud.datastore.helpers import _new_value_pb - pb = entity_pb2.Value() - entity_pb = pb.entity_value + value = entity_pb2.Value() + entity_pb = value.entity_value entity_pb._pb.key.path.add(kind="KIND") entity_pb.key.partition_id.project_id = "PROJECT" value_pb = _new_value_pb(entity_pb, "foo") value_pb.string_value = "Foo" - entity = self._call_fut(pb) + entity = self._call_fut(value._pb) self.assertIsInstance(entity, Entity) self.assertEqual(entity["foo"], "Foo") def test_array(self): from google.cloud.datastore_v1.types import entity as entity_pb2 - pb = entity_pb2.Value() - array_pb = pb.array_value.values + value = entity_pb2.Value() + array_pb = value.array_value.values item_pb = array_pb._pb.add() item_pb.string_value = "Foo" item_pb = array_pb._pb.add() item_pb.string_value = "Bar" - items = self._call_fut(pb) + items = self._call_fut(value._pb) self.assertEqual(items, ["Foo", "Bar"]) def test_geo_point(self): @@ -754,8 +730,8 @@ def test_geo_point(self): lat = -3.14 lng = 13.37 geo_pt_pb = latlng_pb2.LatLng(latitude=lat, longitude=lng) - pb = entity_pb2.Value(geo_point_value=geo_pt_pb) - result = self._call_fut(pb) + value = entity_pb2.Value(geo_point_value=geo_pt_pb) + result = self._call_fut(value._pb) self.assertIsInstance(result, GeoPoint) self.assertEqual(result.latitude, lat) self.assertEqual(result.longitude, lng) @@ -764,16 +740,16 @@ def test_null(self): from google.protobuf import struct_pb2 from google.cloud.datastore_v1.types import entity as entity_pb2 - pb = entity_pb2.Value(null_value=struct_pb2.NULL_VALUE) - result = self._call_fut(pb) + value = entity_pb2.Value(null_value=struct_pb2.NULL_VALUE) + result = self._call_fut(value._pb) self.assertIsNone(result) def test_unknown(self): from google.cloud.datastore_v1.types import entity as entity_pb2 - pb = entity_pb2.Value() + value = entity_pb2.Value() with self.assertRaises(ValueError): - self._call_fut(pb) + self._call_fut(value._pb) class Test_set_protobuf_value(unittest.TestCase): @@ -860,18 +836,16 @@ def test_unicode(self): def test_entity_empty_wo_key(self): from google.cloud.datastore.entity import Entity - from google.cloud.datastore.helpers import _property_tuples pb = self._makePB() entity = Entity() self._call_fut(pb, entity) value = pb.entity_value self.assertEqual(value.key.SerializeToString(), b"") - self.assertEqual(len(list(_property_tuples(value))), 0) + self.assertEqual(len(list(value.properties.items())), 0) def test_entity_w_key(self): from google.cloud.datastore.entity import Entity - from google.cloud.datastore.helpers import _property_tuples from google.cloud.datastore.key import Key name = "foo" @@ -884,7 +858,7 @@ def test_entity_w_key(self): entity_pb = pb.entity_value self.assertEqual(entity_pb.key, key.to_protobuf()._pb) - prop_dict = dict(_property_tuples(entity_pb)) + prop_dict = dict(entity_pb.properties.items()) self.assertEqual(len(prop_dict), 1) self.assertEqual(list(prop_dict.keys()), [name]) self.assertEqual(prop_dict[name].string_value, value) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 89bc7e2c..dcb4e9f5 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -15,6 +15,7 @@ import unittest import mock +import pytest class TestQuery(unittest.TestCase): @@ -527,6 +528,7 @@ def test__process_query_results_done(self): self.assertIsNone(iterator.next_page_token) self.assertFalse(iterator._more_results) + @pytest.mark.filterwarnings("ignore") def test__process_query_results_bad_enum(self): iterator = self._make_one(None, None) more_results_enum = 999 From 30008c9cbc6053cc408e56ca3a771f8a6dbab6fd Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 9 Jul 2021 18:46:23 +0000 Subject: [PATCH 10/10] chore: release 2.1.4 (#184) :robot: I have created a release \*beep\* \*boop\* --- ### [2.1.4](https://www.github.com/googleapis/python-datastore/compare/v2.1.3...v2.1.4) (2021-07-09) ### Performance Improvements * further avoid using proto-plus wrapper when unmarshalling entities ([#190](https://www.github.com/googleapis/python-datastore/issues/190)) ([d0481bf](https://www.github.com/googleapis/python-datastore/commit/d0481bf8caa84a829808e7f512fda8709f38d0cc)) ### Documentation * omit mention of Python 2.7 in 'CONTRIBUTING.rst' ([#1127](https://www.github.com/googleapis/python-datastore/issues/1127)) ([#181](https://www.github.com/googleapis/python-datastore/issues/181)) ([6efde70](https://www.github.com/googleapis/python-datastore/commit/6efde70db751bf708091b24a932ab8571bd981a6)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 12 ++++++++++++ google/cloud/datastore/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 880add6e..a8c63647 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ [1]: https://pypi.org/project/google-cloud-datastore/#history +### [2.1.4](https://www.github.com/googleapis/python-datastore/compare/v2.1.3...v2.1.4) (2021-07-09) + + +### Performance Improvements + +* further avoid using proto-plus wrapper when unmarshalling entities ([#190](https://www.github.com/googleapis/python-datastore/issues/190)) ([d0481bf](https://www.github.com/googleapis/python-datastore/commit/d0481bf8caa84a829808e7f512fda8709f38d0cc)) + + +### Documentation + +* omit mention of Python 2.7 in 'CONTRIBUTING.rst' ([#1127](https://www.github.com/googleapis/python-datastore/issues/1127)) ([#181](https://www.github.com/googleapis/python-datastore/issues/181)) ([6efde70](https://www.github.com/googleapis/python-datastore/commit/6efde70db751bf708091b24a932ab8571bd981a6)) + ### [2.1.3](https://www.github.com/googleapis/python-datastore/compare/v2.1.2...v2.1.3) (2021-05-25) diff --git a/google/cloud/datastore/version.py b/google/cloud/datastore/version.py index 28ad2844..a1c74674 100644 --- a/google/cloud/datastore/version.py +++ b/google/cloud/datastore/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.1.3" +__version__ = "2.1.4"