From eed6f496aa06bd8539b69af784442e7184eb4af6 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Sun, 20 Jun 2021 14:46:06 +0000 Subject: [PATCH 1/3] chore: use templated noxfile.py --- owlbot.py | 128 ++++++++++++++++++++++++++++++++++++++++++++++++- tests/.gitkeep | 0 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/.gitkeep diff --git a/owlbot.py b/owlbot.py index d8a447a..c94f6f6 100644 --- a/owlbot.py +++ b/owlbot.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re + import synthtool as s import synthtool.gcp as gcp from synthtool import tmp @@ -24,7 +26,131 @@ common = gcp.CommonTemplates() templated_files = common.py_library() s.move( - templated_files, excludes=["noxfile.py", ".coveragerc", ".gitignore",], + templated_files, excludes=[".coveragerc", ".gitignore",], +) + +# ---------------------------------------------------------------------------- +# Customize noxfile.py +# ---------------------------------------------------------------------------- + +old_sessions = """ + "unit", + "system", + "cover", + "lint", +""" + +new_sessions = """ + "system", + "lint", + "test", + "generate_protos", +""" + +# Remove `unit` and `cover` sessions. +# Add `test` and `generate_protos` sessions. +s.replace("noxfile.py", old_sessions, new_sessions) + +# Remove unit* sessions +s.replace("noxfile.py", + """@nox.session\(python=UNIT_TEST_PYTHON_VERSIONS\) +def unit\(session\):""", + """def unit(session):""", +) + +# Remove cover session +s.replace("noxfile.py", + """@nox.session\(python=DEFAULT_PYTHON_VERSION\) +def cover.* + session.run\("coverage", "erase"\)""", + """""", + flags=re.MULTILINE | re.DOTALL, +) + +def place_before(path, text, *before_text, escape=None): + replacement = "\n".join(before_text) + "\n" + text + if escape: + for c in escape: + text = text.replace(c, '\\' + c) + s.replace([path], text, replacement) + +custom_nox_sessions = """ +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +@nox.parametrize( + "library", ["python-asset"], ids=["asset"], +) +def test(session, library): + \"\"\"Run tests from a downstream libraries. + To verify that any changes we make here will not break downstream libraries, clone + a few and run their unit and system tests. + NOTE: The unit and system test functions above are copied from the templates. + They will need to be updated when the templates change. + \"\"\" + try: + session.run("git", "-C", library, "pull", external=True) + except nox.command.CommandFailed: + session.run( + "git", + "clone", + "--single-branch", + f"https://github.com/googleapis/{library}", + external=True, + ) + + session.cd(library) + unit(session) + # system tests are run 3.7 only + if session.python == "3.7": + system(session) + + +@nox.session(python="3.8") +def generate_protos(session): + \"\"\"Generates the protos using protoc. + Some notes on the `google` directory: + 1. The `_pb2.py` files are produced by protoc. + 2. The .proto files are non-functional but are left in the repository + to make it easier to understand diffs. + 3. The `google` directory also has `__init__.py` files to create proper modules. + If a new subdirectory is added, you will need to create more `__init__.py` + files. + \"\"\" + session.install("grpcio-tools") + protos = [str(p) for p in (pathlib.Path(".").glob("google/**/*.proto"))] + + # Clone googleapis/api-common-protos + api_common_protos = "api-common-protos" + try: + session.run("git", "-C", api_common_protos, "pull", external=True) + except nox.command.CommandFailed: + session.run( + "git", + "clone", + "--single-branch", + f"https://github.com/googleapis/{api_common_protos}", + external=True, + ) + + session.run( + "python", + "-m", + "grpc_tools.protoc", + "--proto_path=api-common-protos", + "--proto_path=.", + "--python_out=.", + *protos, + ) +""" + +# `google-cloud-context-manager` contains only protos +# and has a customized `test` session to test +# downstream client `google-cloud-asset` +place_before( + "noxfile.py", + "@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)\n" + "def system(session):", + custom_nox_sessions, + escape="()" ) s.shell.run(["nox", "-s", "generate_protos"]) diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29 From 60e7316a04739d307faba22c6cd9cc77659c6588 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 22 Jun 2021 13:47:28 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md --- docs/conf.py | 6 +- .../type/device_resources_pb2.py | 2 +- .../v1/access_level_pb2.py | 6 +- .../v1/service_perimeter_pb2.py | 4 +- noxfile.py | 249 +++++++++++------- 5 files changed, 166 insertions(+), 101 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3362feb..968cc6e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -360,14 +360,10 @@ intersphinx_mapping = { "python": ("https://python.readthedocs.org/en/latest/", None), "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), - "google.api_core": ( - "https://googleapis.dev/python/google-api-core/latest/", - None, - ), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), - } diff --git a/google/identity/accesscontextmanager/type/device_resources_pb2.py b/google/identity/accesscontextmanager/type/device_resources_pb2.py index b79cb5d..6ce0c3e 100644 --- a/google/identity/accesscontextmanager/type/device_resources_pb2.py +++ b/google/identity/accesscontextmanager/type/device_resources_pb2.py @@ -38,7 +38,7 @@ serialized_options=b"\n-com.google.identity.accesscontextmanager.typeB\tTypeProtoP\001ZMgoogle.golang.org/genproto/googleapis/identity/accesscontextmanager/type;type\252\002)Google.Identity.AccessContextManager.Type\312\002)Google\\Identity\\AccessContextManager\\Type\352\002,Google::Identity::AccessContextManager::Type", create_key=_descriptor._internal_create_key, serialized_pb=b"\n@google/identity/accesscontextmanager/type/device_resources.proto\x12)google.identity.accesscontextmanager.type\x1a\x1cgoogle/api/annotations.proto*p\n\x16\x44\x65viceEncryptionStatus\x12\x1a\n\x16\x45NCRYPTION_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x45NCRYPTION_UNSUPPORTED\x10\x01\x12\x0f\n\x0bUNENCRYPTED\x10\x02\x12\r\n\tENCRYPTED\x10\x03*\x82\x01\n\x06OsType\x12\x12\n\x0eOS_UNSPECIFIED\x10\x00\x12\x0f\n\x0b\x44\x45SKTOP_MAC\x10\x01\x12\x13\n\x0f\x44\x45SKTOP_WINDOWS\x10\x02\x12\x11\n\rDESKTOP_LINUX\x10\x03\x12\x15\n\x11\x44\x45SKTOP_CHROME_OS\x10\x06\x12\x0b\n\x07\x41NDROID\x10\x04\x12\x07\n\x03IOS\x10\x05*V\n\x15\x44\x65viceManagementLevel\x12\x1a\n\x16MANAGEMENT_UNSPECIFIED\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\t\n\x05\x42\x41SIC\x10\x02\x12\x0c\n\x08\x43OMPLETE\x10\x03\x42\x92\x02\n-com.google.identity.accesscontextmanager.typeB\tTypeProtoP\x01ZMgoogle.golang.org/genproto/googleapis/identity/accesscontextmanager/type;type\xaa\x02)Google.Identity.AccessContextManager.Type\xca\x02)Google\\Identity\\AccessContextManager\\Type\xea\x02,Google::Identity::AccessContextManager::Typeb\x06proto3", - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR], + dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,], ) _DEVICEENCRYPTIONSTATUS = _descriptor.EnumDescriptor( diff --git a/google/identity/accesscontextmanager/v1/access_level_pb2.py b/google/identity/accesscontextmanager/v1/access_level_pb2.py index 8bb0605..e6d4370 100644 --- a/google/identity/accesscontextmanager/v1/access_level_pb2.py +++ b/google/identity/accesscontextmanager/v1/access_level_pb2.py @@ -240,7 +240,7 @@ containing_type=None, create_key=_descriptor._internal_create_key, fields=[], - ) + ), ], serialized_start=257, serialized_end=569, @@ -296,7 +296,7 @@ ], extensions=[], nested_types=[], - enum_types=[_BASICLEVEL_CONDITIONCOMBININGFUNCTION], + enum_types=[_BASICLEVEL_CONDITIONCOMBININGFUNCTION,], serialized_options=None, is_extendable=False, syntax="proto3", @@ -469,7 +469,7 @@ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key, - ) + ), ], extensions=[], nested_types=[], diff --git a/google/identity/accesscontextmanager/v1/service_perimeter_pb2.py b/google/identity/accesscontextmanager/v1/service_perimeter_pb2.py index a656967..75e069f 100644 --- a/google/identity/accesscontextmanager/v1/service_perimeter_pb2.py +++ b/google/identity/accesscontextmanager/v1/service_perimeter_pb2.py @@ -259,7 +259,7 @@ ], extensions=[], nested_types=[], - enum_types=[_SERVICEPERIMETER_PERIMETERTYPE], + enum_types=[_SERVICEPERIMETER_PERIMETERTYPE,], serialized_options=None, is_extendable=False, syntax="proto3", @@ -415,7 +415,7 @@ ), ], extensions=[], - nested_types=[_SERVICEPERIMETERCONFIG_VPCACCESSIBLESERVICES], + nested_types=[_SERVICEPERIMETERCONFIG_VPCACCESSIBLESERVICES,], enum_types=[], serialized_options=None, is_extendable=False, diff --git a/noxfile.py b/noxfile.py index e21cb54..3409852 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,4 +1,6 @@ -# Copyright 2020 Google LLC +# -*- coding: utf-8 -*- +# +# Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,80 +14,85 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Generated by synthtool. DO NOT EDIT! + +from __future__ import absolute_import import os +import pathlib import shutil -import pathlib -from pathlib import Path import nox -BLACK_VERSION = "black==19.3b0" -CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() +BLACK_VERSION = "black==19.10b0" +BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] +DEFAULT_PYTHON_VERSION = "3.8" +SYSTEM_TEST_PYTHON_VERSIONS = ["2.7", "3.8"] +UNIT_TEST_PYTHON_VERSIONS = ["2.7", "3.6", "3.7", "3.8", "3.9"] -@nox.session(python="3.8") -def blacken(session): - """Run black. - Format code to uniform standard. - This currently uses Python 3.6 due to the automated Kokoro run of synthtool. - That run uses an image that doesn't have 3.6 installed. Before updating this - check the state of the `gcp_ubuntu_config` we use for that Kokoro run. - """ - session.install(BLACK_VERSION) - session.run("black", "google", "setup.py") +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() +# 'docfx' is excluded since it only needs to run in 'docs-presubmit' +nox.options.sessions = [ + "system", + "lint", + "test", + "generate_protos", + "lint_setup_py", + "blacken", + "docs", +] -@nox.session(python="3.8") -def lint_setup_py(session): - """Verify that setup.py is valid""" - session.install("docutils", "pygments") - session.run("python", "setup.py", "check", "--strict") +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True -@nox.session(python="3.7") -def docs(session): - """Build the docs for this library.""" +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. - session.install("-e", ".") - session.install("sphinx<3.0.0", "alabaster", "recommonmark") + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install("flake8", BLACK_VERSION) + session.run( + "black", "--check", *BLACK_PATHS, + ) + session.run("flake8", "google", "tests") - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), + "black", *BLACK_PATHS, ) +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("docutils", "pygments") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + def default(session): # Install all test dependencies, then install this package in-place. - session.install("asyncmock", "pytest-asyncio") - - session.install( - "mock", "pytest", "pytest-cov", - ) - session.install("-e", ".") constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) + session.install("mock", "pytest", "pytest-cov", "-c", constraints_path) - # Install this package - # This *must* be the last install command to get the package from source. - session.install("e", "..", "-c", constraints_path) + session.install("-e", ".", "-c", constraints_path) # Run py.test against the unit tests. session.run( "py.test", "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", "--cov=google/cloud", "--cov=tests/unit", "--cov-append", @@ -102,49 +109,6 @@ def unit(session): default(session) -def system(session): - """Run the system test suite.""" - system_test_path = os.path.join("tests", "system.py") - system_test_folder_path = os.path.join("tests", "system") - - # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. - if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": - session.skip("RUN_SYSTEM_TESTS is set to false, skipping") - # Sanity check: Only run tests if the environment variable is set. - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable") - - system_test_exists = os.path.exists(system_test_path) - system_test_folder_exists = os.path.exists(system_test_folder_path) - # Sanity check: only run tests if found. - if not system_test_exists and not system_test_folder_exists: - session.skip("System tests were not found") - - # Use pre-release gRPC for system tests. - session.install("--pre", "grpcio") - - # Install all test dependencies, then install this package into the - # virtualenv's dist-packages. - session.install( - "mock", "pytest", "google-cloud-testutils", - ) - session.install("-e", ".") - - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - - # Install this package - # This *must* be the last install command to get the package from source. - session.install("e", "..", "-c", constraints_path) - - # Run py.test against the system tests. - if system_test_exists: - session.run("py.test", "--quiet", system_test_path, *session.posargs) - if system_test_folder_exists: - session.run("py.test", "--quiet", system_test_folder_path, *session.posargs) - - @nox.session(python=["3.6", "3.7", "3.8", "3.9"]) @nox.parametrize( "library", ["python-asset"], ids=["asset"], @@ -177,7 +141,6 @@ def test(session, library): @nox.session(python="3.8") def generate_protos(session): """Generates the protos using protoc. - Some notes on the `google` directory: 1. The `_pb2.py` files are produced by protoc. 2. The .proto files are non-functional but are left in the repository @@ -187,7 +150,7 @@ def generate_protos(session): files. """ session.install("grpcio-tools") - protos = [str(p) for p in (Path(".").glob("google/**/*.proto"))] + protos = [str(p) for p in (pathlib.Path(".").glob("google/**/*.proto"))] # Clone googleapis/api-common-protos api_common_protos = "api-common-protos" @@ -211,3 +174,109 @@ def generate_protos(session): "--python_out=.", *protos, ) + + +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + # Use pre-release gRPC for system tests. + session.install("--pre", "grpcio") + + # Install all test dependencies, then install this package into the + # virtualenv's dist-packages. + session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) + session.install("-e", ".", "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if system_test_folder_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx==4.0.1", "alabaster", "recommonmark") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docfx(session): + """Build the docfx yaml files for this library.""" + + session.install("-e", ".") + session.install( + "sphinx==4.0.1", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" + ) + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-T", # show full traceback on exception + "-N", # no colors + "-D", + ( + "extensions=sphinx.ext.autodoc," + "sphinx.ext.autosummary," + "docfx_yaml.extension," + "sphinx.ext.intersphinx," + "sphinx.ext.coverage," + "sphinx.ext.napoleon," + "sphinx.ext.todo," + "sphinx.ext.viewcode," + "recommonmark" + ), + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) From a41618e829714aa08d6711cafdcab5c45536758f Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 22 Jun 2021 15:09:48 +0000 Subject: [PATCH 3/3] chore: remove unused imports --- google/identity/accesscontextmanager/type/__init__.py | 1 - google/identity/accesscontextmanager/v1/__init__.py | 1 - 2 files changed, 2 deletions(-) diff --git a/google/identity/accesscontextmanager/type/__init__.py b/google/identity/accesscontextmanager/type/__init__.py index e4ae888..436b8cc 100644 --- a/google/identity/accesscontextmanager/type/__init__.py +++ b/google/identity/accesscontextmanager/type/__init__.py @@ -16,7 +16,6 @@ from __future__ import absolute_import -import sys from google.api_core.protobuf_helpers import get_messages diff --git a/google/identity/accesscontextmanager/v1/__init__.py b/google/identity/accesscontextmanager/v1/__init__.py index b28d7fc..90c8729 100644 --- a/google/identity/accesscontextmanager/v1/__init__.py +++ b/google/identity/accesscontextmanager/v1/__init__.py @@ -16,7 +16,6 @@ from __future__ import absolute_import -import sys from google.api_core.protobuf_helpers import get_messages