From 3c8da54de9a6837d6d96b6befd3b0ef084d8ff7f Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 21 Apr 2021 19:05:56 -0400 Subject: [PATCH 1/7] chore: preserve gRPC-disabled systests during synth (#141) See #133, #136 --- synth.py | 50 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/synth.py b/synth.py index 96709221..818a9e80 100644 --- a/synth.py +++ b/synth.py @@ -86,17 +86,53 @@ s.replace("noxfile.py", """["']sphinx['"]""", '''"sphinx<3.0.0"''') -# Add the `sphinx-ext-doctest` extenaion +# Preserve system tests w/ GOOGLE_DISABLE_GRPC set (#133, PR #136) s.replace( - "docs/conf.py", + "noxfile.py", """\ - "sphinx.ext.coverage", - "sphinx.ext.napoleon", +@nox.session\(python=SYSTEM_TEST_PYTHON_VERSIONS\) +def system\(session\): """, """\ - "sphinx.ext.coverage", - "sphinx.ext.doctest", - "sphinx.ext.napoleon", +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +@nox.parametrize("disable_grpc", [False, True]) +def system(session, disable_grpc): +""", +) + +s.replace( + "noxfile.py", + """\ + # Run py.test against the system tests. +""", + """\ + env = {} + if disable_grpc: + env["GOOGLE_CLOUD_DISABLE_GRPC"] = "True" + + # Run py.test against the system tests. +""", +) + +s.replace( + "noxfile.py", + """\ + session.run\("py.test", "--quiet", system_test_path, \*session.posargs\) +""", + """\ + session.run("py.test", "--quiet", system_test_path, env=env, *session.posargs) +""", +) + +s.replace( + "noxfile.py", + """\ + session.run\("py.test", "--quiet", system_test_folder_path, \*session.posargs\) +""", + """\ + session.run( + "py.test", "--quiet", system_test_folder_path, env=env, *session.posargs + ) """, ) From c18104c549cb8def3f1d8a0790862ef3e30258f3 Mon Sep 17 00:00:00 2001 From: Dan Lee <71398022+dandhlee@users.noreply.github.com> Date: Wed, 28 Apr 2021 11:56:09 -0400 Subject: [PATCH 2/7] chore: use `gcp-sphinx-docfx-yaml` (#161) makes use of the updated plugin for generating DocFX YAMLs --- noxfile.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 1737a22d..a6c9bf64 100644 --- a/noxfile.py +++ b/noxfile.py @@ -180,9 +180,7 @@ def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") - # sphinx-docfx-yaml supports up to sphinx version 1.5.5. - # https://github.com/docascode/sphinx-docfx-yaml/issues/97 - session.install("sphinx==1.5.5", "alabaster", "recommonmark", "sphinx-docfx-yaml") + session.install("sphinx", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( From d8f7d994d8e7ec456849428f377e52c0ce1e027f Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Mon, 3 May 2021 08:54:04 -0700 Subject: [PATCH 3/7] chore: switch from flakybot to buildcop (via synth) (#159) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/fdb52ec7-8162-4605-a0be-a825994accd2/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) --- .kokoro/test-samples.sh | 8 ++++---- .kokoro/trampoline_v2.sh | 2 +- synth.metadata | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index 4971100d..c4163d79 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -87,11 +87,11 @@ for file in samples/**/requirements.txt; do python3.6 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? - # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. + # If this is a periodic build, send the test log to the Build Cop Bot. + # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/buildcop. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot + chmod +x $KOKORO_GFILE_DIR/linux_amd64/buildcop + $KOKORO_GFILE_DIR/linux_amd64/buildcop fi if [[ $EXIT -ne 0 ]]; then diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh index 4af6cdc2..719bcd5b 100755 --- a/.kokoro/trampoline_v2.sh +++ b/.kokoro/trampoline_v2.sh @@ -159,7 +159,7 @@ if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then "KOKORO_GITHUB_COMMIT" "KOKORO_GITHUB_PULL_REQUEST_NUMBER" "KOKORO_GITHUB_PULL_REQUEST_COMMIT" - # For FlakyBot + # For Build Cop Bot "KOKORO_GITHUB_COMMIT_URL" "KOKORO_GITHUB_PULL_REQUEST_URL" ) diff --git a/synth.metadata b/synth.metadata index a8bf8ba9..adc5da10 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-datastore.git", - "sha": "dd6c0ee43fe6763528374a3494de0124de0ee652" + "sha": "3c8da54de9a6837d6d96b6befd3b0ef084d8ff7f" } }, { From 1582cab488e7886dcdad06a98d05f26e2471b875 Mon Sep 17 00:00:00 2001 From: "google-cloud-policy-bot[bot]" <80869356+google-cloud-policy-bot[bot]@users.noreply.github.com> Date: Mon, 3 May 2021 16:06:03 +0000 Subject: [PATCH 4/7] chore: add SECURITY.md (#162) chore: add SECURITY.md --- SECURITY.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..8b58ae9c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). + +The Google Security Team will respond within 5 working days of your report on g.co/vulnz. + +We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. From 4f90d04c81aacdbaf83f5a9dc996898fa9c7ba26 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Mon, 3 May 2021 09:18:04 -0700 Subject: [PATCH 5/7] docs: update intersphinx URLs for grpc and auth (#93) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/78f53313-0c78-4a29-8841-f031665a4c6a/targets - [ ] To automatically regenerate this PR, check this box. Source-Link: https://github.com/googleapis/synthtool/commit/a073c873f3928c561bdf87fdfbf1d081d1998984 Source-Link: https://github.com/googleapis/synthtool/commit/9a7d9fbb7045c34c9d3d22c1ff766eeae51f04c9 --- docs/conf.py | 6 +++--- synth.metadata | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2aff2fc7..46cba6ca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -345,10 +345,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "python": ("http://python.readthedocs.org/en/latest/", None), - "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "python": ("https://python.readthedocs.org/en/latest/", None), + "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), - "grpc": ("https://grpc.io/grpc/python/", None), + "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), } diff --git a/synth.metadata b/synth.metadata index adc5da10..c8b752ae 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "b19b401571e77192f8dd38eab5fb2300a0de9324" + "sha": "a073c873f3928c561bdf87fdfbf1d081d1998984" } } ], From 924b10b11eb7ff52367f388cf5c8e16aa9b2e32e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 3 May 2021 12:30:09 -0400 Subject: [PATCH 6/7] fix: pass transaction's options to API in 'begin' (#143) Closes #135. --- google/cloud/datastore/transaction.py | 8 +- tests/unit/test_transaction.py | 135 +++++++++++++++++--------- 2 files changed, 96 insertions(+), 47 deletions(-) diff --git a/google/cloud/datastore/transaction.py b/google/cloud/datastore/transaction.py index a1eabed5..21cac1a7 100644 --- a/google/cloud/datastore/transaction.py +++ b/google/cloud/datastore/transaction.py @@ -176,10 +176,12 @@ def Entity(*args, **kwargs): def __init__(self, client, read_only=False): super(Transaction, self).__init__(client) self._id = None + if read_only: options = TransactionOptions(read_only=TransactionOptions.ReadOnly()) else: options = TransactionOptions() + self._options = options @property @@ -231,9 +233,13 @@ def begin(self, retry=None, timeout=None): kwargs = _make_retry_timeout_kwargs(retry, timeout) + request = { + "project_id": self.project, + "transaction_options": self._options, + } try: response_pb = self._client._datastore_api.begin_transaction( - request={"project_id": self.project}, **kwargs + request=request, **kwargs ) self._id = response_pb.transaction except: # noqa: E722 do not use bare except, specify exception instead diff --git a/tests/unit/test_transaction.py b/tests/unit/test_transaction.py index 1bc355cc..bae419df 100644 --- a/tests/unit/test_transaction.py +++ b/tests/unit/test_transaction.py @@ -24,21 +24,25 @@ def _get_target_class(): return Transaction - def _get_options_class(self, **kw): + def _make_one(self, client, **kw): + return self._get_target_class()(client, **kw) + + def _make_options(self, read_only=False, previous_transaction=None): from google.cloud.datastore_v1.types import TransactionOptions - return TransactionOptions + kw = {} - def _make_one(self, client, **kw): - return self._get_target_class()(client, **kw) + if read_only: + kw["read_only"] = TransactionOptions.ReadOnly() - def _make_options(self, **kw): - return self._get_options_class()(**kw) + return TransactionOptions(**kw) def test_ctor_defaults(self): project = "PROJECT" client = _Client(project) + xact = self._make_one(client) + self.assertEqual(xact.project, project) self.assertIs(xact._client, client) self.assertIsNone(xact.id) @@ -46,6 +50,24 @@ def test_ctor_defaults(self): self.assertEqual(xact._mutations, []) self.assertEqual(len(xact._partial_key_entities), 0) + def test_constructor_read_only(self): + project = "PROJECT" + id_ = 850302 + ds_api = _make_datastore_api(xact=id_) + client = _Client(project, datastore_api=ds_api) + options = self._make_options(read_only=True) + + xact = self._make_one(client, read_only=True) + + self.assertEqual(xact._options, options) + + def _make_begin_request(self, project, read_only=False): + expected_options = self._make_options(read_only=read_only) + return { + "project_id": project, + "transaction_options": expected_options, + } + def test_current(self): from google.cloud.datastore_v1.types import datastore as datastore_pb2 @@ -57,24 +79,34 @@ def test_current(self): xact2 = self._make_one(client) self.assertIsNone(xact1.current()) self.assertIsNone(xact2.current()) + with xact1: self.assertIs(xact1.current(), xact1) self.assertIs(xact2.current(), xact1) + with _NoCommitBatch(client): self.assertIsNone(xact1.current()) self.assertIsNone(xact2.current()) + with xact2: self.assertIs(xact1.current(), xact2) self.assertIs(xact2.current(), xact2) + with _NoCommitBatch(client): self.assertIsNone(xact1.current()) self.assertIsNone(xact2.current()) + self.assertIs(xact1.current(), xact1) self.assertIs(xact2.current(), xact1) + self.assertIsNone(xact1.current()) self.assertIsNone(xact2.current()) - ds_api.rollback.assert_not_called() + begin_txn = ds_api.begin_transaction + self.assertEqual(begin_txn.call_count, 2) + expected_request = self._make_begin_request(project) + begin_txn.assert_called_with(request=expected_request) + commit_method = ds_api.commit self.assertEqual(commit_method.call_count, 2) mode = datastore_pb2.CommitRequest.Mode.TRANSACTIONAL @@ -87,9 +119,7 @@ def test_current(self): } ) - begin_txn = ds_api.begin_transaction - self.assertEqual(begin_txn.call_count, 2) - begin_txn.assert_called_with(request={"project_id": project}) + ds_api.rollback.assert_not_called() def test_begin(self): project = "PROJECT" @@ -97,11 +127,27 @@ def test_begin(self): ds_api = _make_datastore_api(xact_id=id_) client = _Client(project, datastore_api=ds_api) xact = self._make_one(client) + xact.begin() + self.assertEqual(xact.id, id_) - ds_api.begin_transaction.assert_called_once_with( - request={"project_id": project} - ) + + expected_request = self._make_begin_request(project) + ds_api.begin_transaction.assert_called_once_with(request=expected_request) + + def test_begin_w_readonly(self): + project = "PROJECT" + id_ = 889 + ds_api = _make_datastore_api(xact_id=id_) + client = _Client(project, datastore_api=ds_api) + xact = self._make_one(client, read_only=True) + + xact.begin() + + self.assertEqual(xact.id, id_) + + expected_request = self._make_begin_request(project, read_only=True) + ds_api.begin_transaction.assert_called_once_with(request=expected_request) def test_begin_w_retry_w_timeout(self): project = "PROJECT" @@ -116,8 +162,10 @@ def test_begin_w_retry_w_timeout(self): xact.begin(retry=retry, timeout=timeout) self.assertEqual(xact.id, id_) + + expected_request = self._make_begin_request(project) ds_api.begin_transaction.assert_called_once_with( - request={"project_id": project}, retry=retry, timeout=timeout + request=expected_request, retry=retry, timeout=timeout, ) def test_begin_tombstoned(self): @@ -126,19 +174,23 @@ def test_begin_tombstoned(self): ds_api = _make_datastore_api(xact_id=id_) client = _Client(project, datastore_api=ds_api) xact = self._make_one(client) + xact.begin() + self.assertEqual(xact.id, id_) - ds_api.begin_transaction.assert_called_once_with( - request={"project_id": project} - ) + + expected_request = self._make_begin_request(project) + ds_api.begin_transaction.assert_called_once_with(request=expected_request) xact.rollback() + client._datastore_api.rollback.assert_called_once_with( request={"project_id": project, "transaction": id_} ) self.assertIsNone(xact.id) - self.assertRaises(ValueError, xact.begin) + with self.assertRaises(ValueError): + xact.begin() def test_begin_w_begin_transaction_failure(self): project = "PROJECT" @@ -152,9 +204,9 @@ def test_begin_w_begin_transaction_failure(self): xact.begin() self.assertIsNone(xact.id) - ds_api.begin_transaction.assert_called_once_with( - request={"project_id": project} - ) + + expected_request = self._make_begin_request(project) + ds_api.begin_transaction.assert_called_once_with(request=expected_request) def test_rollback(self): project = "PROJECT" @@ -256,11 +308,14 @@ def test_context_manager_no_raise(self): ds_api = _make_datastore_api(xact_id=id_) client = _Client(project, datastore_api=ds_api) xact = self._make_one(client) + with xact: - self.assertEqual(xact.id, id_) - ds_api.begin_transaction.assert_called_once_with( - request={"project_id": project} - ) + self.assertEqual(xact.id, id_) # only set between begin / commit + + self.assertIsNone(xact.id) + + expected_request = self._make_begin_request(project) + ds_api.begin_transaction.assert_called_once_with(request=expected_request) mode = datastore_pb2.CommitRequest.Mode.TRANSACTIONAL client._datastore_api.commit.assert_called_once_with( @@ -272,9 +327,6 @@ def test_context_manager_no_raise(self): }, ) - self.assertIsNone(xact.id) - self.assertEqual(ds_api.begin_transaction.call_count, 1) - def test_context_manager_w_raise(self): class Foo(Exception): pass @@ -288,29 +340,20 @@ class Foo(Exception): try: with xact: self.assertEqual(xact.id, id_) - ds_api.begin_transaction.assert_called_once_with( - request={"project_id": project} - ) raise Foo() except Foo: - self.assertIsNone(xact.id) - client._datastore_api.rollback.assert_called_once_with( - request={"project_id": project, "transaction": id_} - ) + pass - client._datastore_api.commit.assert_not_called() self.assertIsNone(xact.id) - self.assertEqual(ds_api.begin_transaction.call_count, 1) - def test_constructor_read_only(self): - project = "PROJECT" - id_ = 850302 - ds_api = _make_datastore_api(xact=id_) - client = _Client(project, datastore_api=ds_api) - read_only = self._get_options_class().ReadOnly() - options = self._make_options(read_only=read_only) - xact = self._make_one(client, read_only=True) - self.assertEqual(xact._options, options) + expected_request = self._make_begin_request(project) + ds_api.begin_transaction.assert_called_once_with(request=expected_request) + + client._datastore_api.commit.assert_not_called() + + client._datastore_api.rollback.assert_called_once_with( + request={"project_id": project, "transaction": id_} + ) def test_put_read_only(self): project = "PROJECT" From 155fb72b5d1f5d097053151a878eb9e5163d57f4 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 3 May 2021 16:40:06 +0000 Subject: [PATCH 7/7] chore: release 2.1.2 (#164) :robot: I have created a release \*beep\* \*boop\* --- ### [2.1.2](https://www.github.com/googleapis/python-datastore/compare/v2.1.1...v2.1.2) (2021-05-03) ### Bug Fixes * pass transaction's options to API in 'begin' ([#143](https://www.github.com/googleapis/python-datastore/issues/143)) ([924b10b](https://www.github.com/googleapis/python-datastore/commit/924b10b11eb7ff52367f388cf5c8e16aa9b2e32e)) ### Documentation * update intersphinx URLs for grpc and auth ([#93](https://www.github.com/googleapis/python-datastore/issues/93)) ([4f90d04](https://www.github.com/googleapis/python-datastore/commit/4f90d04c81aacdbaf83f5a9dc996898fa9c7ba26)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 12 ++++++++++++ google/cloud/datastore/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5535af0..df56b8e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ [1]: https://pypi.org/project/google-cloud-datastore/#history +### [2.1.2](https://www.github.com/googleapis/python-datastore/compare/v2.1.1...v2.1.2) (2021-05-03) + + +### Bug Fixes + +* pass transaction's options to API in 'begin' ([#143](https://www.github.com/googleapis/python-datastore/issues/143)) ([924b10b](https://www.github.com/googleapis/python-datastore/commit/924b10b11eb7ff52367f388cf5c8e16aa9b2e32e)) + + +### Documentation + +* update intersphinx URLs for grpc and auth ([#93](https://www.github.com/googleapis/python-datastore/issues/93)) ([4f90d04](https://www.github.com/googleapis/python-datastore/commit/4f90d04c81aacdbaf83f5a9dc996898fa9c7ba26)) + ### [2.1.1](https://www.github.com/googleapis/python-datastore/compare/v2.1.0...v2.1.1) (2021-04-20) diff --git a/google/cloud/datastore/version.py b/google/cloud/datastore/version.py index 7945f6f4..b6c2aa1a 100644 --- a/google/cloud/datastore/version.py +++ b/google/cloud/datastore/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.1.1" +__version__ = "2.1.2"