From d616f2a89c05e35d1d9dd7e8350a1ac20b63bfb1 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 May 2021 13:18:26 -0400 Subject: [PATCH 1/6] chore: fix synth replacment for index creation (#166) Upstream changes broke the previous anchor for the fragment. Closes #160. --- synth.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/synth.py b/synth.py index 818a9e80..6a59fc5b 100644 --- a/synth.py +++ b/synth.py @@ -142,9 +142,12 @@ def system(session, disable_grpc): # tests. num = s.replace( "CONTRIBUTING.rst", - 'app_credentials.json"', - """app_credentials.json" - + """\ +\*\*\*\*\*\*\*\*\*\*\*\*\* +Test Coverage +\*\*\*\*\*\*\*\*\*\*\*\*\* +""", + """\ - You'll need to create composite `indexes `__ with the ``gcloud`` command line @@ -169,7 +172,12 @@ def system(session, disable_grpc): prevents clean-up) you can clear all system test data from your datastore instance via:: - $ python tests/system/utils/clear_datastore.py""") + $ python tests/system/utils/clear_datastore.py + +************* +Test Coverage +************* +""") if num != 1: raise Exception("Required replacement not made.") From 81fad043b6ee34a3789df55b691ff7c492185573 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 12 May 2021 16:18:02 -0400 Subject: [PATCH 2/6] chore: add library type to .repo-metadata.json (#169) --- .repo-metadata.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.repo-metadata.json b/.repo-metadata.json index 10661c59..5f4bae89 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -6,6 +6,7 @@ "issue_tracker": "https://issuetracker.google.com/savedsearches/559768", "release_level": "ga", "language": "python", + "library_type": "GAPIC_COMBO", "repo": "googleapis/python-datastore", "distribution_name": "google-cloud-datastore", "api_id": "datastore.googleapis.com" From 0da3e22d90ace79a44d752af4b8ec94ddb30e53f Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 18 May 2021 15:27:59 -0400 Subject: [PATCH 3/6] tests: add coverage for skip < offset case (#165) Closes #158 --- google/cloud/datastore/query.py | 5 +++ tests/unit/test_query.py | 60 +++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/google/cloud/datastore/query.py b/google/cloud/datastore/query.py index 2f455b6f..b4b24ca7 100644 --- a/google/cloud/datastore/query.py +++ b/google/cloud/datastore/query.py @@ -593,8 +593,12 @@ def _next_page(self): # skipped all of the results yet. Don't return any results. # Instead, rerun query, adjusting offsets. Datastore doesn't process # more than 1000 skipped results in a query. + old_query_pb = query_pb + query_pb = query_pb2.Query() + query_pb._pb.CopyFrom(old_query_pb._pb) # copy for testability query_pb.start_cursor = response_pb.batch.skipped_cursor query_pb.offset -= response_pb.batch.skipped_results + response_pb = self.client._datastore_api.run_query( request={ "project_id": self._query.project, @@ -604,6 +608,7 @@ def _next_page(self): }, **kwargs, ) + entity_pbs = self._process_query_results(response_pb) return page_iterator.Page(self, entity_pbs, self.item_to_value) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 97c1db88..89bc7e2c 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -609,6 +609,58 @@ def test__next_page_no_more(self): self.assertIsNone(page) ds_api.run_query.assert_not_called() + def test__next_page_w_skipped_lt_offset(self): + from google.api_core import page_iterator + from google.cloud.datastore_v1.types import datastore as datastore_pb2 + from google.cloud.datastore_v1.types import entity as entity_pb2 + from google.cloud.datastore_v1.types import query as query_pb2 + from google.cloud.datastore.query import Query + + project = "prujekt" + skipped_1 = 100 + skipped_cursor_1 = b"DEADBEEF" + skipped_2 = 50 + skipped_cursor_2 = b"FACEDACE" + + more_enum = query_pb2.QueryResultBatch.MoreResultsType.NOT_FINISHED + + result_1 = _make_query_response([], b"", more_enum, skipped_1) + result_1.batch.skipped_cursor = skipped_cursor_1 + result_2 = _make_query_response([], b"", more_enum, skipped_2) + result_2.batch.skipped_cursor = skipped_cursor_2 + + ds_api = _make_datastore_api(result_1, result_2) + client = _Client(project, datastore_api=ds_api) + + query = Query(client) + offset = 150 + iterator = self._make_one(query, client, offset=offset) + + page = iterator._next_page() + + self.assertIsInstance(page, page_iterator.Page) + self.assertIs(page._parent, iterator) + + partition_id = entity_pb2.PartitionId(project_id=project) + read_options = datastore_pb2.ReadOptions() + + query_1 = query_pb2.Query(offset=offset) + query_2 = query_pb2.Query( + start_cursor=skipped_cursor_1, offset=(offset - skipped_1) + ) + expected_calls = [ + mock.call( + request={ + "project_id": project, + "partition_id": partition_id, + "read_options": read_options, + "query": query, + } + ) + for query in [query_1, query_2] + ] + self.assertEqual(ds_api.run_query.call_args_list, expected_calls) + class Test__item_to_entity(unittest.TestCase): def _call_fut(self, iterator, entity_pb): @@ -789,6 +841,10 @@ def _make_query_response( ) -def _make_datastore_api(result=None): - run_query = mock.Mock(return_value=result, spec=[]) +def _make_datastore_api(*results): + if len(results) == 0: + run_query = mock.Mock(return_value=None, spec=[]) + else: + run_query = mock.Mock(side_effect=results, spec=[]) + return mock.Mock(run_query=run_query, spec=["run_query"]) From 57b0fe6c2b3f72ddcc6ec4a874a0e1c0532d4ff1 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 25 May 2021 09:34:02 -0400 Subject: [PATCH 4/6] chore: delete unused protos (#174) --- .../proto/datastore_admin.proto | 425 ------------------ .../datastore_admin_v1/proto/index.proto | 115 ----- .../cloud/datastore_v1/proto/datastore.proto | 410 ----------------- google/cloud/datastore_v1/proto/entity.proto | 205 --------- google/cloud/datastore_v1/proto/query.proto | 313 ------------- 5 files changed, 1468 deletions(-) delete mode 100644 google/cloud/datastore_admin_v1/proto/datastore_admin.proto delete mode 100644 google/cloud/datastore_admin_v1/proto/index.proto delete mode 100644 google/cloud/datastore_v1/proto/datastore.proto delete mode 100644 google/cloud/datastore_v1/proto/entity.proto delete mode 100644 google/cloud/datastore_v1/proto/query.proto diff --git a/google/cloud/datastore_admin_v1/proto/datastore_admin.proto b/google/cloud/datastore_admin_v1/proto/datastore_admin.proto deleted file mode 100644 index c0f47076..00000000 --- a/google/cloud/datastore_admin_v1/proto/datastore_admin.proto +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright 2019 Google LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.datastore.admin.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/datastore/admin/v1/index.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/timestamp.proto"; - -option csharp_namespace = "Google.Cloud.Datastore.Admin.V1"; -option go_package = "google.golang.org/genproto/googleapis/datastore/admin/v1;admin"; -option java_multiple_files = true; -option java_outer_classname = "DatastoreAdminProto"; -option java_package = "com.google.datastore.admin.v1"; -option ruby_package = "Google::Cloud::Datastore::Admin::V1"; - -// Google Cloud Datastore Admin API -// -// -// The Datastore Admin API provides several admin services for Cloud Datastore. -// -// ----------------------------------------------------------------------------- -// ## Concepts -// -// Project, namespace, kind, and entity as defined in the Google Cloud Datastore -// API. -// -// Operation: An Operation represents work being performed in the background. -// -// EntityFilter: Allows specifying a subset of entities in a project. This is -// specified as a combination of kinds and namespaces (either or both of which -// may be all). -// -// ----------------------------------------------------------------------------- -// ## Services -// -// # Export/Import -// -// The Export/Import service provides the ability to copy all or a subset of -// entities to/from Google Cloud Storage. -// -// Exported data may be imported into Cloud Datastore for any Google Cloud -// Platform project. It is not restricted to the export source project. It is -// possible to export from one project and then import into another. -// -// Exported data can also be loaded into Google BigQuery for analysis. -// -// Exports and imports are performed asynchronously. An Operation resource is -// created for each export/import. The state (including any errors encountered) -// of the export/import may be queried via the Operation resource. -// -// # Index -// -// The index service manages Cloud Datastore composite indexes. -// -// Index creation and deletion are performed asynchronously. -// An Operation resource is created for each such asynchronous operation. -// The state of the operation (including any errors encountered) -// may be queried via the Operation resource. -// -// # Operation -// -// The Operations collection provides a record of actions performed for the -// specified project (including any operations in progress). Operations are not -// created directly but through calls on other collections or resources. -// -// An operation that is not yet done may be cancelled. The request to cancel is -// asynchronous and the operation may continue to run for some time after the -// request to cancel is made. -// -// An operation that is done may be deleted so that it is no longer listed as -// part of the Operation collection. -// -// ListOperations returns all pending operations, but not completed operations. -// -// Operations are created by service DatastoreAdmin, -// but are accessed via service google.longrunning.Operations. -service DatastoreAdmin { - option (google.api.default_host) = "datastore.googleapis.com"; - option (google.api.oauth_scopes) = - "https://www.googleapis.com/auth/cloud-platform," - "https://www.googleapis.com/auth/datastore"; - - // Exports a copy of all or a subset of entities from Google Cloud Datastore - // to another storage system, such as Google Cloud Storage. Recent updates to - // entities may not be reflected in the export. The export occurs in the - // background and its progress can be monitored and managed via the - // Operation resource that is created. The output of an export may only be - // used once the associated operation is done. If an export operation is - // cancelled before completion it may leave partial data behind in Google - // Cloud Storage. - rpc ExportEntities(ExportEntitiesRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:export" - body: "*" - }; - option (google.api.method_signature) = "project_id,labels,entity_filter,output_url_prefix"; - option (google.longrunning.operation_info) = { - response_type: "ExportEntitiesResponse" - metadata_type: "ExportEntitiesMetadata" - }; - } - - // Imports entities into Google Cloud Datastore. Existing entities with the - // same key are overwritten. The import occurs in the background and its - // progress can be monitored and managed via the Operation resource that is - // created. If an ImportEntities operation is cancelled, it is possible - // that a subset of the data has already been imported to Cloud Datastore. - rpc ImportEntities(ImportEntitiesRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:import" - body: "*" - }; - option (google.api.method_signature) = "project_id,labels,input_url,entity_filter"; - option (google.longrunning.operation_info) = { - response_type: "google.protobuf.Empty" - metadata_type: "ImportEntitiesMetadata" - }; - } - - // Gets an index. - rpc GetIndex(GetIndexRequest) returns (Index) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/indexes/{index_id}" - }; - } - - // Lists the indexes that match the specified filters. Datastore uses an - // eventually consistent query to fetch the list of indexes and may - // occasionally return stale results. - rpc ListIndexes(ListIndexesRequest) returns (ListIndexesResponse) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/indexes" - }; - } -} - -// Metadata common to all Datastore Admin operations. -message CommonMetadata { - // The various possible states for an ongoing Operation. - enum State { - // Unspecified. - STATE_UNSPECIFIED = 0; - - // Request is being prepared for processing. - INITIALIZING = 1; - - // Request is actively being processed. - PROCESSING = 2; - - // Request is in the process of being cancelled after user called - // google.longrunning.Operations.CancelOperation on the operation. - CANCELLING = 3; - - // Request has been processed and is in its finalization stage. - FINALIZING = 4; - - // Request has completed successfully. - SUCCESSFUL = 5; - - // Request has finished being processed, but encountered an error. - FAILED = 6; - - // Request has finished being cancelled after user called - // google.longrunning.Operations.CancelOperation. - CANCELLED = 7; - } - - // The time that work began on the operation. - google.protobuf.Timestamp start_time = 1; - - // The time the operation ended, either successfully or otherwise. - google.protobuf.Timestamp end_time = 2; - - // The type of the operation. Can be used as a filter in - // ListOperationsRequest. - OperationType operation_type = 3; - - // The client-assigned labels which were provided when the operation was - // created. May also include additional labels. - map labels = 4; - - // The current state of the Operation. - State state = 5; -} - -// Operation types. -enum OperationType { - // Unspecified. - OPERATION_TYPE_UNSPECIFIED = 0; - - // ExportEntities. - EXPORT_ENTITIES = 1; - - // ImportEntities. - IMPORT_ENTITIES = 2; - - // CreateIndex. - CREATE_INDEX = 3; - - // DeleteIndex. - DELETE_INDEX = 4; -} - -// Measures the progress of a particular metric. -message Progress { - // The amount of work that has been completed. Note that this may be greater - // than work_estimated. - int64 work_completed = 1; - - // An estimate of how much work needs to be performed. May be zero if the - // work estimate is unavailable. - int64 work_estimated = 2; -} - -// The request for -// [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. -message ExportEntitiesRequest { - // Required. Project ID against which to make the request. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Client-assigned labels. - map labels = 2; - - // Description of what data from the project is included in the export. - EntityFilter entity_filter = 3; - - // Required. Location for the export metadata and data files. - // - // The full resource URL of the external storage location. Currently, only - // Google Cloud Storage is supported. So output_url_prefix should be of the - // form: `gs://BUCKET_NAME[/NAMESPACE_PATH]`, where `BUCKET_NAME` is the - // name of the Cloud Storage bucket and `NAMESPACE_PATH` is an optional Cloud - // Storage namespace path (this is not a Cloud Datastore namespace). For more - // information about Cloud Storage namespace paths, see - // [Object name - // considerations](https://cloud.google.com/storage/docs/naming#object-considerations). - // - // The resulting files will be nested deeper than the specified URL prefix. - // The final output URL will be provided in the - // [google.datastore.admin.v1.ExportEntitiesResponse.output_url][google.datastore.admin.v1.ExportEntitiesResponse.output_url] field. That - // value should be used for subsequent ImportEntities operations. - // - // By nesting the data files deeper, the same Cloud Storage bucket can be used - // in multiple ExportEntities operations without conflict. - string output_url_prefix = 4 [(google.api.field_behavior) = REQUIRED]; -} - -// The request for -// [google.datastore.admin.v1.DatastoreAdmin.ImportEntities][google.datastore.admin.v1.DatastoreAdmin.ImportEntities]. -message ImportEntitiesRequest { - // Required. Project ID against which to make the request. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Client-assigned labels. - map labels = 2; - - // Required. The full resource URL of the external storage location. Currently, only - // Google Cloud Storage is supported. So input_url should be of the form: - // `gs://BUCKET_NAME[/NAMESPACE_PATH]/OVERALL_EXPORT_METADATA_FILE`, where - // `BUCKET_NAME` is the name of the Cloud Storage bucket, `NAMESPACE_PATH` is - // an optional Cloud Storage namespace path (this is not a Cloud Datastore - // namespace), and `OVERALL_EXPORT_METADATA_FILE` is the metadata file written - // by the ExportEntities operation. For more information about Cloud Storage - // namespace paths, see - // [Object name - // considerations](https://cloud.google.com/storage/docs/naming#object-considerations). - // - // For more information, see - // [google.datastore.admin.v1.ExportEntitiesResponse.output_url][google.datastore.admin.v1.ExportEntitiesResponse.output_url]. - string input_url = 3 [(google.api.field_behavior) = REQUIRED]; - - // Optionally specify which kinds/namespaces are to be imported. If provided, - // the list must be a subset of the EntityFilter used in creating the export, - // otherwise a FAILED_PRECONDITION error will be returned. If no filter is - // specified then all entities from the export are imported. - EntityFilter entity_filter = 4; -} - -// The response for -// [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. -message ExportEntitiesResponse { - // Location of the output metadata file. This can be used to begin an import - // into Cloud Datastore (this project or another project). See - // [google.datastore.admin.v1.ImportEntitiesRequest.input_url][google.datastore.admin.v1.ImportEntitiesRequest.input_url]. - // Only present if the operation completed successfully. - string output_url = 1; -} - -// Metadata for ExportEntities operations. -message ExportEntitiesMetadata { - // Metadata common to all Datastore Admin operations. - CommonMetadata common = 1; - - // An estimate of the number of entities processed. - Progress progress_entities = 2; - - // An estimate of the number of bytes processed. - Progress progress_bytes = 3; - - // Description of which entities are being exported. - EntityFilter entity_filter = 4; - - // Location for the export metadata and data files. This will be the same - // value as the - // [google.datastore.admin.v1.ExportEntitiesRequest.output_url_prefix][google.datastore.admin.v1.ExportEntitiesRequest.output_url_prefix] - // field. The final output location is provided in - // [google.datastore.admin.v1.ExportEntitiesResponse.output_url][google.datastore.admin.v1.ExportEntitiesResponse.output_url]. - string output_url_prefix = 5; -} - -// Metadata for ImportEntities operations. -message ImportEntitiesMetadata { - // Metadata common to all Datastore Admin operations. - CommonMetadata common = 1; - - // An estimate of the number of entities processed. - Progress progress_entities = 2; - - // An estimate of the number of bytes processed. - Progress progress_bytes = 3; - - // Description of which entities are being imported. - EntityFilter entity_filter = 4; - - // The location of the import metadata file. This will be the same value as - // the [google.datastore.admin.v1.ExportEntitiesResponse.output_url][google.datastore.admin.v1.ExportEntitiesResponse.output_url] field. - string input_url = 5; -} - -// Identifies a subset of entities in a project. This is specified as -// combinations of kinds and namespaces (either or both of which may be all, as -// described in the following examples). -// Example usage: -// -// Entire project: -// kinds=[], namespace_ids=[] -// -// Kinds Foo and Bar in all namespaces: -// kinds=['Foo', 'Bar'], namespace_ids=[] -// -// Kinds Foo and Bar only in the default namespace: -// kinds=['Foo', 'Bar'], namespace_ids=[''] -// -// Kinds Foo and Bar in both the default and Baz namespaces: -// kinds=['Foo', 'Bar'], namespace_ids=['', 'Baz'] -// -// The entire Baz namespace: -// kinds=[], namespace_ids=['Baz'] -message EntityFilter { - // If empty, then this represents all kinds. - repeated string kinds = 1; - - // An empty list represents all namespaces. This is the preferred - // usage for projects that don't use namespaces. - // - // An empty string element represents the default namespace. This should be - // used if the project has data in non-default namespaces, but doesn't want to - // include them. - // Each namespace in this list must be unique. - repeated string namespace_ids = 2; -} - -// The request for [google.datastore.admin.v1.DatastoreAdmin.GetIndex][google.datastore.admin.v1.DatastoreAdmin.GetIndex]. -message GetIndexRequest { - // Project ID against which to make the request. - string project_id = 1; - - // The resource ID of the index to get. - string index_id = 3; -} - -// The request for -// [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. -message ListIndexesRequest { - // Project ID against which to make the request. - string project_id = 1; - - string filter = 3; - - // The maximum number of items to return. If zero, then all results will be - // returned. - int32 page_size = 4; - - // The next_page_token value returned from a previous List request, if any. - string page_token = 5; -} - -// The response for -// [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. -message ListIndexesResponse { - // The indexes. - repeated Index indexes = 1; - - // The standard List next-page token. - string next_page_token = 2; -} - -// Metadata for Index operations. -message IndexOperationMetadata { - // Metadata common to all Datastore Admin operations. - CommonMetadata common = 1; - - // An estimate of the number of entities processed. - Progress progress_entities = 2; - - // The index resource ID that this operation is acting on. - string index_id = 3; -} diff --git a/google/cloud/datastore_admin_v1/proto/index.proto b/google/cloud/datastore_admin_v1/proto/index.proto deleted file mode 100644 index 96c2278b..00000000 --- a/google/cloud/datastore_admin_v1/proto/index.proto +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2019 Google LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.datastore.admin.v1; - -import "google/api/field_behavior.proto"; -import "google/api/annotations.proto"; - -option csharp_namespace = "Google.Cloud.Datastore.Admin.V1"; -option go_package = "google.golang.org/genproto/googleapis/datastore/admin/v1;admin"; -option java_multiple_files = true; -option java_outer_classname = "IndexProto"; -option java_package = "com.google.datastore.admin.v1"; -option ruby_package = "Google::Cloud::Datastore::Admin::V1"; - -// A minimal index definition. -message Index { - // A property of an index. - message IndexedProperty { - // Required. The property name to index. - string name = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The indexed property's direction. Must not be DIRECTION_UNSPECIFIED. - Direction direction = 2 [(google.api.field_behavior) = REQUIRED]; - } - - // For an ordered index, specifies whether each of the entity's ancestors - // will be included. - enum AncestorMode { - // The ancestor mode is unspecified. - ANCESTOR_MODE_UNSPECIFIED = 0; - - // Do not include the entity's ancestors in the index. - NONE = 1; - - // Include all the entity's ancestors in the index. - ALL_ANCESTORS = 2; - } - - // The direction determines how a property is indexed. - enum Direction { - // The direction is unspecified. - DIRECTION_UNSPECIFIED = 0; - - // The property's values are indexed so as to support sequencing in - // ascending order and also query by <, >, <=, >=, and =. - ASCENDING = 1; - - // The property's values are indexed so as to support sequencing in - // descending order and also query by <, >, <=, >=, and =. - DESCENDING = 2; - } - - // The possible set of states of an index. - enum State { - // The state is unspecified. - STATE_UNSPECIFIED = 0; - - // The index is being created, and cannot be used by queries. - // There is an active long-running operation for the index. - // The index is updated when writing an entity. - // Some index data may exist. - CREATING = 1; - - // The index is ready to be used. - // The index is updated when writing an entity. - // The index is fully populated from all stored entities it applies to. - READY = 2; - - // The index is being deleted, and cannot be used by queries. - // There is an active long-running operation for the index. - // The index is not updated when writing an entity. - // Some index data may exist. - DELETING = 3; - - // The index was being created or deleted, but something went wrong. - // The index cannot by used by queries. - // There is no active long-running operation for the index, - // and the most recently finished long-running operation failed. - // The index is not updated when writing an entity. - // Some index data may exist. - ERROR = 4; - } - - // Output only. Project ID. - string project_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The resource ID of the index. - string index_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Required. The entity kind to which this index applies. - string kind = 4 [(google.api.field_behavior) = REQUIRED]; - - // Required. The index's ancestor mode. Must not be ANCESTOR_MODE_UNSPECIFIED. - AncestorMode ancestor = 5 [(google.api.field_behavior) = REQUIRED]; - - // Required. An ordered sequence of property names and their index attributes. - repeated IndexedProperty properties = 6 [(google.api.field_behavior) = REQUIRED]; - - // Output only. The state of the index. - State state = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; -} diff --git a/google/cloud/datastore_v1/proto/datastore.proto b/google/cloud/datastore_v1/proto/datastore.proto deleted file mode 100644 index ad016194..00000000 --- a/google/cloud/datastore_v1/proto/datastore.proto +++ /dev/null @@ -1,410 +0,0 @@ -// Copyright 2019 Google LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.datastore.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/datastore/v1/entity.proto"; -import "google/datastore/v1/query.proto"; - -option csharp_namespace = "Google.Cloud.Datastore.V1"; -option go_package = "google.golang.org/genproto/googleapis/datastore/v1;datastore"; -option java_multiple_files = true; -option java_outer_classname = "DatastoreProto"; -option java_package = "com.google.datastore.v1"; -option php_namespace = "Google\\Cloud\\Datastore\\V1"; -option ruby_package = "Google::Cloud::Datastore::V1"; - -// Each RPC normalizes the partition IDs of the keys in its input entities, -// and always returns entities with keys with normalized partition IDs. -// This applies to all keys and entities, including those in values, except keys -// with both an empty path and an empty or unset partition ID. Normalization of -// input keys sets the project ID (if not already set) to the project ID from -// the request. -// -service Datastore { - option (google.api.default_host) = "datastore.googleapis.com"; - option (google.api.oauth_scopes) = - "https://www.googleapis.com/auth/cloud-platform," - "https://www.googleapis.com/auth/datastore"; - - // Looks up entities by key. - rpc Lookup(LookupRequest) returns (LookupResponse) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:lookup" - body: "*" - }; - option (google.api.method_signature) = "project_id,read_options,keys"; - } - - // Queries for entities. - rpc RunQuery(RunQueryRequest) returns (RunQueryResponse) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:runQuery" - body: "*" - }; - } - - // Begins a new transaction. - rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:beginTransaction" - body: "*" - }; - option (google.api.method_signature) = "project_id"; - } - - // Commits a transaction, optionally creating, deleting or modifying some - // entities. - rpc Commit(CommitRequest) returns (CommitResponse) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:commit" - body: "*" - }; - option (google.api.method_signature) = "project_id,mode,transaction,mutations"; - option (google.api.method_signature) = "project_id,mode,mutations"; - } - - // Rolls back a transaction. - rpc Rollback(RollbackRequest) returns (RollbackResponse) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:rollback" - body: "*" - }; - option (google.api.method_signature) = "project_id,transaction"; - } - - // Allocates IDs for the given keys, which is useful for referencing an entity - // before it is inserted. - rpc AllocateIds(AllocateIdsRequest) returns (AllocateIdsResponse) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:allocateIds" - body: "*" - }; - option (google.api.method_signature) = "project_id,keys"; - } - - // Prevents the supplied keys' IDs from being auto-allocated by Cloud - // Datastore. - rpc ReserveIds(ReserveIdsRequest) returns (ReserveIdsResponse) { - option (google.api.http) = { - post: "/v1/projects/{project_id}:reserveIds" - body: "*" - }; - option (google.api.method_signature) = "project_id,keys"; - } -} - -// The request for [Datastore.Lookup][google.datastore.v1.Datastore.Lookup]. -message LookupRequest { - // Required. The ID of the project against which to make the request. - string project_id = 8 [(google.api.field_behavior) = REQUIRED]; - - // The options for this lookup request. - ReadOptions read_options = 1; - - // Required. Keys of entities to look up. - repeated Key keys = 3 [(google.api.field_behavior) = REQUIRED]; -} - -// The response for [Datastore.Lookup][google.datastore.v1.Datastore.Lookup]. -message LookupResponse { - // Entities found as `ResultType.FULL` entities. The order of results in this - // field is undefined and has no relation to the order of the keys in the - // input. - repeated EntityResult found = 1; - - // Entities not found as `ResultType.KEY_ONLY` entities. The order of results - // in this field is undefined and has no relation to the order of the keys - // in the input. - repeated EntityResult missing = 2; - - // A list of keys that were not looked up due to resource constraints. The - // order of results in this field is undefined and has no relation to the - // order of the keys in the input. - repeated Key deferred = 3; -} - -// The request for [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. -message RunQueryRequest { - // Required. The ID of the project against which to make the request. - string project_id = 8 [(google.api.field_behavior) = REQUIRED]; - - // Entities are partitioned into subsets, identified by a partition ID. - // Queries are scoped to a single partition. - // This partition ID is normalized with the standard default context - // partition ID. - PartitionId partition_id = 2; - - // The options for this query. - ReadOptions read_options = 1; - - // The type of query. - oneof query_type { - // The query to run. - Query query = 3; - - // The GQL query to run. - GqlQuery gql_query = 7; - } -} - -// The response for [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. -message RunQueryResponse { - // A batch of query results (always present). - QueryResultBatch batch = 1; - - // The parsed form of the `GqlQuery` from the request, if it was set. - Query query = 2; -} - -// The request for [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. -message BeginTransactionRequest { - // Required. The ID of the project against which to make the request. - string project_id = 8 [(google.api.field_behavior) = REQUIRED]; - - // Options for a new transaction. - TransactionOptions transaction_options = 10; -} - -// The response for [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. -message BeginTransactionResponse { - // The transaction identifier (always present). - bytes transaction = 1; -} - -// The request for [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. -message RollbackRequest { - // Required. The ID of the project against which to make the request. - string project_id = 8 [(google.api.field_behavior) = REQUIRED]; - - // Required. The transaction identifier, returned by a call to - // [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. - bytes transaction = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// The response for [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. -// (an empty message). -message RollbackResponse { - -} - -// The request for [Datastore.Commit][google.datastore.v1.Datastore.Commit]. -message CommitRequest { - // The modes available for commits. - enum Mode { - // Unspecified. This value must not be used. - MODE_UNSPECIFIED = 0; - - // Transactional: The mutations are either all applied, or none are applied. - // Learn about transactions - // [here](https://cloud.google.com/datastore/docs/concepts/transactions). - TRANSACTIONAL = 1; - - // Non-transactional: The mutations may not apply as all or none. - NON_TRANSACTIONAL = 2; - } - - // Required. The ID of the project against which to make the request. - string project_id = 8 [(google.api.field_behavior) = REQUIRED]; - - // The type of commit to perform. Defaults to `TRANSACTIONAL`. - Mode mode = 5; - - // Must be set when mode is `TRANSACTIONAL`. - oneof transaction_selector { - // The identifier of the transaction associated with the commit. A - // transaction identifier is returned by a call to - // [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. - bytes transaction = 1; - } - - // The mutations to perform. - // - // When mode is `TRANSACTIONAL`, mutations affecting a single entity are - // applied in order. The following sequences of mutations affecting a single - // entity are not permitted in a single `Commit` request: - // - // - `insert` followed by `insert` - // - `update` followed by `insert` - // - `upsert` followed by `insert` - // - `delete` followed by `update` - // - // When mode is `NON_TRANSACTIONAL`, no two mutations may affect a single - // entity. - repeated Mutation mutations = 6; -} - -// The response for [Datastore.Commit][google.datastore.v1.Datastore.Commit]. -message CommitResponse { - // The result of performing the mutations. - // The i-th mutation result corresponds to the i-th mutation in the request. - repeated MutationResult mutation_results = 3; - - // The number of index entries updated during the commit, or zero if none were - // updated. - int32 index_updates = 4; -} - -// The request for [Datastore.AllocateIds][google.datastore.v1.Datastore.AllocateIds]. -message AllocateIdsRequest { - // Required. The ID of the project against which to make the request. - string project_id = 8 [(google.api.field_behavior) = REQUIRED]; - - // Required. A list of keys with incomplete key paths for which to allocate IDs. - // No key may be reserved/read-only. - repeated Key keys = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// The response for [Datastore.AllocateIds][google.datastore.v1.Datastore.AllocateIds]. -message AllocateIdsResponse { - // The keys specified in the request (in the same order), each with - // its key path completed with a newly allocated ID. - repeated Key keys = 1; -} - -// The request for [Datastore.ReserveIds][google.datastore.v1.Datastore.ReserveIds]. -message ReserveIdsRequest { - // Required. The ID of the project against which to make the request. - string project_id = 8 [(google.api.field_behavior) = REQUIRED]; - - // If not empty, the ID of the database against which to make the request. - string database_id = 9; - - // Required. A list of keys with complete key paths whose numeric IDs should not be - // auto-allocated. - repeated Key keys = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// The response for [Datastore.ReserveIds][google.datastore.v1.Datastore.ReserveIds]. -message ReserveIdsResponse { - -} - -// A mutation to apply to an entity. -message Mutation { - // The mutation operation. - // - // For `insert`, `update`, and `upsert`: - // - The entity's key must not be reserved/read-only. - // - No property in the entity may have a reserved name, - // not even a property in an entity in a value. - // - No value in the entity may have meaning 18, - // not even a value in an entity in another value. - oneof operation { - // The entity to insert. The entity must not already exist. - // The entity key's final path element may be incomplete. - Entity insert = 4; - - // The entity to update. The entity must already exist. - // Must have a complete key path. - Entity update = 5; - - // The entity to upsert. The entity may or may not already exist. - // The entity key's final path element may be incomplete. - Entity upsert = 6; - - // The key of the entity to delete. The entity may or may not already exist. - // Must have a complete key path and must not be reserved/read-only. - Key delete = 7; - } - - // When set, the server will detect whether or not this mutation conflicts - // with the current version of the entity on the server. Conflicting mutations - // are not applied, and are marked as such in MutationResult. - oneof conflict_detection_strategy { - // The version of the entity that this mutation is being applied to. If this - // does not match the current version on the server, the mutation conflicts. - int64 base_version = 8; - } -} - -// The result of applying a mutation. -message MutationResult { - // The automatically allocated key. - // Set only when the mutation allocated a key. - Key key = 3; - - // The version of the entity on the server after processing the mutation. If - // the mutation doesn't change anything on the server, then the version will - // be the version of the current entity or, if no entity is present, a version - // that is strictly greater than the version of any previous entity and less - // than the version of any possible future entity. - int64 version = 4; - - // Whether a conflict was detected for this mutation. Always false when a - // conflict detection strategy field is not set in the mutation. - bool conflict_detected = 5; -} - -// The options shared by read requests. -message ReadOptions { - // The possible values for read consistencies. - enum ReadConsistency { - // Unspecified. This value must not be used. - READ_CONSISTENCY_UNSPECIFIED = 0; - - // Strong consistency. - STRONG = 1; - - // Eventual consistency. - EVENTUAL = 2; - } - - // If not specified, lookups and ancestor queries default to - // `read_consistency`=`STRONG`, global queries default to - // `read_consistency`=`EVENTUAL`. - oneof consistency_type { - // The non-transactional read consistency to use. - // Cannot be set to `STRONG` for global queries. - ReadConsistency read_consistency = 1; - - // The identifier of the transaction in which to read. A - // transaction identifier is returned by a call to - // [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. - bytes transaction = 2; - } -} - -// Options for beginning a new transaction. -// -// Transactions can be created explicitly with calls to -// [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction] or implicitly by setting -// [ReadOptions.new_transaction][google.datastore.v1.ReadOptions.new_transaction] in read requests. -message TransactionOptions { - // Options specific to read / write transactions. - message ReadWrite { - // The transaction identifier of the transaction being retried. - bytes previous_transaction = 1; - } - - // Options specific to read-only transactions. - message ReadOnly { - - } - - // The `mode` of the transaction, indicating whether write operations are - // supported. - oneof mode { - // The transaction should allow both reads and writes. - ReadWrite read_write = 1; - - // The transaction should only allow reads. - ReadOnly read_only = 2; - } -} diff --git a/google/cloud/datastore_v1/proto/entity.proto b/google/cloud/datastore_v1/proto/entity.proto deleted file mode 100644 index 61286cd7..00000000 --- a/google/cloud/datastore_v1/proto/entity.proto +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2019 Google LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.datastore.v1; - -import "google/api/annotations.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/timestamp.proto"; -import "google/type/latlng.proto"; - -option csharp_namespace = "Google.Cloud.Datastore.V1"; -option go_package = "google.golang.org/genproto/googleapis/datastore/v1;datastore"; -option java_multiple_files = true; -option java_outer_classname = "EntityProto"; -option java_package = "com.google.datastore.v1"; -option php_namespace = "Google\\Cloud\\Datastore\\V1"; -option ruby_package = "Google::Cloud::Datastore::V1"; - -// A partition ID identifies a grouping of entities. The grouping is always -// by project and namespace, however the namespace ID may be empty. -// -// A partition ID contains several dimensions: -// project ID and namespace ID. -// -// Partition dimensions: -// -// - May be `""`. -// - Must be valid UTF-8 bytes. -// - Must have values that match regex `[A-Za-z\d\.\-_]{1,100}` -// If the value of any dimension matches regex `__.*__`, the partition is -// reserved/read-only. -// A reserved/read-only partition ID is forbidden in certain documented -// contexts. -// -// Foreign partition IDs (in which the project ID does -// not match the context project ID ) are discouraged. -// Reads and writes of foreign partition IDs may fail if the project is not in -// an active state. -message PartitionId { - // The ID of the project to which the entities belong. - string project_id = 2; - - // If not empty, the ID of the namespace to which the entities belong. - string namespace_id = 4; -} - -// A unique identifier for an entity. -// If a key's partition ID or any of its path kinds or names are -// reserved/read-only, the key is reserved/read-only. -// A reserved/read-only key is forbidden in certain documented contexts. -message Key { - // A (kind, ID/name) pair used to construct a key path. - // - // If either name or ID is set, the element is complete. - // If neither is set, the element is incomplete. - message PathElement { - // The kind of the entity. - // A kind matching regex `__.*__` is reserved/read-only. - // A kind must not contain more than 1500 bytes when UTF-8 encoded. - // Cannot be `""`. - string kind = 1; - - // The type of ID. - oneof id_type { - // The auto-allocated ID of the entity. - // Never equal to zero. Values less than zero are discouraged and may not - // be supported in the future. - int64 id = 2; - - // The name of the entity. - // A name matching regex `__.*__` is reserved/read-only. - // A name must not be more than 1500 bytes when UTF-8 encoded. - // Cannot be `""`. - string name = 3; - } - } - - // Entities are partitioned into subsets, currently identified by a project - // ID and namespace ID. - // Queries are scoped to a single partition. - PartitionId partition_id = 1; - - // The entity path. - // An entity path consists of one or more elements composed of a kind and a - // string or numerical identifier, which identify entities. The first - // element identifies a _root entity_, the second element identifies - // a _child_ of the root entity, the third element identifies a child of the - // second entity, and so forth. The entities identified by all prefixes of - // the path are called the element's _ancestors_. - // - // An entity path is always fully complete: *all* of the entity's ancestors - // are required to be in the path along with the entity identifier itself. - // The only exception is that in some documented cases, the identifier in the - // last path element (for the entity) itself may be omitted. For example, - // the last path element of the key of `Mutation.insert` may have no - // identifier. - // - // A path can never be empty, and a path can have at most 100 elements. - repeated PathElement path = 2; -} - -// An array value. -message ArrayValue { - // Values in the array. - // The order of values in an array is preserved as long as all values have - // identical settings for 'exclude_from_indexes'. - repeated Value values = 1; -} - -// A message that can hold any of the supported value types and associated -// metadata. -message Value { - // Must have a value set. - oneof value_type { - // A null value. - google.protobuf.NullValue null_value = 11; - - // A boolean value. - bool boolean_value = 1; - - // An integer value. - int64 integer_value = 2; - - // A double value. - double double_value = 3; - - // A timestamp value. - // When stored in the Datastore, precise only to microseconds; - // any additional precision is rounded down. - google.protobuf.Timestamp timestamp_value = 10; - - // A key value. - Key key_value = 5; - - // A UTF-8 encoded string value. - // When `exclude_from_indexes` is false (it is indexed), may have at most - // 1500 bytes. Otherwise, may be set to at most 1,000,000 bytes. - string string_value = 17; - - // A blob value. - // May have at most 1,000,000 bytes. - // When `exclude_from_indexes` is false, may have at most 1500 bytes. - // In JSON requests, must be base64-encoded. - bytes blob_value = 18; - - // A geo point value representing a point on the surface of Earth. - google.type.LatLng geo_point_value = 8; - - // An entity value. - // - // - May have no key. - // - May have a key with an incomplete key path. - // - May have a reserved/read-only key. - Entity entity_value = 6; - - // An array value. - // Cannot contain another array value. - // A `Value` instance that sets field `array_value` must not set fields - // `meaning` or `exclude_from_indexes`. - ArrayValue array_value = 9; - } - - // The `meaning` field should only be populated for backwards compatibility. - int32 meaning = 14; - - // If the value should be excluded from all indexes including those defined - // explicitly. - bool exclude_from_indexes = 19; -} - -// A Datastore data object. -// -// An entity is limited to 1 megabyte when stored. That _roughly_ -// corresponds to a limit of 1 megabyte for the serialized form of this -// message. -message Entity { - // The entity's key. - // - // An entity must have a key, unless otherwise documented (for example, - // an entity in `Value.entity_value` may have no key). - // An entity's kind is its key path's last element's kind, - // or null if it has no key. - Key key = 1; - - // The entity's properties. - // The map's keys are property names. - // A property name matching regex `__.*__` is reserved. - // A reserved property name is forbidden in certain documented contexts. - // The name must not contain more than 500 characters. - // The name cannot be `""`. - map properties = 3; -} diff --git a/google/cloud/datastore_v1/proto/query.proto b/google/cloud/datastore_v1/proto/query.proto deleted file mode 100644 index 4cb3ef99..00000000 --- a/google/cloud/datastore_v1/proto/query.proto +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright 2019 Google LLC. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.datastore.v1; - -import "google/api/annotations.proto"; -import "google/datastore/v1/entity.proto"; -import "google/protobuf/wrappers.proto"; -import "google/type/latlng.proto"; - -option csharp_namespace = "Google.Cloud.Datastore.V1"; -option go_package = "google.golang.org/genproto/googleapis/datastore/v1;datastore"; -option java_multiple_files = true; -option java_outer_classname = "QueryProto"; -option java_package = "com.google.datastore.v1"; -option php_namespace = "Google\\Cloud\\Datastore\\V1"; -option ruby_package = "Google::Cloud::Datastore::V1"; - -// The result of fetching an entity from Datastore. -message EntityResult { - // Specifies what data the 'entity' field contains. - // A `ResultType` is either implied (for example, in `LookupResponse.missing` - // from `datastore.proto`, it is always `KEY_ONLY`) or specified by context - // (for example, in message `QueryResultBatch`, field `entity_result_type` - // specifies a `ResultType` for all the values in field `entity_results`). - enum ResultType { - // Unspecified. This value is never used. - RESULT_TYPE_UNSPECIFIED = 0; - - // The key and properties. - FULL = 1; - - // A projected subset of properties. The entity may have no key. - PROJECTION = 2; - - // Only the key. - KEY_ONLY = 3; - } - - // The resulting entity. - Entity entity = 1; - - // The version of the entity, a strictly positive number that monotonically - // increases with changes to the entity. - // - // This field is set for [`FULL`][google.datastore.v1.EntityResult.ResultType.FULL] entity - // results. - // - // For [missing][google.datastore.v1.LookupResponse.missing] entities in `LookupResponse`, this - // is the version of the snapshot that was used to look up the entity, and it - // is always set except for eventually consistent reads. - int64 version = 4; - - // A cursor that points to the position after the result entity. - // Set only when the `EntityResult` is part of a `QueryResultBatch` message. - bytes cursor = 3; -} - -// A query for entities. -message Query { - // The projection to return. Defaults to returning all properties. - repeated Projection projection = 2; - - // The kinds to query (if empty, returns entities of all kinds). - // Currently at most 1 kind may be specified. - repeated KindExpression kind = 3; - - // The filter to apply. - Filter filter = 4; - - // The order to apply to the query results (if empty, order is unspecified). - repeated PropertyOrder order = 5; - - // The properties to make distinct. The query results will contain the first - // result for each distinct combination of values for the given properties - // (if empty, all results are returned). - repeated PropertyReference distinct_on = 6; - - // A starting point for the query results. Query cursors are - // returned in query result batches and - // [can only be used to continue the same - // query](https://cloud.google.com/datastore/docs/concepts/queries#cursors_limits_and_offsets). - bytes start_cursor = 7; - - // An ending point for the query results. Query cursors are - // returned in query result batches and - // [can only be used to limit the same - // query](https://cloud.google.com/datastore/docs/concepts/queries#cursors_limits_and_offsets). - bytes end_cursor = 8; - - // The number of results to skip. Applies before limit, but after all other - // constraints. Optional. Must be >= 0 if specified. - int32 offset = 10; - - // The maximum number of results to return. Applies after all other - // constraints. Optional. - // Unspecified is interpreted as no limit. - // Must be >= 0 if specified. - google.protobuf.Int32Value limit = 12; -} - -// A representation of a kind. -message KindExpression { - // The name of the kind. - string name = 1; -} - -// A reference to a property relative to the kind expressions. -message PropertyReference { - // The name of the property. - // If name includes "."s, it may be interpreted as a property name path. - string name = 2; -} - -// A representation of a property in a projection. -message Projection { - // The property to project. - PropertyReference property = 1; -} - -// The desired order for a specific property. -message PropertyOrder { - // The sort direction. - enum Direction { - // Unspecified. This value must not be used. - DIRECTION_UNSPECIFIED = 0; - - // Ascending. - ASCENDING = 1; - - // Descending. - DESCENDING = 2; - } - - // The property to order by. - PropertyReference property = 1; - - // The direction to order by. Defaults to `ASCENDING`. - Direction direction = 2; -} - -// A holder for any type of filter. -message Filter { - // The type of filter. - oneof filter_type { - // A composite filter. - CompositeFilter composite_filter = 1; - - // A filter on a property. - PropertyFilter property_filter = 2; - } -} - -// A filter that merges multiple other filters using the given operator. -message CompositeFilter { - // A composite filter operator. - enum Operator { - // Unspecified. This value must not be used. - OPERATOR_UNSPECIFIED = 0; - - // The results are required to satisfy each of the combined filters. - AND = 1; - } - - // The operator for combining multiple filters. - Operator op = 1; - - // The list of filters to combine. - // Must contain at least one filter. - repeated Filter filters = 2; -} - -// A filter on a specific property. -message PropertyFilter { - // A property filter operator. - enum Operator { - // Unspecified. This value must not be used. - OPERATOR_UNSPECIFIED = 0; - - // Less than. - LESS_THAN = 1; - - // Less than or equal. - LESS_THAN_OR_EQUAL = 2; - - // Greater than. - GREATER_THAN = 3; - - // Greater than or equal. - GREATER_THAN_OR_EQUAL = 4; - - // Equal. - EQUAL = 5; - - // Has ancestor. - HAS_ANCESTOR = 11; - } - - // The property to filter by. - PropertyReference property = 1; - - // The operator to filter by. - Operator op = 2; - - // The value to compare the property to. - Value value = 3; -} - -// A [GQL -// query](https://cloud.google.com/datastore/docs/apis/gql/gql_reference). -message GqlQuery { - // A string of the format described - // [here](https://cloud.google.com/datastore/docs/apis/gql/gql_reference). - string query_string = 1; - - // When false, the query string must not contain any literals and instead must - // bind all values. For example, - // `SELECT * FROM Kind WHERE a = 'string literal'` is not allowed, while - // `SELECT * FROM Kind WHERE a = @value` is. - bool allow_literals = 2; - - // For each non-reserved named binding site in the query string, there must be - // a named parameter with that name, but not necessarily the inverse. - // - // Key must match regex `[A-Za-z_$][A-Za-z_$0-9]*`, must not match regex - // `__.*__`, and must not be `""`. - map named_bindings = 5; - - // Numbered binding site @1 references the first numbered parameter, - // effectively using 1-based indexing, rather than the usual 0. - // - // For each binding site numbered i in `query_string`, there must be an i-th - // numbered parameter. The inverse must also be true. - repeated GqlQueryParameter positional_bindings = 4; -} - -// A binding parameter for a GQL query. -message GqlQueryParameter { - // The type of parameter. - oneof parameter_type { - // A value parameter. - Value value = 2; - - // A query cursor. Query cursors are returned in query - // result batches. - bytes cursor = 3; - } -} - -// A batch of results produced by a query. -message QueryResultBatch { - // The possible values for the `more_results` field. - enum MoreResultsType { - // Unspecified. This value is never used. - MORE_RESULTS_TYPE_UNSPECIFIED = 0; - - // There may be additional batches to fetch from this query. - NOT_FINISHED = 1; - - // The query is finished, but there may be more results after the limit. - MORE_RESULTS_AFTER_LIMIT = 2; - - // The query is finished, but there may be more results after the end - // cursor. - MORE_RESULTS_AFTER_CURSOR = 4; - - // The query is finished, and there are no more results. - NO_MORE_RESULTS = 3; - } - - // The number of results skipped, typically because of an offset. - int32 skipped_results = 6; - - // A cursor that points to the position after the last skipped result. - // Will be set when `skipped_results` != 0. - bytes skipped_cursor = 3; - - // The result type for every entity in `entity_results`. - EntityResult.ResultType entity_result_type = 1; - - // The results for this batch. - repeated EntityResult entity_results = 2; - - // A cursor that points to the position after the last result in the batch. - bytes end_cursor = 4; - - // The state of the query after the current batch. - MoreResultsType more_results = 5; - - // The version number of the snapshot this batch was returned from. - // This applies to the range of results from the query's `start_cursor` (or - // the beginning of the query if no cursor was given) to this batch's - // `end_cursor` (not the query's `end_cursor`). - // - // In a single transaction, subsequent query result batches for the same query - // can have a greater snapshot version number. Each batch's snapshot version - // is valid for all preceding batches. - // The value will be zero for eventually consistent queries. - int64 snapshot_version = 7; -} From 0e5b718a70368f656ede3a27174ef74ca324ab65 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 25 May 2021 09:58:10 -0700 Subject: [PATCH 5/6] fix(perf): improve performance unmarshalling entities from protobuf2 (#175) Uses direct access of pb2 object when iterating over Entity.properties. This reduces the time required to hydrate query results by 50% --- google/cloud/datastore/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/datastore/helpers.py b/google/cloud/datastore/helpers.py index c1d022e3..5627d8a3 100644 --- a/google/cloud/datastore/helpers.py +++ b/google/cloud/datastore/helpers.py @@ -138,7 +138,7 @@ def entity_from_protobuf(pb): entity_meanings = {} exclude_from_indexes = [] - for prop_name, value_pb in _property_tuples(proto_pb): + for prop_name, value_pb in _property_tuples(proto_pb._pb): value = _get_value_from_value_pb(value_pb) entity_props[prop_name] = value @@ -154,7 +154,7 @@ def entity_from_protobuf(pb): if is_list and len(value) > 0: exclude_values = set( value_pb.exclude_from_indexes - for value_pb in value_pb._pb.array_value.values + for value_pb in value_pb.array_value.values ) if len(exclude_values) != 1: raise ValueError( From 4f7bee21c2505b9e6b133546e79ead0b6aa4286d Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 25 May 2021 17:08:10 +0000 Subject: [PATCH 6/6] chore: release 2.1.3 (#176) :robot: I have created a release \*beep\* \*boop\* --- ### [2.1.3](https://www.github.com/googleapis/python-datastore/compare/v2.1.2...v2.1.3) (2021-05-25) ### Bug Fixes * **perf:** improve performance unmarshalling entities from protobuf2 ([#175](https://www.github.com/googleapis/python-datastore/issues/175)) ([0e5b718](https://www.github.com/googleapis/python-datastore/commit/0e5b718a70368f656ede3a27174ef74ca324ab65)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 7 +++++++ google/cloud/datastore/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df56b8e5..880add6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-datastore/#history +### [2.1.3](https://www.github.com/googleapis/python-datastore/compare/v2.1.2...v2.1.3) (2021-05-25) + + +### Bug Fixes + +* **perf:** improve performance unmarshalling entities from protobuf2 ([#175](https://www.github.com/googleapis/python-datastore/issues/175)) ([0e5b718](https://www.github.com/googleapis/python-datastore/commit/0e5b718a70368f656ede3a27174ef74ca324ab65)) + ### [2.1.2](https://www.github.com/googleapis/python-datastore/compare/v2.1.1...v2.1.2) (2021-05-03) diff --git a/google/cloud/datastore/version.py b/google/cloud/datastore/version.py index b6c2aa1a..28ad2844 100644 --- a/google/cloud/datastore/version.py +++ b/google/cloud/datastore/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.1.2" +__version__ = "2.1.3"