From 06a127b42677c91e8d5297253fd973e35b4823cc Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 10:21:25 -0400 Subject: [PATCH 1/3] feat: Max concurrent backfill tasks (#168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Max concurrent backfill tasks You can now set the number of maximum concurrent backfill tasks for a stream using the Datastream API. PiperOrigin-RevId: 530067890 Source-Link: https://github.com/googleapis/googleapis/commit/b2c290faf99f388efb1e080333cba620f38e5afc Source-Link: https://github.com/googleapis/googleapis-gen/commit/83c5413e535cb6fa710541dc6e169a8a44b8a65d Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiODNjNTQxM2U1MzVjYjZmYTcxMDU0MWRjNmUxNjlhOGE0NGI4YTY1ZCJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- .../cloud/datastream_v1/types/datastream.py | 7 +++ .../types/datastream_resources.py | 55 +++++++++++++++---- ...t_metadata_google.cloud.datastream.v1.json | 2 +- ...data_google.cloud.datastream.v1alpha1.json | 2 +- scripts/fixup_datastream_v1_keywords.py | 2 +- .../gapic/datastream_v1/test_datastream.py | 14 +++++ 6 files changed, 69 insertions(+), 13 deletions(-) diff --git a/google/cloud/datastream_v1/types/datastream.py b/google/cloud/datastream_v1/types/datastream.py index 59890f4..fac92d0 100644 --- a/google/cloud/datastream_v1/types/datastream.py +++ b/google/cloud/datastream_v1/types/datastream.py @@ -1056,6 +1056,9 @@ class CreatePrivateConnectionRequest(proto.Message): The request ID must be a valid UUID with the exception that zero UUID is not supported (00000000-0000-0000-0000-000000000000). + force (bool): + Optional. If set to true, will skip + validations. """ parent: str = proto.Field( @@ -1075,6 +1078,10 @@ class CreatePrivateConnectionRequest(proto.Message): proto.STRING, number=4, ) + force: bool = proto.Field( + proto.BOOL, + number=6, + ) class ListPrivateConnectionsRequest(proto.Message): diff --git a/google/cloud/datastream_v1/types/datastream_resources.py b/google/cloud/datastream_v1/types/datastream_resources.py index a49ab02..ff62d8e 100644 --- a/google/cloud/datastream_v1/types/datastream_resources.py +++ b/google/cloud/datastream_v1/types/datastream_resources.py @@ -226,7 +226,12 @@ class BigQueryProfile(proto.Message): class StaticServiceIpConnectivity(proto.Message): - r"""Static IP address connectivity.""" + r"""Static IP address connectivity. Used when the source database + is configured to allow incoming connections from the Datastream + public IP addresses for the region specified in the connection + profile. + + """ class ForwardSshTunnelConnectivity(proto.Message): @@ -792,15 +797,20 @@ class OracleSourceConfig(proto.Message): Oracle objects to exclude from the stream. max_concurrent_cdc_tasks (int): Maximum number of concurrent CDC tasks. The - number should be non negative. If not set (or - set to 0), the system's default value will be + number should be non-negative. If not set (or + set to 0), the system's default value is used. + max_concurrent_backfill_tasks (int): + Maximum number of concurrent backfill tasks. + The number should be non-negative. If not set + (or set to 0), the system's default value is used. drop_large_objects (google.cloud.datastream_v1.types.OracleSourceConfig.DropLargeObjects): Drop large object values. This field is a member of `oneof`_ ``large_objects_handling``. stream_large_objects (google.cloud.datastream_v1.types.OracleSourceConfig.StreamLargeObjects): - Stream large object values. + Stream large object values. NOTE: This + feature is currently experimental. This field is a member of `oneof`_ ``large_objects_handling``. """ @@ -825,6 +835,10 @@ class StreamLargeObjects(proto.Message): proto.INT32, number=3, ) + max_concurrent_backfill_tasks: int = proto.Field( + proto.INT32, + number=4, + ) drop_large_objects: DropLargeObjects = proto.Field( proto.MESSAGE, number=100, @@ -967,12 +981,18 @@ class PostgresqlSourceConfig(proto.Message): PostgreSQL objects to exclude from the stream. replication_slot (str): - Required. The name of the logical replication - slot that's configured with the pgoutput plugin. + Required. Immutable. The name of the logical + replication slot that's configured with the + pgoutput plugin. publication (str): Required. The name of the publication that includes the set of all tables that are defined in the stream's include_objects. + max_concurrent_backfill_tasks (int): + Maximum number of concurrent backfill tasks. + The number should be non negative. If not set + (or set to 0), the system's default value will + be used. """ include_objects: "PostgresqlRdbms" = proto.Field( @@ -993,6 +1013,10 @@ class PostgresqlSourceConfig(proto.Message): proto.STRING, number=4, ) + max_concurrent_backfill_tasks: int = proto.Field( + proto.INT32, + number=5, + ) class MysqlColumn(proto.Message): @@ -1122,6 +1146,11 @@ class MysqlSourceConfig(proto.Message): number should be non negative. If not set (or set to 0), the system's default value will be used. + max_concurrent_backfill_tasks (int): + Maximum number of concurrent backfill tasks. + The number should be non negative. If not set + (or set to 0), the system's default value will + be used. """ include_objects: "MysqlRdbms" = proto.Field( @@ -1138,6 +1167,10 @@ class MysqlSourceConfig(proto.Message): proto.INT32, number=3, ) + max_concurrent_backfill_tasks: int = proto.Field( + proto.INT32, + number=4, + ) class SourceConfig(proto.Message): @@ -1268,7 +1301,8 @@ class GcsDestinationConfig(proto.Message): file_rotation_interval (google.protobuf.duration_pb2.Duration): The maximum duration for which new events are added before a file is closed and a new file is - created. + created. Values within the range of 15-60 + seconds are allowed. avro_file_format (google.cloud.datastream_v1.types.AvroFileFormat): AVRO file format configuration. @@ -1307,7 +1341,7 @@ class GcsDestinationConfig(proto.Message): class BigQueryDestinationConfig(proto.Message): - r""" + r"""BigQuery destination configuration This message has `oneof`_ fields (mutually exclusive fields). For each oneof, at most one member field can be set at the same time. @@ -1340,7 +1374,7 @@ class SingleTargetDataset(proto.Message): Attributes: dataset_id (str): - + The dataset ID of the target dataset. """ dataset_id: str = proto.Field( @@ -1354,7 +1388,8 @@ class SourceHierarchyDatasets(proto.Message): Attributes: dataset_template (google.cloud.datastream_v1.types.BigQueryDestinationConfig.SourceHierarchyDatasets.DatasetTemplate): - + The dataset template to use for dynamic + dataset creation. """ class DatasetTemplate(proto.Message): diff --git a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json index dd58d51..8a69b90 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-datastream", - "version": "1.5.1" + "version": "0.1.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json index afccdda..09dea00 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-datastream", - "version": "1.5.1" + "version": "0.1.0" }, "snippets": [ { diff --git a/scripts/fixup_datastream_v1_keywords.py b/scripts/fixup_datastream_v1_keywords.py index f390fcd..cf28155 100644 --- a/scripts/fixup_datastream_v1_keywords.py +++ b/scripts/fixup_datastream_v1_keywords.py @@ -40,7 +40,7 @@ class datastreamCallTransformer(cst.CSTTransformer): CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { 'create_connection_profile': ('parent', 'connection_profile_id', 'connection_profile', 'request_id', 'validate_only', 'force', ), - 'create_private_connection': ('parent', 'private_connection_id', 'private_connection', 'request_id', ), + 'create_private_connection': ('parent', 'private_connection_id', 'private_connection', 'request_id', 'force', ), 'create_route': ('parent', 'route_id', 'route', 'request_id', ), 'create_stream': ('parent', 'stream_id', 'stream', 'request_id', 'validate_only', 'force', ), 'delete_connection_profile': ('name', 'request_id', ), diff --git a/tests/unit/gapic/datastream_v1/test_datastream.py b/tests/unit/gapic/datastream_v1/test_datastream.py index 66b7935..acdaec2 100644 --- a/tests/unit/gapic/datastream_v1/test_datastream.py +++ b/tests/unit/gapic/datastream_v1/test_datastream.py @@ -10421,6 +10421,7 @@ def test_create_stream_rest(request_type): }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, "drop_large_objects": {}, "stream_large_objects": {}, }, @@ -10450,6 +10451,7 @@ def test_create_stream_rest(request_type): }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, }, "postgresql_source_config": { "include_objects": { @@ -10479,6 +10481,7 @@ def test_create_stream_rest(request_type): "exclude_objects": {}, "replication_slot": "replication_slot_value", "publication": "publication_value", + "max_concurrent_backfill_tasks": 3076, }, }, "destination_config": { @@ -10766,6 +10769,7 @@ def test_create_stream_rest_bad_request( }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, "drop_large_objects": {}, "stream_large_objects": {}, }, @@ -10795,6 +10799,7 @@ def test_create_stream_rest_bad_request( }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, }, "postgresql_source_config": { "include_objects": { @@ -10824,6 +10829,7 @@ def test_create_stream_rest_bad_request( "exclude_objects": {}, "replication_slot": "replication_slot_value", "publication": "publication_value", + "max_concurrent_backfill_tasks": 3076, }, }, "destination_config": { @@ -10996,6 +11002,7 @@ def test_update_stream_rest(request_type): }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, "drop_large_objects": {}, "stream_large_objects": {}, }, @@ -11025,6 +11032,7 @@ def test_update_stream_rest(request_type): }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, }, "postgresql_source_config": { "include_objects": { @@ -11054,6 +11062,7 @@ def test_update_stream_rest(request_type): "exclude_objects": {}, "replication_slot": "replication_slot_value", "publication": "publication_value", + "max_concurrent_backfill_tasks": 3076, }, }, "destination_config": { @@ -11319,6 +11328,7 @@ def test_update_stream_rest_bad_request( }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, "drop_large_objects": {}, "stream_large_objects": {}, }, @@ -11348,6 +11358,7 @@ def test_update_stream_rest_bad_request( }, "exclude_objects": {}, "max_concurrent_cdc_tasks": 2550, + "max_concurrent_backfill_tasks": 3076, }, "postgresql_source_config": { "include_objects": { @@ -11377,6 +11388,7 @@ def test_update_stream_rest_bad_request( "exclude_objects": {}, "replication_slot": "replication_slot_value", "publication": "publication_value", + "max_concurrent_backfill_tasks": 3076, }, }, "destination_config": { @@ -13579,6 +13591,7 @@ def test_create_private_connection_rest_required_fields( # Check that path parameters and body parameters are not mixing in. assert not set(unset_fields) - set( ( + "force", "private_connection_id", "request_id", ) @@ -13645,6 +13658,7 @@ def test_create_private_connection_rest_unset_required_fields(): assert set(unset_fields) == ( set( ( + "force", "privateConnectionId", "requestId", ) From 37cf12b7b06c89aa9948a6ef24d412a75b96fbc0 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 16:22:33 +0000 Subject: [PATCH 2/3] build(deps): bump requests to 2.31.0 [autoapprove] (#170) Source-Link: https://togithub.com/googleapis/synthtool/commit/30bd01b4ab78bf1b2a425816e15b3e7e090993dd Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:9bc5fa3b62b091f60614c08a7fb4fd1d3e1678e326f34dd66ce1eefb5dc3267b --- .github/.OwlBot.lock.yaml | 3 ++- .kokoro/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b8edda5..32b3c48 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2e247c7bf5154df7f98cce087a20ca7605e236340c7d6d1a14447e5c06791bd6 + digest: sha256:9bc5fa3b62b091f60614c08a7fb4fd1d3e1678e326f34dd66ce1eefb5dc3267b +# created: 2023-05-25T14:56:16.294623272Z diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 66a2172..3b8d7ee 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -419,9 +419,9 @@ readme-renderer==37.3 \ --hash=sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273 \ --hash=sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343 # via twine -requests==2.28.1 \ - --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ - --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 # via # gcp-releasetool # google-api-core From d0a5307aa98eb51f26d3315ed13a4607281f3ac8 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 13:56:50 -0400 Subject: [PATCH 3/3] chore(main): release 1.6.0 (#169) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 7 +++++++ google/cloud/datastream/gapic_version.py | 2 +- google/cloud/datastream_v1/gapic_version.py | 2 +- google/cloud/datastream_v1alpha1/gapic_version.py | 2 +- .../snippet_metadata_google.cloud.datastream.v1.json | 2 +- .../snippet_metadata_google.cloud.datastream.v1alpha1.json | 2 +- 7 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index e20d7e8..0d1bebe 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.5.1" + ".": "1.6.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 76e8258..bdd8392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.6.0](https://github.com/googleapis/python-datastream/compare/v1.5.1...v1.6.0) (2023-05-25) + + +### Features + +* Max concurrent backfill tasks ([#168](https://github.com/googleapis/python-datastream/issues/168)) ([06a127b](https://github.com/googleapis/python-datastream/commit/06a127b42677c91e8d5297253fd973e35b4823cc)) + ## [1.5.1](https://github.com/googleapis/python-datastream/compare/v1.5.0...v1.5.1) (2023-03-23) diff --git a/google/cloud/datastream/gapic_version.py b/google/cloud/datastream/gapic_version.py index 69ff013..a016bdf 100644 --- a/google/cloud/datastream/gapic_version.py +++ b/google/cloud/datastream/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "1.5.1" # {x-release-please-version} +__version__ = "1.6.0" # {x-release-please-version} diff --git a/google/cloud/datastream_v1/gapic_version.py b/google/cloud/datastream_v1/gapic_version.py index 69ff013..a016bdf 100644 --- a/google/cloud/datastream_v1/gapic_version.py +++ b/google/cloud/datastream_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "1.5.1" # {x-release-please-version} +__version__ = "1.6.0" # {x-release-please-version} diff --git a/google/cloud/datastream_v1alpha1/gapic_version.py b/google/cloud/datastream_v1alpha1/gapic_version.py index 69ff013..a016bdf 100644 --- a/google/cloud/datastream_v1alpha1/gapic_version.py +++ b/google/cloud/datastream_v1alpha1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "1.5.1" # {x-release-please-version} +__version__ = "1.6.0" # {x-release-please-version} diff --git a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json index 8a69b90..57a3985 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-datastream", - "version": "0.1.0" + "version": "1.6.0" }, "snippets": [ { diff --git a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json index 09dea00..3c94b61 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-datastream", - "version": "0.1.0" + "version": "1.6.0" }, "snippets": [ {