diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 6ffef5f..3815c98 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:a31c2fd2f28e994254f6367b44e112722f3263247798ae222d7986bb12328ec3 + digest: sha256:7a40313731a7cb1454eef6b33d3446ebb121836738dc3ab3d2d3ded5268c35b6 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 385f2d4..d15994b 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -325,31 +325,30 @@ platformdirs==2.5.2 \ --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 # via virtualenv -protobuf==3.20.1 \ - --hash=sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf \ - --hash=sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f \ - --hash=sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f \ - --hash=sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7 \ - --hash=sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996 \ - --hash=sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067 \ - --hash=sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c \ - --hash=sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7 \ - --hash=sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9 \ - --hash=sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c \ - --hash=sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739 \ - --hash=sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91 \ - --hash=sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c \ - --hash=sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153 \ - --hash=sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9 \ - --hash=sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388 \ - --hash=sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e \ - --hash=sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab \ - --hash=sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde \ - --hash=sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531 \ - --hash=sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8 \ - --hash=sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7 \ - --hash=sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20 \ - --hash=sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3 +protobuf==3.20.2 \ + --hash=sha256:03d76b7bd42ac4a6e109742a4edf81ffe26ffd87c5993126d894fe48a120396a \ + --hash=sha256:09e25909c4297d71d97612f04f41cea8fa8510096864f2835ad2f3b3df5a5559 \ + --hash=sha256:18e34a10ae10d458b027d7638a599c964b030c1739ebd035a1dfc0e22baa3bfe \ + --hash=sha256:291fb4307094bf5ccc29f424b42268640e00d5240bf0d9b86bf3079f7576474d \ + --hash=sha256:2c0b040d0b5d5d207936ca2d02f00f765906622c07d3fa19c23a16a8ca71873f \ + --hash=sha256:384164994727f274cc34b8abd41a9e7e0562801361ee77437099ff6dfedd024b \ + --hash=sha256:3cb608e5a0eb61b8e00fe641d9f0282cd0eedb603be372f91f163cbfbca0ded0 \ + --hash=sha256:5d9402bf27d11e37801d1743eada54372f986a372ec9679673bfcc5c60441151 \ + --hash=sha256:712dca319eee507a1e7df3591e639a2b112a2f4a62d40fe7832a16fd19151750 \ + --hash=sha256:7a5037af4e76c975b88c3becdf53922b5ffa3f2cddf657574a4920a3b33b80f3 \ + --hash=sha256:8228e56a865c27163d5d1d1771d94b98194aa6917bcfb6ce139cbfa8e3c27334 \ + --hash=sha256:84a1544252a933ef07bb0b5ef13afe7c36232a774affa673fc3636f7cee1db6c \ + --hash=sha256:84fe5953b18a383fd4495d375fe16e1e55e0a3afe7b4f7b4d01a3a0649fcda9d \ + --hash=sha256:9c673c8bfdf52f903081816b9e0e612186684f4eb4c17eeb729133022d6032e3 \ + --hash=sha256:9f876a69ca55aed879b43c295a328970306e8e80a263ec91cf6e9189243c613b \ + --hash=sha256:a9e5ae5a8e8985c67e8944c23035a0dff2c26b0f5070b2f55b217a1c33bbe8b1 \ + --hash=sha256:b4fdb29c5a7406e3f7ef176b2a7079baa68b5b854f364c21abe327bbeec01cdb \ + --hash=sha256:c184485e0dfba4dfd451c3bd348c2e685d6523543a0f91b9fd4ae90eb09e8422 \ + --hash=sha256:c9cdf251c582c16fd6a9f5e95836c90828d51b0069ad22f463761d27c6c19019 \ + --hash=sha256:e39cf61bb8582bda88cdfebc0db163b774e7e03364bbf9ce1ead13863e81e359 \ + --hash=sha256:e8fbc522303e09036c752a0afcc5c0603e917222d8bedc02813fd73b4b4ed804 \ + --hash=sha256:f34464ab1207114e73bba0794d1257c150a2b89b7a9faf504e00af7c9fd58978 \ + --hash=sha256:f52dabc96ca99ebd2169dadbe018824ebda08a795c7684a0b7d203a290f3adb0 # via # gcp-docuploader # gcp-releasetool diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c22a4..dfd7167 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.3.1](https://github.com/googleapis/python-batch/compare/v0.3.0...v0.3.1) (2022-10-03) + + +### Bug Fixes + +* **deps:** Require protobuf >= 3.20.2 ([#47](https://github.com/googleapis/python-batch/issues/47)) ([aa83e55](https://github.com/googleapis/python-batch/commit/aa83e556112d4649a7de59a91ae942830dde4688)) + + +### Documentation + +* **samples:** Adding sample for bucket mounting ([#43](https://github.com/googleapis/python-batch/issues/43)) ([af33ed9](https://github.com/googleapis/python-batch/commit/af33ed9ab12e7d72a21dab4f4fefb5f5104d0595)) +* **samples:** Adding samples for list and get tasks ([#50](https://github.com/googleapis/python-batch/issues/50)) ([9401da1](https://github.com/googleapis/python-batch/commit/9401da162fcde57dcbf9aff97f289e2cffb3dc9f)) +* **samples:** Adding samples for template usage ([#41](https://github.com/googleapis/python-batch/issues/41)) ([7376708](https://github.com/googleapis/python-batch/commit/73767084e1f63e68cb1ade22f390ef208017a6ac)) + ## [0.3.0](https://github.com/googleapis/python-batch/compare/v0.2.0...v0.3.0) (2022-09-16) diff --git a/samples/requirements-test.txt b/samples/requirements-test.txt index 75ceb43..90982db 100644 --- a/samples/requirements-test.txt +++ b/samples/requirements-test.txt @@ -1,2 +1,4 @@ pytest==7.1.3 -pytest-parallel==0.1.1 +google-cloud-compute==1.5.2 +google-cloud-resource-manager==1.6.1 +google-cloud-storage==2.5.0 \ No newline at end of file diff --git a/samples/requirements.txt b/samples/requirements.txt index e88c9b0..0deda5c 100644 --- a/samples/requirements.txt +++ b/samples/requirements.txt @@ -1,3 +1,3 @@ isort==5.10.1 black==22.8.0 -google-cloud-batch==0.2.0 +google-cloud-batch==0.3.0 diff --git a/samples/snippets/create/create_with_container_no_mounting.py b/samples/snippets/create/create_with_container_no_mounting.py index 9c54318..fd7f655 100644 --- a/samples/snippets/create/create_with_container_no_mounting.py +++ b/samples/snippets/create/create_with_container_no_mounting.py @@ -54,6 +54,7 @@ def create_container_job(project_id: str, region: str, job_name: str) -> batch_v task.max_run_duration = "3600s" # Tasks are grouped inside a job using TaskGroups. + # Currently, it's possible to have only one task group. group = batch_v1.TaskGroup() group.task_count = 4 group.task_spec = task diff --git a/samples/snippets/create/create_with_mounted_bucket.py b/samples/snippets/create/create_with_mounted_bucket.py new file mode 100644 index 0000000..d64821c --- /dev/null +++ b/samples/snippets/create/create_with_mounted_bucket.py @@ -0,0 +1,91 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START batch_create_script_job_with_bucket] +from google.cloud import batch_v1 + + +def create_script_job_with_bucket(project_id: str, region: str, job_name: str, bucket_name: str) -> batch_v1.Job: + """ + This method shows how to create a sample Batch Job that will run + a simple command on Cloud Compute instances. + + Args: + project_id: project ID or project number of the Cloud project you want to use. + region: name of the region you want to use to run the job. Regions that are + available for Batch are listed on: https://cloud.google.com/batch/docs/get-started#locations + job_name: the name of the job that will be created. + It needs to be unique for each project and region pair. + bucket_name: name of the bucket to be mounted for your Job. + + Returns: + A job object representing the job created. + """ + client = batch_v1.BatchServiceClient() + + # Define what will be done as part of the job. + task = batch_v1.TaskSpec() + runnable = batch_v1.Runnable() + runnable.script = batch_v1.Runnable.Script() + runnable.script.text = "echo Hello world from task ${BATCH_TASK_INDEX}. >> /mnt/share/output_task_${BATCH_TASK_INDEX}.txt" + task.runnables = [runnable] + + gcs_bucket = batch_v1.GCS() + gcs_bucket.remote_path = bucket_name + gcs_volume = batch_v1.Volume() + gcs_volume.gcs = gcs_bucket + gcs_volume.mount_path = '/mnt/share' + task.volumes = [gcs_volume] + + # We can specify what resources are requested by each task. + resources = batch_v1.ComputeResource() + resources.cpu_milli = 500 # in milliseconds per cpu-second. This means the task requires 50% of a single CPUs. + resources.memory_mib = 16 + task.compute_resource = resources + + task.max_retry_count = 2 + task.max_run_duration = "3600s" + + # Tasks are grouped inside a job using TaskGroups. + # Currently, it's possible to have only one task group. + group = batch_v1.TaskGroup() + group.task_count = 4 + group.task_spec = task + + # Policies are used to define on what kind of virtual machines the tasks will run on. + # In this case, we tell the system to use "e2-standard-4" machine type. + # Read more about machine types here: https://cloud.google.com/compute/docs/machine-types + allocation_policy = batch_v1.AllocationPolicy() + policy = batch_v1.AllocationPolicy.InstancePolicy() + policy.machine_type = "e2-standard-4" + instances = batch_v1.AllocationPolicy.InstancePolicyOrTemplate() + instances.policy = policy + allocation_policy.instances = [instances] + + job = batch_v1.Job() + job.task_groups = [group] + job.allocation_policy = allocation_policy + job.labels = {"env": "testing", "type": "script", "mount": "bucket"} + # We use Cloud Logging as it's an out of the box available option + job.logs_policy = batch_v1.LogsPolicy() + job.logs_policy.destination = batch_v1.LogsPolicy.Destination.CLOUD_LOGGING + + create_request = batch_v1.CreateJobRequest() + create_request.job = job + create_request.job_id = job_name + # The job's parent is the region in which the job will run + create_request.parent = f"projects/{project_id}/locations/{region}" + + return client.create_job(create_request) +# [END batch_create_script_job_with_bucket] diff --git a/samples/snippets/create/create_with_script_no_mounting.py b/samples/snippets/create/create_with_script_no_mounting.py index 29942cb..87a1dfe 100644 --- a/samples/snippets/create/create_with_script_no_mounting.py +++ b/samples/snippets/create/create_with_script_no_mounting.py @@ -54,6 +54,7 @@ def create_script_job(project_id: str, region: str, job_name: str) -> batch_v1.J task.max_run_duration = "3600s" # Tasks are grouped inside a job using TaskGroups. + # Currently, it's possible to have only one task group. group = batch_v1.TaskGroup() group.task_count = 4 group.task_spec = task diff --git a/samples/snippets/create/create_with_template.py b/samples/snippets/create/create_with_template.py new file mode 100644 index 0000000..c37a94f --- /dev/null +++ b/samples/snippets/create/create_with_template.py @@ -0,0 +1,88 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START batch_create_job_with_template] +from google.cloud import batch_v1 + + +def create_script_job_with_template(project_id: str, region: str, job_name: str, template_link: str) -> batch_v1.Job: + """ + This method shows how to create a sample Batch Job that will run + a simple command on Cloud Compute instances created using a provided Template. + + Args: + project_id: project ID or project number of the Cloud project you want to use. + region: name of the region you want to use to run the job. Regions that are + available for Batch are listed on: https://cloud.google.com/batch/docs/get-started#locations + job_name: the name of the job that will be created. + It needs to be unique for each project and region pair. + template_link: a link to an existing Instance Template. Acceptable formats: + * "projects/{project_id}/global/instanceTemplates/{template_name}" + * "{template_name}" - if the template is defined in the same project as used to create the Job. + + Returns: + A job object representing the job created. + """ + client = batch_v1.BatchServiceClient() + + # Define what will be done as part of the job. + task = batch_v1.TaskSpec() + runnable = batch_v1.Runnable() + runnable.script = batch_v1.Runnable.Script() + runnable.script.text = "echo Hello world! This is task ${BATCH_TASK_INDEX}. This job has a total of ${BATCH_TASK_COUNT} tasks." + # You can also run a script from a file. Just remember, that needs to be a script that's + # already on the VM that will be running the job. Using runnable.script.text and runnable.script.path is mutually + # exclusive. + # runnable.script.path = '/tmp/test.sh' + task.runnables = [runnable] + + # We can specify what resources are requested by each task. + resources = batch_v1.ComputeResource() + resources.cpu_milli = 2000 # in milliseconds per cpu-second. This means the task requires 2 whole CPUs. + resources.memory_mib = 16 + task.compute_resource = resources + + task.max_retry_count = 2 + task.max_run_duration = "3600s" + + # Tasks are grouped inside a job using TaskGroups. + # Currently, it's possible to have only one task group. + group = batch_v1.TaskGroup() + group.task_count = 4 + group.task_spec = task + + # Policies are used to define on what kind of virtual machines the tasks will run on. + # In this case, we tell the system to use an instance template that defines all the + # required parameters. + allocation_policy = batch_v1.AllocationPolicy() + instances = batch_v1.AllocationPolicy.InstancePolicyOrTemplate() + instances.instance_template = template_link + allocation_policy.instances = [instances] + + job = batch_v1.Job() + job.task_groups = [group] + job.allocation_policy = allocation_policy + job.labels = {"env": "testing", "type": "script"} + # We use Cloud Logging as it's an out of the box available option + job.logs_policy = batch_v1.LogsPolicy() + job.logs_policy.destination = batch_v1.LogsPolicy.Destination.CLOUD_LOGGING + + create_request = batch_v1.CreateJobRequest() + create_request.job = job + create_request.job_id = job_name + # The job's parent is the region in which the job will run + create_request.parent = f"projects/{project_id}/locations/{region}" + + return client.create_job(create_request) +# [END batch_create_job_with_template] diff --git a/samples/snippets/get/get_task.py b/samples/snippets/get/get_task.py new file mode 100644 index 0000000..ea5fc5c --- /dev/null +++ b/samples/snippets/get/get_task.py @@ -0,0 +1,38 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START batch_get_task] + +from google.cloud import batch_v1 + + +def get_task(project_id: str, region: str, job_name: str, group_name: str, task_number: int) -> batch_v1.Task: + """ + Retrieve information about a Task. + + Args: + project_id: project ID or project number of the Cloud project you want to use. + region: name of the region hosts the job. + job_name: the name of the job you want to retrieve information about. + group_name: the name of the group that owns the task you want to check. Usually it's `group0`. + task_number: number of the task you want to look up. + + Returns: + A Task object representing the specified task. + """ + client = batch_v1.BatchServiceClient() + + return client.get_task(name=f"projects/{project_id}/locations/{region}/jobs/{job_name}" + f"/taskGroups/{group_name}/tasks/{task_number}") +# [END batch_get_task] diff --git a/samples/snippets/list/list_jobs.py b/samples/snippets/list/list_jobs.py index 77c39b5..e52f4de 100644 --- a/samples/snippets/list/list_jobs.py +++ b/samples/snippets/list/list_jobs.py @@ -11,9 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Iterable # [START batch_list_jobs] +from typing import Iterable + from google.cloud import batch_v1 diff --git a/samples/snippets/list/list_tasks.py b/samples/snippets/list/list_tasks.py new file mode 100644 index 0000000..9ef6674 --- /dev/null +++ b/samples/snippets/list/list_tasks.py @@ -0,0 +1,37 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START batch_list_tasks] +from typing import Iterable + +from google.cloud import batch_v1 + + +def list_tasks(project_id: str, region: str, job_name: str, group_name: str) -> Iterable[batch_v1.Task]: + """ + Get a list of all jobs defined in given region. + + Args: + project_id: project ID or project number of the Cloud project you want to use. + region: name of the region hosting the jobs. + job_name: name of the job which tasks you want to list. + group_name: name of the group of tasks. Usually it's `group0`. + + Returns: + An iterable collection of Task objects. + """ + client = batch_v1.BatchServiceClient() + + return client.list_tasks(parent=f"projects/{project_id}/locations/{region}/jobs/{job_name}/taskGroups/{group_name}") +# [END batch_list_tasks] diff --git a/samples/snippets/tests/test_basics.py b/samples/snippets/tests/test_basics.py index 88dfa75..980314d 100644 --- a/samples/snippets/tests/test_basics.py +++ b/samples/snippets/tests/test_basics.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import time +from typing import Callable import uuid - import google.auth from google.cloud import batch_v1 import pytest @@ -24,7 +24,9 @@ from ..delete.delete_job import delete_job from ..get.get_job import get_job +from ..get.get_task import get_task from ..list.list_jobs import list_jobs +from ..list.list_tasks import list_tasks PROJECT = google.auth.default()[1] REGION = 'europe-north1' @@ -36,6 +38,7 @@ batch_v1.JobStatus.State.QUEUED, batch_v1.JobStatus.State.RUNNING, batch_v1.JobStatus.State.SCHEDULED, + batch_v1.JobStatus.State.DELETION_IN_PROGRESS } @@ -44,7 +47,7 @@ def job_name(): return f"test-job-{uuid.uuid4().hex[:10]}" -def _test_body(test_job: batch_v1.Job): +def _test_body(test_job: batch_v1.Job, additional_test: Callable = None): start_time = time.time() try: while test_job.status.state in WAIT_STATES: @@ -53,14 +56,16 @@ def _test_body(test_job: batch_v1.Job): test_job = get_job(PROJECT, REGION, test_job.name.rsplit('/', maxsplit=1)[1]) time.sleep(5) - assert test_job.status.state in (batch_v1.JobStatus.State.SUCCEEDED, - batch_v1.JobStatus.State.DELETION_IN_PROGRESS) + assert test_job.status.state == batch_v1.JobStatus.State.SUCCEEDED for job in list_jobs(PROJECT, REGION): if test_job.uid == job.uid: break else: pytest.fail(f"Couldn't find job {test_job.uid} on the list of jobs.") + + if additional_test: + additional_test() finally: delete_job(PROJECT, REGION, test_job.name.rsplit('/', maxsplit=1)[1]).result() @@ -69,9 +74,17 @@ def _test_body(test_job: batch_v1.Job): pytest.fail("The test job should be deleted at this point!") +def _check_tasks(job_name): + tasks = list_tasks(PROJECT, REGION, job_name, 'group0') + assert len(list(tasks)) == 4 + for i in range(4): + assert get_task(PROJECT, REGION, job_name, 'group0', i) is not None + print('Tasks tested') + + def test_script_job(job_name): job = create_script_job(PROJECT, REGION, job_name) - _test_body(job) + _test_body(job, additional_test=lambda: _check_tasks(job_name)) def test_container_job(job_name): diff --git a/samples/snippets/tests/test_bucket.py b/samples/snippets/tests/test_bucket.py new file mode 100644 index 0000000..ad8a347 --- /dev/null +++ b/samples/snippets/tests/test_bucket.py @@ -0,0 +1,70 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import uuid + + +import google.auth +from google.cloud import batch_v1 +from google.cloud import storage +import pytest + +from .test_basics import _test_body +from ..create.create_with_mounted_bucket import create_script_job_with_bucket + +PROJECT = google.auth.default()[1] +REGION = 'europe-north1' + +TIMEOUT = 600 # 10 minutes + +WAIT_STATES = { + batch_v1.JobStatus.State.STATE_UNSPECIFIED, + batch_v1.JobStatus.State.QUEUED, + batch_v1.JobStatus.State.RUNNING, + batch_v1.JobStatus.State.SCHEDULED, +} + + +@pytest.fixture +def job_name(): + return f"test-job-{uuid.uuid4().hex[:10]}" + + +@pytest.fixture() +def test_bucket(): + bucket_name = f"test-bucket-{uuid.uuid4().hex[:8]}" + client = storage.Client() + client.create_bucket(bucket_name, location="eu") + + yield bucket_name + + bucket = client.get_bucket(bucket_name) + bucket.delete(force=True) + + +def _test_bucket_content(test_bucket): + client = storage.Client() + bucket = client.get_bucket(test_bucket) + + file_name_template = "output_task_{task_number}.txt" + file_content_template = "Hello world from task {task_number}.\n" + + for i in range(4): + blob = bucket.blob(file_name_template.format(task_number=i)) + content = blob.download_as_bytes().decode() + assert content == file_content_template.format(task_number=i) + + +def test_bucket_job(job_name, test_bucket): + job = create_script_job_with_bucket(PROJECT, REGION, job_name, test_bucket) + _test_body(job, lambda: _test_bucket_content(test_bucket)) diff --git a/samples/snippets/tests/test_template.py b/samples/snippets/tests/test_template.py new file mode 100644 index 0000000..5728117 --- /dev/null +++ b/samples/snippets/tests/test_template.py @@ -0,0 +1,114 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import google.auth +from google.cloud import batch_v1 +from google.cloud import compute_v1 +from google.cloud import resourcemanager_v3 +import pytest + + +from .test_basics import _test_body + +from ..create.create_with_template import create_script_job_with_template + +PROJECT = google.auth.default()[1] + +PROJECT_NUMBER = resourcemanager_v3.ProjectsClient().get_project(name=f"projects/{PROJECT}").name.split("/")[1] + +REGION = 'europe-north1' + +TIMEOUT = 600 # 10 minutes + +WAIT_STATES = { + batch_v1.JobStatus.State.STATE_UNSPECIFIED, + batch_v1.JobStatus.State.QUEUED, + batch_v1.JobStatus.State.RUNNING, + batch_v1.JobStatus.State.SCHEDULED, +} + + +@pytest.fixture +def job_name(): + return f"test-job-{uuid.uuid4().hex[:10]}" + + +@pytest.fixture +def instance_template(): + disk = compute_v1.AttachedDisk() + initialize_params = compute_v1.AttachedDiskInitializeParams() + initialize_params.source_image = ( + "projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts" + ) + initialize_params.disk_size_gb = 25 + initialize_params.disk_type = 'pd-balanced' + disk.initialize_params = initialize_params + disk.auto_delete = True + disk.boot = True + + network_interface = compute_v1.NetworkInterface() + network_interface.name = "global/networks/default" + + access = compute_v1.AccessConfig() + access.type_ = compute_v1.AccessConfig.Type.ONE_TO_ONE_NAT.name + access.name = "External NAT" + access.network_tier = access.NetworkTier.PREMIUM.name + network_interface.access_configs = [access] + + template = compute_v1.InstanceTemplate() + template.name = "test-template-" + uuid.uuid4().hex[:10] + template.properties = compute_v1.InstanceProperties() + template.properties.disks = [disk] + template.properties.machine_type = "e2-standard-16" + template.properties.network_interfaces = [network_interface] + + template.properties.scheduling = compute_v1.Scheduling() + template.properties.scheduling.on_host_maintenance = compute_v1.Scheduling.OnHostMaintenance.MIGRATE.name + template.properties.scheduling.provisioning_model = compute_v1.Scheduling.ProvisioningModel.STANDARD.name + template.properties.scheduling.automatic_restart = True + + template.properties.service_accounts = [ + { + "email": f"{PROJECT_NUMBER}-compute@developer.gserviceaccount.com", + "scopes": [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append" + ] + } + ] + + template_client = compute_v1.InstanceTemplatesClient() + operation_client = compute_v1.GlobalOperationsClient() + op = template_client.insert_unary( + project=PROJECT, instance_template_resource=template + ) + operation_client.wait(project=PROJECT, operation=op.name) + + template = template_client.get(project=PROJECT, instance_template=template.name) + + yield template + + op = template_client.delete_unary(project=PROJECT, instance_template=template.name) + operation_client.wait(project=PROJECT, operation=op.name) + + +def test_template_job(job_name, instance_template): + job = create_script_job_with_template(PROJECT, REGION, job_name, instance_template.self_link) + _test_body(job) diff --git a/setup.py b/setup.py index b53521b..64e0709 100644 --- a/setup.py +++ b/setup.py @@ -21,12 +21,12 @@ name = "google-cloud-batch" description = "Cloud Batch API client library" -version = "0.3.0" +version = "0.3.1" release_status = "Development Status :: 4 - Beta" dependencies = [ "google-api-core[grpc] >= 1.33.1, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*", "proto-plus >= 1.22.0, <2.0.0dev", - "protobuf >= 3.20.1, <5.0.0dev", + "protobuf >= 3.20.2, <5.0.0dev", "grpc-google-iam-v1 >=0.12.4, <1.0.0dev", ] url = "https://github.com/googleapis/python-batch" diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 1384af6..ea9ee0d 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,5 +6,5 @@ # Then this file should have google-cloud-foo==1.14.0 google-api-core==1.33.1 proto-plus==1.22.0 -protobuf==3.20.1 +protobuf==3.20.2 grpc-google-iam-v1==0.12.4