From c240ecb29ddbad2e0118246b858266aded7b5d6f Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Mon, 20 Apr 2020 11:11:01 -0700 Subject: [PATCH 1/7] [datalabeling] testing: mark some tests as flaky --- datalabeling/create_annotation_spec_set_test.py | 7 ++++--- datalabeling/create_instruction.py | 1 + datalabeling/create_instruction_test.py | 5 +++-- datalabeling/import_data_test.py | 5 +++-- datalabeling/label_image_test.py | 9 +++++---- datalabeling/label_text_test.py | 9 +++++---- datalabeling/label_video_test.py | 9 +++++---- datalabeling/manage_dataset_test.py | 3 ++- 8 files changed, 28 insertions(+), 20 deletions(-) diff --git a/datalabeling/create_annotation_spec_set_test.py b/datalabeling/create_annotation_spec_set_test.py index 0618f2dab9e..6f396e40120 100644 --- a/datalabeling/create_annotation_spec_set_test.py +++ b/datalabeling/create_annotation_spec_set_test.py @@ -16,15 +16,16 @@ import os -import create_annotation_spec_set -from google.cloud import datalabeling_v1beta1 as datalabeling from google.api_core.client_options import ClientOptions +from google.cloud import datalabeling_v1beta1 as datalabeling import pytest +import create_annotation_spec_set + PROJECT_ID = os.getenv('GCLOUD_PROJECT') -@pytest.mark.slow +@pytest.mark.flaky(max_runs=3) def test_create_annotation_spec_set(capsys): response = create_annotation_spec_set.create_annotation_spec_set( PROJECT_ID) diff --git a/datalabeling/create_instruction.py b/datalabeling/create_instruction.py index c60f55d6e4a..5495acbaf18 100644 --- a/datalabeling/create_instruction.py +++ b/datalabeling/create_instruction.py @@ -16,6 +16,7 @@ import argparse import os + from google.api_core.client_options import ClientOptions diff --git a/datalabeling/create_instruction_test.py b/datalabeling/create_instruction_test.py index 09315761764..1d2930b2f41 100644 --- a/datalabeling/create_instruction_test.py +++ b/datalabeling/create_instruction_test.py @@ -16,17 +16,18 @@ import os -import create_instruction from google.api_core.client_options import ClientOptions from google.cloud import datalabeling_v1beta1 as datalabeling import pytest +import create_instruction + PROJECT_ID = os.getenv('GCLOUD_PROJECT') INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' '/instruction/test.pdf') -@pytest.mark.slow +@pytest.mark.flaky(max_runs=3) def test_create_instruction(capsys): result = create_instruction.create_instruction( PROJECT_ID, diff --git a/datalabeling/import_data_test.py b/datalabeling/import_data_test.py index 6a389e94204..f3a63c90460 100644 --- a/datalabeling/import_data_test.py +++ b/datalabeling/import_data_test.py @@ -16,9 +16,10 @@ import os +import pytest + import import_data import manage_dataset -import pytest PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' @@ -35,7 +36,7 @@ def dataset(): manage_dataset.delete_dataset(dataset.name) -@pytest.mark.slow +@pytest.mark.flaky(max_runs=3) def test_import_data(capsys, dataset): import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) out, _ = capsys.readouterr() diff --git a/datalabeling/label_image_test.py b/datalabeling/label_image_test.py index 3917ef0481a..b7fb13b6809 100644 --- a/datalabeling/label_image_test.py +++ b/datalabeling/label_image_test.py @@ -16,14 +16,15 @@ import os -import create_annotation_spec_set -import create_instruction from google.api_core.client_options import ClientOptions from google.cloud import datalabeling_v1beta1 as datalabeling +import pytest + +import create_annotation_spec_set +import create_instruction import import_data import label_image import manage_dataset -import pytest PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' @@ -86,7 +87,7 @@ def instruction(): # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.slow +@pytest.mark.flaky(max_runs=3) def test_label_image(capsys, annotation_spec_set, instruction, dataset): # Start labeling. diff --git a/datalabeling/label_text_test.py b/datalabeling/label_text_test.py index e65fc187478..5b2cc51bc93 100644 --- a/datalabeling/label_text_test.py +++ b/datalabeling/label_text_test.py @@ -16,14 +16,15 @@ import os +from google.api_core.client_options import ClientOptions +from google.cloud import datalabeling_v1beta1 as datalabeling +import pytest + import create_annotation_spec_set import create_instruction -from google.cloud import datalabeling_v1beta1 as datalabeling -from google.api_core.client_options import ClientOptions import import_data import label_text import manage_dataset -import pytest PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/text/input.csv' @@ -86,7 +87,7 @@ def instruction(): # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.slow +@pytest.mark.flaky(max_runs=3) def test_label_text(capsys, annotation_spec_set, instruction, dataset): # Start labeling. diff --git a/datalabeling/label_video_test.py b/datalabeling/label_video_test.py index 606b28c1c9a..2c8b79de756 100644 --- a/datalabeling/label_video_test.py +++ b/datalabeling/label_video_test.py @@ -16,14 +16,15 @@ import os -import create_annotation_spec_set -import create_instruction from google.api_core.client_options import ClientOptions from google.cloud import datalabeling_v1beta1 as datalabeling +import pytest + +import create_annotation_spec_set +import create_instruction import import_data import label_video import manage_dataset -import pytest PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/videos/video_dataset.csv' @@ -86,7 +87,7 @@ def instruction(): # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.slow +@pytest.mark.flaky(max_runs=3) def test_label_video(capsys, annotation_spec_set, instruction, dataset): # Start labeling. diff --git a/datalabeling/manage_dataset_test.py b/datalabeling/manage_dataset_test.py index 02f3c934577..6b8fd3828a2 100644 --- a/datalabeling/manage_dataset_test.py +++ b/datalabeling/manage_dataset_test.py @@ -16,9 +16,10 @@ import os -import manage_dataset import pytest +import manage_dataset + PROJECT_ID = os.getenv("GCLOUD_PROJECT") From 4e7f1d7834fe9638ad0145f155c63eb1428c4552 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Tue, 21 Apr 2020 22:15:48 +0000 Subject: [PATCH 2/7] added backoff to one test --- .../create_annotation_spec_set_test.py | 38 +++++++++++-------- datalabeling/requirements-test.txt | 1 + datalabeling/testing_lib.py | 37 ++++++++++++++++++ 3 files changed, 60 insertions(+), 16 deletions(-) create mode 100644 datalabeling/testing_lib.py diff --git a/datalabeling/create_annotation_spec_set_test.py b/datalabeling/create_annotation_spec_set_test.py index 6f396e40120..4bf89c2bc15 100644 --- a/datalabeling/create_annotation_spec_set_test.py +++ b/datalabeling/create_annotation_spec_set_test.py @@ -16,30 +16,36 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest import create_annotation_spec_set +import testing_lib + PROJECT_ID = os.getenv('GCLOUD_PROJECT') -@pytest.mark.flaky(max_runs=3) +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.delete_annotation_spec_set(resource_name) + + def test_create_annotation_spec_set(capsys): - response = create_annotation_spec_set.create_annotation_spec_set( - PROJECT_ID) - out, _ = capsys.readouterr() - assert 'The annotation_spec_set resource name:' in out - # Delete the created annotation spec set. - annotation_spec_set_name = response.name - client = datalabeling.DataLabelingServiceClient() + @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) + def run_sample(): + return create_annotation_spec_set.create_annotation_spec_set(PROJECT_ID) - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) + response = run_sample() + out, _ = capsys.readouterr() + assert 'The annotation_spec_set resource name:' in out - client.delete_annotation_spec_set(annotation_spec_set_name) + # For cleanup + cleaner.append(response.name) diff --git a/datalabeling/requirements-test.txt b/datalabeling/requirements-test.txt index 1b569cb4f2c..40857e816e6 100644 --- a/datalabeling/requirements-test.txt +++ b/datalabeling/requirements-test.txt @@ -1,2 +1,3 @@ +backoff==1.10.0 pytest==5.3.2 flaky==3.6.1 diff --git a/datalabeling/testing_lib.py b/datalabeling/testing_lib.py new file mode 100644 index 00000000000..8f7628e4bc4 --- /dev/null +++ b/datalabeling/testing_lib.py @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import backoff +from google.api_core.client_options import ClientOptions +from google.api_core.exceptions import DeadlineExceeded +from google.cloud import datalabeling_v1beta1 as datalabeling + + +def create_client(): + # If provided, use a provided test endpoint - this will prevent tests on + # this snippet from triggering any action by a real human + if 'DATALABELING_ENDPOINT' in os.environ: + opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) + client = datalabeling.DataLabelingServiceClient(client_options=opts) + else: + client = datalabeling.DataLabelingServiceClient() + return client + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) +def delete_annotation_spec_set(name): + client = create_client() + client.delete_annotation_spec_set(name) From 9fa6ea47f6abbe032d09e3dd652a5a8813f27c34 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Tue, 21 Apr 2020 22:36:25 +0000 Subject: [PATCH 3/7] inject the cleaner --- datalabeling/create_annotation_spec_set_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/datalabeling/create_annotation_spec_set_test.py b/datalabeling/create_annotation_spec_set_test.py index 4bf89c2bc15..6ee2d31e71d 100644 --- a/datalabeling/create_annotation_spec_set_test.py +++ b/datalabeling/create_annotation_spec_set_test.py @@ -37,15 +37,16 @@ def cleaner(): testing_lib.delete_annotation_spec_set(resource_name) -def test_create_annotation_spec_set(capsys): +def test_create_annotation_spec_set(cleaner, capsys): @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) def run_sample(): return create_annotation_spec_set.create_annotation_spec_set(PROJECT_ID) response = run_sample() - out, _ = capsys.readouterr() - assert 'The annotation_spec_set resource name:' in out # For cleanup cleaner.append(response.name) + + out, _ = capsys.readouterr() + assert 'The annotation_spec_set resource name:' in out From aaff0be1b78bcf9a91274ad85bf0dbeeec394b8a Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Wed, 22 Apr 2020 00:08:25 +0000 Subject: [PATCH 4/7] use backoff instead of flaky --- .../create_annotation_spec_set_test.py | 3 +- datalabeling/create_instruction_test.py | 44 ++++----- datalabeling/import_data_test.py | 32 +++++-- datalabeling/label_image_test.py | 90 +++++++------------ datalabeling/label_text_test.py | 86 +++++++----------- datalabeling/label_video_test.py | 87 +++++++----------- datalabeling/manage_dataset_test.py | 64 +++++++++---- datalabeling/requirements-test.txt | 1 - datalabeling/testing_lib.py | 46 +++++++++- 9 files changed, 243 insertions(+), 210 deletions(-) diff --git a/datalabeling/create_annotation_spec_set_test.py b/datalabeling/create_annotation_spec_set_test.py index 6ee2d31e71d..bcbfb7cb339 100644 --- a/datalabeling/create_annotation_spec_set_test.py +++ b/datalabeling/create_annotation_spec_set_test.py @@ -39,7 +39,8 @@ def cleaner(): def test_create_annotation_spec_set(cleaner, capsys): - @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) def run_sample(): return create_annotation_spec_set.create_annotation_spec_set(PROJECT_ID) diff --git a/datalabeling/create_instruction_test.py b/datalabeling/create_instruction_test.py index 1d2930b2f41..183704b2bd7 100644 --- a/datalabeling/create_instruction_test.py +++ b/datalabeling/create_instruction_test.py @@ -16,35 +16,39 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest import create_instruction +import testing_lib + PROJECT_ID = os.getenv('GCLOUD_PROJECT') INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' '/instruction/test.pdf') -@pytest.mark.flaky(max_runs=3) -def test_create_instruction(capsys): - result = create_instruction.create_instruction( - PROJECT_ID, - 'IMAGE', - INSTRUCTION_GCS_URI - ) - out, _ = capsys.readouterr() - assert 'The instruction resource name: ' in out +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.delete_instruction(resource_name) - # Delete the created instruction. - instruction_name = result.name - client = datalabeling.DataLabelingServiceClient() - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) +def test_create_instruction(cleaner, capsys): - client.delete_instruction(instruction_name) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + return create_instruction.create_instruction( + PROJECT_ID, 'IMAGE', INSTRUCTION_GCS_URI) + + instruction = run_sample() + cleaner.append(instruction.name) + + out, _ = capsys.readouterr() + assert 'The instruction resource name: ' in out diff --git a/datalabeling/import_data_test.py b/datalabeling/import_data_test.py index f3a63c90460..009349b8067 100644 --- a/datalabeling/import_data_test.py +++ b/datalabeling/import_data_test.py @@ -16,10 +16,14 @@ import os +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest import import_data import manage_dataset +import testing_lib + PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' @@ -27,17 +31,33 @@ @pytest.fixture(scope='function') def dataset(): - # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def create_dataset(): + # create a temporary dataset + return manage_dataset.create_dataset(PROJECT_ID) + + dataset = create_dataset() yield dataset - # tear down - manage_dataset.delete_dataset(dataset.name) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def delete_dataset(): + # tear down + manage_dataset.delete_dataset(dataset.name) + + delete_dataset() -@pytest.mark.flaky(max_runs=3) def test_import_data(capsys, dataset): - import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + + run_sample() out, _ = capsys.readouterr() assert 'Dataset resource name: ' in out diff --git a/datalabeling/label_image_test.py b/datalabeling/label_image_test.py index b7fb13b6809..6fb0e66495b 100644 --- a/datalabeling/label_image_test.py +++ b/datalabeling/label_image_test.py @@ -16,101 +16,77 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest -import create_annotation_spec_set -import create_instruction -import import_data import label_image -import manage_dataset +import testing_lib + PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' +INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' + '/instruction/test.pdf') @pytest.fixture(scope='function') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) - - # import some data to it - import_data.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) + dataset = testing_lib.create_dataset(PROJECT_ID) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) @pytest.fixture(scope='function') def annotation_spec_set(): # create a temporary annotation_spec_set - response = create_annotation_spec_set.create_annotation_spec_set( - PROJECT_ID) + response = testing_lib.create_annotation_spec_set(PROJECT_ID) yield response - # tear down - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.delete_annotation_spec_set(response.name) + testing_lib.delete_annotation_spec_set(response.name) @pytest.fixture(scope='function') def instruction(): # create a temporary instruction - instruction = create_instruction.create_instruction( - PROJECT_ID, 'IMAGE', - 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + instruction = testing_lib.create_instruction( + PROJECT_ID, 'IMAGE', INSTRUCTION_GCS_URI) yield instruction # tear down - client = datalabeling.DataLabelingServiceClient() + testing_lib.delete_instruction(instruction.name) + - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] - client.delete_instruction(instruction.name) + yield resource_names + + for resource_name in resource_names: + testing_lib.cancel_operation(resource_name) # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.flaky(max_runs=3) -def test_label_image(capsys, annotation_spec_set, instruction, dataset): - - # Start labeling. - response = label_image.label_image( - dataset.name, - instruction.name, - annotation_spec_set.name - ) - out, _ = capsys.readouterr() - assert 'Label_image operation name: ' in out - operation_name = response.operation.name +def test_label_image( + capsys, annotation_spec_set, instruction, dataset, cleaner): - # Cancels the labeling operation. - response.cancel() - assert response.cancelled() is True + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + # Start labeling. + return label_image.label_image( + dataset.name, instruction.name, annotation_spec_set.name) - client = datalabeling.DataLabelingServiceClient() + response = run_sample() + cleaner.append(response.operation.name) - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.transport._operations_client.cancel_operation( - operation_name) + out, _ = capsys.readouterr() + assert 'Label_image operation name: ' in out diff --git a/datalabeling/label_text_test.py b/datalabeling/label_text_test.py index 5b2cc51bc93..1c0c28601d0 100644 --- a/datalabeling/label_text_test.py +++ b/datalabeling/label_text_test.py @@ -16,101 +16,81 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest -import create_annotation_spec_set -import create_instruction -import import_data import label_text -import manage_dataset +import testing_lib PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/text/input.csv' +INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' + '/instruction/test.pdf') @pytest.fixture(scope='function') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + dataset = testing_lib.create_dataset(PROJECT_ID) - # import some data to it - import_data.import_data(dataset.name, 'TEXT', INPUT_GCS_URI) + testing_lib.import_data(dataset.name, 'TEXT', INPUT_GCS_URI) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) @pytest.fixture(scope='function') def annotation_spec_set(): # create a temporary annotation_spec_set - response = create_annotation_spec_set.create_annotation_spec_set( - PROJECT_ID) + response = testing_lib.create_annotation_spec_set(PROJECT_ID) yield response - # tear down - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.delete_annotation_spec_set(response.name) + testing_lib.delete_annotation_spec_set(response.name) @pytest.fixture(scope='function') def instruction(): # create a temporary instruction - instruction = create_instruction.create_instruction( - PROJECT_ID, 'TEXT', - 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + instruction = testing_lib.create_instruction( + PROJECT_ID, 'IMAGE', INSTRUCTION_GCS_URI) yield instruction # tear down - client = datalabeling.DataLabelingServiceClient() + testing_lib.delete_instruction(instruction.name) - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - client.delete_instruction(instruction.name) +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.cancel_operation(resource_name) # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.flaky(max_runs=3) -def test_label_text(capsys, annotation_spec_set, instruction, dataset): - - # Start labeling. - response = label_text.label_text( - dataset.name, - instruction.name, - annotation_spec_set.name - ) +def test_label_text(capsys, annotation_spec_set, instruction, dataset, cleaner): + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + # Start labeling. + return label_text.label_text( + dataset.name, instruction.name, annotation_spec_set.name) + + response = run_sample() + cleaner.append(response.operation.name) + out, _ = capsys.readouterr() assert 'Label_text operation name: ' in out - operation_name = response.operation.name # Cancels the labeling operation. response.cancel() assert response.cancelled() is True - - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.transport._operations_client.cancel_operation( - operation_name) diff --git a/datalabeling/label_video_test.py b/datalabeling/label_video_test.py index 2c8b79de756..b98849cca4b 100644 --- a/datalabeling/label_video_test.py +++ b/datalabeling/label_video_test.py @@ -16,101 +16,82 @@ import os -from google.api_core.client_options import ClientOptions -from google.cloud import datalabeling_v1beta1 as datalabeling +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest -import create_annotation_spec_set -import create_instruction -import import_data import label_video -import manage_dataset +import testing_lib PROJECT_ID = os.getenv('GCLOUD_PROJECT') INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/videos/video_dataset.csv' +INSTRUCTION_GCS_URI = ('gs://cloud-samples-data/datalabeling' + '/instruction/test.pdf') @pytest.fixture(scope='function') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + dataset = testing_lib.create_dataset(PROJECT_ID) - # import some data to it - import_data.import_data(dataset.name, 'VIDEO', INPUT_GCS_URI) + testing_lib.import_data(dataset.name, 'VIDEO', INPUT_GCS_URI) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) @pytest.fixture(scope='function') def annotation_spec_set(): # create a temporary annotation_spec_set - response = create_annotation_spec_set.create_annotation_spec_set( - PROJECT_ID) + response = testing_lib.create_annotation_spec_set(PROJECT_ID) yield response - # tear down - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.delete_annotation_spec_set(response.name) + testing_lib.delete_annotation_spec_set(response.name) @pytest.fixture(scope='function') def instruction(): # create a temporary instruction - instruction = create_instruction.create_instruction( - PROJECT_ID, 'VIDEO', - 'gs://cloud-samples-data/datalabeling/instruction/test.pdf') + instruction = testing_lib.create_instruction( + PROJECT_ID, 'VIDEO', INSTRUCTION_GCS_URI) yield instruction # tear down - client = datalabeling.DataLabelingServiceClient() + testing_lib.delete_instruction(instruction.name) - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - client.delete_instruction(instruction.name) +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.cancel_operation(resource_name) # Passing in dataset as the last argument in test_label_image since it needs # to be deleted before the annotation_spec_set can be deleted. -@pytest.mark.flaky(max_runs=3) -def test_label_video(capsys, annotation_spec_set, instruction, dataset): - - # Start labeling. - response = label_video.label_video( - dataset.name, - instruction.name, - annotation_spec_set.name - ) +def test_label_video( + capsys, annotation_spec_set, instruction, dataset, cleaner): + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + # Start labeling. + return label_video.label_video( + dataset.name, instruction.name, annotation_spec_set.name) + + response = run_sample() + cleaner.append(response) + out, _ = capsys.readouterr() assert 'Label_video operation name: ' in out - operation_name = response.operation.name # Cancels the labeling operation. response.cancel() assert response.cancelled() is True - - client = datalabeling.DataLabelingServiceClient() - - # If provided, use a provided test endpoint - this will prevent tests on - # this snippet from triggering any action by a real human - if 'DATALABELING_ENDPOINT' in os.environ: - opts = ClientOptions(api_endpoint=os.getenv('DATALABELING_ENDPOINT')) - client = datalabeling.DataLabelingServiceClient(client_options=opts) - - client.transport._operations_client.cancel_operation( - operation_name) diff --git a/datalabeling/manage_dataset_test.py b/datalabeling/manage_dataset_test.py index 6b8fd3828a2..eb8824c3faa 100644 --- a/datalabeling/manage_dataset_test.py +++ b/datalabeling/manage_dataset_test.py @@ -16,53 +16,81 @@ import os +import backoff +from google.api_core.exceptions import DeadlineExceeded import pytest import manage_dataset +import testing_lib + PROJECT_ID = os.getenv("GCLOUD_PROJECT") -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def dataset(): # create a temporary dataset - dataset = manage_dataset.create_dataset(PROJECT_ID) + dataset = testing_lib.create_dataset(PROJECT_ID) yield dataset # tear down - manage_dataset.delete_dataset(dataset.name) + testing_lib.delete_dataset(dataset.name) + + +@pytest.fixture(scope='module') +def cleaner(): + resource_names = [] + + yield resource_names + + for resource_name in resource_names: + testing_lib.delete_dataset(resource_name) + +def test_create_dataset(cleaner, capsys): + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + return manage_dataset.create_dataset(PROJECT_ID) + + response = run_sample() + cleaner.append(response.name) -@pytest.mark.flaky(max_runs=3) -def test_create_dataset(capsys): - response = manage_dataset.create_dataset(PROJECT_ID) out, _ = capsys.readouterr() assert "The dataset resource name:" in out - # clean up - manage_dataset.delete_dataset(response.name) - -@pytest.mark.flaky(max_runs=3) def test_list_dataset(capsys, dataset): - manage_dataset.list_datasets(PROJECT_ID) + + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + manage_dataset.list_datasets(PROJECT_ID) + + run_sample() out, _ = capsys.readouterr() assert dataset.name in out -@pytest.mark.flaky(max_runs=3) def test_get_dataset(capsys, dataset): - manage_dataset.get_dataset(dataset.name) + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + manage_dataset.get_dataset(dataset.name) + + run_sample() out, _ = capsys.readouterr() assert "The dataset resource name:" in out -@pytest.mark.flaky(max_runs=3) -def test_delete_dataset(capsys): - # Creates a dataset. - response = manage_dataset.create_dataset(PROJECT_ID) +def test_delete_dataset(capsys, dataset): + @backoff.on_exception( + backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) + def run_sample(): + manage_dataset.delete_dataset(dataset.name) - manage_dataset.delete_dataset(response.name) + run_sample() out, _ = capsys.readouterr() assert "Dataset deleted." in out diff --git a/datalabeling/requirements-test.txt b/datalabeling/requirements-test.txt index 40857e816e6..8855f3cf1f8 100644 --- a/datalabeling/requirements-test.txt +++ b/datalabeling/requirements-test.txt @@ -1,3 +1,2 @@ backoff==1.10.0 pytest==5.3.2 -flaky==3.6.1 diff --git a/datalabeling/testing_lib.py b/datalabeling/testing_lib.py index 8f7628e4bc4..e28da194a6a 100644 --- a/datalabeling/testing_lib.py +++ b/datalabeling/testing_lib.py @@ -19,6 +19,13 @@ from google.api_core.exceptions import DeadlineExceeded from google.cloud import datalabeling_v1beta1 as datalabeling +import create_annotation_spec_set as annotation_spec_set_sample +import create_instruction as instruction_sample +import manage_dataset as dataset_sample +import import_data as import_sample + +RETRY_DEADLINE = 60 + def create_client(): # If provided, use a provided test endpoint - this will prevent tests on @@ -31,7 +38,44 @@ def create_client(): return client -@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=60) +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def create_dataset(project_id): + return dataset_sample.create_dataset(project_id) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def delete_dataset(name): + return dataset_sample.delete_dataset(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def create_annotation_spec_set(project_id): + return annotation_spec_set_sample.create_annotation_spec_set(project_id) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) def delete_annotation_spec_set(name): client = create_client() client.delete_annotation_spec_set(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def create_instruction(project_id, data_type, gcs_uri): + return instruction_sample.create_instruction(project_id, data_type, gcs_uri) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def delete_instruction(name): + client = create_client() + client.delete_instruction(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def cancel_operation(name): + client = create_client() + client.transport._operations_client.cancel_operation(name) + + +@backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) +def import_data(dataset_name, data_type, gcs_uri): + import_sample.import_data(dataset_name, data_type, gcs_uri) From 56ad4b430d9d1b4dd0a0c2aa49ea570e5b4a6f4e Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Wed, 22 Apr 2020 00:36:13 +0000 Subject: [PATCH 5/7] fix builds --- datalabeling/label_image_test.py | 1 + datalabeling/testing_lib.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/datalabeling/label_image_test.py b/datalabeling/label_image_test.py index 6fb0e66495b..3ee43bc6b27 100644 --- a/datalabeling/label_image_test.py +++ b/datalabeling/label_image_test.py @@ -35,6 +35,7 @@ def dataset(): # create a temporary dataset dataset = testing_lib.create_dataset(PROJECT_ID) + testing_lib.import_data(dataset.name, 'IMAGE', INPUT_GCS_URI) yield dataset # tear down diff --git a/datalabeling/testing_lib.py b/datalabeling/testing_lib.py index e28da194a6a..5f97876cb0b 100644 --- a/datalabeling/testing_lib.py +++ b/datalabeling/testing_lib.py @@ -72,6 +72,8 @@ def delete_instruction(name): @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) def cancel_operation(name): + if not name: + return client = create_client() client.transport._operations_client.cancel_operation(name) From b9777c02a13238bc3427be7bd12031f458127029 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Wed, 22 Apr 2020 00:38:39 +0000 Subject: [PATCH 6/7] really fix builds --- datalabeling/label_video_test.py | 2 +- datalabeling/testing_lib.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/datalabeling/label_video_test.py b/datalabeling/label_video_test.py index b98849cca4b..47936cb23a6 100644 --- a/datalabeling/label_video_test.py +++ b/datalabeling/label_video_test.py @@ -87,7 +87,7 @@ def run_sample(): dataset.name, instruction.name, annotation_spec_set.name) response = run_sample() - cleaner.append(response) + cleaner.append(response.operation.name) out, _ = capsys.readouterr() assert 'Label_video operation name: ' in out diff --git a/datalabeling/testing_lib.py b/datalabeling/testing_lib.py index 5f97876cb0b..e28da194a6a 100644 --- a/datalabeling/testing_lib.py +++ b/datalabeling/testing_lib.py @@ -72,8 +72,6 @@ def delete_instruction(name): @backoff.on_exception(backoff.expo, DeadlineExceeded, max_time=RETRY_DEADLINE) def cancel_operation(name): - if not name: - return client = create_client() client.transport._operations_client.cancel_operation(name) From 5ff21a34dd373da01279531050e5ff6c23d2385f Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Wed, 22 Apr 2020 01:08:42 +0000 Subject: [PATCH 7/7] correct scope for the fixtures --- datalabeling/import_data_test.py | 22 +++++----------------- datalabeling/label_image_test.py | 6 +++--- datalabeling/label_text_test.py | 6 +++--- datalabeling/label_video_test.py | 6 +++--- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/datalabeling/import_data_test.py b/datalabeling/import_data_test.py index 009349b8067..c5b111748a6 100644 --- a/datalabeling/import_data_test.py +++ b/datalabeling/import_data_test.py @@ -21,7 +21,6 @@ import pytest import import_data -import manage_dataset import testing_lib @@ -29,26 +28,15 @@ INPUT_GCS_URI = 'gs://cloud-samples-data/datalabeling/image/image_dataset.csv' -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def dataset(): - - @backoff.on_exception( - backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) - def create_dataset(): - # create a temporary dataset - return manage_dataset.create_dataset(PROJECT_ID) - - dataset = create_dataset() + # create a temporary dataset + dataset = testing_lib.create_dataset(PROJECT_ID) yield dataset - @backoff.on_exception( - backoff.expo, DeadlineExceeded, max_time=testing_lib.RETRY_DEADLINE) - def delete_dataset(): - # tear down - manage_dataset.delete_dataset(dataset.name) - - delete_dataset() + # tear down + testing_lib.delete_dataset(dataset.name) def test_import_data(capsys, dataset): diff --git a/datalabeling/label_image_test.py b/datalabeling/label_image_test.py index 3ee43bc6b27..f50606623e0 100644 --- a/datalabeling/label_image_test.py +++ b/datalabeling/label_image_test.py @@ -30,7 +30,7 @@ '/instruction/test.pdf') -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def dataset(): # create a temporary dataset dataset = testing_lib.create_dataset(PROJECT_ID) @@ -42,7 +42,7 @@ def dataset(): testing_lib.delete_dataset(dataset.name) -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def annotation_spec_set(): # create a temporary annotation_spec_set response = testing_lib.create_annotation_spec_set(PROJECT_ID) @@ -52,7 +52,7 @@ def annotation_spec_set(): testing_lib.delete_annotation_spec_set(response.name) -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def instruction(): # create a temporary instruction instruction = testing_lib.create_instruction( diff --git a/datalabeling/label_text_test.py b/datalabeling/label_text_test.py index 1c0c28601d0..3e4a075447c 100644 --- a/datalabeling/label_text_test.py +++ b/datalabeling/label_text_test.py @@ -29,7 +29,7 @@ '/instruction/test.pdf') -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def dataset(): # create a temporary dataset dataset = testing_lib.create_dataset(PROJECT_ID) @@ -42,7 +42,7 @@ def dataset(): testing_lib.delete_dataset(dataset.name) -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def annotation_spec_set(): # create a temporary annotation_spec_set response = testing_lib.create_annotation_spec_set(PROJECT_ID) @@ -52,7 +52,7 @@ def annotation_spec_set(): testing_lib.delete_annotation_spec_set(response.name) -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def instruction(): # create a temporary instruction instruction = testing_lib.create_instruction( diff --git a/datalabeling/label_video_test.py b/datalabeling/label_video_test.py index 47936cb23a6..fd9bb846de3 100644 --- a/datalabeling/label_video_test.py +++ b/datalabeling/label_video_test.py @@ -29,7 +29,7 @@ '/instruction/test.pdf') -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def dataset(): # create a temporary dataset dataset = testing_lib.create_dataset(PROJECT_ID) @@ -42,7 +42,7 @@ def dataset(): testing_lib.delete_dataset(dataset.name) -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def annotation_spec_set(): # create a temporary annotation_spec_set response = testing_lib.create_annotation_spec_set(PROJECT_ID) @@ -52,7 +52,7 @@ def annotation_spec_set(): testing_lib.delete_annotation_spec_set(response.name) -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def instruction(): # create a temporary instruction instruction = testing_lib.create_instruction(