fix: create separate bucket for iot gcs_file_to_device test (GoogleCloudPlatform#6732)

busunkim96 · nicain · web-flow · commit 597763f0cbca · 2021-09-29T08:48:45.000-06:00
Co-authored-by: nicain &lt;nicain.seattle@gmail.com&gt;
diff --git a/appengine/flexible/storage/main.py b/appengine/flexible/storage/main.py
@@ -15,6 +15,7 @@
 # [START gae_flex_storage_app]
 import logging
 import os
+from typing import Union
 
 from flask import Flask, request
 from google.cloud import storage
@@ -26,7 +27,7 @@
 
 
 @app.route('/')
-def index():
+def index() -> str:
     return """
 <form method="POST" action="/upload" enctype="multipart/form-data">
     <input type="file" name="file">
@@ -36,7 +37,7 @@ def index():
 
 
 @app.route('/upload', methods=['POST'])
-def upload():
+def upload() -> str:
     """Process the uploaded file and upload it to Google Cloud Storage."""
     uploaded_file = request.files.get('file')
 
@@ -57,12 +58,17 @@ def upload():
         content_type=uploaded_file.content_type
     )
 
+    # Make the blob public. This is not necessary if the
+    # entire bucket is public.
+    # See https://cloud.google.com/storage/docs/access-control/making-data-public.
+    blob.make_public()
+
     # The public URL can be used to directly access the uploaded file via HTTP.
     return blob.public_url
 
 
 @app.errorhandler(500)
-def server_error(e):
+def server_error(e: Union[Exception, int]) -> str:
     logging.exception('An error occurred during a request.')
     return """
     An internal error occurred: <pre>{}</pre>
diff --git a/appengine/flexible/storage/main_test.py b/appengine/flexible/storage/main_test.py
@@ -12,6 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import uuid
+
+import flask
+import flask.testing
+from google.cloud import storage
 import pytest
 import requests
 from six import BytesIO
@@ -20,24 +26,34 @@
 
 
 @pytest.fixture
-def client():
+def client() -> flask.testing.FlaskClient:
     main.app.testing = True
     return main.app.test_client()
 
 
-def test_index(client):
+def test_index(client: flask.testing.FlaskClient) -> None:
     r = client.get('/')
     assert r.status_code == 200
 
 
-def test_upload(client):
+@pytest.fixture(scope="module")
+def blob_name() -> str:
+    name = f"gae-flex-storage-{uuid.uuid4()}"
+    yield name
+
+    bucket = storage.Client().bucket(os.environ["CLOUD_STORAGE_BUCKET"])
+    blob = bucket.blob(name)
+    blob.delete()
+
+
+def test_upload(client: flask.testing.FlaskClient, blob_name: str) -> None:
     # Upload a simple file
     file_content = b"This is some test content."
 
     r = client.post(
         '/upload',
         data={
-            'file': (BytesIO(file_content), 'example.txt')
+            'file': (BytesIO(file_content), blob_name)
         }
     )
 
diff --git a/appengine/flexible/storage/noxfile_config.py b/appengine/flexible/storage/noxfile_config.py
@@ -0,0 +1,44 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default TEST_CONFIG_OVERRIDE for python repos.
+
+# You can copy this file into your directory, then it will be imported from
+# the noxfile.py.
+
+# The source of truth:
+# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py
+
+TEST_CONFIG_OVERRIDE = {
+    # You can opt out from the test for specific Python versions.
+    "ignored_versions": ["2.7"],
+    # Old samples are opted out of enforcing Python type hints
+    # All new samples should feature them
+    "enforce_type_hints": True,
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    "gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
+    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+    # If you need to use a specific version of pip,
+    # change pip_version_override to the string representation
+    # of the version number, for example, "20.2.4"
+    "pip_version_override": None,
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    "envs": {
+        "CLOUD_STORAGE_BUCKET": "python-docs-samples-tests-public"
+    },
+}
diff --git a/appengine/flexible/storage/requirements-test.txt b/appengine/flexible/storage/requirements-test.txt
@@ -1 +1,2 @@
 pytest==6.2.4
+google-cloud-storage==1.42.2
diff --git a/iot/api-client/gcs_file_to_device/gcs_send_to_device_test.py b/iot/api-client/gcs_file_to_device/gcs_send_to_device_test.py
@@ -20,7 +20,6 @@
 
 from google.cloud import pubsub
 from google.cloud import storage
-import mock
 import pytest
 import requests
 
@@ -30,8 +29,6 @@
 sys.path.append(os.path.join(os.path.dirname(__file__), "..", "manager"))  # noqa
 import manager  # noqa
 
-
-gcs_bucket = os.environ["CLOUD_STORAGE_BUCKET"]
 project_id = os.environ["GOOGLE_CLOUD_PROJECT"]
 service_account_json = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
 
@@ -44,13 +41,36 @@
 destination_file_name = "destination-file.bin"
 gcs_file_name = "my-config"
 
+storage_client = storage.Client()
+
+
+@pytest.fixture(scope="module")
+def test_bucket_name():
+    bucket_name = "python-docs-samples-iot-{}".format(uuid.uuid4())
+
+    yield bucket_name
+
+    bucket = storage_client.bucket(bucket_name)
+    bucket.delete(force=True)
+
 
 @pytest.fixture(scope="module")
-def test_blob():
+def test_bucket(test_bucket_name):
+    """Yields a bucket that is deleted after the test completes."""
+    bucket = storage_client.bucket(test_bucket_name)
+
+    if not bucket.exists():
+        bucket = storage_client.create_bucket(test_bucket_name)
+
+    yield bucket.name
+
+
+@pytest.fixture(scope="module")
+def test_blob(test_bucket):
     """Provides a pre-existing blob in the test bucket."""
-    bucket = storage.Client().bucket(gcs_bucket)
+    bucket = storage_client.bucket(test_bucket)
     # Name of the blob
-    blob = bucket.blob("iot_core_store_file_gcs")
+    blob = bucket.blob("iot_core_store_file_gcs-{}".format(uuid.uuid4()))
     # Text in the blob
     blob.upload_from_string("This file on GCS will go to a device.")
 
@@ -63,39 +83,35 @@ def test_blob():
         pass
 
 
-@mock.patch("google.cloud.storage.client.Client.create_bucket")
-def test_create_bucket(create_bucket_mock, capsys):
-    # Unlike other tests for sending a config, this one mocks out the creation
-    # because buckets are expensive, globally-namespaced objects.
-    create_bucket_mock.return_value = mock.sentinel.bucket
-
-    gcs_to_device.create_bucket(gcs_bucket)
+def test_create_bucket(test_bucket_name, capsys):
+    gcs_to_device.create_bucket(test_bucket_name)
 
-    create_bucket_mock.assert_called_with(gcs_bucket)
+    out, _ = capsys.readouterr()
+    assert f"Bucket {test_bucket_name} created" in out
 
 
-def test_upload_local_file(capsys):
+def test_upload_local_file(test_bucket, capsys):
     # Creates a temporary source file that gets uploaded
     # to GCS. All other tests use the blob in test_blob().
     with tempfile.NamedTemporaryFile() as source_file:
         source_file.write(b"This is a source file.")
 
-        gcs_to_device.upload_local_file(gcs_bucket, gcs_file_name, source_file.name)
+        gcs_to_device.upload_local_file(test_bucket, gcs_file_name, source_file.name)
 
     out, _ = capsys.readouterr()
     assert "File {} uploaded as {}.".format(source_file.name, gcs_file_name) in out
 
 
-def test_make_file_public(test_blob):
-    gcs_to_device.make_file_public(gcs_bucket, test_blob.name)
+def test_make_file_public(test_bucket, test_blob):
+    gcs_to_device.make_file_public(test_bucket, test_blob.name)
 
     r = requests.get(test_blob.public_url)
     # Test for the content of the file to verify that
     # it's publicly accessible.
     assert r.text == "This file on GCS will go to a device."
 
 
-def test_send_to_device(capsys):
+def test_send_to_device(test_bucket, capsys):
     manager.create_iot_topic(project_id, topic_id)
     manager.open_registry(
         service_account_json, project_id, cloud_region, pubsub_topic, registry_id
@@ -106,7 +122,7 @@ def test_send_to_device(capsys):
     )
 
     gcs_to_device.send_to_device(
-        gcs_bucket,
+        test_bucket,
         gcs_file_name,
         destination_file_name,
         project_id,

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
`1`	`1`	`pytest==6.2.4`
	`2`	`+google-cloud-storage==1.42.2`