Skip to content

Commit 057c5cf

Browse files
shen-chaochalmerlowe
shen-chao
and
chalmerlowe
authored
Remote function translate (GoogleCloudPlatform#9117)
* Add remote function example to use translate api * Add remote function example to use translate api * Fix lint errors * Update README to include translate sample * Addressing some comments * Fix a bug where userDefinedContext was assumed always in request but it's optional * Fix lint * Fix type hint * Exclude 3.7 for fstring * Not using new fstring feature to make it work for Python 3.7 * Update bigquery/remote-function/translate/main.py Co-authored-by: Chalmer Lowe <chalmer.lowe@gmail.com> * Update bigquery/remote-function/translate/main.py Co-authored-by: Chalmer Lowe <chalmer.lowe@gmail.com> * Update bigquery/remote-function/translate/main.py Co-authored-by: Chalmer Lowe <chalmer.lowe@gmail.com> * add comment and reformat * fix bug * Add a test for client error --------- Co-authored-by: Chalmer Lowe <chalmer.lowe@gmail.com>
1 parent 67c4076 commit 057c5cf

File tree

6 files changed

+252
-0
lines changed

6 files changed

+252
-0
lines changed

‎bigquery/remote-function/README.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,12 @@ Samples
6868

6969
- `Vision`_: this sample can detect and extract objects from input images.
7070
- `Document`_: this sample can extract text from input documents.
71+
- `Translate`_: this sample can translate input text to other languages.
7172

7273

7374
.. _Vision: vision/
7475
.. _Document: document/
76+
.. _Translate: translate/
7577

7678
The client library
7779
-------------------------------------------------------------------------------

‎bigquery/remote-function/translate/main.py

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Copyright 2023 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START bigquery_remote_function_translation]
16+
from typing import List
17+
18+
import flask
19+
import functions_framework
20+
from google.api_core.retry import Retry
21+
from google.cloud import translate
22+
23+
# Construct a Translation Client object
24+
translate_client = translate.TranslationServiceClient()
25+
26+
27+
# Register an HTTP function with the Functions Framework
28+
@functions_framework.http
29+
def handle_translation(request: flask.Request) -> flask.Response:
30+
"""BigQuery remote function to translate input text.
31+
32+
Args:
33+
request: HTTP request from BigQuery
34+
https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#input_format
35+
36+
Returns:
37+
HTTP response to BigQuery
38+
https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#output_format
39+
"""
40+
try:
41+
# Parse request data as JSON
42+
request_json = request.get_json()
43+
# Get the project of the query
44+
caller = request_json["caller"]
45+
project = extract_project_from_caller(caller)
46+
if project is None:
47+
return flask.make_response(
48+
flask.jsonify(
49+
{
50+
"errorMessage": (
51+
'project can\'t be extracted from "caller":'
52+
f" {caller}."
53+
)
54+
}
55+
),
56+
400,
57+
)
58+
# Get the target language code, default is Spanish ("es")
59+
context = request_json.get("userDefinedContext", {})
60+
target = context.get("target_language", "es")
61+
62+
calls = request_json["calls"]
63+
translated = translate_text(
64+
[call[0] for call in calls], project, target
65+
)
66+
67+
return flask.jsonify({"replies": translated})
68+
except Exception as err:
69+
return flask.make_response(
70+
flask.jsonify(
71+
{"errorMessage": f"Unexpected error {type(err)}:{err}"}
72+
),
73+
400,
74+
)
75+
76+
77+
def extract_project_from_caller(job: str) -> str:
78+
"""Extract project id from full resource name of a BigQuery job.
79+
80+
Args:
81+
job: full resource name of a BigQuery job, like
82+
"//bigquery.googleapi.com/projects/<project>/jobs/<job_id>"
83+
84+
Returns:
85+
project id which is contained in the full resource name of the job.
86+
"""
87+
path = job.split("/")
88+
return path[4] if len(path) > 4 else None
89+
90+
91+
def translate_text(
92+
calls: List[str], project: str, target_language_code: str
93+
) -> List[str]:
94+
"""Translates the input text to specified language using Translation API.
95+
96+
Args:
97+
calls: a list of input text to translate.
98+
project: the project where the translate service will be used.
99+
target_language_code: The ISO-639 language code to use for translation
100+
of the input text. See
101+
https://cloud.google.com/translate/docs/advanced/discovering-supported-languages-v3#supported-target
102+
for the supported language list.
103+
104+
Returns:
105+
a list of translated text.
106+
"""
107+
location = "<your location>"
108+
parent = f"projects/{project}/locations/{location}"
109+
# Call the Translation API, passing a list of values and the target language
110+
response = translate_client.translate_text(
111+
request={
112+
"parent": parent,
113+
"contents": calls,
114+
"target_language_code": target_language_code,
115+
"mime_type": "text/plain",
116+
},
117+
retry=Retry(),
118+
)
119+
# Convert the translated value to a list and return it
120+
return [
121+
translation.translated_text for translation in response.translations
122+
]
123+
124+
125+
# [END bigquery_remote_function_translation]

‎bigquery/remote-function/translate/main_test.py

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Copyright 2023 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from unittest import mock
15+
16+
17+
import flask
18+
from google.cloud import translate
19+
import pytest
20+
21+
22+
# Create a fake "app" for generating test request contexts.
23+
@pytest.fixture(scope='module')
24+
def app() -> flask.Flask:
25+
return flask.Flask(__name__)
26+
27+
28+
@mock.patch('main.translate_client')
29+
def test_main(mock_translate: object, app: flask.Flask) -> None:
30+
import main
31+
32+
mock_translate.translate_text.return_value = (
33+
translate.TranslateTextResponse(
34+
{
35+
'translations': (
36+
[
37+
{'translated_text': 'Hola'},
38+
{'translated_text': 'Mundo'},
39+
]
40+
)
41+
}
42+
)
43+
)
44+
45+
with app.test_request_context(
46+
json={
47+
'caller': (
48+
'//bigquery.googleapis.com/projects/test-project/jobs/job-id'
49+
),
50+
'userDefinedContext': {},
51+
'calls': [
52+
['Hello'],
53+
['World'],
54+
],
55+
}
56+
):
57+
response = main.handle_translation(flask.request)
58+
assert response.status_code == 200
59+
assert response.get_json()['replies'] == ['Hola', 'Mundo']
60+
61+
62+
@mock.patch('main.translate_client')
63+
def test_translate_client_error(
64+
mock_translate: object, app: flask.Flask
65+
) -> None:
66+
import main
67+
68+
mock_translate.translate_text.side_effect = Exception('API error')
69+
70+
with app.test_request_context(
71+
json={
72+
'caller': (
73+
'//bigquery.googleapis.com/projects/test-project/jobs/job-id'
74+
),
75+
'userDefinedContext': {},
76+
'calls': [
77+
['Hello'],
78+
['World'],
79+
],
80+
}
81+
):
82+
response = main.handle_translation(flask.request)
83+
assert response.status_code == 400
84+
assert 'errorMessage' in response.get_json()
85+
assert response.get_json()['errorMessage'].endswith('API error')

‎bigquery/remote-function/translate/noxfile_config.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# TEST_CONFIG_OVERRIDE copied from the source of truth:
16+
# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py
17+
18+
TEST_CONFIG_OVERRIDE = {
19+
# You can opt out from the test for specific Python versions.
20+
"ignored_versions": ["2.7", "3.6"],
21+
# Old samples are opted out of enforcing Python type hints
22+
# All new samples should feature them
23+
"enforce_type_hints": True,
24+
# An envvar key for determining the project id to use. Change it
25+
# to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
26+
# build specific Cloud project. You can also use your own string
27+
# to use your own Cloud project.
28+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
29+
# 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
30+
# A dictionary you want to inject into your test. Don't put any
31+
# secrets here. These values will override predefined values.
32+
"envs": {},
33+
}

‎bigquery/remote-function/translate/requirements-test.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Flask==2.2.2
2+
functions-framework==3.3.0
3+
google-cloud-translate==3.10.1
4+
pytest==7.2.1

‎bigquery/remote-function/translate/requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Flask==2.2.2
2+
functions-framework==3.3.0
3+
google-cloud-translate==3.10.1

0 commit comments

Comments
 (0)