Skip to content

Commit 957019f

Browse files
engelkeengelke
committed
Initial background processing sample
1 parent 53698af commit 957019f

File tree

9 files changed

+383
-0
lines changed

9 files changed

+383
-0
lines changed

‎background/README.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
Background Processing
2+
---------------------
3+
4+
This directory contains an example of doing background processing with App
5+
Engine, Cloud Pub/Sub, Cloud Functions, and Firestore.
6+
7+
Deploy commands:
8+
9+
From the app directory:
10+
```
11+
$ gcloud app deploy
12+
```
13+
14+
From the function directory, after creating the PubSub topic:
15+
```
16+
$ gcloud functions deploy --runtime=go111 --trigger-topic=translate Translate --set-env-vars GOOGLE_CLOUD_PROJECT=my-project
17+
```

‎background/app/app.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2019 Google LLC All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
runtime: python37

‎background/app/main.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Copyright 2019 Google LLC All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
""" This web app shows translations that have been previously requested, and
16+
provides a form to request a new translation.
17+
"""
18+
19+
import json
20+
import os
21+
22+
from google.cloud import firestore
23+
from google.cloud import pubsub
24+
25+
from flask import Flask, render_template, redirect, request
26+
app = Flask(__name__)
27+
28+
# Get client objects to reuse over multiple invocations
29+
db = firestore.Client()
30+
publisher = pubsub.PublisherClient()
31+
32+
# Keep this list of supported languages up to date
33+
ACCEPTABLE_LANGUAGES = ('de', 'en', 'es', 'fr', 'ja', 'sw')
34+
35+
36+
@app.route('/', methods=['GET'])
37+
def index():
38+
""" The home page has a list of prior translations and a form to
39+
ask for a new translation.
40+
"""
41+
42+
doc_list = []
43+
docs = db.collection('translations').stream()
44+
for doc in docs:
45+
doc_list.append(doc.to_dict())
46+
47+
return render_template('index.html', translations=doc_list)
48+
49+
50+
@app.route('/request-translation', methods=['POST'])
51+
def translate():
52+
""" Handle a request to translate a string (form field 'v') to a given
53+
language (form field 'lang'), by sending a PubSub message to a topic.
54+
"""
55+
source_string = request.form.get('v', '')
56+
to_language = request.form.get('lang', '')
57+
58+
if source_string == '':
59+
error_message = 'Empty value'
60+
return error_message, 400
61+
62+
if to_language not in ACCEPTABLE_LANGUAGES:
63+
error_message = 'Unsupported language: {}'.format(to_language)
64+
return error_message, 400
65+
66+
message = {
67+
'Original': source_string,
68+
'Language': to_language,
69+
'Translated': '',
70+
'OriginalLanguage': '',
71+
}
72+
73+
topic_name = 'projects/{}/topics/{}'.format(
74+
os.getenv('GOOGLE_CLOUD_PROJECT'), 'translate'
75+
)
76+
publisher.publish(topic_name, json.dumps(message).encode('utf8'))
77+
return redirect('/')

‎background/app/main_test.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright 2019 Google LLC All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.

‎background/app/requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
google-cloud-firestore==1.4.0
2+
google-cloud-pubsub==0.45.0
3+
flask>=1.0.0

‎background/app/templates/index.html

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
<!DOCTYPE html>
2+
<!-- Copyright 2019 Google LLC
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
https://www.apache.org/licenses/LICENSE-2.0
7+
Unless required by applicable law or agreed to in writing, software
8+
distributed under the License is distributed on an "AS IS" BASIS,
9+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
See the License for the specific language governing permissions and
11+
limitations under the License. -->
12+
13+
<!-- [START getting_started_background_js] -->
14+
<html>
15+
16+
<head>
17+
<meta charset="UTF-8">
18+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
19+
<title>Translations</title>
20+
21+
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
22+
<link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.indigo-pink.min.css">
23+
<script defer src="https://code.getmdl.io/1.3.0/material.min.js"></script>
24+
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
25+
<script>
26+
$(document).ready(function() {
27+
$("#translate-form").submit(function(e) {
28+
e.preventDefault();
29+
// Get value, make sure it's not empty.
30+
if ($("#v").val() == "") {
31+
return;
32+
}
33+
$.ajax({
34+
type: "POST",
35+
url: "/request-translation",
36+
data: $(this).serialize(),
37+
success: function(data) {
38+
// Show snackbar.
39+
console.log(data);
40+
var notification = document.querySelector('.mdl-js-snackbar');
41+
$("#snackbar").removeClass("mdl-color--red-100");
42+
$("#snackbar").addClass("mdl-color--green-100");
43+
notification.MaterialSnackbar.showSnackbar({
44+
message: 'Translation requested'
45+
});
46+
},
47+
error: function(data) {
48+
// Show snackbar.
49+
console.log("Error requesting translation");
50+
var notification = document.querySelector('.mdl-js-snackbar');
51+
$("#snackbar").removeClass("mdl-color--green-100");
52+
$("#snackbar").addClass("mdl-color--red-100");
53+
notification.MaterialSnackbar.showSnackbar({
54+
message: 'Translation request failed'
55+
});
56+
}
57+
});
58+
});
59+
});
60+
</script>
61+
<style>
62+
.lang {
63+
width: 50px;
64+
}
65+
.translate-form {
66+
display: inline;
67+
}
68+
</style>
69+
</head>
70+
<!-- [END getting_started_background_js] -->
71+
<!-- [START getting_started_background_html] -->
72+
<body>
73+
<div class="mdl-layout mdl-js-layout mdl-layout--fixed-header">
74+
<header class="mdl-layout__header">
75+
<div class="mdl-layout__header-row">
76+
<!-- Title -->
77+
<span class="mdl-layout-title">Translate with Background Processing</span>
78+
</div>
79+
</header>
80+
<main class="mdl-layout__content">
81+
<div class="page-content">
82+
<div class="mdl-grid">
83+
<div class="mdl-cell mdl-cell--1-col"></div>
84+
<div class="mdl-cell mdl-cell--3-col">
85+
<form id="translate-form" class="translate-form">
86+
<div class="mdl-textfield mdl-js-textfield mdl-textfield--floating-label">
87+
<input class="mdl-textfield__input" type="text" id="v" name="v">
88+
<label class="mdl-textfield__label" for="v">Text to translate...</label>
89+
</div>
90+
<select class="mdl-textfield__input lang" name="lang">
91+
<option value="de">de</option>
92+
<option value="en">en</option>
93+
<option value="es">es</option>
94+
<option value="fr">fr</option>
95+
<option value="ja">ja</option>
96+
<option value="sw">sw</option>
97+
</select>
98+
<button class="mdl-button mdl-js-button mdl-button--raised mdl-button--accent" type="submit"
99+
name="submit">Submit</button>
100+
</form>
101+
</div>
102+
<div class="mdl-cell mdl-cell--8-col">
103+
<table class="mdl-data-table mdl-js-data-table mdl-shadow--2dp">
104+
<thead>
105+
<tr>
106+
<th class="mdl-data-table__cell--non-numeric"><strong>Original</strong></th>
107+
<th class="mdl-data-table__cell--non-numeric"><strong>Translation</strong></th>
108+
</tr>
109+
</thead>
110+
<tbody>
111+
{% for translation in translations %}
112+
<tr>
113+
<td class="mdl-data-table__cell--non-numeric">
114+
<span class="mdl-chip mdl-color--primary">
115+
<span class="mdl-chip__text mdl-color-text--white">{{ translation['OriginalLanguage'] }} </span>
116+
</span>
117+
{{ translation['Original'] }}
118+
</td>
119+
<td class="mdl-data-table__cell--non-numeric">
120+
<span class="mdl-chip mdl-color--accent">
121+
<span class="mdl-chip__text mdl-color-text--white">{{ translation['Language'] }} </span>
122+
</span>
123+
{{ translation['Translated'] }}
124+
</td>
125+
</tr>
126+
{% endfor %}
127+
</tbody>
128+
</table>
129+
<br/>
130+
<button class="mdl-button mdl-js-button mdl-button--raised" type="button" onClick="window.location.reload();">
131+
Refresh
132+
</button>
133+
</div>
134+
</div>
135+
</div>
136+
<div aria-live="assertive" aria-atomic="true" aria-relevant="text" class="mdl-snackbar mdl-js-snackbar" id="snackbar">
137+
<div class="mdl-snackbar__text mdl-color-text--black"></div>
138+
<button type="button" class="mdl-snackbar__action"></button>
139+
</div>
140+
</main>
141+
</div>
142+
</body>
143+
144+
</html>
145+
<!-- [END getting_started_background_html] -->

‎background/function/main.py

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright 2019 Google LLC All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
""" This function handles messages posted to a pubsub topic by translating
16+
the data in the message as requested. The message must be a JSON encoded
17+
dictionary with fields:
18+
19+
Original - the string to translate
20+
Language - the language to translate the string to
21+
22+
The dictionary may have other fields, which will be ignored.
23+
"""
24+
25+
import base64
26+
import hashlib
27+
import json
28+
29+
from google.cloud import firestore
30+
from google.cloud import translate
31+
32+
# Get client objects once to reuse over multiple invocations.
33+
xlate = translate.Client()
34+
db = firestore.Client()
35+
36+
37+
def document_name(message):
38+
""" Messages are saved in a Firestore database with document IDs generated
39+
from the original string and destination language. If the exact same
40+
translation is requested a second time, the result will overwrite the
41+
prior result.
42+
43+
message - a dictionary with fields named Language and Original, and
44+
optionally other fields with any names
45+
46+
Returns a unique name that is an allowed Firestore document ID
47+
"""
48+
key = '{}/{}'.format(message['Language'], message['Original'])
49+
hashed = hashlib.sha512(key.encode()).digest()
50+
51+
# Note that document IDs should not contain the '/' character
52+
name = base64.b64encode(hashed, altchars=b'+-').decode('utf-8')
53+
54+
55+
@firestore.transactional
56+
def update_database(transaction, message):
57+
name = document_name(message)
58+
doc_ref = db.collection("translations").document(document_id=name)
59+
60+
try:
61+
doc_ref.get(transaction=transaction)
62+
except NotFound:
63+
return # Don't replace an existing translation
64+
65+
transaction.set(doc_ref, message)
66+
67+
68+
def translate_string(from_string, to_language):
69+
""" Translates a string to a specified language.
70+
71+
from_string - the original string before translation
72+
73+
to_language - the language to translate to, as a two-letter code (e.g.,
74+
'en' for english, 'de' for german)
75+
76+
Returns the translated string and the code for original language
77+
"""
78+
result = xlate.translate(from_string, target_language=to_language)
79+
db = firestore.Client()
80+
return result['translatedText'], result['detectedSourceLanguage']
81+
82+
83+
def translate_message(event, context):
84+
""" Process a pubsub message requesting a translation
85+
"""
86+
message_data = base64.b64decode(event['data']).decode('utf-8')
87+
message = json.loads(message_data)
88+
89+
from_string = message['Original']
90+
to_language = message['Language']
91+
92+
to_string, from_language = translate_string(from_string, to_language)
93+
94+
message['Translated'] = to_string
95+
message['OriginalLanguage'] = from_language
96+
97+
transaction = db.transaction()
98+
update_database(transaction, message)

‎background/function/main_test.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright 2019 Google LLC All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.

‎background/function/requirements.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
google-cloud-translate==1.6.0
2+
google-cloud-firestore==1.4.0

0 commit comments

Comments
 (0)