Skip to content

Commit 3cdd870

Browse files
committed
urlfetch async sample Working in Python 3 App Engine
1 parent 77355ae commit 3cdd870

File tree

6 files changed

+138
-0
lines changed

6 files changed

+138
-0
lines changed
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
runtime: python27
2+
api_version: 1
3+
threadsafe: yes
4+
5+
handlers:
6+
- url: .*
7+
script: main.app
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
runtime: python37
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from google.appengine.ext import vendor
16+
17+
# Add any libraries installed in the "lib" folder.
18+
vendor.add('lib')
19+
20+
import requests
21+
import requests_toolbelt.adapters.appengine
22+
23+
# Use the App Engine Requests adapter. This makes sure that Requests uses
24+
# URLFetch.
25+
requests_toolbelt.adapters.appengine.monkeypatch()
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START app]
16+
import logging
17+
18+
from flask import Flask, make_response
19+
20+
# [START imports]
21+
import requests
22+
from requests_futures.sessions import FuturesSession
23+
from time import sleep
24+
# [END imports]
25+
26+
27+
TIMEOUT = 10 # Wait this many seconds for background calls to finish
28+
app = Flask(__name__)
29+
30+
31+
@app.route('/') # Fetch and return remote page asynchronously
32+
def get_async():
33+
# [START requests_get]
34+
session = FuturesSession()
35+
url = 'http://www.google.com/humans.txt'
36+
37+
rpc = session.get(url)
38+
39+
# ... do other things ...
40+
41+
resp = make_response(rpc.result().text)
42+
resp.headers['Content-type'] = 'text/plain'
43+
return resp
44+
# [END requests_get]
45+
46+
47+
@app.route('/callback') # Fetch and return remote pages using callback
48+
def get_callback():
49+
global response_text
50+
global counter
51+
52+
response_text = ''
53+
counter = 0
54+
55+
def cb(resp, *args, **kwargs):
56+
global response_text
57+
global counter
58+
59+
if 300 <= resp.status_code < 400:
60+
return # ignore intermediate redirection responses
61+
62+
counter += 1
63+
response_text += 'Response number {} is {} bytes from {}\n'.format(
64+
counter, len(resp.text), resp.url)
65+
66+
67+
session = FuturesSession()
68+
urls = [
69+
'https://google.com/',
70+
'https://www.google.com/humans.txt',
71+
'https://www.github.com',
72+
'https://www.travis-ci.org'
73+
]
74+
75+
futures = [session.get(url, hooks={'response': cb}) for url in urls]
76+
77+
# No wait functionality in requests_futures, so check every second to
78+
# see if all callbacks are done, up to TIMEOUT seconds
79+
for elapsed_time in range(TIMEOUT+1):
80+
all_done = True
81+
for future in futures:
82+
if not future.done():
83+
all_done = False
84+
break
85+
if all_done:
86+
break
87+
sleep(1)
88+
89+
resp = make_response(response_text)
90+
resp.headers['Content-type'] = 'text/plain'
91+
return resp
92+
93+
94+
@app.errorhandler(500)
95+
def server_error(e):
96+
logging.exception('An error occurred during a request.')
97+
return """
98+
An internal error occurred: <pre>{}</pre>
99+
See logs for full stacktrace.
100+
""".format(e), 500
101+
# [END app]

appengine/standard/migration/urlfetch/async/main_test.py

Whitespace-only changes.
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Flask==1.1.1
2+
requests==2.22.0
3+
requests-futures==1.0.0
4+
requests-toolbelt==0.9.1

0 commit comments

Comments
 (0)