diff --git a/.gitignore b/.gitignore index ecb44b9..e833757 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ +.vscode/ +.venv3?/ +*.sublime-project +*.sublime-workspace +.env* +.DS_Store concurrency/flags/img/*.gif concurrency/charfinder/charfinder_index.pickle 18-asyncio/charfinder/charfinder_index.pickle diff --git a/03-dict-set/index.py b/03-dict-set/index.py index 3eac1fb..f8641d2 100644 --- a/03-dict-set/index.py +++ b/03-dict-set/index.py @@ -8,7 +8,7 @@ import sys import re -WORD_RE = re.compile('\w+') +WORD_RE = re.compile(r'\w+') index = {} with open(sys.argv[1], encoding='utf-8') as fp: diff --git a/03-dict-set/index0.py b/03-dict-set/index0.py index e1fa28f..b41af0e 100644 --- a/03-dict-set/index0.py +++ b/03-dict-set/index0.py @@ -8,7 +8,7 @@ import sys import re -WORD_RE = re.compile('\w+') +WORD_RE = re.compile(r'\w+') index = {} with open(sys.argv[1], encoding='utf-8') as fp: diff --git a/03-dict-set/index_default.py b/03-dict-set/index_default.py index 521b2d5..8d3ae58 100644 --- a/03-dict-set/index_default.py +++ b/03-dict-set/index_default.py @@ -9,7 +9,7 @@ import re import collections -WORD_RE = re.compile('\w+') +WORD_RE = re.compile(r'\w+') index = collections.defaultdict(list) # <1> with open(sys.argv[1], encoding='utf-8') as fp: diff --git a/11-iface-abc/lotto.py b/11-iface-abc/lotto.py index 2295b71..da8c2de 100644 --- a/11-iface-abc/lotto.py +++ b/11-iface-abc/lotto.py @@ -17,7 +17,7 @@ def pick(self): try: position = random.randrange(len(self._balls)) # <2> except ValueError: - raise LookupError('pick from empty BingoCage') + raise LookupError('pick from empty LotteryBlower') return self._balls.pop(position) # <3> def loaded(self): # <4> diff --git a/14-it-generator/sentence.py b/14-it-generator/sentence.py index fb866c4..447a192 100644 --- a/14-it-generator/sentence.py +++ b/14-it-generator/sentence.py @@ -5,7 +5,7 @@ import re import reprlib -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') class Sentence: @@ -17,7 +17,7 @@ def __init__(self, text): def __getitem__(self, index): return self.words[index] # <2> - def __len__(self, index): # <3> + def __len__(self): # <3> return len(self.words) def __repr__(self): diff --git a/14-it-generator/sentence_gen.py b/14-it-generator/sentence_gen.py index a17c48f..32a8225 100644 --- a/14-it-generator/sentence_gen.py +++ b/14-it-generator/sentence_gen.py @@ -5,7 +5,7 @@ import re import reprlib -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') class Sentence: diff --git a/14-it-generator/sentence_gen2.py b/14-it-generator/sentence_gen2.py index 8b0f355..b308100 100644 --- a/14-it-generator/sentence_gen2.py +++ b/14-it-generator/sentence_gen2.py @@ -5,7 +5,7 @@ import re import reprlib -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') class Sentence: diff --git a/14-it-generator/sentence_genexp.py b/14-it-generator/sentence_genexp.py index 2919c29..52228de 100644 --- a/14-it-generator/sentence_genexp.py +++ b/14-it-generator/sentence_genexp.py @@ -6,7 +6,7 @@ import re import reprlib -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') class Sentence: diff --git a/14-it-generator/sentence_iter.py b/14-it-generator/sentence_iter.py index 938d5b4..11b8179 100644 --- a/14-it-generator/sentence_iter.py +++ b/14-it-generator/sentence_iter.py @@ -9,7 +9,7 @@ import re import reprlib -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') class Sentence: diff --git a/14-it-generator/sentence_iter2.py b/14-it-generator/sentence_iter2.py index 8597b32..2663f3f 100644 --- a/14-it-generator/sentence_iter2.py +++ b/14-it-generator/sentence_iter2.py @@ -8,7 +8,7 @@ import re import reprlib -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') class Sentence: diff --git a/16-coroutine/taxi_sim.py b/16-coroutine/taxi_sim.py index 705418a..e9c4cc1 100644 --- a/16-coroutine/taxi_sim.py +++ b/16-coroutine/taxi_sim.py @@ -53,7 +53,6 @@ import collections import queue import argparse -import time DEFAULT_NUMBER_OF_TAXIS = 3 DEFAULT_END_TIME = 180 diff --git a/17-futures-py3.7/README.rst b/17-futures-py3.7/README.rst new file mode 100644 index 0000000..7167b33 --- /dev/null +++ b/17-futures-py3.7/README.rst @@ -0,0 +1,10 @@ +Updated sample code for Chapter 17 - "Concurrency with futures" + +From the book "Fluent Python" by Luciano Ramalho (O'Reilly, 2015) +http://shop.oreilly.com/product/0636920032519.do + + This directory contains code updated to run with Python 3.7 and + **aiohttp** 3.5. When the first edition of "Fluent Python" was + written, the **asyncio** package was provisional, and the latest + version of **aiohttp** was 0.13.1. The API for both packages had + significant breaking changes. diff --git a/17-futures-py3.7/countries/.gitignore b/17-futures-py3.7/countries/.gitignore new file mode 100644 index 0000000..8ea4ee7 --- /dev/null +++ b/17-futures-py3.7/countries/.gitignore @@ -0,0 +1 @@ +flags/ diff --git a/17-futures-py3.7/countries/README.rst b/17-futures-py3.7/countries/README.rst new file mode 100644 index 0000000..0f29b01 --- /dev/null +++ b/17-futures-py3.7/countries/README.rst @@ -0,0 +1,178 @@ +============================ +Setting up Nginx and Vaurien +============================ + +This text explains how to configure Nginx and Vaurien to test HTTP client code while avoiding network traffic and introducing simulated delays and errors. This setup is necessary if you want to experiment with the ``flags2*.py`` image download examples in this directory -- covered in chapters 17 and 18 of Fluent Python. + + +Overview +======== + +The flag download examples are designed to compare the performance of different approaches to finding and downloading files from the Web. However, we don't want to hit a public server with multiple requests per second while testing, and we want to be able to simulate high latency and random network errors. + +For this setup I chose Nginx as the HTTP server because it is very fast and easy to configure, and Toxiproxy — designed by Shopify to introduce delays and network errors for testing distributed systems. + +The archive ``flags.zip``, contains a directory ``flags/`` with 194 subdirectories, each containing a ``.gif`` image and a ``metadata.json`` file. These are public-domain images copied from the `CIA World Fact Book `_. + +Once these files are unpacked to the ``flags/`` directory and Nginx is configured, you can experiment with the ``flags2*.py`` examples without hitting the network. + + +Procedure +========= + +1. Unpack test data +------------------- + +The instructions in this section are for GNU/Linux or OSX using the command line. Windows users should have no difficulty doing the same operations with the Windows Exporer GUI. + +Unpack the initial data in the ``fancy_flags/`` directory:: + + $ unzip flags.zip + ... many lines omitted ... + creating: flags/zw/ + inflating: flags/zw/metadata.json + inflating: flags/zw/zw.gif + + +Verify that 194 directories are created in ``fancy_flags/flags/``, each with a ``.gif`` and a ``metadata.json`` file:: + + + $ ls flags | wc -w + 194 + $ find flags | grep .gif | wc -l + 194 + $ find flags | grep .json | wc -l + 194 + $ ls flags/ad + ad.gif metadata.json + + +2. Install Nginx +---------------- + +Download and install Nginx. I used version 1.6.2 -- the latest stable version as I write this. + +* Download page: http://nginx.org/en/download.html + +* Beginner's guide: http://nginx.org/en/docs/beginners_guide.html + + +3. Configure Nginx +------------------ + +Edit the the ``nginx.conf`` file to set the port and document root. You can determine which ``nginx.conf`` is in use by running:: + + + $ nginx -V + + +The output starts with:: + + nginx version: nginx/1.6.2 + built by clang 6.0 (clang-600.0.51) (based on LLVM 3.5svn) + TLS SNI support enabled + configure arguments:... + + +Among the configure arguments you'll see ``--conf-path=``. That's the file you will edit. + +Most of the content in ``nginx.conf`` is within a block labeled ``http`` and enclosed in curly braces. Within that block there can be multiple blocks labeled ``server``. Add another ``server`` block like this one:: + + + server { + listen 8001; + + location /flags/ { + root /full-path-to.../countries/; + } + } + + +After editing ``nginx.conf`` the server must be started (if it's not running) or told to reload the configuration file:: + + + $ nginx # to start, if necessary + $ nginx -s reload # to reload the configuration + + +To test the configuration, open the URL http://localhost:8001/flags/ad/ad.gif in a browser. You should see the blue, yellow and red flag of Andorra. + +If the test fails, please double check the procedure just described and refer to the Nginx documentation. + +At this point you may run the ``flags_*2.py`` examples against the Nginx install by providing the ``--server LOCAL`` command line option. For example:: + + + $ python3 flags2_threadpool.py -s LOCAL + LOCAL site: http://localhost:8001/flags + Searching for 20 flags: from BD to VN + 20 concurrent connections will be used. + -------------------- + 20 flags downloaded. + Elapsed time: 0.09s + + +Note that Nginx is so fast that you will not see much difference in run time between the sequential and the concurrent versions. For more realistic testing with simulated network lag, we need to set up Toxiproxy. + + +4. Install and run Toxiproxy +---------------------------- + +Install... + +In one terminal: + + $ toxiproxy-server + +In another terminal: + + $ toxiproxy-cli create nginx_flags_delay -l localhost:8002 -u localhost:8001 + Created new proxy nginx_flags_delay + $ toxiproxy-cli toxic add nginx_flags_delay -t latency -a latency=500 + Added downstream latency toxic 'latency_downstream' on proxy 'nginx_flags_delay' + + +This creates an HTTP proxy on port 8002 which adds a 0.5s delay to every response. You can test it with a browser on port 8002: http://localhost:8002/flags/ad/ad.gif -- the flag of Andorra should appear after ½ second. + +TODO: UPDATE NEXT PARAGRAPH + +There is also the ``XXX`` script which runs a proxy on port 8003 producing errors in 25% of the responses and a .5 s delay to 50% of the responses. You can also test it with the browser on port 8003, but rememeber that errors are expected. + + +Platform-specific instructions +============================== + + +Nginx setup on Mac OS X +------------------------ + +Homebrew (copy & paste code at the bottom of http://brew.sh/):: + + + $ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + $ brew doctor + $ brew install nginx + + +Download and unpack:: + +Docroot is: /usr/local/var/www +/usr/local/etc/nginx/nginx.conf + + +:: + + To have launchd start nginx at login: + ln -sfv /usr/local/opt/nginx/*.plist ~/Library/LaunchAgents + Then to load nginx now: + launchctl load ~/Library/LaunchAgents/homebrew.mxcl.nginx.plist + Or, if you don't want/need launchctl, you can just run: + nginx + + + +Nginx setup on Lubuntu 14.04.1 LTS +---------------------------------- + +Docroot is: /usr/share/nginx/html + + diff --git a/17-futures-py3.7/countries/country_codes.txt b/17-futures-py3.7/countries/country_codes.txt new file mode 100644 index 0000000..72c37f0 --- /dev/null +++ b/17-futures-py3.7/countries/country_codes.txt @@ -0,0 +1,8 @@ +AD AE AF AG AL AM AO AR AT AU AZ BA BB BD BE BF BG BH BI BJ BN BO BR BS BT +BW BY BZ CA CD CF CG CH CI CL CM CN CO CR CU CV CY CZ DE DJ DK DM DZ EC EE +EG ER ES ET FI FJ FM FR GA GB GD GE GH GM GN GQ GR GT GW GY HN HR HT HU ID +IE IL IN IQ IR IS IT JM JO JP KE KG KH KI KM KN KP KR KW KZ LA LB LC LI LK +LR LS LT LU LV LY MA MC MD ME MG MH MK ML MM MN MR MT MU MV MW MX MY MZ NA +NE NG NI NL NO NP NR NZ OM PA PE PG PH PK PL PT PW PY QA RO RS RU RW SA SB +SC SD SE SG SI SK SL SM SN SO SR SS ST SV SY SZ TD TG TH TJ TL TM TN TO TR +TT TV TW TZ UA UG US UY UZ VA VC VE VN VU WS YE ZA ZM ZW diff --git a/17-futures-py3.7/countries/flags.py b/17-futures-py3.7/countries/flags.py new file mode 100644 index 0000000..7a7f854 --- /dev/null +++ b/17-futures-py3.7/countries/flags.py @@ -0,0 +1,63 @@ +"""Download flags of top 20 countries by population + +Sequential version + +Sample run:: + + $ python3 flags.py + BD BR CD CN DE EG ET FR ID IN IR JP MX NG PH PK RU TR US VN + 20 flags downloaded in 5.49s + +""" +# BEGIN FLAGS_PY +import os +import time +import sys + +import requests # <1> + +POP20_CC = ('CN IN US ID BR PK NG BD RU JP ' + 'MX PH VN ET EG DE IR TR CD FR').split() # <2> + +BASE_URL = 'http://flupy.org/data/flags' # <3> + +DEST_DIR = 'downloads/' # <4> + + +def save_flag(img, filename): # <5> + path = os.path.join(DEST_DIR, filename) + with open(path, 'wb') as fp: + fp.write(img) + + +def get_flag(cc): # <6> + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + resp = requests.get(url) + return resp.content + + +def show(text): # <7> + print(text, end=' ') + sys.stdout.flush() + + +def download_many(cc_list): # <8> + for cc in sorted(cc_list): # <9> + image = get_flag(cc) + show(cc) + save_flag(image, cc.lower() + '.gif') + + return len(cc_list) + + +def main(): # <10> + t0 = time.time() + count = download_many(POP20_CC) + elapsed = time.time() - t0 + msg = '\n{} flags downloaded in {:.2f}s' + print(msg.format(count, elapsed)) + + +if __name__ == '__main__': + main() +# END FLAGS_PY diff --git a/17-futures-py3.7/countries/flags2_asyncio.py b/17-futures-py3.7/countries/flags2_asyncio.py new file mode 100644 index 0000000..2635155 --- /dev/null +++ b/17-futures-py3.7/countries/flags2_asyncio.py @@ -0,0 +1,103 @@ +"""Download flags of countries (with error handling). + +asyncio async/await version + +""" +# BEGIN FLAGS2_ASYNCIO_TOP +import asyncio +import collections + +import aiohttp +from aiohttp import web +import tqdm + +from flags2_common import main, HTTPStatus, Result, save_flag + +# default set low to avoid errors from remote site, such as +# 503 - Service Temporarily Unavailable +DEFAULT_CONCUR_REQ = 5 +MAX_CONCUR_REQ = 1000 + + +class FetchError(Exception): # <1> + def __init__(self, country_code): + self.country_code = country_code + + +async def get_flag(session, base_url, cc): # <2> + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + async with session.get(url) as resp: + if resp.status == 200: + return await resp.read() + elif resp.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.HttpProcessingError( + code=resp.status, message=resp.reason, + headers=resp.headers) + + +async def download_one(session, cc, base_url, semaphore, verbose): # <3> + try: + async with semaphore: # <4> + image = await get_flag(session, base_url, cc) # <5> + except web.HTTPNotFound: # <6> + status = HTTPStatus.not_found + msg = 'not found' + except Exception as exc: + raise FetchError(cc) from exc # <7> + else: + save_flag(image, cc.lower() + '.gif') # <8> + status = HTTPStatus.ok + msg = 'OK' + + if verbose and msg: + print(cc, msg) + + return Result(status, cc) +# END FLAGS2_ASYNCIO_TOP + +# BEGIN FLAGS2_ASYNCIO_DOWNLOAD_MANY +async def downloader_coro(cc_list, base_url, verbose, concur_req): # <1> + counter = collections.Counter() + semaphore = asyncio.Semaphore(concur_req) # <2> + async with aiohttp.ClientSession() as session: # <8> + to_do = [download_one(session, cc, base_url, semaphore, verbose) + for cc in sorted(cc_list)] # <3> + + to_do_iter = asyncio.as_completed(to_do) # <4> + if not verbose: + to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> + for future in to_do_iter: # <6> + try: + res = await future # <7> + except FetchError as exc: # <8> + country_code = exc.country_code # <9> + try: + error_msg = exc.__cause__.args[0] # <10> + except IndexError: + error_msg = exc.__cause__.__class__.__name__ # <11> + if verbose and error_msg: + msg = '*** Error for {}: {}' + print(msg.format(country_code, error_msg)) + status = HTTPStatus.error + else: + status = res.status + + counter[status] += 1 # <12> + + return counter # <13> + + +def download_many(cc_list, base_url, verbose, concur_req): + loop = asyncio.get_event_loop() + coro = downloader_coro(cc_list, base_url, verbose, concur_req) + counts = loop.run_until_complete(coro) # <14> + loop.close() # <15> + + return counts + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) +# END FLAGS2_ASYNCIO_DOWNLOAD_MANY diff --git a/17-futures-py3.7/countries/flags2_common.py b/17-futures-py3.7/countries/flags2_common.py new file mode 100644 index 0000000..bfa40fb --- /dev/null +++ b/17-futures-py3.7/countries/flags2_common.py @@ -0,0 +1,149 @@ +"""Utilities for second set of flag examples. +""" + +import os +import time +import sys +import string +import argparse +from collections import namedtuple +from enum import Enum + + +Result = namedtuple('Result', 'status data') + +HTTPStatus = Enum('Status', 'ok not_found error') + +POP20_CC = ('CN IN US ID BR PK NG BD RU JP ' + 'MX PH VN ET EG DE IR TR CD FR').split() + +DEFAULT_CONCUR_REQ = 1 +MAX_CONCUR_REQ = 1 + +SERVERS = { + 'REMOTE': 'http://flupy.org/data/flags', + 'LOCAL': 'http://localhost:8001/flags', + 'DELAY': 'http://localhost:8002/flags', + 'ERROR': 'http://localhost:8003/flags', +} +DEFAULT_SERVER = 'LOCAL' + +DEST_DIR = 'downloads/' +COUNTRY_CODES_FILE = 'country_codes.txt' + + +def save_flag(img, filename): + path = os.path.join(DEST_DIR, filename) + with open(path, 'wb') as fp: + fp.write(img) + + +def initial_report(cc_list, actual_req, server_label): + if len(cc_list) <= 10: + cc_msg = ', '.join(cc_list) + else: + cc_msg = 'from {} to {}'.format(cc_list[0], cc_list[-1]) + print('{} site: {}'.format(server_label, SERVERS[server_label])) + msg = 'Searching for {} flag{}: {}' + plural = 's' if len(cc_list) != 1 else '' + print(msg.format(len(cc_list), plural, cc_msg)) + plural = 's' if actual_req != 1 else '' + msg = '{} concurrent connection{} will be used.' + print(msg.format(actual_req, plural)) + + +def final_report(cc_list, counter, start_time): + elapsed = time.time() - start_time + print('-' * 20) + msg = '{} flag{} downloaded.' + plural = 's' if counter[HTTPStatus.ok] != 1 else '' + print(msg.format(counter[HTTPStatus.ok], plural)) + if counter[HTTPStatus.not_found]: + print(counter[HTTPStatus.not_found], 'not found.') + if counter[HTTPStatus.error]: + plural = 's' if counter[HTTPStatus.error] != 1 else '' + print('{} error{}.'.format(counter[HTTPStatus.error], plural)) + print('Elapsed time: {:.2f}s'.format(elapsed)) + + +def expand_cc_args(every_cc, all_cc, cc_args, limit): + codes = set() + A_Z = string.ascii_uppercase + if every_cc: + codes.update(a+b for a in A_Z for b in A_Z) + elif all_cc: + with open(COUNTRY_CODES_FILE) as fp: + text = fp.read() + codes.update(text.split()) + else: + for cc in (c.upper() for c in cc_args): + if len(cc) == 1 and cc in A_Z: + codes.update(cc+c for c in A_Z) + elif len(cc) == 2 and all(c in A_Z for c in cc): + codes.add(cc) + else: + msg = 'each CC argument must be A to Z or AA to ZZ.' + raise ValueError('*** Usage error: '+msg) + return sorted(codes)[:limit] + + +def process_args(default_concur_req): + server_options = ', '.join(sorted(SERVERS)) + parser = argparse.ArgumentParser( + description='Download flags for country codes. ' + 'Default: top 20 countries by population.') + parser.add_argument('cc', metavar='CC', nargs='*', + help='country code or 1st letter (eg. B for BA...BZ)') + parser.add_argument('-a', '--all', action='store_true', + help='get all available flags (AD to ZW)') + parser.add_argument('-e', '--every', action='store_true', + help='get flags for every possible code (AA...ZZ)') + parser.add_argument('-l', '--limit', metavar='N', type=int, + help='limit to N first codes', default=sys.maxsize) + parser.add_argument('-m', '--max_req', metavar='CONCURRENT', type=int, + default=default_concur_req, + help='maximum concurrent requests (default={})' + .format(default_concur_req)) + parser.add_argument('-s', '--server', metavar='LABEL', + default=DEFAULT_SERVER, + help='Server to hit; one of {} (default={})' + .format(server_options, DEFAULT_SERVER)) + parser.add_argument('-v', '--verbose', action='store_true', + help='output detailed progress info') + args = parser.parse_args() + if args.max_req < 1: + print('*** Usage error: --max_req CONCURRENT must be >= 1') + parser.print_usage() + sys.exit(1) + if args.limit < 1: + print('*** Usage error: --limit N must be >= 1') + parser.print_usage() + sys.exit(1) + args.server = args.server.upper() + if args.server not in SERVERS: + print('*** Usage error: --server LABEL must be one of', + server_options) + parser.print_usage() + sys.exit(1) + try: + cc_list = expand_cc_args(args.every, args.all, args.cc, args.limit) + except ValueError as exc: + print(exc.args[0]) + parser.print_usage() + sys.exit(1) + + if not cc_list: + cc_list = sorted(POP20_CC) + return args, cc_list + + +def main(download_many, default_concur_req, max_concur_req): + args, cc_list = process_args(default_concur_req) + actual_req = min(args.max_req, max_concur_req, len(cc_list)) + initial_report(cc_list, actual_req, args.server) + base_url = SERVERS[args.server] + t0 = time.time() + counter = download_many(cc_list, base_url, args.verbose, actual_req) + assert sum(counter.values()) == len(cc_list), \ + 'some downloads are unaccounted for' + final_report(cc_list, counter, t0) diff --git a/17-futures-py3.7/countries/flags2_sequential.py b/17-futures-py3.7/countries/flags2_sequential.py new file mode 100644 index 0000000..65a7e43 --- /dev/null +++ b/17-futures-py3.7/countries/flags2_sequential.py @@ -0,0 +1,87 @@ +"""Download flags of countries (with error handling). + +Sequential version + +Sample run:: + + $ python3 flags2_sequential.py -s DELAY b + DELAY site: http://localhost:8002/flags + Searching for 26 flags: from BA to BZ + 1 concurrent connection will be used. + -------------------- + 17 flags downloaded. + 9 not found. + Elapsed time: 13.36s + +""" + +import collections + +import requests +import tqdm + +from flags2_common import main, save_flag, HTTPStatus, Result + + +DEFAULT_CONCUR_REQ = 1 +MAX_CONCUR_REQ = 1 + +# BEGIN FLAGS2_BASIC_HTTP_FUNCTIONS +def get_flag(base_url, cc): + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + resp = requests.get(url) + if resp.status_code != 200: # <1> + resp.raise_for_status() + return resp.content + + +def download_one(cc, base_url, verbose=False): + try: + image = get_flag(base_url, cc) + except requests.exceptions.HTTPError as exc: # <2> + res = exc.response + if res.status_code == 404: + status = HTTPStatus.not_found # <3> + msg = 'not found' + else: # <4> + raise + else: + save_flag(image, cc.lower() + '.gif') + status = HTTPStatus.ok + msg = 'OK' + + if verbose: # <5> + print(cc, msg) + + return Result(status, cc) # <6> +# END FLAGS2_BASIC_HTTP_FUNCTIONS + +# BEGIN FLAGS2_DOWNLOAD_MANY_SEQUENTIAL +def download_many(cc_list, base_url, verbose, max_req): + counter = collections.Counter() # <1> + cc_iter = sorted(cc_list) # <2> + if not verbose: + cc_iter = tqdm.tqdm(cc_iter) # <3> + for cc in cc_iter: # <4> + try: + res = download_one(cc, base_url, verbose) # <5> + except requests.exceptions.HTTPError as exc: # <6> + error_msg = 'HTTP error {res.status_code} - {res.reason}' + error_msg = error_msg.format(res=exc.response) + except requests.exceptions.ConnectionError as exc: # <7> + error_msg = 'Connection error' + else: # <8> + error_msg = '' + status = res.status + + if error_msg: + status = HTTPStatus.error # <9> + counter[status] += 1 # <10> + if verbose and error_msg: # <11> + print('*** Error for {}: {}'.format(cc, error_msg)) + + return counter # <12> +# END FLAGS2_DOWNLOAD_MANY_SEQUENTIAL + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) diff --git a/17-futures-py3.7/countries/flags2_threadpool.py b/17-futures-py3.7/countries/flags2_threadpool.py new file mode 100644 index 0000000..069d4ff --- /dev/null +++ b/17-futures-py3.7/countries/flags2_threadpool.py @@ -0,0 +1,68 @@ +"""Download flags of countries (with error handling). + +ThreadPool version + +Sample run:: + + $ python3 flags2_threadpool.py -s ERROR -e + ERROR site: http://localhost:8003/flags + Searching for 676 flags: from AA to ZZ + 30 concurrent connections will be used. + -------------------- + 150 flags downloaded. + 361 not found. + 165 errors. + Elapsed time: 7.46s + +""" + +# BEGIN FLAGS2_THREADPOOL +import collections +from concurrent import futures + +import requests +import tqdm # <1> + +from flags2_common import main, HTTPStatus # <2> +from flags2_sequential import download_one # <3> + +DEFAULT_CONCUR_REQ = 30 # <4> +MAX_CONCUR_REQ = 1000 # <5> + + +def download_many(cc_list, base_url, verbose, concur_req): + counter = collections.Counter() + with futures.ThreadPoolExecutor(max_workers=concur_req) as executor: # <6> + to_do_map = {} # <7> + for cc in sorted(cc_list): # <8> + future = executor.submit(download_one, + cc, base_url, verbose) # <9> + to_do_map[future] = cc # <10> + done_iter = futures.as_completed(to_do_map) # <11> + if not verbose: + done_iter = tqdm.tqdm(done_iter, total=len(cc_list)) # <12> + for future in done_iter: # <13> + try: + res = future.result() # <14> + except requests.exceptions.HTTPError as exc: # <15> + error_msg = 'HTTP {res.status_code} - {res.reason}' + error_msg = error_msg.format(res=exc.response) + except requests.exceptions.ConnectionError as exc: + error_msg = 'Connection error' + else: + error_msg = '' + status = res.status + + if error_msg: + status = HTTPStatus.error + counter[status] += 1 + if verbose and error_msg: + cc = to_do_map[future] # <16> + print('*** Error for {}: {}'.format(cc, error_msg)) + + return counter + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) +# END FLAGS2_THREADPOOL diff --git a/17-futures-py3.7/countries/flags_asyncio.py b/17-futures-py3.7/countries/flags_asyncio.py new file mode 100644 index 0000000..89421f8 --- /dev/null +++ b/17-futures-py3.7/countries/flags_asyncio.py @@ -0,0 +1,72 @@ +"""Download flags of top 20 countries by population + +asyncio + aiottp version + +Sample run:: + + $ python3 flags_asyncio.py + CN EG BR IN ID RU NG VN JP DE TR PK FR ET MX PH US IR CD BD + 20 flags downloaded in 0.35s + +""" +# BEGIN FLAGS_ASYNCIO +import os +import time +import sys +import asyncio # <1> + +import aiohttp # <2> + + +POP20_CC = ('CN IN US ID BR PK NG BD RU JP ' + 'MX PH VN ET EG DE IR TR CD FR').split() + +BASE_URL = 'http://flupy.org/data/flags' + +DEST_DIR = 'downloads/' + + +def save_flag(img, filename): + path = os.path.join(DEST_DIR, filename) + with open(path, 'wb') as fp: + fp.write(img) + + +async def get_flag(session, cc): # <3> + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + async with session.get(url) as resp: # <4> + return await resp.read() # <5> + + +def show(text): + print(text, end=' ') + sys.stdout.flush() + + +async def download_one(session, cc): # <6> + image = await get_flag(session, cc) # <7> + show(cc) + save_flag(image, cc.lower() + '.gif') + return cc + + +async def download_many(cc_list): + async with aiohttp.ClientSession() as session: # <8> + res = await asyncio.gather( # <9> + *[asyncio.create_task(download_one(session, cc)) + for cc in sorted(cc_list)]) + + return len(res) + + +def main(): # <10> + t0 = time.time() + count = asyncio.run(download_many(POP20_CC)) + elapsed = time.time() - t0 + msg = '\n{} flags downloaded in {:.2f}s' + print(msg.format(count, elapsed)) + + +if __name__ == '__main__': + main() +# END FLAGS_ASYNCIO diff --git a/17-futures-py3.7/countries/flags_threadpool.py b/17-futures-py3.7/countries/flags_threadpool.py new file mode 100644 index 0000000..47a5ee6 --- /dev/null +++ b/17-futures-py3.7/countries/flags_threadpool.py @@ -0,0 +1,71 @@ +"""Download flags of top 20 countries by population + +ThreadPoolExecutor version + +Sample run:: + + $ python3 flags_threadpool.py + DE FR BD CN EG RU IN TR VN ID JP BR NG MX PK ET PH CD US IR + 20 flags downloaded in 0.35s + +""" +# BEGIN FLAGS_THREADPOOL +import os +import time +import sys +from concurrent import futures # <1> + +import requests + +POP20_CC = ('CN IN US ID BR PK NG BD RU JP ' + 'MX PH VN ET EG DE IR TR CD FR').split() + +BASE_URL = 'http://flupy.org/data/flags' + +DEST_DIR = 'downloads/' + +MAX_WORKERS = 20 # <2> + +def save_flag(img, filename): + path = os.path.join(DEST_DIR, filename) + with open(path, 'wb') as fp: + fp.write(img) + + +def get_flag(cc): + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + resp = requests.get(url) + return resp.content + + +def show(text): + print(text, end=' ') + sys.stdout.flush() + + +def download_one(cc): # <3> + image = get_flag(cc) + show(cc) + save_flag(image, cc.lower() + '.gif') + return cc + + +def download_many(cc_list): + workers = min(MAX_WORKERS, len(cc_list)) # <4> + with futures.ThreadPoolExecutor(workers) as executor: # <5> + res = executor.map(download_one, sorted(cc_list)) # <6> + + return len(list(res)) # <7> + + +def main(): # <10> + t0 = time.time() + count = download_many(POP20_CC) + elapsed = time.time() - t0 + msg = '\n{} flags downloaded in {:.2f}s' + print(msg.format(count, elapsed)) + + +if __name__ == '__main__': + main() +# END FLAGS_THREADPOOL diff --git a/17-futures-py3.7/countries/requirements.txt b/17-futures-py3.7/countries/requirements.txt new file mode 100644 index 0000000..aa7a6de --- /dev/null +++ b/17-futures-py3.7/countries/requirements.txt @@ -0,0 +1,2 @@ +requests==2.21.0 +aiohttp==3.5.4 diff --git a/17-futures-py3.7/demo_executor_map.py b/17-futures-py3.7/demo_executor_map.py new file mode 100644 index 0000000..f3625cf --- /dev/null +++ b/17-futures-py3.7/demo_executor_map.py @@ -0,0 +1,34 @@ +""" +Experiment with ``ThreadPoolExecutor.map`` +""" +# BEGIN EXECUTOR_MAP +from time import sleep, strftime +from concurrent import futures + + +def display(*args): # <1> + print(strftime('[%H:%M:%S]'), end=' ') + print(*args) + + +def loiter(n): # <2> + msg = '{}loiter({}): doing nothing for {}s...' + display(msg.format('\t'*n, n, n)) + sleep(n) + msg = '{}loiter({}): done.' + display(msg.format('\t'*n, n)) + return n * 10 # <3> + + +def main(): + display('Script starting.') + executor = futures.ThreadPoolExecutor(max_workers=3) # <4> + results = executor.map(loiter, range(5)) # <5> + display('results:', results) # <6>. + display('Waiting for individual results:') + for i, result in enumerate(results): # <7> + display('result {}: {}'.format(i, result)) + + +main() +# END EXECUTOR_MAP diff --git a/17-futures/countries/README.rst b/17-futures/countries/README.rst index e4f2c6f..3785ffd 100644 --- a/17-futures/countries/README.rst +++ b/17-futures/countries/README.rst @@ -151,7 +151,7 @@ Using that same shell with the ``.env27`` environment active, run the ``vaurien_ The ``vaurien_delay.sh`` creates an HTTP proxy on port 8002 which adds a 1s delay to every response. You can test it with a browser on port 8002: http://localhost:8002/flags/ad/ad.gif -- the flag of Andorra should appear after 1 second. -There is also the ``vaurien_error_delay.sh`` script which runs a proxy on port 8003 producing errors in 25% of the responses and a .5 se delay to 50% of the responses. You can also test it with the browser on port 8003, but rememeber that errors are expected. +There is also the ``vaurien_error_delay.sh`` script which runs a proxy on port 8003 producing errors in 25% of the responses and a .5 s delay to 50% of the responses. You can also test it with the browser on port 8003, but rememeber that errors are expected. Platform-specific instructions diff --git a/17-futures/countries/flags2_asyncio.py b/17-futures/countries/flags2_asyncio.py index a74c975..d3c78f8 100644 --- a/17-futures/countries/flags2_asyncio.py +++ b/17-futures/countries/flags2_asyncio.py @@ -1,6 +1,6 @@ """Download flags of countries (with error handling). -asyncio version +asyncio yield-from version Sample run:: @@ -18,6 +18,7 @@ # BEGIN FLAGS2_ASYNCIO_TOP import asyncio import collections +import contextlib import aiohttp from aiohttp import web @@ -40,15 +41,16 @@ def __init__(self, country_code): def get_flag(base_url, cc): # <2> url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) resp = yield from aiohttp.request('GET', url) - if resp.status == 200: - image = yield from resp.read() - return image - elif resp.status == 404: - raise web.HTTPNotFound() - else: - raise aiohttp.HttpProcessingError( - code=resp.status, message=resp.reason, - headers=resp.headers) + with contextlib.closing(resp): + if resp.status == 200: + image = yield from resp.read() + return image + elif resp.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.HttpProcessingError( + code=resp.status, message=resp.reason, + headers=resp.headers) @asyncio.coroutine diff --git a/17-futures/countries/flags2_asyncio_executor.py b/17-futures/countries/flags2_asyncio_executor.py index 60102a0..53ade4b 100644 --- a/17-futures/countries/flags2_asyncio_executor.py +++ b/17-futures/countries/flags2_asyncio_executor.py @@ -2,14 +2,11 @@ asyncio version using thread pool to save files -Sample run:: - - $ - """ import asyncio import collections +import contextlib import aiohttp from aiohttp import web @@ -32,15 +29,17 @@ def __init__(self, country_code): def get_flag(base_url, cc): url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) resp = yield from aiohttp.request('GET', url) - if resp.status == 200: - image = yield from resp.read() - return image - elif resp.status == 404: - raise web.HTTPNotFound() - else: - raise aiohttp.HttpProcessingError( - code=resp.status, message=resp.reason, - headers=resp.headers) + with contextlib.closing(resp): + if resp.status == 200: + image = yield from resp.read() + return image + elif resp.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.HttpProcessingError( + code=resp.status, message=resp.reason, + headers=resp.headers) + # BEGIN FLAGS2_ASYNCIO_EXECUTOR @asyncio.coroutine @@ -85,16 +84,14 @@ def downloader_coro(cc_list, base_url, verbose, concur_req): error_msg = exc.__cause__.args[0] except IndexError: error_msg = exc.__cause__.__class__.__name__ + if verbose and error_msg: + msg = '*** Error for {}: {}' + print(msg.format(country_code, error_msg)) + status = HTTPStatus.error else: - error_msg = '' status = res.status - if error_msg: - status = HTTPStatus.error counter[status] += 1 - if verbose and error_msg: - msg = '*** Error for {}: {}' - print(msg.format(country_code, error_msg)) return counter diff --git a/17-futures/countries/flags2_await.py b/17-futures/countries/flags2_await.py new file mode 100644 index 0000000..8443484 --- /dev/null +++ b/17-futures/countries/flags2_await.py @@ -0,0 +1,104 @@ +"""Download flags of countries (with error handling). + +asyncio async/await version + +""" +# BEGIN FLAGS2_ASYNCIO_TOP +import asyncio +import collections +from contextlib import closing + +import aiohttp +from aiohttp import web +import tqdm + +from flags2_common import main, HTTPStatus, Result, save_flag + +# default set low to avoid errors from remote site, such as +# 503 - Service Temporarily Unavailable +DEFAULT_CONCUR_REQ = 5 +MAX_CONCUR_REQ = 1000 + + +class FetchError(Exception): # <1> + def __init__(self, country_code): + self.country_code = country_code + + +async def get_flag(base_url, cc): # <2> + url = '{}/{cc}/{cc}.gif'.format(base_url, cc=cc.lower()) + with closing(await aiohttp.request('GET', url)) as resp: + if resp.status == 200: + image = await resp.read() + return image + elif resp.status == 404: + raise web.HTTPNotFound() + else: + raise aiohttp.HttpProcessingError( + code=resp.status, message=resp.reason, + headers=resp.headers) + + +async def download_one(cc, base_url, semaphore, verbose): # <3> + try: + with (await semaphore): # <4> + image = await get_flag(base_url, cc) # <5> + except web.HTTPNotFound: # <6> + status = HTTPStatus.not_found + msg = 'not found' + except Exception as exc: + raise FetchError(cc) from exc # <7> + else: + save_flag(image, cc.lower() + '.gif') # <8> + status = HTTPStatus.ok + msg = 'OK' + + if verbose and msg: + print(cc, msg) + + return Result(status, cc) +# END FLAGS2_ASYNCIO_TOP + +# BEGIN FLAGS2_ASYNCIO_DOWNLOAD_MANY +async def downloader_coro(cc_list, base_url, verbose, concur_req): # <1> + counter = collections.Counter() + semaphore = asyncio.Semaphore(concur_req) # <2> + to_do = [download_one(cc, base_url, semaphore, verbose) + for cc in sorted(cc_list)] # <3> + + to_do_iter = asyncio.as_completed(to_do) # <4> + if not verbose: + to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list)) # <5> + for future in to_do_iter: # <6> + try: + res = await future # <7> + except FetchError as exc: # <8> + country_code = exc.country_code # <9> + try: + error_msg = exc.__cause__.args[0] # <10> + except IndexError: + error_msg = exc.__cause__.__class__.__name__ # <11> + if verbose and error_msg: + msg = '*** Error for {}: {}' + print(msg.format(country_code, error_msg)) + status = HTTPStatus.error + else: + status = res.status + + counter[status] += 1 # <12> + + return counter # <13> + + +def download_many(cc_list, base_url, verbose, concur_req): + loop = asyncio.get_event_loop() + coro = downloader_coro(cc_list, base_url, verbose, concur_req) + counts = loop.run_until_complete(coro) # <14> + loop.close() # <15> + + return counts + + +if __name__ == '__main__': + main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) +# END FLAGS2_ASYNCIO_DOWNLOAD_MANY diff --git a/17-futures/countries/flags3_asyncio.py b/17-futures/countries/flags3_asyncio.py index f3961a1..3340367 100644 --- a/17-futures/countries/flags3_asyncio.py +++ b/17-futures/countries/flags3_asyncio.py @@ -1,11 +1,6 @@ -"""Download flags of countries (with error handling). - -asyncio version using thread pool to save files - -Sample run:: - - $ +"""Download flags and names of countries. +asyncio version """ import asyncio diff --git a/17-futures/countries/flags3_threadpool.py b/17-futures/countries/flags3_threadpool.py index 6bca9b1..e5e4b78 100644 --- a/17-futures/countries/flags3_threadpool.py +++ b/17-futures/countries/flags3_threadpool.py @@ -1,10 +1,6 @@ """Download flags and names of countries. ThreadPool version - -Sample run:: - - """ import collections diff --git a/17-futures/countries/flags_await.py b/17-futures/countries/flags_await.py new file mode 100644 index 0000000..b02dde9 --- /dev/null +++ b/17-futures/countries/flags_await.py @@ -0,0 +1,46 @@ +"""Download flags of top 20 countries by population + +asyncio + aiottp version + +Sample run:: + + $ python3 flags_asyncio.py + EG VN IN TR RU ID US DE CN MX JP BD NG ET FR BR PH PK CD IR + 20 flags downloaded in 1.07s + +""" +# BEGIN FLAGS_ASYNCIO +import asyncio + +import aiohttp # <1> + +from flags import BASE_URL, save_flag, show, main # <2> + + +async def get_flag(cc): # <3> + url = '{}/{cc}/{cc}.gif'.format(BASE_URL, cc=cc.lower()) + resp = await aiohttp.request('GET', url) # <4> + image = await resp.read() # <5> + return image + + +async def download_one(cc): # <6> + image = await get_flag(cc) # <7> + show(cc) + save_flag(image, cc.lower() + '.gif') + return cc + + +def download_many(cc_list): + loop = asyncio.get_event_loop() # <8> + to_do = [download_one(cc) for cc in sorted(cc_list)] # <9> + wait_coro = asyncio.wait(to_do) # <10> + res, _ = loop.run_until_complete(wait_coro) # <11> + loop.close() # <12> + + return len(res) + + +if __name__ == '__main__': + main(download_many) +# END FLAGS_ASYNCIO diff --git a/17-futures/countries/requirements.txt b/17-futures/countries/requirements.txt new file mode 100644 index 0000000..6f29576 --- /dev/null +++ b/17-futures/countries/requirements.txt @@ -0,0 +1,3 @@ +aiohttp==0.13.1 +requests==2.21.0 +tqdm==1.0 diff --git a/18-asyncio-py3.7/README.rst b/18-asyncio-py3.7/README.rst new file mode 100644 index 0000000..aad52a3 --- /dev/null +++ b/18-asyncio-py3.7/README.rst @@ -0,0 +1,11 @@ +Refactored sample code for Chapter 18 - "Concurrency with asyncio" + +From the book "Fluent Python" by Luciano Ramalho (O'Reilly, 2015) +http://shop.oreilly.com/product/0636920032519.do + +################################################################## +NOTE: this directory contains the examples of chapter 18 +rewritten using the new async/await syntax available from Python +3.5+, instead of the "yield-from" syntax of Python 3.3 and 3.4. +The code was tested with Python 3.7 +################################################################## diff --git a/18-asyncio-py3.7/charfinder/.gitignore b/18-asyncio-py3.7/charfinder/.gitignore new file mode 100644 index 0000000..f93ecca --- /dev/null +++ b/18-asyncio-py3.7/charfinder/.gitignore @@ -0,0 +1 @@ +charfinder_index.pickle diff --git a/18-asyncio-py3.7/charfinder/charfinder.py b/18-asyncio-py3.7/charfinder/charfinder.py new file mode 100755 index 0000000..64e4949 --- /dev/null +++ b/18-asyncio-py3.7/charfinder/charfinder.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 + +""" +Unicode character finder utility: +find characters based on words in their official names. + +This can be used from the command line, just pass words as arguments. + +Here is the ``main`` function which makes it happen:: + + >>> main('rook') # doctest: +NORMALIZE_WHITESPACE + U+2656 ♖ WHITE CHESS ROOK + U+265C ♜ BLACK CHESS ROOK + (2 matches for 'rook') + >>> main('rook', 'black') # doctest: +NORMALIZE_WHITESPACE + U+265C ♜ BLACK CHESS ROOK + (1 match for 'rook black') + >>> main('white bishop') # doctest: +NORMALIZE_WHITESPACE + U+2657 ♗ WHITE CHESS BISHOP + (1 match for 'white bishop') + >>> main("jabberwocky's vest") + (No match for "jabberwocky's vest") + + +For exploring words that occur in the character names, there is the +``word_report`` function:: + + >>> index = UnicodeNameIndex(sample_chars) + >>> index.word_report() + 3 SIGN + 2 A + 2 EURO + 2 LATIN + 2 LETTER + 1 CAPITAL + 1 CURRENCY + 1 DOLLAR + 1 SMALL + >>> index = UnicodeNameIndex() + >>> index.word_report(10) + 75821 CJK + 75761 IDEOGRAPH + 74656 UNIFIED + 13196 SYLLABLE + 11735 HANGUL + 7616 LETTER + 2232 WITH + 2180 SIGN + 2122 SMALL + 1709 CAPITAL + +Note: characters with names starting with 'CJK UNIFIED IDEOGRAPH' +are indexed with those three words only, excluding the hexadecimal +codepoint at the end of the name. + +""" + +import sys +import re +import unicodedata +import pickle +import warnings +import itertools +import functools +from collections import namedtuple + +RE_WORD = re.compile(r'\w+') +RE_UNICODE_NAME = re.compile('^[A-Z0-9 -]+$') +RE_CODEPOINT = re.compile(r'U\+([0-9A-F]{4,6})') + +INDEX_NAME = 'charfinder_index.pickle' +MINIMUM_SAVE_LEN = 10000 +CJK_UNI_PREFIX = 'CJK UNIFIED IDEOGRAPH' +CJK_CMP_PREFIX = 'CJK COMPATIBILITY IDEOGRAPH' + +sample_chars = [ + '$', # DOLLAR SIGN + 'A', # LATIN CAPITAL LETTER A + 'a', # LATIN SMALL LETTER A + '\u20a0', # EURO-CURRENCY SIGN + '\u20ac', # EURO SIGN +] + +CharDescription = namedtuple('CharDescription', 'code_str char name') + +QueryResult = namedtuple('QueryResult', 'count items') + + +def tokenize(text): + """return iterable of uppercased words""" + for match in RE_WORD.finditer(text): + yield match.group().upper() + + +def query_type(text): + text_upper = text.upper() + if 'U+' in text_upper: + return 'CODEPOINT' + elif RE_UNICODE_NAME.match(text_upper): + return 'NAME' + else: + return 'CHARACTERS' + + +class UnicodeNameIndex: + + def __init__(self, chars=None): + self.load(chars) + + def load(self, chars=None): + self.index = None + if chars is None: + try: + with open(INDEX_NAME, 'rb') as fp: + self.index = pickle.load(fp) + except OSError: + pass + if self.index is None: + self.build_index(chars) + if len(self.index) > MINIMUM_SAVE_LEN: + try: + self.save() + except OSError as exc: + warnings.warn('Could not save {!r}: {}' + .format(INDEX_NAME, exc)) + + def save(self): + with open(INDEX_NAME, 'wb') as fp: + pickle.dump(self.index, fp) + + def build_index(self, chars=None): + if chars is None: + chars = (chr(i) for i in range(32, sys.maxunicode)) + index = {} + for char in chars: + try: + name = unicodedata.name(char) + except ValueError: + continue + if name.startswith(CJK_UNI_PREFIX): + name = CJK_UNI_PREFIX + elif name.startswith(CJK_CMP_PREFIX): + name = CJK_CMP_PREFIX + + for word in tokenize(name): + index.setdefault(word, set()).add(char) + + self.index = index + + def word_rank(self, top=None): + res = [(len(self.index[key]), key) for key in self.index] + res.sort(key=lambda item: (-item[0], item[1])) + if top is not None: + res = res[:top] + return res + + def word_report(self, top=None): + for postings, key in self.word_rank(top): + print('{:5} {}'.format(postings, key)) + + def find_chars(self, query, start=0, stop=None): + stop = sys.maxsize if stop is None else stop + result_sets = [] + for word in tokenize(query): + chars = self.index.get(word) + if chars is None: # shorcut: no such word + result_sets = [] + break + result_sets.append(chars) + + if not result_sets: + return QueryResult(0, ()) + + result = functools.reduce(set.intersection, result_sets) + result = sorted(result) # must sort to support start, stop + result_iter = itertools.islice(result, start, stop) + return QueryResult(len(result), + (char for char in result_iter)) + + def describe(self, char): + code_str = 'U+{:04X}'.format(ord(char)) + name = unicodedata.name(char) + return CharDescription(code_str, char, name) + + def find_descriptions(self, query, start=0, stop=None): + for char in self.find_chars(query, start, stop).items: + yield self.describe(char) + + def get_descriptions(self, chars): + for char in chars: + yield self.describe(char) + + def describe_str(self, char): + return '{:7}\t{}\t{}'.format(*self.describe(char)) + + def find_description_strs(self, query, start=0, stop=None): + for char in self.find_chars(query, start, stop).items: + yield self.describe_str(char) + + @staticmethod # not an instance method due to concurrency + def status(query, counter): + if counter == 0: + msg = 'No match' + elif counter == 1: + msg = '1 match' + else: + msg = '{} matches'.format(counter) + return '{} for {!r}'.format(msg, query) + + +def main(*args): + index = UnicodeNameIndex() + query = ' '.join(args) + n = 0 + for n, line in enumerate(index.find_description_strs(query), 1): + print(line) + print('({})'.format(index.status(query, n))) + +if __name__ == '__main__': + if len(sys.argv) > 1: + main(*sys.argv[1:]) + else: + print('Usage: {} word1 [word2]...'.format(sys.argv[0])) diff --git a/18-asyncio-py3.7/charfinder/tcp_charfinder.py b/18-asyncio-py3.7/charfinder/tcp_charfinder.py new file mode 100755 index 0000000..4980b92 --- /dev/null +++ b/18-asyncio-py3.7/charfinder/tcp_charfinder.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +# BEGIN TCP_CHARFINDER_TOP +import sys +import asyncio + +from charfinder import UnicodeNameIndex # <1> + +CRLF = b'\r\n' +PROMPT = b'?> ' + +index = UnicodeNameIndex() # <2> + +async def handle_queries(reader, writer): # <3> + while True: # <4> + writer.write(PROMPT) # can't await! # <5> + await writer.drain() # must await! # <6> + data = await reader.readline() # <7> + try: + query = data.decode().strip() + except UnicodeDecodeError: # <8> + query = '\x00' + client = writer.get_extra_info('peername') # <9> + print('Received from {}: {!r}'.format(client, query)) # <10> + if query: + if ord(query[:1]) < 32: # <11> + break + lines = list(index.find_description_strs(query)) # <12> + if lines: + writer.writelines(line.encode() + CRLF for line in lines) # <13> + writer.write(index.status(query, len(lines)).encode() + CRLF) # <14> + + await writer.drain() # <15> + print('Sent {} results'.format(len(lines))) # <16> + + print('Close the client socket') # <17> + writer.close() # <18> +# END TCP_CHARFINDER_TOP + +# BEGIN TCP_CHARFINDER_MAIN +async def main(address='127.0.0.1', port=2323): # <1> + port = int(port) + server = await asyncio.start_server(handle_queries, address, port) # <2> + + host = server.sockets[0].getsockname() # <3> + print('Serving on {}. Hit CTRL-C to stop.'.format(host)) # <4> + + async with server: + await server.serve_forever() + + +if __name__ == '__main__': + asyncio.run(main(*sys.argv[1:])) # <5> +# END TCP_CHARFINDER_MAIN diff --git a/18-asyncio-py3.7/charfinder/test_charfinder.py b/18-asyncio-py3.7/charfinder/test_charfinder.py new file mode 100644 index 0000000..a6fe729 --- /dev/null +++ b/18-asyncio-py3.7/charfinder/test_charfinder.py @@ -0,0 +1,115 @@ +import pytest + +from charfinder import UnicodeNameIndex, tokenize, sample_chars, query_type +from unicodedata import name + + +@pytest.fixture +def sample_index(): + return UnicodeNameIndex(sample_chars) + + +@pytest.fixture(scope="module") +def full_index(): + return UnicodeNameIndex() + + +def test_query_type(): + assert query_type('blue') == 'NAME' + + +def test_tokenize(): + assert list(tokenize('')) == [] + assert list(tokenize('a b')) == ['A', 'B'] + assert list(tokenize('a-b')) == ['A', 'B'] + assert list(tokenize('abc')) == ['ABC'] + assert list(tokenize('café')) == ['CAFÉ'] + + +def test_index(): + sample_index = UnicodeNameIndex(sample_chars) + assert len(sample_index.index) == 9 + + +def test_find_word_no_match(sample_index): + res = sample_index.find_chars('qwertyuiop') + assert len(res.items) == 0 + + +def test_find_word_1_match(sample_index): + res = [(ord(char), name(char)) + for char in sample_index.find_chars('currency').items] + assert res == [(8352, 'EURO-CURRENCY SIGN')] + + +def test_find_word_1_match_character_result(sample_index): + res = [name(char) for char in + sample_index.find_chars('currency').items] + assert res == ['EURO-CURRENCY SIGN'] + + +def test_find_word_2_matches(sample_index): + res = [(ord(char), name(char)) + for char in sample_index.find_chars('Euro').items] + assert res == [(8352, 'EURO-CURRENCY SIGN'), + (8364, 'EURO SIGN')] + + +def test_find_2_words_no_matches(sample_index): + res = sample_index.find_chars('Euro letter') + assert res.count == 0 + + +def test_find_2_words_no_matches_because_one_not_found(sample_index): + res = sample_index.find_chars('letter qwertyuiop') + assert res.count == 0 + + +def test_find_2_words_1_match(sample_index): + res = sample_index.find_chars('sign dollar') + assert res.count == 1 + + +def test_find_2_words_2_matches(sample_index): + res = sample_index.find_chars('latin letter') + assert res.count == 2 + + +def test_find_chars_many_matches_full(full_index): + res = full_index.find_chars('letter') + assert res.count > 7000 + + +def test_find_1_word_1_match_full(full_index): + res = [(ord(char), name(char)) + for char in full_index.find_chars('registered').items] + assert res == [(174, 'REGISTERED SIGN')] + + +def test_find_1_word_2_matches_full(full_index): + res = full_index.find_chars('rook') + assert res.count == 2 + + +def test_find_3_words_no_matches_full(full_index): + res = full_index.find_chars('no such character') + assert res.count == 0 + + +def test_find_with_start(sample_index): + res = [(ord(char), name(char)) + for char in sample_index.find_chars('sign', 1).items] + assert res == [(8352, 'EURO-CURRENCY SIGN'), (8364, 'EURO SIGN')] + + +def test_find_with_stop(sample_index): + res = [(ord(char), name(char)) + for char in sample_index.find_chars('sign', 0, 2).items] + assert res == [(36, 'DOLLAR SIGN'), (8352, 'EURO-CURRENCY SIGN')] + + +def test_find_with_start_stop(sample_index): + res = [(ord(char), name(char)) + for char in sample_index.find_chars('sign', 1, 2).items] + assert res == [(8352, 'EURO-CURRENCY SIGN')] + diff --git a/18-asyncio-py3.7/countdown.py b/18-asyncio-py3.7/countdown.py new file mode 100644 index 0000000..5f09000 --- /dev/null +++ b/18-asyncio-py3.7/countdown.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +# Inspired by +# https://snarky.ca/how-the-heck-does-async-await-work-in-python-3-5/ + +import asyncio +import time + + +async def countdown(label, delay): + tabs = (ord(label) - ord('A')) * '\t' + n = 3 + while n > 0: + await asyncio.sleep(delay) # <---- + dt = time.perf_counter() - t0 + print('━' * 50) + print(f'{dt:7.4f}s \t{tabs}{label} = {n}') + n -= 1 + +loop = asyncio.get_event_loop() +tasks = [ + loop.create_task(countdown('A', .7)), + loop.create_task(countdown('B', 2)), + loop.create_task(countdown('C', .3)), + loop.create_task(countdown('D', 1)), +] +t0 = time.perf_counter() +loop.run_until_complete(asyncio.wait(tasks)) +loop.close() +print('━' * 50) diff --git a/18-asyncio-py3.7/countries/README.rst b/18-asyncio-py3.7/countries/README.rst new file mode 100644 index 0000000..4945806 --- /dev/null +++ b/18-asyncio-py3.7/countries/README.rst @@ -0,0 +1,3 @@ +The ``asyncio`` flag download examples are in the +``../../17-futures/countries/`` directory together +with the sequential and threadpool examples. diff --git a/18-asyncio-py3.7/spinner_asyncio.py b/18-asyncio-py3.7/spinner_asyncio.py new file mode 100755 index 0000000..369a8f0 --- /dev/null +++ b/18-asyncio-py3.7/spinner_asyncio.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +# spinner_asyncio.py + +# credits: Example by Luciano Ramalho inspired by +# Michele Simionato's multiprocessing example in the python-list: +# https://mail.python.org/pipermail/python-list/2009-February/538048.html + +# BEGIN SPINNER_ASYNCIO +import asyncio +import itertools + + +async def spin(msg): # <1> + for char in itertools.cycle('|/-\\'): + status = char + ' ' + msg + print(status, flush=True, end='\r') + try: + await asyncio.sleep(.1) # <2> + except asyncio.CancelledError: # <3> + break + print(' ' * len(status), end='\r') + + +async def slow_function(): # <4> + # pretend waiting a long time for I/O + await asyncio.sleep(3) # <5> + return 42 + + +async def supervisor(): # <6> + spinner = asyncio.create_task(spin('thinking!')) # <7> + print('spinner object:', spinner) # <8> + result = await slow_function() # <9> + spinner.cancel() # <10> + return result + + +def main(): + result = asyncio.run(supervisor()) # <11> + print('Answer:', result) + + +if __name__ == '__main__': + main() +# END SPINNER_ASYNCIO diff --git a/18-asyncio-py3.7/spinner_thread.py b/18-asyncio-py3.7/spinner_thread.py new file mode 100755 index 0000000..bffc921 --- /dev/null +++ b/18-asyncio-py3.7/spinner_thread.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# spinner_thread.py + +# credits: Adapted from Michele Simionato's +# multiprocessing example in the python-list: +# https://mail.python.org/pipermail/python-list/2009-February/538048.html + +# BEGIN SPINNER_THREAD +import threading +import itertools +import time + + +def spin(msg, done): # <1> + for char in itertools.cycle('|/-\\'): # <3> + status = char + ' ' + msg + print(status, flush=True, end='\r') + if done.wait(.1): # <5> + break + print(' ' * len(status), end='\r') + +def slow_function(): # <7> + # pretend waiting a long time for I/O + time.sleep(3) # <8> + return 42 + + +def supervisor(): # <9> + done = threading.Event() + spinner = threading.Thread(target=spin, + args=('thinking!', done)) + print('spinner object:', spinner) # <10> + spinner.start() # <11> + result = slow_function() # <12> + done.set() # <13> + spinner.join() # <14> + return result + + +def main(): + result = supervisor() # <15> + print('Answer:', result) + + +if __name__ == '__main__': + main() +# END SPINNER_THREAD diff --git a/18-asyncio/charfinder/charfinder.py b/18-asyncio/charfinder/charfinder.py index 7e06792..c061f90 100755 --- a/18-asyncio/charfinder/charfinder.py +++ b/18-asyncio/charfinder/charfinder.py @@ -64,7 +64,7 @@ import functools from collections import namedtuple -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') RE_UNICODE_NAME = re.compile('^[A-Z0-9 -]+$') RE_CODEPOINT = re.compile('U\+([0-9A-F]{4,6})') @@ -163,7 +163,7 @@ def find_chars(self, query, start=0, stop=None): result_sets = [] for word in tokenize(query): chars = self.index.get(word) - if chars is None: # shorcut: no such word + if chars is None: # shortcut: no such word result_sets = [] break result_sets.append(chars) diff --git a/18-asyncio/charfinder/http_charfinder.py b/18-asyncio/charfinder/http_charfinder.py index a11e0ed..d7e8ca5 100755 --- a/18-asyncio/charfinder/http_charfinder.py +++ b/18-asyncio/charfinder/http_charfinder.py @@ -29,7 +29,7 @@ def home(request): # <1> print('Query: {!r}'.format(query)) # <3> if query: # <4> descriptions = list(index.find_descriptions(query)) - res = '\n'.join(ROW_TPL.format(**vars(descr)) + res = '\n'.join(ROW_TPL.format(**descr._asdict()) for descr in descriptions) msg = index.status(query, len(descriptions)) else: diff --git a/18-asyncio/charfinder/requirements.txt b/18-asyncio/charfinder/requirements.txt new file mode 100644 index 0000000..2d16731 --- /dev/null +++ b/18-asyncio/charfinder/requirements.txt @@ -0,0 +1 @@ +aiohttp==0.13.1 diff --git a/18-asyncio/spinner_asyncio.py b/18-asyncio/spinner_asyncio.py old mode 100644 new mode 100755 index 3366998..aeb2b55 --- a/18-asyncio/spinner_asyncio.py +++ b/18-asyncio/spinner_asyncio.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # spinner_asyncio.py # credits: Example by Luciano Ramalho inspired by diff --git a/18-asyncio/spinner_await.py b/18-asyncio/spinner_await.py new file mode 100755 index 0000000..ad9c9b0 --- /dev/null +++ b/18-asyncio/spinner_await.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# spinner_await.py + +# credits: Example by Luciano Ramalho inspired by +# Michele Simionato's multiprocessing example in the python-list: +# https://mail.python.org/pipermail/python-list/2009-February/538048.html + +import asyncio +import itertools +import sys + + +async def spin(msg): # <1> + write, flush = sys.stdout.write, sys.stdout.flush + for char in itertools.cycle('|/-\\'): + status = char + ' ' + msg + write(status) + flush() + write('\x08' * len(status)) + try: + await asyncio.sleep(.1) # <2> + except asyncio.CancelledError: # <3> + break + write(' ' * len(status) + '\x08' * len(status)) + + +async def slow_function(): # <4> + # pretend waiting a long time for I/O + await asyncio.sleep(3) # <5> + return 42 + + +async def supervisor(): # <6> + spinner = asyncio.ensure_future(spin('thinking!')) # <7> + print('spinner object:', spinner) # <8> + result = await slow_function() # <9> + spinner.cancel() # <10> + return result + + +def main(): + loop = asyncio.get_event_loop() # <11> + result = loop.run_until_complete(supervisor()) # <12> + loop.close() + print('Answer:', result) + + +if __name__ == '__main__': + main() diff --git a/18-asyncio/spinner_curio.py b/18-asyncio/spinner_curio.py new file mode 100755 index 0000000..9475b7c --- /dev/null +++ b/18-asyncio/spinner_curio.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# spinner_curio.py + +# credits: Example by Luciano Ramalho inspired by +# Michele Simionato's multiprocessing example in the python-list: +# https://mail.python.org/pipermail/python-list/2009-February/538048.html + +import curio + +import itertools +import sys + + +async def spin(msg): # <1> + write, flush = sys.stdout.write, sys.stdout.flush + for char in itertools.cycle('|/-\\'): + status = char + ' ' + msg + write(status) + flush() + write('\x08' * len(status)) + try: + await curio.sleep(.1) # <2> + except curio.CancelledError: # <3> + break + write(' ' * len(status) + '\x08' * len(status)) + + +async def slow_function(): # <4> + # pretend waiting a long time for I/O + await curio.sleep(3) # <5> + return 42 + + +async def supervisor(): # <6> + spinner = await curio.spawn(spin('thinking!')) # <7> + print('spinner object:\n ', repr(spinner)) # <8> + result = await slow_function() # <9> + await spinner.cancel() # <10> + return result + + +def main(): + result = curio.run(supervisor) # <12> + print('Answer:', result) + + +if __name__ == '__main__': + main() diff --git a/18-asyncio/spinner_thread.py b/18-asyncio/spinner_thread.py old mode 100644 new mode 100755 index 23feb8f..dffcca6 --- a/18-asyncio/spinner_thread.py +++ b/18-asyncio/spinner_thread.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # spinner_thread.py # credits: Adapted from Michele Simionato's @@ -11,19 +13,14 @@ import sys -class Signal: # <1> - go = True - - -def spin(msg, signal): # <2> +def spin(msg, done): # <2> write, flush = sys.stdout.write, sys.stdout.flush for char in itertools.cycle('|/-\\'): # <3> status = char + ' ' + msg write(status) flush() write('\x08' * len(status)) # <4> - time.sleep(.1) - if not signal.go: # <5> + if done.wait(.1): # <5> break write(' ' * len(status) + '\x08' * len(status)) # <6> @@ -35,13 +32,13 @@ def slow_function(): # <7> def supervisor(): # <9> - signal = Signal() + done = threading.Event() spinner = threading.Thread(target=spin, - args=('thinking!', signal)) + args=('thinking!', done)) print('spinner object:', spinner) # <10> spinner.start() # <11> result = slow_function() # <12> - signal.go = False # <13> + done.set() # <13> spinner.join() # <14> return result diff --git a/19-dyn-attr-prop/oscon/test_schedule1.py b/19-dyn-attr-prop/oscon/test_schedule1.py index dbaacc9..ba5dfd1 100644 --- a/19-dyn-attr-prop/oscon/test_schedule1.py +++ b/19-dyn-attr-prop/oscon/test_schedule1.py @@ -1,15 +1,20 @@ +import os import shelve + import pytest import schedule1 as schedule +DB_NAME = 'data/test_db' + -@pytest.yield_fixture +@pytest.fixture(scope='module') def db(): - with shelve.open(schedule.DB_NAME) as the_db: + with shelve.open(DB_NAME) as the_db: if schedule.CONFERENCE not in the_db: schedule.load_db(the_db) yield the_db + os.remove(DB_NAME) def test_record_class(): diff --git a/19-dyn-attr-prop/oscon/test_schedule2.py b/19-dyn-attr-prop/oscon/test_schedule2.py index de09d32..ab1c79c 100644 --- a/19-dyn-attr-prop/oscon/test_schedule2.py +++ b/19-dyn-attr-prop/oscon/test_schedule2.py @@ -1,15 +1,18 @@ +import os import shelve + import pytest import schedule2 as schedule -@pytest.yield_fixture +@pytest.fixture(scope='module') def db(): - with shelve.open(schedule.DB_NAME) as the_db: + with shelve.open(DB_NAME) as the_db: if schedule.CONFERENCE not in the_db: schedule.load_db(the_db) yield the_db + os.remove(DB_NAME) def test_record_attr_access(): diff --git a/21-class-metaprog/evalsupport.py b/21-class-metaprog/evalsupport.py index ce490fa..51bb890 100644 --- a/21-class-metaprog/evalsupport.py +++ b/21-class-metaprog/evalsupport.py @@ -9,7 +9,7 @@ def inner_1(self): cls.method_y = inner_1 return cls -# BEGIN META_ALEPH + class MetaAleph(type): print('<[400]> MetaAleph body') @@ -20,6 +20,6 @@ def inner_2(self): print('<[600]> MetaAleph.__init__:inner_2') cls.method_z = inner_2 -# END META_ALEPH + print('<[700]> evalsupport module end') diff --git a/README.rst b/README.rst index 99a0ede..39b95c4 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ -Fluent Python: example code -=========================== +Fluent Python, First Edition: example code +========================================== -Example code for the book `Fluent Python`_ by Luciano Ramalho (O'Reilly, 2014). +**This repository is archived and will not be updated. Please visit https://github.com/fluentpython/example-code-2e** - **BEWARE**: This is a work in progress, like the book itself. +Example code for the book `Fluent Python, First Edition` by Luciano Ramalho (O'Reilly, 2015). * Code here may change and disappear without warning. diff --git a/attic/concurrency/charfinder/charfinder.py b/attic/concurrency/charfinder/charfinder.py index d73f60b..72e21a4 100755 --- a/attic/concurrency/charfinder/charfinder.py +++ b/attic/concurrency/charfinder/charfinder.py @@ -63,9 +63,9 @@ import itertools from collections import namedtuple -RE_WORD = re.compile('\w+') +RE_WORD = re.compile(r'\w+') RE_UNICODE_NAME = re.compile('^[A-Z0-9 -]+$') -RE_CODEPOINT = re.compile('U\+([0-9A-F]{4,6})') +RE_CODEPOINT = re.compile(r'U\+([0-9A-F]{4,6})') INDEX_NAME = 'charfinder_index.pickle' MINIMUM_SAVE_LEN = 10000 @@ -165,7 +165,7 @@ def find_chars(self, query, start=0, stop=None): for word in tokenize(query): if word in self.index: result_sets.append(self.index[word]) - else: # shorcut: no such word + else: # shortcut: no such word result_sets = [] break if result_sets: diff --git a/attic/concurrency/flags/fixture.tar.gz b/attic/concurrency/flags/fixture.tar.gz new file mode 100644 index 0000000..be32b40 Binary files /dev/null and b/attic/concurrency/flags/fixture.tar.gz differ diff --git a/attic/dicts/index_alex.py b/attic/dicts/index_alex.py index 27d7175..73db8c6 100644 --- a/attic/dicts/index_alex.py +++ b/attic/dicts/index_alex.py @@ -8,7 +8,7 @@ import sys import re -NONWORD_RE = re.compile('\W+') +NONWORD_RE = re.compile(r'\W+') idx = {} with open(sys.argv[1], encoding='utf-8') as fp: diff --git a/attic/sequences/sentence_slice.py b/attic/sequences/sentence_slice.py index d275989..918338d 100644 --- a/attic/sequences/sentence_slice.py +++ b/attic/sequences/sentence_slice.py @@ -6,9 +6,9 @@ import reprlib -RE_TOKEN = re.compile('\w+|\s+|[^\w\s]+') +RE_TOKEN = re.compile(r'\w+|\s+|[^\w\s]+') RE_WORD = re.compile('\w+') -RE_PUNCTUATION = re.compile('[^\w\s]+') +RE_PUNCTUATION = re.compile(r'[^\w\s]+') class SentenceSlice: