diff --git a/.gitignore b/.gitignore index a81c6be..13ba244 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,11 @@ dist/ *.egg-info/ .eggs/ build/ +*.so +.tox/ +__pycache__/ +*.pyc +*.log +.idea/ +docs/_book/ +docs/node_modules/ diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/CNAME b/CNAME new file mode 100644 index 0000000..587dc14 --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +litefs.leafcoder.cn \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 31eb359..537f7f2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ -include LICENSE requirements.txt -recursive-include demo *.* +include LICENSE requirements.txt README.md +recursive-include demo * +recursive-include test *.* \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2ac34cd --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +PYTHON=`which python` + +install: + $(PYTHON) setup.py install + +# 构建源码包 +build: wheel + $(PYTHON) setup.py build sdist + +build_ext: + $(PYTHON) setup.py build_ext --inplace + +# 构建 wheel 包 +wheel: + $(PYTHON) setup.py bdist_wheel + +upload-test: + pip install twine; \ + twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +upload: + pip install twine; \ + twine upload dist/* + +clean: + rm -rf build dist *.egg-info __pycache__ tests/__pycache__ tests/*.pyc + +.PHONY: test upload upload-test build wheel install clean diff --git a/README.md b/README.md index 3725335..8c373c3 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,34 @@ - -# 1. 快速启动 - -1.1 启动脚本 ---------------- - - import sys - sys.dont_write_bytecode = True - - import litefs - litefs = litefs.Litefs( - address='0.0.0.0:8080', webroot='./site', debug=True - ) - litefs.run(timeout=2.) - -将上面的代码保存为 run.py 文件。 - -1.2 页面脚本 ---------------- - -在网站目录(注:启动脚本中 webroot 的目录)中,添加一个后缀名为 **.py** 的文件,如 example.py,代码如下: - - def handler(self): - self.start_response(200, headers=[]) - return 'Hello world!' - - or - - def handler(self): - return 'Hello world!' - -1.3 启动网站 ------------ - - $ python run.py - Server is running at 0.0.0.0:8080 - Hit Ctrl-C to quit. - -运行启动脚本后,访问 http://0.0.0.0:8080/example,您会看到 `Hello world!`。 - - -# 2. CGI 规则 - -2.1 httpfile 对象是服务器上下文接口,接口如下: --------------------------------------- - -接口类型 | 接口使用 | 接口描述 ----- | --- | ---- -环境变量(只读) | httpfile.environ | 环境变量 -某环境变量 | httpfile.environ`[`_*envname*_`]` | 获取某环境变量 -Session | httpfile.session | session 对象,可临时保存或获取内存数据 -Session ID | httpfile.session_id | session 对象 ID,将通过 SET_COOKIE 环境变量返回给客户端浏览器 -Form | httpfile.form | form 为字典对象,保存您提交到服务器的数据 -Config | httpfile.config | 服务器的配置对象,可获取初始化服务器的配置信息 -files | httpfile.files | 字典对象,保存上传的文件,格式为:{ *filename1*: *\*, *filename2*: *\* } -cookie | httpfile.cookie | SimpleCookie 对象,获取 Cookie 数据 -页面跳转 | httpfile.redirect(url=None) | 跳转到某一页面 -HTTP头部 | httpfile.start_response(status_code=200, headers=None) | HTTP 返回码和头部 - -2.2 以下为 httpfile 对象中环境变量(environ)包含的变量对应表 ---------------------------------------------------- - -环境变量 | 描述 | 例子 -------- | ------ | ---- -REQUEST_METHOD | 请求方法 | GET、POST、PUT、HEAD等 -SERVER_PROTOCOL | 请求协议/版本 | HTTP/1.1" -REMOTE_ADDR | 请求客户端的IP地址 | 192.168.1.5 -REMOTE_PORT | 请求客户端的端口 | 9999 -REQUEST_URI | 完整 uri | /user_info?name=li&age=20 -PATH_INFO | 页面地址 | /user_info -QUERY_STRING | 请求参数 | name=li&age=20 -CONTENT_TYPE | POST 等报文类型 | application/x-www-form-urlencoded 或 text/html;charset=utf-8 -CONTENT_LENGTH | POST 等报文长度 | 1024 -HTTP_*_HEADERNAME_* | 其他请求头部 | 如 HTTP_REFERER:https://www.baidu.com/ - -2.3 部分环境变量也可以使用 httpfile 属性的方式获取 ------------------------------------------- - -环境变量 | 对应属性 -------- | ------- -PATH_INFO | httpfile.path_Info -QUERY_STRING | httpfile.query_string -REQUEST_URI | httpfile.request_uri -REFERER | httpfile.referer -REQUEST_METHOD | httpfile.request_method -SERVER_PROTOCOL | httpfile.server_protocol - -# 3. Mako 文件支持 - -TODO - -# 4. CGI 支持 - -TODO \ No newline at end of file +
+ +# Litefs + +

+ + + GitHub forks + + + GitHub forks + + + GitHub forks + +

+ +

+ GitHub release (latest by date) + GitHub top language + GitHub code size in bytes + GitHub commit activity + PyPI - Downloads +

+ +
+ +Litefs is a lite python web framework. + +Build a web server framework using Python. Litefs was developed to implement +a server framework that can quickly, securely, and flexibly build Web +projects. Litefs is a high-performance HTTP server. Litefs has the +characteristics of high stability, rich functions, and low system +consumption. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..ffe0591 --- /dev/null +++ b/TODO.md @@ -0,0 +1,9 @@ +* WSGI 支持 +* HTTP Method 限制 +* 更多的 HTTP 错误页 +* cookies 属性 +* query 参数和 post 参数分为不同的属性 +* 数据库模块 +* 搜索模块 +* 配置模块 +* 尝试整个框架不依赖其他模块来实现 \ No newline at end of file diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000..3397c9a --- /dev/null +++ b/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-architect \ No newline at end of file diff --git a/demo/example.py b/demo/example.py old mode 100755 new mode 100644 index ea02092..0c999e3 --- a/demo/example.py +++ b/demo/example.py @@ -1,11 +1,7 @@ -#!/usr/bin/python +#!/usr/bin/env python import sys sys.dont_write_bytecode = True -sys.path.insert(0, '..') import litefs -litefs = litefs.Litefs( - address='localhost:8080', webroot='./site', debug=True -) -litefs.run(timeout=2.) +litefs.test_server() \ No newline at end of file diff --git a/demo/site/environ.py b/demo/site/environ.py deleted file mode 100644 index d871ded..0000000 --- a/demo/site/environ.py +++ /dev/null @@ -1,2 +0,0 @@ -def handler(self): - return self.environ \ No newline at end of file diff --git a/demo/site/form.html b/demo/site/form.html deleted file mode 100644 index fef28ce..0000000 --- a/demo/site/form.html +++ /dev/null @@ -1,23 +0,0 @@ - - - - - 表单提交、支持jQuery提交格式 - - -

表单提交一

-
-
-
-
-
-

表单提交二

-
-
-
-
-
-
-
- - diff --git a/demo/site/form.py b/demo/site/form.py deleted file mode 100644 index d8e1568..0000000 --- a/demo/site/form.py +++ /dev/null @@ -1,3 +0,0 @@ -def handler(self): - print self.form - return 'ok' diff --git a/demo/site/index.html.py b/demo/site/index.html.py index 559ac60..9a23965 100644 --- a/demo/site/index.html.py +++ b/demo/site/index.html.py @@ -1,11 +1,3 @@ -#-*- coding: utf-8 -*- - def handler(self): - return ''' - - ''' \ No newline at end of file + self.start_response(200) + return ['Hello World'] \ No newline at end of file diff --git a/demo/site/not_found b/demo/site/not_found new file mode 100644 index 0000000..750e0c6 --- /dev/null +++ b/demo/site/not_found @@ -0,0 +1 @@ +

Not Found

The requested resource was not found on this server.

\ No newline at end of file diff --git a/demo/site/test.py b/demo/site/test.py new file mode 100644 index 0000000..bbd1700 --- /dev/null +++ b/demo/site/test.py @@ -0,0 +1,3 @@ +def handler(self): + self.start_response(200) + return ['hello'.encode('utf-8'), b' ', b'world'] \ No newline at end of file diff --git a/demo/site/test_iter.py b/demo/site/test_iter.py new file mode 100644 index 0000000..774b089 --- /dev/null +++ b/demo/site/test_iter.py @@ -0,0 +1,23 @@ +#-*- coding: utf-8 -*- + +def handler(self): + self.start_response(200, [('Content-Type', 'text/html; charset=utf-8')]) + return iter_response(self) + +def iter_response(self): + yield '

Environ

' + for k, v in self.environ.items(): + yield '{}: {}'.format(k, v) + yield '
' + yield '

Form

' + yield 'Params', str(self.params) + yield 'data', str(self.data) + yield '
' + + yield '

Files

' + files = self.files + for fp in files.values(): + yield '
' + yield '
' diff --git a/demo/site/helloworld.mako b/demo/site/test_mako.mako similarity index 99% rename from demo/site/helloworld.mako rename to demo/site/test_mako.mako index dd1b6ff..86d0b8f 100644 --- a/demo/site/helloworld.mako +++ b/demo/site/test_mako.mako @@ -1,4 +1,3 @@ -
 PATH_INFO   : ${http.path_info}
 QUERY_STRING: ${http.query_string}
diff --git a/demo/site/upload.html b/demo/site/upload.html
deleted file mode 100644
index 0c0abf6..0000000
--- a/demo/site/upload.html
+++ /dev/null
@@ -1,17 +0,0 @@
-
-
-
-    
-    文件上传例子
-
-
-    

上传文件

-
-
-
-
-
-
-
- - diff --git a/demo/site/upload.py b/demo/site/upload.py deleted file mode 100644 index 11425fb..0000000 --- a/demo/site/upload.py +++ /dev/null @@ -1,4 +0,0 @@ -def handler(self): - files = self.files - for fobj in files.values(): - yield fobj.read() \ No newline at end of file diff --git a/demo/site/user-agent.py b/demo/site/user-agent.py deleted file mode 100644 index 37fc383..0000000 --- a/demo/site/user-agent.py +++ /dev/null @@ -1,7 +0,0 @@ -#coding: utf-8 - -def handler(self): - print self.environ - user_agent = self.environ.get('HTTP_USER_AGENT') - print user_agent - return '你的浏览器是:%s' % user_agent diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..5f1f03c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,194 @@ +
+ +# Litefs {docsify-ignore} + +

+ + + GitHub forks + + + GitHub forks + + + GitHub forks + +

+ +

+ GitHub release (latest by date) + GitHub top language + GitHub code size in bytes + GitHub commit activity + GitHub All Releases +

+ +
+ +# Introduction + +Litefs is a lite python web framework. + +Build a web server framework using Python. Litefs was developed to implement +a server framework that can quickly, securely, and flexibly build Web +projects. Litefs is a high-performance HTTP server. Litefs has the +characteristics of high stability, rich functions, and low system +consumption. + +# Installation + +It can be installed via pip: + + $ pip install litefs + +It can be installed via source code: + + $ git clone https://github.com/leafcoder/litefs.git litefs + $ cd litefs + $ python setup.py install + +# Quickstart: "Hello world" + +Firstly, let's write a basci example via litefs. Save it to "example.py". + + # /usr/bin/env python + + import litefs + litefs.test_server() + +Secondly, you should create a directory named "site" (or any other name +which is same as __"--webroot"__). + + $ mkdir ./site + +Thirdly, you can copy the below code into a new file "./site/helloworld.py". + + def handler(self): + return "Hello World!" + +Run "example.py", visit "http://localhost:9090/helloworld" via your browser. +You can see "Hello World!" in your browser. + + $ ./example.py + Litefs 0.3.0 - January 15, 2020 - 10:46:39 + Starting server at http://localhost:9090/ + Quit the server with CONTROL-C. + +# URL ROUTES + +The relative path of the python scripts in the "site" folder will be url routes. + +For example, we create tow files in "site" folder like below. + +## Script route + # Python route + # ./site/hello.py => matches /hello + def handler(self): + return 'Hello world' + +## Template route + +Note: "http" means "self" in handler function. + + # Template route + # ./site/cn/hello.mako => matches /cn/hello +
+        PATH_INFO   : ${http.path_info}
+        QUERY_STRING: ${http.query_string}
+        REQUEST_URI : ${http.request_uri}
+        REFERER     : ${http.referer}
+        COOKIE      : ${http.cookie}
+
+        hello world
+    
+ +## Static file + + # Other static route => matches /hello.txt + # ./site/hello.txt + Hello world + +# HTTP Methods + +Litefs handlers all methods such as GET, POST, PUT, DELETE or PATCH in +"handler" function. + + def handler(self): + request_method = self.request_method + logger.info(request_method) + return 'request_method: %s' % request_method + +# Error Pages + +You can set a default 404 page when you start a litefs server. + + $ ./example.py --not-found=not_found.html + +# Help + + $ ./example.py --help + usage: example.py [-h] [--host HOST] [--port PORT] [--webroot WEBROOT] + [--debug] [--not-found NOT_FOUND] + [--default-page DEFAULT_PAGE] [--cgi-dir CGI_DIR] + [--log LOG] [--listen LISTEN] + + Build a web server framework using Python. Litefs was developed to implement a + server framework that can quickly, securely, and flexibly build Web projects. + Litefs is a high-performance HTTP server. Litefs has the characteristics of + high stability, rich functions, and low system consumption. Author: leafcoder + Email: leafcoder@gmail.com Copyright (c) 2017, Leafcoder. License: MIT (see + LICENSE for details) + + optional arguments: + -h, --help show this help message and exit + --host HOST bind server to HOST + --port PORT bind server to PORT + --webroot WEBROOT use WEBROOT as root directory + --debug start server in debug mode + --not-found NOT_FOUND + use NOT_FOUND as 404 page + --default-page DEFAULT_PAGE + use DEFAULT_PAGE as web default page + --cgi-dir CGI_DIR use CGI_DIR as cgi scripts directory + --log LOG save log to LOG + --listen LISTEN server LISTEN + + +## Context + +List attributes of "self". + +Attributes | Description +---------------------------------------------------- | ----------- +self.environ | 环境变量(只读) +self.environ[*envname*] | 获取某环境变量 +self.session | session 对象,可临时保存或获取内存数据 +self.session_id | session 对象 ID,将通过 SET_COOKIE 环境变量返回给客户端浏览器 +self.form | form 为字典对象,保存您提交到服务器的数据 +self.config | 服务器的配置对象,可获取初始化服务器的配置信息 +self.files | 字典对象,保存上传的文件,格式为:{ *filename1*: *\*, *filename2*: *\* } +self.cookie | SimpleCookie 对象,获取 Cookie 数据 +self.redirect(url=None) | 跳转到某一页面 +self.start_response(status_code=200, headers=None) | HTTP 返回码和头部 +self.path_Info | PATH_INFO +self.query_string | QUERY_STRING +self.request_uri | REQUEST_URI +self.referer | REFERER +self.request_method | REQUEST_METHOD +self.server_protocol | SERVER_PROTOCOL + + +## Environ + +环境变量 | 描述 | 例子 +------------------- | --------------------- | ---- +REQUEST_METHOD | 请求方法 | GET、POST、PUT、HEAD等 +SERVER_PROTOCOL | 请求协议/版本 | HTTP/1.1" +REMOTE_ADDR | 请求客户端的IP地址 | 192.168.1.5 +REMOTE_PORT | 请求客户端的端口 | 9999 +REQUEST_URI | 完整 uri | /user_info?name=li&age=20 +PATH_INFO | 页面地址 | /user_info +QUERY_STRING | 请求参数 | name=li&age=20 +CONTENT_TYPE | POST 等报文类型 | application/x-www-form-urlencoded 或 text/html;charset=utf-8 +CONTENT_LENGTH | POST 等报文长度 | 1024 +HTTP_*_HEADERNAME_* | 其他请求头部 | 如 HTTP_REFERER:https://www.baidu.com/ \ No newline at end of file diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..748a047 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,55 @@ + + + + + litefs - leafcoder + + + + + + + + + +
+ + + + + + + + + + + + + + + + + diff --git a/index.html b/index.html new file mode 100644 index 0000000..80d8096 --- /dev/null +++ b/index.html @@ -0,0 +1,56 @@ + + + + + litefs - leafcoder + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + diff --git a/litefs.py b/litefs.py index 6a64d15..5ca671b 100644 --- a/litefs.py +++ b/litefs.py @@ -1,85 +1,119 @@ #!/usr/bin/env python -#-*- coding: utf-8 -*- - -'''使用 Python 从零开始构建一个 Web 服务器框架。 开发 Litefs 的是为了实现一个能快速、安\ -全、灵活的构建 Web 项目的服务器框架。 Litefs 是一个高性能的 HTTP 服务器。Litefs 具有高\ -稳定性、丰富的功能、系统消耗低的特点。 - -Name: leafcoder -Email: leafcoder@gmail.com - -Copyright (c) 2017, Leafcoder. -License: MIT (see LICENSE for details) -''' - -__version__ = '0.2.5' -__author__ = 'Leafcoder' -__license__ = 'MIT' +# coding: utf-8 +import argparse +import cgi +import itertools +import json import logging import re +import sqlite3 import sys -import _socket as socket -from collections import deque, Iterable -from Cookie import SimpleCookie -from cStringIO import StringIO + +from collections import deque +from datetime import datetime from errno import ENOTCONN, EMFILE, EWOULDBLOCK, EAGAIN, EPIPE from functools import partial from greenlet import greenlet, getcurrent, GreenletExit from gzip import GzipFile from hashlib import sha1 -from httplib import responses as http_status_codes from imp import find_module, load_module, new_module as imp_new_module +from io import RawIOBase, BufferedRWPair, DEFAULT_BUFFER_SIZE from mako import exceptions from mako.lookup import TemplateLookup from mimetypes import guess_type from os import urandom, stat -from platform import platform from posixpath import join as path_join, splitext as path_splitext, \ split as path_split, realpath as path_realpath, \ abspath as path_abspath, isfile as path_isfile, \ isdir as path_isdir, exists as path_exists from select import EPOLLIN, EPOLLOUT, EPOLLHUP, EPOLLERR, EPOLLET, \ epoll as select_epoll -from sqlite3 import connect as sqlite3_connect from subprocess import Popen, PIPE from tempfile import NamedTemporaryFile, TemporaryFile from time import time, strftime, gmtime -from urllib import splitport, unquote_plus -from UserDict import UserDict +from traceback import print_exc from uuid import uuid4 from watchdog.events import * from watchdog.observers import Observer -from weakref import proxy as weakref_proxy -from zlib import compress as zlib_compress -from io import RawIOBase, BufferedRWPair, DEFAULT_BUFFER_SIZE +from weakref import proxy +from zlib import compress + +PY3 = sys.version_info.major > 2 + +if PY3: + # Import modules in py3 + import socket + from collections import UserDict + from http.client import responses as http_status_codes + from http.cookies import SimpleCookie + from io import BytesIO as StringIO + from urllib.parse import splitport, unquote_plus + + def is_unicode(s): + return isinstance(s, str) + + def is_bytes(s): + return isinstance(s, bytes) + + imap = map +else: + # Import modules in py2 + import _socket as socket + from cStringIO import StringIO + from httplib import responses as http_status_codes + from itertools import imap + from urllib import splitport, unquote_plus + from Cookie import SimpleCookie + from UserDict import UserDict + + def is_unicode(s): + return isinstance(s, unicode) + + def is_bytes(s): + return isinstance(s, basestring) + +__doc__ = """\ +Build a web server framework using Python. Litefs was developed to imple\ +ment a server framework that can quickly, securely, and flexibly build Web \ +projects. Litefs is a high-performance HTTP server. Litefs has the characte\ +ristics of high stability, rich functions, and low system consumption. + +Author: leafcoder +Email: leafcoder@gmail.com + +Copyright (c) 2020, Leafcoder. +License: MIT (see LICENSE for details) +""" -default_404 = '404' -default_port = 9090 -default_host = 'localhost' -default_prefix = 'litefs' -default_page = 'index.html' -default_webroot = './site' -default_cgi_dir = '/cgi-bin' -default_request_size = 10240 -default_litefs_sid = '%s.sid' % default_prefix - -server_name = 'litefs %s' % __version__ -EOFS = ('', '\n', '\r\n') -FILES_HEADER_NAME = 'litefs.files' -date_format = '%Y/%m/%d %H:%M:%S' +__version__ = "0.3.0" +__license__ = "MIT" +__author__ = "Leafcoder" + +server_software = "litefs/%s" % __version__ + +default_page = "index.html" +default_404 = "not_found" +default_sid = "litefs.sid" +default_content_type = "text/plain; charset=utf-8" + +EOFS = ("", "\n", "\r\n") +POSTS_HEADER_NAME = "litefs.posts" +FILES_HEADER_NAME = "litefs.files" +date_format = "%Y/%m/%d %H:%M:%S" should_retry_error = (EWOULDBLOCK, EAGAIN) -double_slash_sub = re.compile(r'\/{2,}').sub -startswith_dot_sub = re.compile(r'\/\.+').sub -suffixes = ('.py', '.pyc', '.pyo', '.so', '.mako') -form_dict_match = re.compile(r'(.+)\[([^\[\]]+)\]').match -server_info = 'litefs/%s python/%s' % (__version__, sys.version.split()[0]) +double_slash_sub = re.compile(r"\/{2,}").sub +startswith_dot_sub = re.compile(r"\/\.+").sub +suffixes = (".py", ".pyc", ".pyo", ".so", ".mako") +cgi_suffixes = (".pl", ".py", ".pyc", ".pyo", ".php") +form_dict_match = re.compile(r"(.+)\[([^\[\]]+)\]").match +server_info = "litefs/%s python/%s" % (__version__, sys.version.split()[0]) cgi_runners = { - '.pl' : '/usr/bin/perl', - '.py' : '/usr/bin/python', - '.pyc': '/usr/bin/python', - '.pyo': '/usr/bin/python', - '.php': '/usr/bin/php', + ".pl": "/usr/bin/perl", + ".py": "/usr/bin/python", + ".pyc": "/usr/bin/python", + ".pyo": "/usr/bin/python", + ".php": "/usr/bin/php" } DEFAULT_STATUS_MESSAGE = """\ @@ -96,179 +130,244 @@ """ + +def log_error(logger, message=None): + if message is None: + message = "error occured" + logger.error(message, exc_info=True) + + +def log_info(logger, message=None): + if message is None: + message = "info" + logger.info(message) + + +def log_debug(logger, message=None): + if message is None: + message = "debug" + logger.debug(message) + + class HttpError(Exception): pass + +def render_error(): + return exceptions.html_error_template().render() + + def gmt_date(timestamp=None): if timestamp is None: timestamp = time() - return strftime('%a, %d %b %Y %H:%M:%S GMT', gmtime(timestamp)) + return strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime(timestamp)) + def new_module(**kwargs): - '''创建新模块 + """创建新模块 新创建的模块不会加入到 sys.path 中,并导入自定义属性。 - ''' - mod_name = ''.join((default_prefix, uuid4().hex)) - mod = imp_new_module(mod_name) - mod.__dict__.update(kwargs) - return mod + """ + name = "".join(("litefs", uuid4().hex)) + module = imp_new_module(name) + module.__dict__.update(kwargs) + return module + def make_config(**kwargs): - default_config = { - 'address' : '%s:%d' % (default_host, default_port), - 'webroot' : default_webroot, - 'cgi_dir' : default_cgi_dir, - 'default_page': default_page, - 'not_found' : default_404, - 'debug' : False, - 'request_size': default_request_size - } + default_config = vars(_cmd_args([])) default_config.update(kwargs) config = new_module(**default_config) config.webroot = path_abspath(config.webroot) return config -def make_logger(name, level=logging.DEBUG): - '''创建日志对象 + +def make_logger(name, log=None, level=logging.DEBUG): + """创建日志对象 输出 HTTP 访问日志和错误异常。 - ''' + """ logger = logging.getLogger(name) - logger.setLevel(level) - handler = logging.StreamHandler() fmt = logging.Formatter( - '%(asctime)s - %(message)s', datefmt=date_format + ("%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message" + ")s"), + datefmt=date_format ) + logger.setLevel(level) + if log: + handler = logging.FileHandler(log) + handler.setFormatter(fmt) + logger.addHandler(handler) + handler = logging.StreamHandler() handler.setFormatter(fmt) logger.addHandler(handler) return logger -def log_error(logger, message=None): - if message is None: - message = 'error occured' - logger.error(message, exc_info=True) - -def log_debug(logger, message=None): - if message is None: - message = 'debug' - logger.debug(message) -def make_server(address, request_size=-1): - host, port = splitport(address) - port = 80 if port is None else int(port) +def make_server(host, port, request_size=-1): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((host, port)) if -1 == request_size: - request_size = default_request_size + request_size = 1024 sock.listen(request_size) sock.setblocking(0) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - return { - 'address': '%s:%d' % (host, port), - 'fileno' : sock.fileno(), - 'socket' : sock, - 'host' : host, - 'port' : port - } + return sock -def make_environ(app, rw, address): - environ = {} - environ['SERVER_NAME'] = server_name - environ['SERVER_PORT'] = int(app.server_info['port']) - s = rw.readline(DEFAULT_BUFFER_SIZE) - if not s: - # 注意:读出来为空字符串时,代表着服务器在等待读 - raise HttpError('invalid http headers') - request_method, path_info, protocol = s.strip().split() - if '?' in path_info: - path_info, query_string = path_info.split('?') - else: - path_info, query_string = path_info, '' - base_uri, script_name = path_info.split('/', 1) - if '' == script_name: - script_name = app.config.default_page - environ['REMOTE_ADDR'] = address - environ['REMOTE_HOST'] = address[0] - environ['REMOTE_PORT'] = address[1] - environ['REQUEST_METHOD'] = request_method - environ['QUERY_STRING'] = query_string - environ['SERVER_PROTOCOL'] = protocol - environ['SCRIPT_NAME'] = script_name - environ['PATH_INFO'] = path_info + +def make_headers(rw): + """Read and parse HTTP headers from RWPair object""" + headers = {} s = rw.readline(DEFAULT_BUFFER_SIZE) + if PY3: + s = s.decode("utf-8") while True: if s in EOFS: break - k, v = s.split(':', 1) - k, v = k.strip(), v.strip() - k = k.replace('-', '_').upper() + k, v = s.split(":", 1) + k, v = k.lower().strip(), v.strip() + headers[k] = v + s = rw.readline(DEFAULT_BUFFER_SIZE) + if PY3: + s = s.decode("utf-8") + return headers + + +def make_environ(server, rw, client_address): + environ = dict() + environ["SERVER_NAME"] = server.server_name + environ["SERVER_SOFTWARE"] = server_software + environ["SERVER_PORT"] = server.server_port + environ["REMOTE_ADDR"] = client_address + environ["REMOTE_HOST"] = client_address[0] + environ["REMOTE_PORT"] = client_address[1] + # Read first line + s = rw.readline(DEFAULT_BUFFER_SIZE) + if PY3: + s = s.decode("utf-8") + if not s: + # It means server is waiting for reading when getting empty string. + raise HttpError("invalid http headers") + request_method, path_info, protocol = s.strip().split() + if "?" in path_info: + path_info, query_string = path_info.split("?", 1) + else: + path_info, query_string = path_info, "" + path_info = unquote_plus(path_info) + base_uri, script_name = path_info.split("/", 1) + if "" == script_name: + script_name = default_page + environ["REQUEST_METHOD"] = request_method.upper() + environ["QUERY_STRING"] = unquote_plus(query_string) + environ["SERVER_PROTOCOL"] = protocol + environ["SCRIPT_NAME"] = script_name + environ["PATH_INFO"] = path_info + headers = make_headers(rw) + length = headers.get("content-length") + if length: + environ["CONTENT_LENGTH"] = length = int(length) + content_type = headers.get("content-type") + if content_type: + environ["CONTENT_TYPE"] = content_type + else: + environ["CONTENT_TYPE"] = content_type = default_content_type + _, params = cgi.parse_header(content_type) + charset = params.get('charset') + environ['CHARSET'] = charset + for k, v in headers.items(): + k = k.replace("-", "_").upper() + # Skip content_length, content_type, etc. if k in environ: continue - if k.startswith('_'): - environ[k] = v + k = "HTTP_%s" % k + if k in environ: + environ[k] += ",%s" % v # Comma-separate multiple headers else: - environ['HTTP_%s' % k] = v - s = rw.readline(DEFAULT_BUFFER_SIZE) - size = environ.pop('HTTP_CONTENT_LENGTH', None) - if not size: + environ[k] = v + if not length: return environ - size = int(size) - content_type = environ.get('HTTP_CONTENT_TYPE', '') - if content_type.startswith('multipart/form-data'): - boundary = content_type.split('=')[1] - begin_boundary = ('--%s' % boundary) - end_boundary = ('--%s--' % boundary) - files = {} + content_type = environ.get("CONTENT_TYPE", "") + if content_type.startswith("multipart/form-data"): + posts, files = parse_multipart(rw, content_type) + environ[POSTS_HEADER_NAME] = posts + environ[FILES_HEADER_NAME] = files + else: + environ["POST_CONTENT"] = post_content = rw.read(int(length)) + if PY3: + environ["POST_CONTENT"] \ + = unquote_plus(post_content.decode("utf-8")) + # Body will be empty when CONTENT_LENGTH equals to -1 + environ.setdefault('CONTENT_LENGTH', -1) + for k in ("HTTP_USER_AGENT", "HTTP_COOKIE", "HTTP_REFERER"): + environ.setdefault(k, "") + return environ + + +def parse_multipart(rw, content_type): + boundary = content_type.split("=")[1].strip() + if PY3: + boundary = boundary.encode("utf-8") + begin_boundary = (b"--%s" % boundary) + end_boundary = (b"--%s--" % boundary) + posts = {} + files = {} + s = rw.readline(DEFAULT_BUFFER_SIZE).strip() + while True: + if s.strip() != begin_boundary: + assert s.strip() == end_boundary + break + headers = {} s = rw.readline(DEFAULT_BUFFER_SIZE).strip() - while True: - if s.strip() != begin_boundary: - assert s.strip() == end_boundary - break - headers = {} + while s: + if PY3: + s = s.decode("utf-8") + k, v = s.split(":", 1) + headers[k.strip().upper()] = v.strip() s = rw.readline(DEFAULT_BUFFER_SIZE).strip() - while s: - k, v = s.split(':', 1) - headers[k.strip().upper()] = v.strip() - s = rw.readline(DEFAULT_BUFFER_SIZE).strip() - disposition = headers['CONTENT-DISPOSITION'] - h, m, t = disposition.split(';') - name = m.split('=')[1].strip() - if size <= 5242880: # <= 5M file save in memory - fp = StringIO() - else: - fp = TemporaryFile(mode='w+b') + disposition = headers["CONTENT-DISPOSITION"] + disposition, params = cgi.parse_header(disposition) + name = params["name"] + filename = params.get("filename") + if filename: + fp = TemporaryFile(mode="w+b") s = rw.readline(DEFAULT_BUFFER_SIZE) while s.strip() != begin_boundary \ and s.strip() != end_boundary: fp.write(s) s = rw.readline(DEFAULT_BUFFER_SIZE) fp.seek(0) - files[name[1:-1]] = fp - environ[FILES_HEADER_NAME] = files - else: - environ['POST_CONTENT'] = rw.read(int(size)) - environ['CONTENT_LENGTH'] = len(environ['POST_CONTENT']) - return environ + files[name] = fp + else: + fp = StringIO() + s = rw.readline(DEFAULT_BUFFER_SIZE) + while s.strip() != begin_boundary \ + and s.strip() != end_boundary: + fp.write(s) + s = rw.readline(DEFAULT_BUFFER_SIZE) + fp.seek(0) + posts[name] = fp.getvalue().strip().decode('utf-8') + return posts, files + -def parse_form(form, qstr): - qstr = unquote_plus(qstr) - for s in qstr.split('&'): +def parse_form(query_string): + form = {} + query_string = unquote_plus(query_string) + for s in query_string.split("&"): if not s: continue - kv = s.split('=', 1) + kv = s.split("=", 1) if 2 == len(kv): k, v = kv else: - k, v = kv[0], '' + k, v = kv[0], "" k, v = unquote_plus(k), unquote_plus(v) - if k.endswith('[]'): + if k.endswith("[]"): k = k[:-2] if k in form: result = form[k] if isinstance(result, dict): - raise ValueError('invalid form data %s' % qstr) + raise ValueError("invalid form data %s" % query_string) if isinstance(result, list): form[k].append(v) else: @@ -281,7 +380,7 @@ def parse_form(form, qstr): if k in form: result = form[k] if isinstance(result, dict): - raise ValueError('invalid form data %s' % qstr) + raise ValueError("invalid form data %s" % query_string) if isinstance(result, list): form[k].append(v) else: @@ -293,28 +392,20 @@ def parse_form(form, qstr): if key in form: result = form[key] if not isinstance(result, dict): - raise ValueError('invalid form data %s' % qstr) + raise ValueError("invalid form data %s" % query_string) if prefix in result: result[prefix] = [result[prefix], v] else: result[prefix] = v else: - form[key] = { prefix: v } - -def make_form(environ): - query_string = environ['QUERY_STRING'] - form = {} - parse_form(form, query_string) - post_content = environ.get('POST_CONTENT') - if post_content: - parse_form(form, post_content) + form[key] = {prefix: v} return form class FileEventHandler(FileSystemEventHandler): def __init__(self, app): FileSystemEventHandler.__init__(self) - self._app = weakref_proxy(app) + self._app = proxy(app) def on_moved(self, event): src_path = event.src_path @@ -322,14 +413,16 @@ def on_moved(self, event): webroot = self._app.config.webroot if webroot == src_path and event.is_directory: return - if not src_path.startswith(webroot+'/'): + if not src_path.startswith(webroot + "/"): return if webroot == dest_path and event.is_directory: return - if not dest_path.startswith(webroot+'/'): + if not dest_path.startswith(webroot + "/"): return - src_path = '/%s' % src_path [len(webroot):].strip('/') - dest_path = '/%s' % dest_path[len(webroot):].strip('/') + log_info(self._app.logger, "%s has been moved to %s" \ + % (src_path, dest_path)) + src_path = "/%s" % src_path[len(webroot):].strip("/") + dest_path = "/%s" % dest_path[len(webroot):].strip("/") caches = self._app.caches files = self._app.files caches.delete(src_path) @@ -350,9 +443,10 @@ def on_created(self, event): webroot = self._app.config.webroot if webroot == src_path and event.is_directory: return - if not src_path.startswith(webroot+'/'): + if not src_path.startswith(webroot + "/"): return - src_path = '/%s' % src_path[len(webroot):].strip('/') + log_info(self._app.logger, "%s has been modified" % src_path) + src_path = "/%s" % src_path[len(webroot):].strip("/") caches = self._app.caches files = self._app.files caches.delete(src_path) @@ -364,64 +458,61 @@ def on_created(self, event): on_modified = on_deleted = on_created + class LiteFile(object): - def __init__(self, path, base, name, text): + def __init__(self, path, base, name, text, status_code=200): + self.status_code = int(status_code) self.path = path self.text = text self.etag = etag = sha1(text).hexdigest() - self.zlib_text = zlib_text = zlib_compress(text, 9)[2:-4] + self.zlib_text = zlib_text = compress(text, 9)[2:-4] self.zlib_etag = sha1(zlib_text).hexdigest() stream = StringIO() - with GzipFile(fileobj=stream, mode="w") as f: + with GzipFile(fileobj=stream, mode="wb") as f: f.write(text) self.gzip_text = gzip_text = stream.getvalue() self.gzip_etag = sha1(gzip_text).hexdigest() self.last_modified = gmt_date(stat(path).st_mtime) mimetype, coding = guess_type(name) - headers = [('Content-Type', 'text/html;charset=utf-8')] + headers = [("Content-Type", "text/html;charset=utf-8")] if mimetype is not None: - headers = [('Content-Type', '%s;charset=utf-8' % mimetype)] - else: - headers = [('Content-Type', 'application/octet-stream;charset=utf-8')] - headers.append(('Last-Modified', self.last_modified)) - headers.append(('Connection', 'close')) + headers = [("Content-Type", "%s;charset=utf-8" % mimetype)] + headers.append(("Last-Modified", self.last_modified)) + headers.append(("Connection", "close")) self.headers = headers - def handler(self, httpfile): - environ = httpfile.environ - if_modified_since = environ.get('HTTP_IF_MODIFIED_SINCE') + def handler(self, request): + environ = request.environ + if_modified_since = environ.get("HTTP_IF_MODIFIED_SINCE") if if_modified_since == self.last_modified: - result = httpfile._response(304) - return httpfile._finish(result) - if_none_match = environ.get('HTTP_IF_NONE_MATCH') + return request._response(304) + if_none_match = environ.get("HTTP_IF_NONE_MATCH") accept_encodings = environ.get( - 'HTTP_ACCEPT_ENCODING', '').split(',') + "HTTP_ACCEPT_ENCODING", "").split(",") accept_encodings = [s.strip().lower() for s in accept_encodings] headers = list(self.headers) - if 'gzip' in accept_encodings: + if "gzip" in accept_encodings: if if_none_match == self.gzip_etag: - result = httpfile._response(304) - return httpfile._finish(result) - headers.append(('Etag', self.gzip_etag)) - headers.append(('Content-Encoding', 'gzip')) + return request._response(304) + headers.append(("Etag", self.gzip_etag)) + headers.append(("Content-Encoding", "gzip")) text = self.gzip_text - elif 'deflate' in accept_encodings: + elif "deflate" in accept_encodings: if if_none_match == self.zlib_etag: - result = httpfile._response(304) - return httpfile._finish(result) - headers.append(('Etag', self.zlib_etag)) - headers.append(('Content-Encoding', 'deflate')) + return request._response(304) + headers.append(("Etag", self.zlib_etag)) + headers.append(("Content-Encoding", "deflate")) text = self.zlib_text else: if if_none_match == self.etag: - result = httpfile._response(304) - return httpfile._finish(result) - headers.append(('Etag', self.etag)) + return request._response(304) + headers.append(("Etag", self.etag)) text = self.text - headers.append(('Content-Length', '%d' % len(text))) - result = httpfile._response(200, headers=headers, content=text) - return httpfile._finish(result) + headers.append(("Content-Length", "%d" % len(text))) + return request._response( + self.status_code, headers=headers, content=text) + class TreeCache(object): @@ -430,15 +521,15 @@ def __init__(self, clean_period=60, expiration_time=3600): self.clean_time = time() self.clean_period = clean_period self.expiration_time = expiration_time - self.conn = sqlite3_connect(':memory:', check_same_thread=False) + self.conn = sqlite3.connect(":memory:", check_same_thread=False) self.conn.text_factory = str - self.conn.executescript(''' + self.conn.executescript(""" CREATE TABLE IF NOT EXISTS cache ( key VARCHAR PRIMARY KEY, timestamp INTEGER ); CREATE INDEX idx_cache ON cache (key); - ''') + """) def __len__(self): return len(self.data) @@ -450,13 +541,13 @@ def put(self, key, val): self.auto_clean() timestamp = int(time()) if key not in data: - conn.execute(''' + conn.execute(""" INSERT INTO cache (key, timestamp) VALUES (?, ?); - ''', (key, timestamp)) + """, (key, timestamp)) else: - conn.execute(''' + conn.execute(""" UPDATE cache SET timestamp=? WHERE key=?; - ''', (timestamp, key)) + """, (timestamp, key)) data[key] = [val, timestamp] def get(self, key): @@ -477,13 +568,13 @@ def delete(self, key): data = self.data if self.clean_time + self.clean_period < time(): self.auto_clean() - curr = conn.execute('''\ + curr = conn.execute("""\ SELECT key FROM cache WHERE key=? OR key LIKE ?; - ''', (key, key + '/%')) + """, (key, key + "/%")) keys = curr.fetchall() - conn.executemany('''\ + conn.executemany("""\ DELETE FROM cache WHERE key=?; - ''', keys) + """, keys) for item in keys: key = item[0] del data[key] @@ -492,17 +583,18 @@ def auto_clean(self): conn = self.conn data = self.data last_expiration_time = int(time() - self.expiration_time) - curr = conn.execute(''' + curr = conn.execute(""" SELECT key FROM cache WHERE timestamp < ?; - ''', (last_expiration_time, )) + """, (last_expiration_time, )) keys = curr.fetchall() - conn.executemany(''' + conn.executemany(""" DELETE FROM cache WHERE key=?; - ''', keys) + """, keys) for item in keys: key = item[0] del data[key] + class MemoryCache(object): def __init__(self, max_size=10000): @@ -537,6 +629,7 @@ def delete(self, key): self._queue.remove(key) del self._cache[key] + class Session(UserDict): def __init__(self, session_id): @@ -544,27 +637,53 @@ def __init__(self, session_id): self.data = {} def __str__(self): - return '' % self.id + return "" % self.id + -class HttpFile(object): +class RequestHandler(object): - def __init__(self, app, rw, environ, address): + default_headers = { + "Content-Type": default_content_type + } + + def __init__(self, app, rw, environ): self._rw = rw self._app = app self._environ = environ - self._address = address self._buffers = StringIO() self._headers_responsed = False - self._form = make_form(environ) + self._response_headers = {} + self._cookies = None + self._status_code = 200 + self._get = parse_form(environ["QUERY_STRING"]) + content_type = environ.get("CONTENT_TYPE", "") + content_type, params = cgi.parse_header(content_type) + self._post = {} + self._body = "" + if content_type == 'application/x-www-form-urlencoded': + post_content = environ.get("POST_CONTENT", "") + self._post = parse_form(post_content) + elif content_type == 'multipart/form-data': + self._post = environ.pop(POSTS_HEADER_NAME, {}) + else: + post_content = environ.get("POST_CONTENT", "") + self._body = post_content self._session_id, self._session = self._get_session(environ) - self._files = environ.pop(FILES_HEADER_NAME, None) + self._files = environ.pop(FILES_HEADER_NAME, {}) + if app.config.debug: + log_debug(app.logger, "%s - \"%s %s %s\"" % ( + environ["REMOTE_HOST"], + environ["SERVER_PROTOCOL"], + environ["REQUEST_METHOD"], + environ["PATH_INFO"] + )) def _get_session(self, environ): app = self._app sessions = app.sessions - cookie = environ.get('HTTP_COOKIE') + cookie = environ.get("HTTP_COOKIE") cookie = SimpleCookie(cookie) - morsel = cookie.get(default_litefs_sid) + morsel = cookie.get(default_sid) if morsel is not None: session_id = morsel.value else: @@ -574,38 +693,68 @@ def _get_session(self, environ): return session_id, session session = Session(session_id) sessions.put(session_id, session) - return session_id, session + # New session + return None, session def _new_session_id(self): app = self._app sessions = app.sessions while 1: - token = '%s%s' % (urandom(24), time()) + token = "%s%s" % (urandom(24), time()) + if PY3: + token = token.encode("utf-8") session_id = sha1(token).hexdigest() session = sessions.get(session_id) if session is None: break return session_id + def set_cookie(self, name, value, **options): + cookies = self._cookies + if not cookies: + cookies = SimpleCookie() + cookies[name] = value + for key, value in options.items(): + if not value: + continue + cookies[name][key] = value + self._cookies = cookies + @property def config(self): return self._app.config @property - def address(self): - return self._address + def files(self): + return self._files or {} @property - def files(self): - return self._files + def body(self): + return self._body + + @property + def json(self): + body = self._body + if not self._body: + return {} + content_type = self.content_type + content_type, _ = cgi.parse_header(content_type) + content_type = content_type.lower() + if content_type not in ('application/json', 'application/json-rpc'): + return {} + return json.loads(body) @property def environ(self): return self._environ @property - def form(self): - return self._form + def params(self): + return self._get + + @property + def data(self): + return self._post @property def session_id(self): @@ -617,103 +766,150 @@ def session(self): @property def request_method(self): - return self.environ['REQUEST_METHOD'] + return self.environ["REQUEST_METHOD"] + method = request_method @property def server_protocol(self): - return self.environ['SERVER_PROTOCOL'] + return self.environ["SERVER_PROTOCOL"] + + @property + def content_type(self): + return self.environ.get("CONTENT_TYPE") + + @property + def charset(self, default="UTF-8"): + _, params = cgi.parse_header(self.content_type) + return params.get("charset", default) + + @property + def content_length(self): + return int(self.environ.get("CONTENT_LENGTH") or -1) @property def path_info(self): - return self.environ['PATH_INFO'] + return self.environ["PATH_INFO"] @property def query_string(self): - return self.environ['QUERY_STRING'] + return self.environ["QUERY_STRING"] @property def request_uri(self): environ = self.environ - path_info = environ['PATH_INFO'] - query_string = environ['QUERY_STRING'] + path_info = environ["PATH_INFO"] + query_string = environ["QUERY_STRING"] if not query_string: return path_info - return '?'.join((path_info, query_string)) + return "?".join((path_info, query_string)) @property def referer(self): - return self.environ.get('HTTP_REFERER') + return self.environ.get("HTTP_REFERER") @property def cookie(self): - cookie_str = self.environ.get('HTTP_COOKIE', '') + cookie_str = self.environ.get("HTTP_COOKIE", "") cookie = SimpleCookie() cookie.load(cookie_str) return cookie def start_response(self, status_code=200, headers=None): - buffers = self._buffers if self._headers_responsed: - raise ValueError('Http headers already responsed.') - response_headers = {} - response_headers['Server'] = server_info - response_headers['Content-Type'] = 'text/html;charset=utf-8' - status_code = int(status_code) - status_text = http_status_codes[status_code] - buffers.write('HTTP/1.1 %d %s\r\n' % (status_code, status_text)) - for name, text in response_headers.items(): - buffers.write('%s: %s\r\n' % (name, text)) + raise ValueError("Http headers already responsed.") + self._status_code = int(status_code) + response_headers = self._response_headers + response_headers["Server"] = server_info + response_headers["Content-Type"] = "text/html;charset=utf-8" if headers is not None: for header in headers: - if isinstance(header, basestring): - buffers.write(header) + if not isinstance(header, (list, tuple)): + if PY3: + header = header.encode("utf-8") + k, v = header.split(":") + k, v = k.strip(), v.strip() else: - buffers.write('%s: %s\r\n' % header) + k, v = header + response_headers[k] = v if self.session_id is None: - cookie = SimpleCookie() - cookie[default_litefs_sid] = self.session.id - cookie[default_litefs_sid]['path'] = '/' - buffers.write('%s\r\n' % cookie.output()) - buffers.write('\r\n') + self.set_cookie(default_sid, self.session.id, path="/") self._headers_responsed = True def redirect(self, url=None): if self._headers_responsed: - raise ValueError('Http headers already responsed.') + raise ValueError("Http headers already responsed.") url = '/' if url is None else url - response_headers = {} - response_headers['Content-Type'] = 'text/html;charset=utf-8' - response_headers['Location'] = 'http://%s%s' % ( - self._environ['HTTP_HOST'], url + response_headers = self._response_headers + response_headers["Content-Type"] = "text/html;charset=utf-8" + response_headers["Location"] = "http://%s%s" % ( + self._environ["HTTP_HOST"], url ) status_code = 302 status_text = http_status_codes[status_code] - content = '%d %s' % (status_code, status_text) + content = "%d %s" % (status_code, status_text) headers = response_headers.items() self.start_response(status_code, headers=headers) return content - def _finish(self, content): - rw = self._rw - if not self._headers_responsed: - self.start_response(200) - rw.write(self._buffers.getvalue()) - if isinstance(content, basestring): - rw.write(content) - elif isinstance(content, dict): - rw.write(repr(content)) - elif isinstance(content, Iterable): - for s in content: - if isinstance(s, basestring): - rw.write(s) - else: - rw.write(repr(s)) - else: - rw.write(repr(content)) + def _cast(self, s=None): + response_headers = self._response_headers + if not s: + if "Content-Length" not in response_headers: + response_headers["Content-Length"] = 0 + return [] + if isinstance(s, (tuple, list)) \ + and (is_unicode(s[0]) or is_bytes(s[0])): + join_chr = s[0][:0] + s = join_chr.join(s) + if is_unicode(s): + s = s.encode("utf-8") + if is_bytes(s): + if "Content-Length" not in response_headers: + response_headers["Content-Length"] = len(s) + return [s] try: - rw.close() - except: - pass + iter_s = iter(s) + first = next(iter_s) + while not first: + first = next(iter_s) + except StopIteration: + return self._cast() + if is_bytes(first): + new_iter_s = itertools.chain([first], iter_s) + elif is_unicode(first): + encoder = lambda item: str(item).encode("utf-8") + new_iter_s = itertools.chain([first], iter_s) + new_iter_s = imap(encoder, new_iter_s) + else: + raise TypeError("response type is not allowd: %s" % type(first)) + return new_iter_s + + def finish(self, content): + rw = self._rw + status_code = self._status_code + status_text = http_status_codes[status_code] + line = "HTTP/1.1 %d %s\r\n" % (status_code, status_text) + if PY3: + line = line.encode("utf-8") + rw.write(line) + headers = self._response_headers + if not headers: + headers = self.default_headers + for header, value in headers.items(): + line = "%s: %s\r\n" % (header, value) + if PY3: + line = line.encode("utf-8") + rw.write(line) + if self._cookies: + for c in self._cookies.values(): + line = "%s: %s\r\n" % ('Set-Cookie', c.OutputString()) + if PY3: + line = line.encode("utf-8") + rw.write(line) + rw.write("\r\n".encode("utf-8")) + for _ in self._cast(content): + rw.write(_) + rw.close() def __del__(self): files = self._environ.get(FILES_HEADER_NAME) @@ -724,92 +920,68 @@ def __del__(self): def _response(self, status_code, headers=None, content=None): if self._headers_responsed: - raise ValueError('Http headers already responsed.') + raise ValueError("Http headers already responsed.") status_code = int(status_code) status_text = http_status_codes[status_code] self.start_response(status_code, headers=headers) if content is None: content = DEFAULT_STATUS_MESSAGE % { - 'code': status_code, - 'message': status_text, - 'explain': status_text + "code": status_code, + "message": status_text, + "explain": status_text } return content - def _handler(self): + def handler(self): app = self._app environ = self.environ - path_info = environ['PATH_INFO'] - path = startswith_dot_sub('/', path_info) - path = double_slash_sub('/', path) + path_info = environ["PATH_INFO"] + path = startswith_dot_sub("/", path_info) + path = double_slash_sub("/", path) if path != path_info: - result = self.redirect(path) - try: - return self._finish(result) - except: - content = exceptions.html_error_template().render() - result = self._response(500, content=content) - return self._finish(result) + return self.redirect(path) base, name = path_split(path) if not name: - name = app.config.default_page + name = default_page path = path_join(base, name) module = app.caches.get(path) if module is not None: try: - result = module.handler(self) + return module.handler(self) except: log_error(app.logger) if app.config.debug: - content = exceptions.html_error_template().render() - result = self._response(500, content=content) - return self._finish(result) - result = self._response(500) - return self._finish(result) - else: - try: - return self._finish(result) - except: - result = exceptions.html_error_template().render() - return self._finish(result) + content = render_error() + return self._response(500, content=content) + return self._response(500) litefile = app.files.get(path) if litefile is not None: return litefile.handler(self) realpath = path_abspath( - path_join(app.config.webroot, path.lstrip('/')) + path_join(app.config.webroot, path.lstrip("/")) ) if path_isdir(realpath): - result = self.redirect(path + '/') - return self._finish(result) + return self.redirect(path + "/") module = self._load_script(base, name) - if module is not None and hasattr(module, 'handler'): + if module is not None and hasattr(module, "handler"): basepath, ext = path_splitext(path) - if ext in ('.mako', ): + if ext in (".mako", ): app.caches.put(basepath, module) else: app.caches.put(path, module) try: - result = module.handler(self) + return module.handler(self) except: log_error(app.logger) if app.config.debug: - content = exceptions.html_error_template().render() - result = self._response(500, content=content) - return self._finish(result) - result = self._response(500) - return self._finish(result) - else: - try: - return self._finish(result) - except: - result = exceptions.html_error_template().render() - return self._finish(result) + content = render_error() + return self._response(500, content=content) + return self._response(500) try: litefile = self._load_static_file(base, name) except IOError: log_error(app.logger) - result = self._response(404) - return self._finish(result) + return self._response(404) if litefile is not None: app.files.put(path, litefile) try: @@ -817,11 +989,9 @@ def _handler(self): except: log_error(app.logger) if app.config.debug: - content = exceptions.html_error_template().render() - result = self._response(500, content=content) - return self._finish(result) - result = self._response(500) - return self._finish(result) + content = render_error() + return self._response(500, content=content) + return self._response(500) path = app.config.not_found base, name = path_split(path) if not name: @@ -832,18 +1002,16 @@ def _handler(self): litefile = None if litefile is not None: app.files.put(path, litefile) + litefile.status_code = 404 try: return litefile.handler(self) except: log_error(app.logger) if app.config.debug: - content = exceptions.html_error_template().render() - result = self._response(500, content=content) - return self._finish(result) - result = self._response(500) - return self._finish(result) - result = self._response(404) - return self._finish(result) + content = render_error() + return self._response(500, content=content) + return self._response(500) + return self._response(404) def _load_static_file(self, base, name): app = self._app @@ -855,29 +1023,29 @@ def _load_static_file(self, base, name): realpath = path_join(realbase, name) if ext in suffixes or not path_isfile(realpath): return None - with open(realpath, 'rb') as fp: + with open(realpath, "rb") as fp: text = fp.read() return LiteFile(realpath, base, name, text) def _load_template(self, base, name): app = self._app webroot = app.config.webroot - script_uri = path_join(base.lstrip('/'), name) - path = path_join('/%s' % base.rstrip('/'), name) + script_uri = path_join(base.lstrip("/"), name) + path = path_join("/%s" % base.rstrip("/"), name) mylookup = TemplateLookup(directories=[webroot]) def handler(mylookup, script_uri): def _handler(self): try: template = mylookup.get_template(script_uri) content = template.render(http=self) - headers = getattr(template.module, 'headers', None) + headers = getattr(template.module, "headers", None) if headers: - headers = '\r\n'.join( - [':'.join(h) for h in headers] + headers = "\r\n".join( + [":".join(h) for h in headers] ) if not headers: headers = [ - ('Content-Type', 'text/html;charset=utf-8') + ("Content-Type", "text/html;charset=utf-8") ] return self._response( 200, headers=headers, content=content @@ -885,7 +1053,7 @@ def _handler(self): except: log_error(app.logger) if app.config.debug: - content = exceptions.html_error_template().render() + content = render_error() return self._response(500, content=content) return self._response(500) return _handler @@ -897,11 +1065,11 @@ def _load_script(self, base, name): app = self._app webroot = app.config.webroot realbase = path_realpath( - path_abspath(path_join(webroot, base.lstrip('/'))) + path_abspath(path_join(webroot, base.lstrip("/"))) ) - script_uri = path_join(base.lstrip('/'), name) + script_uri = path_join(base.lstrip("/"), name) script_name, ext = path_splitext(name) - if ext in ('.pl', '.py', '.pyc', '.pyo', '.php') and \ + if ext in cgi_suffixes and \ base.startswith(app.config.cgi_dir): runner = cgi_runners[ext] script_path = path_join(webroot, script_uri) @@ -914,7 +1082,7 @@ def _handler(self): module.handler = handler() return module return self._load_cgi(runner, script_uri, webroot) - tmplname = '%s.mako' % name + tmplname = "%s.mako" % name tmplpath = path_join(realbase, tmplname) if path_exists(tmplpath): return self._load_template(base, tmplname) @@ -922,7 +1090,7 @@ def _handler(self): fp, pathname, description = find_module(name, [realbase]) except ImportError: return None - module_name = 'litefs_%s' % uuid4().hex + module_name = "litefs_%s" % uuid4().hex sys.dont_write_bytecode = True try: module = load_module(module_name, fp, pathname, description) @@ -930,7 +1098,7 @@ def _handler(self): log_error(app.logger) content = None if app.config.debug: - content = exceptions.html_error_template().render() + content = render_error() def handler(content): def _handler(self): return self._response(500, content=content) @@ -947,7 +1115,7 @@ def _handler(self): def _load_cgi(self, runner, script_uri, webroot): app = self._app - tmpf = NamedTemporaryFile('w+') + tmpf = NamedTemporaryFile("w+") try: p = Popen( [runner, script_uri], @@ -956,10 +1124,10 @@ def _load_cgi(self, runner, script_uri, webroot): stdout, stderr = p.communicate() returncode = p.returncode if 0 == returncode: - log_debug(app.logger, 'CGI script exited OK') + log_debug(app.logger, "CGI script exited OK") else: log_error(app.logger, - 'CGI script exit status %#x' % returncode + "CGI script exit status %#x" % returncode ) p.stderr.close() if stderr: @@ -967,26 +1135,29 @@ def _load_cgi(self, runner, script_uri, webroot): tmpf.seek(0) if not stderr: stdout = tmpf.read() - def handler(stdout, stderr): + + def handler(out, err): def _handler(self): - if stdout is not None: - return self._response(200, content=stdout) + if out is not None: + return self._response(200, content=out) else: - return self._response(500, content=stderr) + return self._response(500, content=err) return _handler + module = new_module() module.handler = handler(stdout, stderr) return module finally: tmpf.close() -class RawIO(RawIOBase): - def __init__(self, app, sock): +class SocketIO(RawIOBase): + + def __init__(self, server, sock): RawIOBase.__init__(self) self._fileno = sock.fileno() self._sock = sock - self._app = app + self._server = server def fileno(self): return self._fileno @@ -998,14 +1169,17 @@ def writable(self): return True def readinto(self, b): - epoll = self._app.epoll + real_epoll = epoll._epoll fileno = self._fileno curr = getcurrent() self.read_gr = curr if self.write_gr is None: - epoll.register(fileno, EPOLLIN | EPOLLET) + real_epoll.register(fileno, EPOLLIN | EPOLLET) else: - epoll.modify(fileno, EPOLLIN | EPOLLOUT | EPOLLET) + real_epoll.modify(fileno, EPOLLIN | EPOLLOUT | EPOLLET) + data = "" + if PY3: + data = b"" try: curr.parent.switch() data = self._sock.recv(len(b)) @@ -1016,9 +1190,9 @@ def readinto(self, b): finally: self.read_gr = None if self.write_gr is None: - epoll.unregister(fileno) + real_epoll.unregister(fileno) else: - epoll.modify(fileno, EPOLLOUT | EPOLLET) + real_epoll.modify(fileno, EPOLLOUT | EPOLLET) n = len(data) try: b[:n] = data @@ -1026,18 +1200,18 @@ def readinto(self, b): import array if not isinstance(b, array.array): raise err - b[:n] = array.array(b'b', data) + b[:n] = array.array(b"b", data) return n def write(self, data): - epoll = self._app.epoll + real_epoll = epoll._epoll fileno = self._fileno curr = getcurrent() self.write_gr = curr if self.read_gr is None: - epoll.register(fileno, EPOLLOUT | EPOLLET) + real_epoll.register(fileno, EPOLLOUT | EPOLLET) else: - epoll.modify(fileno, EPOLLIN | EPOLLOUT | EPOLLET) + real_epoll.modify(fileno, EPOLLIN | EPOLLOUT | EPOLLET) try: curr.parent.switch() return self._sock.send(data) @@ -1048,9 +1222,9 @@ def write(self, data): finally: self.write_gr = None if self.read_gr is None: - epoll.unregister(fileno) + real_epoll.unregister(fileno) else: - epoll.modify(fileno, EPOLLIN | EPOLLET) + real_epoll.modify(fileno, EPOLLIN | EPOLLET) def close(self): if self.closed: @@ -1066,63 +1240,157 @@ def close(self): read_gr = write_gr = None -class Litefs(object): - def __init__(self, **kwargs): - self.config = config = make_config(**kwargs) - level = logging.DEBUG if config.debug else logging.ERROR - self.logger = make_logger(__name__, level=level) - self.server_info = server_info = make_server( - config.address, request_size=config.request_size - ) - self.address = address = server_info['address'] - self.fileno = fileno = server_info['fileno'] - self.socket = server_info['socket'] - self.sessions = MemoryCache(max_size=1000000) - self.caches = TreeCache() - self.files = TreeCache() - self.epoll = select_epoll() - self.epoll.register(fileno, EPOLLIN | EPOLLET) - self.grs = {} - sys.stdout.write(( - ' * Server is running at http://%s/\n' - ' * Press ctrl-c to quit.\n\n' - ) % address) +class Epoll(object): + + def __init__(self): + self._epoll = select_epoll() + self._servers = {} + self._greenlets = {} + + def register(self, server_socket): + servers = self._servers + fileno = server_socket.fileno() + servers[fileno] = server_socket + self._epoll.register(fileno, EPOLLIN | EPOLLET) - def _handler_accept(self): + def close(self): + for fileno, server_socket in self._servers.items(): + self._epoll.unregister(fileno) + server_socket.server_close() + self._epoll.close() + + def poll(self, poll_interval=.2): + servers = self._servers + greenlets = self._greenlets + _poll = self._epoll.poll while True: + events = _poll(poll_interval) + for fileno, event in events: + if fileno in servers: + server = servers[fileno] + try: + server.handle_request() + except KeyboardInterrupt: + break + except socket.error as e: + if e.errno == EMFILE: + raise + print_exc() + except: + print_exc() + elif (event & EPOLLIN) or (event & EPOLLOUT): + try: + greenlets[fileno].switch() + except KeyboardInterrupt: + break + except: + print_exc() + elif event & (EPOLLHUP | EPOLLERR): + try: + greenlets[fileno].throw() + except KeyboardInterrupt: + break + except: + print_exc() + + +class TCPServer(object): + """Classic Python TCPServer""" + + allow_reuse_address = True + request_queue_size = 4194304 + address_family, socket_type = socket.AF_INET, socket.SOCK_STREAM + + def __init__(self, server_address, RequestHandlerClass, + bind_and_activate=True): + self.server_address = server_address + self.RequestHandlerClass = RequestHandlerClass + self.socket = socket.socket(self.address_family, + self.socket_type) + self._started = False + if bind_and_activate: try: - sock, address = self.socket.accept() + self.server_bind() + self.server_activate() + except: + self.server_close() + raise + + def server_bind(self): + if self.allow_reuse_address: + self.socket.setsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR, 1) + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.socket.bind(self.server_address) + self.socket.setblocking(0) + + def server_activate(self): + self.socket.listen(self.request_queue_size) + + def server_close(self): + self.socket.close() + + def fileno(self): + return self.socket.fileno() + + def get_request(self): + return self.socket.accept() + + def handle_request(self): + self._handle_request_noblock() + + def _handle_request_noblock(self): + while True: + try: + request, client_address = self.get_request() except socket.error as e: errno = e.args[0] if EAGAIN == errno or EWOULDBLOCK == errno: return raise - sock.setblocking(0) - fileno = sock.fileno() - raw = RawIO(self, sock) - self.grs[fileno] = gr = greenlet( - partial(self._handler_io, raw, address) - ) - gr.switch() + if self.verify_request(request, client_address): + try: + self.process_request(request, client_address) + except: + self.handle_error(request, client_address) + self.shutdown_request(request) + else: + self.shutdown_request(request) + + def handle_timeout(self): + """Called if no new request arrives within self.timeout. + + Overridden by ForkingMixIn. + """ + pass + + def verify_request(self, request, client_address): + return True + + def process_request(self, request, client_address): + try: + self.finish_request(request, client_address) + except: + self.handle_error(request, client_address) + self.shutdown_request(request) + + def finish_request(self, request, client_address): + """Finish one request by instantiating RequestHandlerClass.""" + request.setblocking(0) + fileno = request.fileno() + epoll._greenlets[fileno] = curr = greenlet( + partial(self._finish_request, request, client_address) + ) + curr.switch() - def _handler_io(self, raw, address): + def _finish_request(self, request, client_address): + raw = SocketIO(self, request) try: - config = self.config - fileno = raw.fileno() rw = BufferedRWPair(raw, raw, DEFAULT_BUFFER_SIZE) - environ = make_environ(self, rw, address) - httpfile = HttpFile(self, rw, environ, address) - if config.debug: - log_debug(self.logger, - '%s - "%s %s %s"' % ( - environ['REMOTE_HOST'], - environ['SERVER_PROTOCOL'], - environ['REQUEST_METHOD'], - environ['PATH_INFO'] - ) - ) - httpfile._handler() + environ = make_environ(self, rw, client_address) + self.RequestHandlerClass(request, environ, self) + self.shutdown_request(request) except socket.error as e: if e.errno == EPIPE: raise GreenletExit @@ -1137,56 +1405,116 @@ def _handler_io(self, raw, address): if raw.write_gr is not None: raw.write_gr.throw() finally: - self.grs.pop(fileno, None) + fileno = raw.fileno() + epoll._greenlets.pop(fileno, None) + if not raw.closed: try: raw.close() except: pass - def _poll(self, timeout=.2): - while True: - events = self.epoll.poll(timeout) - for fileno, event in events: - if fileno == self.fileno: - try: - self._handler_accept() - except KeyboardInterrupt: - break - except socket.error as e: - if e.errno == EMFILE: - raise - log_error(self.logger) - except: - log_error(self.logger) - elif event & EPOLLIN: - try: - self.grs[fileno].switch() - except KeyboardInterrupt: - break - except: - log_error(self.logger) - elif event & EPOLLOUT: - try: - self.grs[fileno].switch() - except KeyboardInterrupt: - break - except: - log_error(self.logger) - elif event & (EPOLLHUP | EPOLLERR): - try: - self.grs[fileno].throw() - except KeyboardInterrupt: - break - except: - log_error(self.logger) + def shutdown_request(self, request): + try: + # explicitly shutdown. socket.close() merely releases + # the socket and waits for GC to perform the actual close. + request.shutdown(socket.SHUT_WR) + except OSError: + pass # some platforms may raise ENOTCONN here + self.close_request(request) + + def close_request(self, request): + request.close() + + def handle_error(self, request, client_address): + import traceback + traceback.print_exc() # XXX But this goes to stderr! + + def server_forever(self, poll_interval=.1): + if not self._started: + epoll.register(self) + mainloop(poll_interval=poll_interval) + + def start(self): + if not self._started: + epoll.register(self) + + def shutdown(self): + if self._started: + epoll.unregister(self) + + +class HTTPServer(TCPServer): + + allow_reuse_address = 1 + + def server_bind(self): + """Override server_bind to store the server name.""" + TCPServer.server_bind(self) + host, port = self.socket.getsockname()[:2] + self.server_name = socket.getfqdn(host) + self.server_port = port + + +class WSGIServer(HTTPServer): + + application = None - def run(self, timeout=.2): + def server_bind(self): + HTTPServer.server_bind(self) + self.setup_environ() + + def setup_environ(self): + # Set up base environment + env = {} + env["SERVER_NAME"] = self.server_name + env["GATEWAY_INTERFACE"] = "CGI/1.1" + env["SERVER_PORT"] = str(self.server_port) + env["REMOTE_HOST"] = "" + env["CONTENT_LENGTH"] = -1 + env["SCRIPT_NAME"] = "" + self.base_environ = env + + def get_app(self): + return self.application + + def set_app(self, application): + self.application = application + + +class Litefs(object): + + def __init__(self, **kwargs): + self.config = config = make_config(**kwargs) + level = logging.DEBUG if config.debug else logging.INFO + self.logger = make_logger(__name__, log=config.log, level=level) + self.host = host = config.host + self.port = port = config.port + self.server = HTTPServer((host, port), self.handler) + self.sessions = MemoryCache(max_size=1000000) + self.caches = TreeCache() + self.files = TreeCache() + now = datetime.now().strftime("%B %d, %Y - %X") + sys.stdout.write(( + "Litefs %s - %s\n" + "Starting server at http://%s:%d/\n" + "Quit the server with CONTROL-C.\n" + ) % (__version__, now, host, port)) + + def handler(self, request, environ, server): + raw = SocketIO(server, request) + rw = BufferedRWPair(raw, raw, DEFAULT_BUFFER_SIZE) + request_handler = RequestHandler(self, rw, environ) + result = request_handler.handler() + return request_handler.finish(result) + + def run(self, poll_interval=.2): observer = Observer() event_handler = FileEventHandler(self) observer.schedule(event_handler, self.config.webroot, True) observer.start() try: - self._poll(timeout=timeout) + self.server.start() + mainloop(poll_interval=poll_interval) except KeyboardInterrupt: pass except: @@ -1194,37 +1522,61 @@ def run(self, timeout=.2): finally: observer.stop() observer.join() - self.epoll.unregister(self.fileno) - self.epoll.close() - self.socket.close() + self.server.server_close() + + +def _cmd_args(args): + title = args[0] if args else "litefs" + parser = argparse.ArgumentParser(title, description=__doc__) + parser.add_argument("-H", "--host", dest="host", + required=False, default="localhost", + help="bind server to HOST") + parser.add_argument("-P", "--port", action="store", dest="port", type=int, + required=False, default=9090, + help="bind server to PORT") + parser.add_argument("--webroot", dest="webroot", + required=False, default="./site", + help="use WEBROOT as root directory") + parser.add_argument("--debug", action="store_true", dest="debug", + required=False, default=False, + help="start server in debug mode") + parser.add_argument("--not-found", dest="not_found", + required=False, default=default_404, + help="use NOT_FOUND as 404 page") + parser.add_argument("--default-page", dest="default_page", + required=False, default="index.html", + help="use DEFAULT_PAGE as web default page") + parser.add_argument("--cgi-dir", dest="cgi_dir", + required=False, default="/cgi-bin", + help="use CGI_DIR as cgi scripts directory") + parser.add_argument("--log", dest="log", + required=False, default="./default.log", + help="save log to LOG") + parser.add_argument("--listen", dest="listen", type=int, + required=False, default=1024, + help="server LISTEN") + args = parser.parse_args(args and args[1:]) + return args + + +def mainloop(poll_interval=.1): + try: + epoll.poll(poll_interval=poll_interval) + except KeyboardInterrupt: + pass + finally: + epoll.close() + +server_forever = mainloop + def test_server(): - import argparse - parser = argparse.ArgumentParser(description='Litefs with epoll') - parser.add_argument('--host', action="store", dest="host", - required=False, default=default_host) - parser.add_argument('--port', action="store", dest="port", type=int, - required=False, default=default_port) - parser.add_argument('--webroot', action="store", dest="webroot", - required=False, default=default_webroot) - parser.add_argument('--debug', action="store", dest="debug", - required=False, default=False) - parser.add_argument('--not-found', action="store", dest="not_found", - required=False, default=default_404) - parser.add_argument('--default-page', action="store", - dest="default_page", required=False, default=default_page) - parser.add_argument('--cgi-dir', action="store", dest="cgi_dir", - required=False, default=default_cgi_dir) - args = parser.parse_args() - litefs = Litefs( - address='%s:%d' % (args.host, args.port), - webroot=args.webroot, - debug=args.debug, - default_page=args.default_page, - not_found=args.not_found, - cgi_dir=args.cgi_dir - ) - litefs.run(timeout=2.) + args = _cmd_args(sys.argv) + kwargs = vars(args) + litefs = Litefs(**kwargs) + litefs.run(poll_interval=.1) + +epoll = Epoll() -if '__main__' == __name__: +if "__main__" == __name__: test_server() diff --git a/requirements.txt b/requirements.txt index 41d35f0..f40f34d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ argh==0.26.2 greenlet==0.4.13 -Mako==1.0.6 -MarkupSafe==1.0 +Mako==1.2.2 +MarkupSafe==1.1.1 pathtools==0.1.2 -PyYAML==3.12 +PyYAML==5.4 watchdog==0.8.3 diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 0d9a9fb..7e29354 --- a/setup.py +++ b/setup.py @@ -4,58 +4,61 @@ import sys sys.dont_write_bytecode = True +import os +import re +import posixpath + try: - from setuptools import setup -except ImportError: - from distutils.core import setup -try: - from Cython.Build import cythonize + from setuptools import setup, Extension except ImportError: - import os - os.system('pip install cython') - from Cython.Build import cythonize + from distutils.core import setup, Extension -long_description = '''\ -使用 Python 从零开始构建一个 Web 服务器框架。 开发 Litefs 的是为了实现一个能快速、安\ -全、灵活的构建 Web 项目的服务器框架。 Litefs 是一个高性能的 HTTP 服务器。Litefs 具有高\ -稳定性、丰富的功能、系统消耗低的特点。 +def get_str(var_name): + src_py = open('litefs.py').read() + return re.search( + r"%s\s*=\s*['\"]([^'\"]+)['\"]" % var_name, src_py).group(1) -Name: leafcoder -Email: leafcoder@gmail.com - -Copyright (c) 2017, Leafcoder. -License: MIT (see LICENSE for details) -''' - -__version__ = '0.2.5' -__author__ = 'Leafcoder' -__license__ = 'MIT' +def get_long_str(var_name): + src_py = open('litefs.py').read() + return re.search( + r"%s\s*=\s*['\"]{3}([^'\"]+)['\"]{3}" % var_name, src_py).group(1) setup( name='litefs', - version=__version__, - description='使用 Python 从零开始构建一个 Web 服务器框架。', - long_description=__doc__, - author=__author__, + version=get_str('__version__'), + description='Build a web server framework using Python.', + long_description=get_long_str('__doc__'), + author=get_str('__author__'), author_email='leafcoder@gmail.com', url='https://github.com/leafcoder/litefs', py_modules=['litefs'], - ext_modules=cythonize('litefs.py'), - license=__license__, + license=get_str('__license__'), platforms='any', package_data={ - '': ["*.txt", 'LICENSE'], - 'demo': ['site/*', '*.py'] + '': ['*.txt', '*.md', 'LICENSE', 'MANIFEST.in'], + 'demo': ['demo/*', '*.py'], + 'test': ['test/*', '*.py'] + }, + install_requires=open('requirements.txt').read().split('\n'), + entry_points={ + 'console_scripts': [ + 'litefs=litefs:test_server', + ] }, - install_requires=[ - 'Mako==1.0.6', - 'MarkupSafe==1.0', - 'greenlet==0.4.13', - 'PyYAML==3.12', - 'argh==0.26.2', - 'argparse==1.2.1', - 'pathtools==0.1.2', - 'watchdog==0.8.3', - 'wsgiref==0.1.2' + classifiers=[ + 'Development Status :: 4 - Beta', + "Operating System :: OS Independent", + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries', + 'Topic :: Internet :: WWW/HTTP :: HTTP Servers', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ] ) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_environ.py b/test/test_environ.py new file mode 100644 index 0000000..2f15df7 --- /dev/null +++ b/test/test_environ.py @@ -0,0 +1,6 @@ +import unittest + +class EnvironTest(unittest.TestCase): + + def test_get_environ(self): + pass \ No newline at end of file diff --git a/test/test_form.py b/test/test_form.py new file mode 100644 index 0000000..501b8a7 --- /dev/null +++ b/test/test_form.py @@ -0,0 +1,50 @@ +#-*- coding: utf-8 -*- + +import unittest +from litefs import parse_form, make_form + +class FormTest(unittest.TestCase): + + def test_make_form(self): + # test case 1 + test_environ = { + 'QUERY_STRING': 'name=tom&name=tom&cn_name=%E6%B1%A4%E5%A7%86', + 'POST_CONTENT': 'name=jerry&cn_name=%E6%9D%B0%E7%91%9E' + } + self.assertEqual( + make_form(test_environ), + { + 'name': ['tom', 'tom', 'jerry'], + 'cn_name': ['汤姆', '杰瑞'] + } + ) + # test case 2 + test_environ = { + 'QUERY_STRING': 'name=tom', + 'POST_CONTENT': 'name=tom&name=tom' + } + self.assertEqual( + make_form(test_environ), { 'name': ['tom', 'tom', 'tom'] }) + # test case 3 + test_environ = { + 'QUERY_STRING': 'name=tom' + } + self.assertEqual(make_form(test_environ), { 'name': 'tom' }) + # test case 4 + test_environ = { + 'QUERY_STRING': '', + 'POST_CONTENT': 'name=jerry' + } + self.assertEqual(make_form(test_environ), { 'name': 'jerry' }) + with self.assertRaises(KeyError): + # test case 5 + test_environ = { + 'POST_CONTENT': 'name=jerry' + } + make_form(test_environ) + + def test_parse_form(self): + pass + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/test/test_memorycache.py b/test/test_memorycache.py new file mode 100644 index 0000000..4e4a3f1 --- /dev/null +++ b/test/test_memorycache.py @@ -0,0 +1,36 @@ +#-*- coding: utf-8 -*- + +import unittest +from litefs import MemoryCache + +class MemoryCacheTest(unittest.TestCase): + + def setUp(self): + self.max_size = 100 + self.cache_key = 'test_key' + self.cache_value = 'test_val' + + def test_put(self): + cache = MemoryCache(max_size=self.max_size) + for i in range(self.max_size): + cache.put('%s-%d' % (self.cache_key, i), self.cache_value) + self.assertEqual(len(cache), self.max_size) + + def test_delete(self): + cache = MemoryCache(max_size=self.max_size) + size = len(cache) + cache.put(self.cache_key, self.cache_value) + cache.delete(self.cache_key) + self.assertEqual(len(cache), size) + + def test_out_of_max_size(self): + cache = MemoryCache(max_size=self.max_size) + for i in range(1000): + cache.put('%s-%d' % (self.cache_key, i), self.cache_value) + self.assertEqual(len(cache), self.max_size) + for i in range(900): + cache_key = '%s-%d' % (self.cache_key, i) + self.assertIsNone(cache.get(cache_key)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/test/test_treecache.py b/test/test_treecache.py new file mode 100644 index 0000000..214feb5 --- /dev/null +++ b/test/test_treecache.py @@ -0,0 +1,32 @@ +#-*- coding: utf-8 -*- + +import unittest +from litefs import TreeCache + +class TestTreeCache(unittest.TestCase): + + def setUp(self): + self.cache = TreeCache(clean_period=60, expiration_time=3600) + + def test_put(self): + caches = { + 'k_int': 1, + 'k_str': 'hello', + 'k_float': .5 + } + cache = TreeCache(clean_period=60, expiration_time=3600) + for k, v in caches.items(): + cache.put(k, v) + self.assertEqual(len(cache), len(caches)) + + def test_delete(self): + cache = self.cache + cache_key = 'delete_key' + cache.put(cache_key, 'delete me') + size_before_delete = len(cache) + cache.delete(cache_key) + size_after_delete = len(cache) + self.assertEqual(size_before_delete, size_after_delete + 1) + +if __name__ == '__main__': + unittest.main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..cbfe628 --- /dev/null +++ b/tox.ini @@ -0,0 +1,22 @@ +# tox (https://tox.readthedocs.io/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +envlist = py27, py35, py36, py37 + +[testenv] +deps = + argh==0.26.2 + argparse==1.2.1 + Cython==0.29.14 + greenlet==0.4.13 + Mako==1.0.6 + MarkupSafe==1.0 + pathtools==0.1.2 + PyYAML==3.12 + watchdog==0.8.3 + +commands = + python -m unittest diff --git a/wsgi.ini b/wsgi.ini new file mode 100644 index 0000000..e324033 --- /dev/null +++ b/wsgi.ini @@ -0,0 +1,6 @@ +[uwsgi] +chdir=/home/zhanglei3/Desktop/dev/litefs +http = :9090 +wsgi-file = wsgi_example.py +workers = 4 +listen = 10240 diff --git a/wsgi_example.py b/wsgi_example.py new file mode 100644 index 0000000..8c55867 --- /dev/null +++ b/wsgi_example.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +import sys +sys.dont_write_bytecode = True + +# uwsgi --http :9090 --wsgi-file wsgi_example.py + +import litefs +print(litefs) +app = litefs.Litefs() +application = app.wsgi()