diff --git a/.github/workflows/compile-test.yml b/.github/workflows/compile-test.yml new file mode 100644 index 0000000000..71678caa4e --- /dev/null +++ b/.github/workflows/compile-test.yml @@ -0,0 +1,58 @@ +name: Compile test + +on: + push: + branches: [ master, uwsgi-2.0 ] + pull_request: + branches: [ master, uwsgi-2.0 ] + +jobs: + build: + strategy: + matrix: + libpcre: ["libpcre3-dev", "libpcre2-dev"] + os: ["ubuntu-22.04"] + cc: [gcc, clang] + include: + - os: ubuntu-22.04 + php: "php8.1" + php-config: "php-config8.1" + + runs-on: ${{ matrix.os }} + + steps: + - name: remove sury php ppa that does not ship libphpX.Y-embed + run: | + sudo add-apt-repository --remove ppa:ondrej/php + sudo apt remove ${{ matrix.php }}-dev ${{ matrix.php }} ${{ matrix.php }}-common + - name: Install dependencies + run: | + sudo apt update -qq + sudo apt install --no-install-recommends -qqyf python3-dev \ + libxml2-dev ${{ matrix.libpcre }} libcap2-dev \ + libargon2-0-dev libsodium-dev \ + ${{ matrix.php }}-dev lib${{ matrix.php }}-embed \ + liblua5.1-0-dev ruby-dev \ + libjansson-dev libldap2-dev libpq-dev \ + libpam0g-dev libsqlite3-dev libyaml-dev \ + libzmq3-dev libmatheval-dev libperl-dev \ + libonig-dev libdb-dev libqdbm-dev libbz2-dev \ + libwrap0-dev libgeoip-dev libv8-dev libxslt1-dev \ + libboost-thread-dev libboost-filesystem-dev \ + libssl-dev libacl1-dev python-greenlet-dev \ + libcurl4-openssl-dev \ + openjdk-11-jdk libgloox-dev gccgo \ + cli-common-dev mono-devel mono-mcs uuid-dev \ + curl check ${{ matrix.cc == 'clang' && 'clang' || '' }} + - uses: actions/checkout@v4 + - name: Build kitchensink uWSGI binary with gcc or default with clang + run: CC=${{ matrix.cc }} UWSGICONFIG_PHPPATH=${{ matrix.php-config }} /usr/bin/python3 uwsgiconfig.py --build ${{ matrix.cc == 'gcc' && 'travis' || '' }} + - name: Build uWSGI binary + run: | + CC=${{ matrix.cc }} /usr/bin/python3 uwsgiconfig.py --build base + - name: Build cgi plugin + run: | + CC=${{ matrix.cc }} /usr/bin/python3 uwsgiconfig.py --plugin plugins/cgi base + - name: Build dummy plugin + run: | + CC=${{ matrix.cc }} /usr/bin/python3 uwsgiconfig.py --plugin plugins/dummy base diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..7f6ae1ba6f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,96 @@ +name: Test + +on: + push: + branches: [ master, uwsgi-2.0 ] + pull_request: + branches: [ master, uwsgi-2.0 ] + +jobs: + + unittest: + runs-on: ubuntu-22.04 + steps: + - name: Install dependencies + run: | + sudo apt update -qq + sudo apt install --no-install-recommends -qqyf \ + libpcre3-dev libjansson-dev libcap2-dev \ + check + - uses: actions/checkout@v4 + - name: Run unit tests + run: make unittests + + test: + runs-on: ubuntu-22.04 + steps: + - name: Install dependencies + run: | + sudo apt update -qq + sudo apt install --no-install-recommends -qqyf \ + libpcre2-dev libjansson-dev libcap2-dev \ + php-dev libphp-embed libargon2-dev libsodium-dev \ + pypy3 default-jdk-headless libperl-dev \ + ruby-dev ruby-rack + - uses: actions/checkout@v4 + - name: Set env + run: echo "PROFILE=integration-tests" >> $GITHUB_ENV + - name: Run integration tests + run: make all tests + + python: + runs-on: ubuntu-22.04 + strategy: + matrix: + python-version: ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + test-suite: [python, deadlocks] + steps: + - name: Add deadnakes ppa + run: sudo add-apt-repository ppa:deadsnakes/ppa -y + - name: Install dependencies + run: | + sudo apt update -qq + sudo apt install --no-install-recommends -qqyf python${{ matrix.python-version }}-dev \ + libpcre2-dev libjansson-dev libcap2-dev \ + curl + - name: Install distutils + if: contains(fromJson('["3.7","3.8","3.9","3.10","3.11"]'), matrix.python-version) + run: | + sudo apt install --no-install-recommends -qqyf python${{ matrix.python-version }}-distutils \ + - uses: actions/checkout@v4 + - name: Build uWSGI binary + run: make + - name: Build python${{ matrix.python-version }} plugin + run: | + PYTHON_VERSION=${{ matrix.python-version }} + PYTHON_VERSION=python${PYTHON_VERSION//.} + /usr/bin/python${{ matrix.python-version }} -V + /usr/bin/python${{ matrix.python-version }} uwsgiconfig.py --plugin plugins/python base $PYTHON_VERSION + - name: run smoke tests + run: | + PYTHON_VERSION=${{ matrix.python-version }} + PYTHON_VERSION=python${PYTHON_VERSION//.} + ./tests/gh-${{ matrix.test-suite }}.sh ${PYTHON_VERSION} + + rack: + runs-on: ubuntu-22.04 + strategy: + matrix: + rack-version: ["300"] + steps: + - name: Install dependencies + run: | + sudo apt update -qq + sudo apt install --no-install-recommends -qqyf python3-dev \ + libpcre2-dev libjansson-dev libcap2-dev ruby3.0-dev \ + curl + - uses: actions/checkout@v4 + - name: Build uWSGI binary + run: make + - name: Build rack plugin + run: | + ruby -v + UWSGICONFIG_RUBYPATH=ruby /usr/bin/python uwsgiconfig.py --plugin plugins/rack base rack${{ matrix.rack-version }} + - name: run smoke tests + run: | + ./tests/gh-rack.sh rack${{ matrix.rack-version}} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ece46d63a6..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,54 +0,0 @@ -language: c - -compiler: - - gcc - -script: - - echo -e "\n\n>>> Building kitchensink uWSGI binary" - - UWSGICONFIG_PHPPATH=php-config7.2 /usr/bin/python uwsgiconfig.py --build travis - - echo -e "\n\n>>> Building uWSGI binary" - - make - - echo -e "\n\n>>> Building python26 plugin" - - /usr/bin/python2.6 -V - - /usr/bin/python2.6 uwsgiconfig.py --plugin plugins/python base python26 - - echo -e "\n\n>>> Building python27 plugin" - - /usr/bin/python2.7 -V - - /usr/bin/python2.7 uwsgiconfig.py --plugin plugins/python base python27 - - echo -e "\n\n>>> Building python34 plugin" - - /usr/bin/python3.4 -V - - /usr/bin/python3.4 uwsgiconfig.py --plugin plugins/python base python34 - - echo -e "\n\n>>> Building python35 plugin" - - /usr/bin/python3.5 -V - - /usr/bin/python3.5 uwsgiconfig.py --plugin plugins/python base python35 - - echo -e "\n\n>>> Building python36 plugin" - - /usr/bin/python3.6 -V - - /usr/bin/python3.6 uwsgiconfig.py --plugin plugins/python base python36 - - echo -e "\n\n>>> Building rack plugin" - - rvm use 2.4 - - ruby -v - - UWSGICONFIG_RUBYPATH=ruby /usr/bin/python uwsgiconfig.py --plugin plugins/rack base rack226 - - echo -e "\n\n>>> Building cgi plugin" - - /usr/bin/python uwsgiconfig.py --plugin plugins/cgi base - - echo -e "\n\n>>> Building dummy plugin" - - /usr/bin/python uwsgiconfig.py --plugin plugins/dummy base - - echo -e "\n\n>>> Building done, starting tests" - - ./tests/travis.sh - -before_install: - - sudo add-apt-repository ppa:deadsnakes/ppa -y - - sudo add-apt-repository ppa:ondrej/php -y - - sudo apt-get update -qq - - sudo apt-get install -qqyf python2.6-dev python3.4-dev python3.5-dev python3.6-dev - - sudo apt-get install -qqyf libxml2-dev libpcre3-dev libcap2-dev - - sudo apt-get install -qqyf php7.2-dev libphp7.2-embed libargon2-0-dev libsodium-dev - - sudo apt-get install -qqyf liblua5.1-0-dev - - sudo apt-get install -qqyf libjansson-dev libldap2-dev libpq-dev - - sudo apt-get install -qqyf libpam0g-dev libsqlite3-dev libyaml-dev - - sudo apt-get install -qqyf libzmq-dev libmatheval-dev libperl-dev - - sudo apt-get install -qqyf libonig-dev libdb-dev libqdbm-dev libbz2-dev - - sudo apt-get install -qqyf libwrap0-dev libgeoip-dev libv8-dev libxslt1-dev - - sudo apt-get install -qqyf libboost-thread-dev libboost-filesystem-dev - - sudo apt-get install -qqyf libssl-dev libacl1-dev python-greenlet-dev - - sudo apt-get install -qqyf openjdk-7-jdk libgloox-dev gccgo - - sudo apt-get install -qqyf cli-common-dev mono-devel mono-mcs uuid-dev - - sudo apt-get install -qqyf curl diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 738110f9dc..91c5eb08d9 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -37,3 +37,5 @@ Vladimir Didenko Alexandre Bonnetain Darvame Hleran Sokolov Yura +Marcin Lulek +Derzsi Dániel diff --git a/Makefile b/Makefile index 2b05ef970a..fd2e3301c6 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ -PYTHON := python +PYTHON := python3 all: $(PYTHON) uwsgiconfig.py --build $(PROFILE) clean: $(PYTHON) uwsgiconfig.py --clean + cd unittest && make clean check: $(PYTHON) uwsgiconfig.py --check @@ -12,11 +13,14 @@ check: plugin.%: $(PYTHON) uwsgiconfig.py --plugin plugins/$* $(PROFILE) -tests: +unittests: $(PYTHON) uwsgiconfig.py --build unittest - cd check && make && make test + cd unittest && make test + +tests: + $(PYTHON) t/runner %: $(PYTHON) uwsgiconfig.py --build $@ -.PHONY: tests +.PHONY: all clean check tests diff --git a/README b/README index 46bea5dc17..03f39da510 100644 --- a/README +++ b/README @@ -2,11 +2,15 @@ The uWSGI project For official documentation check: https://uwsgi-docs.readthedocs.io/en/latest/ -For commercial support check: http://unbit.com/ +Note: The project is in maintenance mode (only bugfixes and updates for new languages apis) -uWSGI development is sponsored by: +Do not expect quick answers on github issues and/or pull requests (sorry for that) +A big thanks to all of the users and contributors since 2009. -http://unbit.com +uWSGI development has been sponsored by: + +http://unbit.it https://www.pythonanywhere.com/ https://lincolnloop.com/ https://yourlabs.io/oss +https://fili.com diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000000..10b7729d5d --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,31 @@ +## How to cut a new stable release + +Stable releases are tagged from the `uwsgi-2.0` branch. Changes to the `uwsgi-2.0` branch are +cherry-picked from the `master` branch. + +Before tagging a new release the release notes should be updated and a file named +`Changelog-2.0.XY.rst` where XY is the number of the release created in +the [uwsgi-docs repository](https://github.com/unbit/uwsgi-docs). The newly created changelog file +should then be referenced from the `index.rst`. The release should also be updated in the `Download.rst` +page. + +In order to cut a new release you have to first bump the version and then tag with the same version +the last commit in git. All the commands are assumed to be run from the `uwsgi-2.0` branch. + +The the tag should be pushed to git and the source distribution created with the following commands. +Please remember to substitute XY with proper version. + +``` +git tag 2.0.XY +git push --tags origin HEAD +git archive HEAD --prefix uwsgi-2.0.XY/ -o uwsgi-2.0.XY.tar.gz +``` + +Then the tarball must be uploaded to pypi with: + +``` +python3 -m twine upload uwsgi-2.0.XY.tar.gz +``` + +Once the file is uploaded the `Download.rst` page of [uwsgi-docs repository](https://github.com/unbit/uwsgi-docs) +should be updated. diff --git a/apache2/mod_Ruwsgi.c b/apache2/mod_Ruwsgi.c index ecdd5d83cd..0bff85356a 100644 --- a/apache2/mod_Ruwsgi.c +++ b/apache2/mod_Ruwsgi.c @@ -1,27 +1,3 @@ -/* - - *** uWSGI/mod_Ruwsgi *** - - Copyright 2009-2010 Roger Florkowski - Copyright 2009-2010 Unbit S.a.s. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - - -*/ - #define MOD_UWSGI_VERSION "1.0" #include "ap_config.h" diff --git a/apache2/mod_proxy_uwsgi.c b/apache2/mod_proxy_uwsgi.c index 68fb8fb557..c28714ceb2 100644 --- a/apache2/mod_proxy_uwsgi.c +++ b/apache2/mod_proxy_uwsgi.c @@ -1,20 +1,6 @@ /* - -*** mod_proxy_uwsgi *** - -Copyright 2009-2017 Unbit S.a.s. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +*** mod_proxy_uwsgi *** To build: @@ -315,18 +301,16 @@ static int uwsgi_response(request_rec *r, proxy_conn_rec *backend, proxy_server_ apr_bucket_brigade *pass_bb = apr_brigade_create(r->pool, c->bucket_alloc); len = ap_getline(buffer, sizeof(buffer), rp, 1); - if (len <= 0) { - // oops + /* invalid or empty */ return HTTP_INTERNAL_SERVER_ERROR; } - backend->worker->s->read += len; - - if (len >= sizeof(buffer)-1) { - // oops + if ((apr_size_t)len >= sizeof(buffer)) { + /* too long */ return HTTP_INTERNAL_SERVER_ERROR; } + /* Position of http status code */ int status_start; if (apr_date_checkmask(buffer, "HTTP/#.# ###*")) { @@ -334,8 +318,8 @@ static int uwsgi_response(request_rec *r, proxy_conn_rec *backend, proxy_server_ } else if (apr_date_checkmask(buffer, "HTTP/# ###*")) { status_start = 7; } else { - // oops - return HTTP_INTERNAL_SERVER_ERROR; + /* not HTTP */ + return HTTP_BAD_GATEWAY; } int status_end = status_start + 3; @@ -354,17 +338,47 @@ static int uwsgi_response(request_rec *r, proxy_conn_rec *backend, proxy_server_ } r->status_line = apr_pstrdup(r->pool, &buffer[status_start]); - // start parsing headers; + /* parse headers */ while ((len = ap_getline(buffer, sizeof(buffer), rp, 1)) > 0) { + if ((apr_size_t)len >= sizeof(buffer)) { + /* too long */ + len = -1; + break; + } value = strchr(buffer, ':'); - // invalid header skip - if (!value) continue; - *value = '\0'; - ++value; + if (!value) { + /* invalid header */ + len = -1; + break; + } + *value++ = '\0'; + if (*ap_scan_http_token(buffer)) { + /* invalid name */ + len = -1; + break; + } while (apr_isspace(*value)) ++value; for (end = &value[strlen(value)-1]; end > value && apr_isspace(*end); --end) *end = '\0'; + if (*ap_scan_http_field_content(value)) { + /* invalid value */ + len = -1; + break; + } apr_table_add(r->headers_out, buffer, value); } + if (len < 0) { + /* Reset headers, but not to NULL because things below the chain expect + * this to be non NULL e.g. the ap_content_length_filter. + */ + r->headers_out = apr_table_make(r->pool, 1); + return HTTP_BAD_GATEWAY; + } + + /* T-E wins over C-L */ + if (apr_table_get(r->headers_out, "Transfer-Encoding")) { + apr_table_unset(r->headers_out, "Content-Length"); + backend->close = 1; + } if ((buf = apr_table_get(r->headers_out, "Content-Type"))) { ap_set_content_type(r, apr_pstrdup(r->pool, buf)); diff --git a/apache2/mod_uwsgi.c b/apache2/mod_uwsgi.c index ff5cad682e..b80b7744f0 100644 --- a/apache2/mod_uwsgi.c +++ b/apache2/mod_uwsgi.c @@ -2,23 +2,6 @@ *** uWSGI/mod_uwsgi *** -Copyright 2009-2014 Unbit S.a.s. - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation; either version 2 -of the License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - - To compile: (Linux) apxs2 -i -c mod_uwsgi.c diff --git a/buildconf/integration-tests.ini b/buildconf/integration-tests.ini new file mode 100644 index 0000000000..cf92e64a02 --- /dev/null +++ b/buildconf/integration-tests.ini @@ -0,0 +1,4 @@ +[uwsgi] +inherit = base +main_plugin = +plugins = notfound,python,php,pypy,jvm,jwsgi,psgi,cgi,rack diff --git a/buildconf/travis.ini b/buildconf/travis.ini index 68b3642b67..9212fd88c9 100644 --- a/buildconf/travis.ini +++ b/buildconf/travis.ini @@ -1,3 +1,3 @@ [uwsgi] -main_plugin = psgi,rack,lua,python,gevent,php,cgi,pty,xslt,msgpack,geoip,v8,pam,ldap,mono,jvm,ring,jwsgi,servlet,pypy,airbrake,alarm_curl,alarm_xmpp,asyncio,cheaper_backlog2,clock_monotonic,clock_realtime,cplusplus,curl_cron,dumbloop,dummy,echo,emperor_amqp,emperor_pg,emperor_zeromq,example,exception_log,fiber,forkptyrouter,graylog2,legion_cache_fetch,libffi,logcrypto,logpipe,logzmq,matheval,notfound,pyuwsgi,rbthreads,router_access,router_radius,router_spnego,router_xmldir,sqlite3,ssi,stats_pusher_file,stats_pusher_statsd,tornado,transformation_toupper,tuntap,webdav,xattr,zabbix +main_plugin = psgi,rack,lua,python,gevent,php,cgi,pty,xslt,msgpack,geoip,pam,ldap,mono,jvm,ring,jwsgi,servlet,pypy,airbrake,alarm_curl,asyncio,cheaper_backlog2,clock_monotonic,clock_realtime,cplusplus,curl_cron,dumbloop,dummy,echo,emperor_amqp,emperor_pg,emperor_zeromq,example,exception_log,fiber,forkptyrouter,graylog2,legion_cache_fetch,libffi,logcrypto,logpipe,logzmq,matheval,notfound,rbthreads,router_access,router_radius,router_spnego,router_xmldir,sqlite3,ssi,stats_pusher_file,stats_pusher_statsd,tornado,transformation_toupper,tuntap,webdav,xattr,zabbix inherit = base diff --git a/check/check_core.c b/check/check_core.c deleted file mode 100644 index 14d0b16ac6..0000000000 --- a/check/check_core.c +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include "../uwsgi.h" - - -START_TEST(test_uwsgi_strncmp) -{ - int result; - result = uwsgi_strncmp("test", 4, "test", 4); - ck_assert(result == 0); - - result = uwsgi_strncmp("test", 4, "tes", 3); - ck_assert(result == 1); - - result = uwsgi_strncmp("tes", 3, "test", 4); - ck_assert(result == 1); - - result = uwsgi_strncmp("aaa", 3, "bbb", 3); - ck_assert_msg(result == -1, "result: %d", result); - - result = uwsgi_strncmp("bbb", 3, "aaa", 3); - ck_assert_msg(result == 1, "result: %d", result); -} -END_TEST - -Suite *check_core_strings(void) -{ - Suite *s = suite_create("uwsgi strings"); - TCase *tc = tcase_create("strings"); - - suite_add_tcase(s, tc); - tcase_add_test(tc, test_uwsgi_strncmp); - return s; -} - -int main(void) -{ - int nf; - Suite *s = check_core_strings(); - SRunner *r = srunner_create(s); - srunner_run_all(r, CK_NORMAL); - nf = srunner_ntests_failed(r); - srunner_free(r); - return (nf == 0) ? EXIT_SUCCESS : EXIT_FAILURE; -} - diff --git a/contrib/spoolqueue/tasks.py b/contrib/spoolqueue/tasks.py index e9048ee89b..bb5b57a353 100644 --- a/contrib/spoolqueue/tasks.py +++ b/contrib/spoolqueue/tasks.py @@ -1,11 +1,13 @@ +from __future__ import print_function + from tasksconsumer import queueconsumer @queueconsumer('fast', 4) def fast_queue(arguments): - print "fast", arguments + print("fast", arguments) @queueconsumer('slow') def slow_queue(arguments): - print "foobar", arguments + print("foobar", arguments) diff --git a/contrib/spoolqueue/tasksconsumer.py b/contrib/spoolqueue/tasksconsumer.py index f2b6a77976..b3987fef2a 100644 --- a/contrib/spoolqueue/tasksconsumer.py +++ b/contrib/spoolqueue/tasksconsumer.py @@ -1,7 +1,8 @@ from uwsgidecorators import spool -import Queue from threading import Thread +from six.moves import queue + queues = {} @@ -10,7 +11,7 @@ class queueconsumer(object): def __init__(self, name, num=1, **kwargs): self.name = name self.num = num - self.queue = Queue.Queue() + self.queue = queue.Queue() self.threads = [] self.func = None queues[self.name] = self @@ -19,7 +20,7 @@ def __init__(self, name, num=1, **kwargs): def consumer(self): while True: req = self.queue.get() - print req + print(req) self.func(req) self.queue.task_done() diff --git a/contrib/uwsgi-cache-monitor.py b/contrib/uwsgi-cache-monitor.py index ee15dab36d..ec543f7b7d 100644 --- a/contrib/uwsgi-cache-monitor.py +++ b/contrib/uwsgi-cache-monitor.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import mmap import os import struct diff --git a/core/alarm.c b/core/alarm.c index 7691a7d0a4..bdd7b00ce4 100644 --- a/core/alarm.c +++ b/core/alarm.c @@ -160,7 +160,7 @@ static struct uwsgi_alarm_instance *uwsgi_alarm_get_instance(char *name) { } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) static int uwsgi_alarm_log_add(char *alarms, char *regexp, int negate) { struct uwsgi_alarm_log *old_ual = NULL, *ual = uwsgi.alarm_logs; @@ -170,7 +170,7 @@ static int uwsgi_alarm_log_add(char *alarms, char *regexp, int negate) { } ual = uwsgi_calloc(sizeof(struct uwsgi_alarm_log)); - if (uwsgi_regexp_build(regexp, &ual->pattern, &ual->pattern_extra)) { + if (uwsgi_regexp_build(regexp, &ual->pattern)) { free(ual); return -1; } @@ -331,7 +331,7 @@ void uwsgi_alarms_init() { usl = usl->next; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) // then map log-alarm usl = uwsgi.alarm_logs_list; while (usl) { @@ -377,14 +377,14 @@ void uwsgi_alarm_trigger_uai(struct uwsgi_alarm_instance *uai, char *msg, size_t } } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) // check if a log should raise an alarm void uwsgi_alarm_log_check(char *msg, size_t len) { if (!uwsgi_strncmp(msg, len, "[uwsgi-alarm", 12)) return; struct uwsgi_alarm_log *ual = uwsgi.alarm_logs; while (ual) { - if (uwsgi_regexp_match(ual->pattern, ual->pattern_extra, msg, len) >= 0) { + if (uwsgi_regexp_match(ual->pattern, msg, len) >= 0) { if (!ual->negate) { struct uwsgi_alarm_ll *uall = ual->alarms; while (uall) { diff --git a/core/async.c b/core/async.c index f03aaf5978..db8949429f 100644 --- a/core/async.c +++ b/core/async.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/cache.c b/core/cache.c index 13a23dbee6..6e9b261493 100644 --- a/core/cache.c +++ b/core/cache.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; #define cache_item(x) (struct uwsgi_cache_item *) (((char *)uc->items) + ((sizeof(struct uwsgi_cache_item)+uc->keysize) * x)) @@ -737,6 +737,12 @@ void uwsgi_cache_fix(struct uwsgi_cache *uc) { if (uci->expires && (!next_scan || next_scan > uci->expires)) { next_scan = uci->expires; } + if (!uc->lru_head && !uci->lru_prev) { + uc->lru_head = i; + } + if (!uc->lru_tail && !uci->lru_next) { + uc->lru_tail = i; + } restored++; } else { diff --git a/core/chunked.c b/core/chunked.c index 3979c20039..fe693aad47 100644 --- a/core/chunked.c +++ b/core/chunked.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/config.c b/core/config.c index 7b05abc6f0..164bd84250 100644 --- a/core/config.c +++ b/core/config.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* @@ -336,7 +336,7 @@ int uwsgi_logic_opt_if_not_hostname(char *key, char *value) { return 0; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) int uwsgi_logic_opt_if_hostname_match(char *key, char *value) { uwsgi.logic_opt_if_failed = 0; if (uwsgi_regexp_match_pattern(uwsgi.logic_opt_data, uwsgi.hostname)) { diff --git a/core/cookie.c b/core/cookie.c index 37350bf020..1664798545 100644 --- a/core/cookie.c +++ b/core/cookie.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* diff --git a/core/cron.c b/core/cron.c index 013ade6919..2c482aaea9 100644 --- a/core/cron.c +++ b/core/cron.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -238,6 +238,7 @@ void uwsgi_manage_signal_cron(time_t now) { if (run_task == 1) { // date match, signal it ? if (now - ucron->last_job >= 60) { + uwsgi_log_verbose("[uwsgi-cron] routing signal %d\n", ucron->sig); uwsgi_route_signal(ucron->sig); ucron->last_job = now; } diff --git a/core/daemons.c b/core/daemons.c index 025dac5a9c..70ff3d904f 100644 --- a/core/daemons.c +++ b/core/daemons.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -254,6 +254,16 @@ void uwsgi_detach_daemons() { // unregister daemon to prevent it from being respawned ud->registered = 0; } + + // smart daemons that have to be notified when master is reloading or stopping + if (ud->notifypid && ud->pid > 0 && ud->pidfile) { + if (uwsgi_instance_is_reloading) { + kill(-(ud->pid), ud->reload_signal > 0 ? ud->reload_signal : SIGHUP); + } + else { + kill(-(ud->pid), ud->stop_signal); + } + } ud = ud->next; } } @@ -520,6 +530,7 @@ void uwsgi_opt_add_daemon2(char *opt, char *value, void *none) { char *d_ns_pid = NULL; char *d_chdir = NULL; char *d_max_throttle = NULL; + char *d_notifypid = NULL; char *arg = uwsgi_str(value); @@ -543,6 +554,7 @@ void uwsgi_opt_add_daemon2(char *opt, char *value, void *none) { "ns_pid", &d_ns_pid, "chdir", &d_chdir, "max_throttle", &d_max_throttle, + "notifypid", &d_notifypid, NULL)) { uwsgi_log("invalid --%s keyval syntax\n", opt); exit(1); @@ -594,6 +606,8 @@ void uwsgi_opt_add_daemon2(char *opt, char *value, void *none) { uwsgi_ud->max_throttle = d_max_throttle ? atoi(d_max_throttle) : 0; + uwsgi_ud->notifypid = d_notifypid ? 1 : 0; + if (d_touch) { size_t i,rlen = 0; char **argv = uwsgi_split_quoted(d_touch, strlen(d_touch), ";", &rlen); diff --git a/core/emperor.c b/core/emperor.c index c878767449..cd99e75500 100644 --- a/core/emperor.c +++ b/core/emperor.c @@ -3,7 +3,7 @@ The uWSGI Emperor */ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; extern char **environ; @@ -918,7 +918,7 @@ void emperor_back_to_ondemand(struct uwsgi_instance *c_ui) { // remove uWSGI instance if (c_ui->pid != -1) { - if (write(c_ui->pipe[0], "\0", 1) != 1) { + if (write(c_ui->pipe[0], uwsgi.emperor_graceful_shutdown ? "\2" : "\0", 1) != 1) { uwsgi_error("emperor_stop()/write()"); } } @@ -941,7 +941,7 @@ void emperor_stop(struct uwsgi_instance *c_ui) { } if (c_ui->pid != -1) { - if (write(c_ui->pipe[0], "\0", 1) != 1) { + if (write(c_ui->pipe[0], uwsgi.emperor_graceful_shutdown ? "\2" : "\0", 1) != 1) { uwsgi_error("emperor_stop()/write()"); } } @@ -2970,8 +2970,9 @@ void uwsgi_master_manage_emperor() { if (byte == 0) { uwsgi_hooks_run(uwsgi.hook_emperor_stop, "emperor-stop", 0); close(uwsgi.emperor_fd); - if (!uwsgi.status.brutally_reloading) + if (!uwsgi.status.brutally_reloading && !uwsgi.status.brutally_destroying) { kill_them_all(0); + } } // reload me else if (byte == 1) { @@ -2982,6 +2983,14 @@ void uwsgi_master_manage_emperor() { grace_them_all(0); uwsgi_unblock_signal(SIGHUP); } + // remove me gracefully + else if (byte == 2) { + uwsgi_hooks_run(uwsgi.hook_emperor_stop, "emperor-stop", 0); + close(uwsgi.emperor_fd); + if (!uwsgi.status.brutally_reloading && !uwsgi.status.brutally_destroying) { + gracefully_kill_them_all(0); + } + } } #ifdef UWSGI_EVENT_USE_PORT // special cose for port event system diff --git a/core/errors.c b/core/errors.c index e4879854e3..deabd1f4ba 100644 --- a/core/errors.c +++ b/core/errors.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/exceptions.c b/core/exceptions.c index a939839de3..1d7196a88b 100644 --- a/core/exceptions.c +++ b/core/exceptions.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/fifo.c b/core/fifo.c index 1e8cd2e94f..172f06dc88 100644 --- a/core/fifo.c +++ b/core/fifo.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -78,25 +78,25 @@ void uwsgi_master_fifo_prepare() { uwsgi_fifo_table['8'] = uwsgi_fifo_set_slot_eight; uwsgi_fifo_table['9'] = uwsgi_fifo_set_slot_nine; - uwsgi_fifo_table['-'] = uwsgi_cheaper_decrease; - uwsgi_fifo_table['+'] = uwsgi_cheaper_increase; - uwsgi_fifo_table['B'] = vassal_sos; - uwsgi_fifo_table['c'] = uwsgi_chain_reload; - uwsgi_fifo_table['C'] = uwsgi_go_cheap; + uwsgi_fifo_table['-'] = (void (*)(int))uwsgi_cheaper_decrease; + uwsgi_fifo_table['+'] = (void (*)(int))uwsgi_cheaper_increase; + uwsgi_fifo_table['B'] = (void (*)(int))vassal_sos; + uwsgi_fifo_table['c'] = (void (*)(int))uwsgi_chain_reload; + uwsgi_fifo_table['C'] = (void (*)(int))uwsgi_go_cheap; uwsgi_fifo_table['E'] = emperor_rescan; - uwsgi_fifo_table['f'] = uwsgi_refork_master; - uwsgi_fifo_table['l'] = uwsgi_log_reopen; - uwsgi_fifo_table['L'] = uwsgi_log_rotate; + uwsgi_fifo_table['f'] = (void (*)(int))uwsgi_refork_master; + uwsgi_fifo_table['l'] = (void (*)(int))uwsgi_log_reopen; + uwsgi_fifo_table['L'] = (void (*)(int))uwsgi_log_rotate; uwsgi_fifo_table['p'] = suspend_resume_them_all; - uwsgi_fifo_table['P'] = uwsgi_update_pidfiles; + uwsgi_fifo_table['P'] = (void (*)(int))uwsgi_update_pidfiles; uwsgi_fifo_table['q'] = gracefully_kill_them_all; uwsgi_fifo_table['Q'] = kill_them_all; uwsgi_fifo_table['r'] = grace_them_all; uwsgi_fifo_table['R'] = reap_them_all; uwsgi_fifo_table['s'] = stats; uwsgi_fifo_table['S'] = subscriptions_blocker; - uwsgi_fifo_table['w'] = uwsgi_reload_workers; - uwsgi_fifo_table['W'] = uwsgi_brutally_reload_workers; + uwsgi_fifo_table['w'] = (void (*)(int))uwsgi_reload_workers; + uwsgi_fifo_table['W'] = (void (*)(int))uwsgi_brutally_reload_workers; } @@ -104,7 +104,9 @@ int uwsgi_master_fifo() { char *path = uwsgi_fifo_by_slot(); - unlink(path); + if (unlink(path) != 0 && errno != ENOENT) { + uwsgi_error("uwsgi_master_fifo()/unlink()"); + } if (mkfifo(path, S_IRUSR|S_IWUSR)) { uwsgi_error("uwsgi_master_fifo()/mkfifo()"); diff --git a/core/fork_server.c b/core/fork_server.c index fb75163c1d..fa2f27f3b6 100644 --- a/core/fork_server.c +++ b/core/fork_server.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/fsmon.c b/core/fsmon.c index cca53d9f70..3d5ee0957d 100644 --- a/core/fsmon.c +++ b/core/fsmon.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/hash.c b/core/hash.c index 86ddf83141..5940f661ee 100644 --- a/core/hash.c +++ b/core/hash.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/hooks.c b/core/hooks.c index 832d6d3193..9329d9ce26 100644 --- a/core/hooks.c +++ b/core/hooks.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/ini.c b/core/ini.c index 61a3c26ac1..9ff18a47a3 100644 --- a/core/ini.c +++ b/core/ini.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -87,8 +87,6 @@ void uwsgi_ini_config(char *file, char *magic_table[]) { char *key; char *val; - int lines = 1; - char *section_asked = "uwsgi"; char *colon; int got_section = 0; @@ -130,7 +128,6 @@ void uwsgi_ini_config(char *file, char *magic_table[]) { if (ini_line == NULL) { break; } - lines++; // skip empty line key = ini_lstrip(ini); diff --git a/core/init.c b/core/init.c index 0c3178a3d9..267a3fe3f2 100644 --- a/core/init.c +++ b/core/init.c @@ -3,7 +3,7 @@ extern struct uwsgi_server uwsgi; struct http_status_codes { - const char key[3]; + const char key[4]; const char *message; int message_size; }; @@ -46,12 +46,21 @@ struct http_status_codes hsc[] = { {"415", "Unsupported Media Type"}, {"416", "Requested Range Not Satisfiable"}, {"417", "Expectation Failed"}, + {"418", "I'm a teapot"}, + {"422", "Unprocessable Entity"}, + {"425", "Too Early"}, + {"426", "Upgrade Required"}, + {"428", "Precondition Required"}, + {"429", "Too Many Requests"}, + {"431", "Request Header Fields Too Large"}, + {"451", "Unavailable For Legal Reasons"}, {"501", "Not Implemented"}, {"502", "Bad Gateway"}, {"503", "Service Unavailable"}, {"504", "Gateway Timeout"}, {"505", "HTTP Version Not Supported"}, {"509", "Bandwidth Limit Exceeded"}, + {"511", "Network Authentication Required"}, {"", NULL}, }; @@ -196,6 +205,8 @@ void uwsgi_init_default() { uwsgi.notify_socket_fd = -1; uwsgi.mule_msg_recv_size = 65536; + + uwsgi.harakiri_graceful_signal = SIGTERM; } void uwsgi_setup_reload() { @@ -457,8 +468,8 @@ void sanitize_args() { uwsgi.cores = uwsgi.async; } + uwsgi.has_threads = 1; if (uwsgi.threads > 1) { - uwsgi.has_threads = 1; uwsgi.cores = uwsgi.threads; } diff --git a/core/lock.c b/core/lock.c index 686e3b106a..eb0de35cfc 100644 --- a/core/lock.c +++ b/core/lock.c @@ -91,11 +91,6 @@ struct uwsgi_lock_item *uwsgi_lock_fast_init(char *id) { } #ifdef EOWNERDEAD -#ifndef PTHREAD_MUTEX_ROBUST -#define PTHREAD_MUTEX_ROBUST PTHREAD_MUTEX_ROBUST_NP -#define pthread_mutexattr_setrobust pthread_mutexattr_setrobust_np -#define pthread_mutex_consistent pthread_mutex_consistent_np -#endif if (uwsgi_pthread_robust_mutexes_enabled) { int ret; if ((ret = pthread_mutexattr_setprotocol(&attr, PTHREAD_PRIO_INHERIT)) != 0) { diff --git a/core/logging.c b/core/logging.c index 9afdc0b730..b174614c6b 100644 --- a/core/logging.c +++ b/core/logging.c @@ -1,5 +1,5 @@ #ifndef __DragonFly__ -#include +#include "uwsgi.h" #endif #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) #include @@ -15,7 +15,7 @@ #endif #ifdef __DragonFly__ -#include +#include "uwsgi.h" #endif extern struct uwsgi_server uwsgi; @@ -325,7 +325,7 @@ void logto(char *logfile) { void uwsgi_setup_log() { - + uwsgi_setup_log_encoders(); if (uwsgi.daemonize) { @@ -414,7 +414,7 @@ void uwsgi_setup_log_master(void) { usl = usl->next; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) // set logger by its id struct uwsgi_regexp_list *url = uwsgi.log_route; while (url) { @@ -498,7 +498,7 @@ void uwsgi_check_logrotate(void) { return; } - if (logstat.st_mode & S_IFIFO) { + if (S_ISFIFO(logstat.st_mode) || S_ISSOCK(logstat.st_mode)) { return; } @@ -688,7 +688,7 @@ void uwsgi_logit_simple(struct wsgi_request *wsgi_req) { via = msg4; break; default: - break; + break; } #if defined(__sun__) && !defined(__clang__) @@ -720,7 +720,7 @@ void uwsgi_logit_simple(struct wsgi_request *wsgi_req) { } if (uwsgi.logging_options.memory_report) { - rlen = snprintf(mempkt, 4096, "{address space usage: %lld bytes/%lluMB} {rss usage: %llu bytes/%lluMB} ", + rlen = snprintf(mempkt, 4096, "{address space usage: %llu bytes/%lluMB} {rss usage: %llu bytes/%lluMB} ", (unsigned long long) uwsgi.workers[uwsgi.mywid].vsz_size, (unsigned long long) uwsgi.workers[uwsgi.mywid].vsz_size / 1024 / 1024, (unsigned long long) uwsgi.workers[uwsgi.mywid].rss_size, @@ -772,7 +772,7 @@ void get_memusage(uint64_t * rss, uint64_t * vsz) { int i; procfile = fopen("/proc/self/stat", "r"); if (procfile) { - i = fscanf(procfile, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %llu %lld", (unsigned long long *) vsz, (unsigned long long *) rss); + i = fscanf(procfile, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %llu %llu", (unsigned long long *) vsz, (unsigned long long *) rss); if (i != 2) { uwsgi_log("warning: invalid record in /proc/self/stat\n"); } else { @@ -786,7 +786,7 @@ void get_memusage(uint64_t * rss, uint64_t * vsz) { int i; procfile = fopen("/proc/self/stat", "r"); if (procfile) { - i = fscanf(procfile, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %llu %lld", (unsigned long long *) vsz, (unsigned long long *) rss); + i = fscanf(procfile, "%*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %*s %llu %llu", (unsigned long long *) vsz, (unsigned long long *) rss); if (i != 2) { uwsgi_log("warning: invalid record in /proc/self/stat\n"); } @@ -1051,7 +1051,7 @@ void uwsgi_logit_lf(struct wsgi_request *wsgi_req) { if (uwsgi.logvectors[wsgi_req->async_id][pos].iov_len == 0 && logchunk->type != 0) { uwsgi.logvectors[wsgi_req->async_id][pos].iov_base = (char *) empty_var; - uwsgi.logvectors[wsgi_req->async_id][pos].iov_len = 1; + uwsgi.logvectors[wsgi_req->async_id][pos].iov_len = 1; } logchunk = logchunk->next; } @@ -1128,19 +1128,18 @@ static ssize_t uwsgi_lf_status(struct wsgi_request *wsgi_req, char **buf) { return strlen(*buf); } - static ssize_t uwsgi_lf_rsize(struct wsgi_request *wsgi_req, char **buf) { - *buf = uwsgi_num2str(wsgi_req->response_size); + *buf = uwsgi_size2str(wsgi_req->response_size); return strlen(*buf); } static ssize_t uwsgi_lf_hsize(struct wsgi_request *wsgi_req, char **buf) { - *buf = uwsgi_num2str(wsgi_req->headers_size); + *buf = uwsgi_size2str(wsgi_req->headers_size); return strlen(*buf); } static ssize_t uwsgi_lf_size(struct wsgi_request *wsgi_req, char **buf) { - *buf = uwsgi_num2str(wsgi_req->headers_size+wsgi_req->response_size); + *buf = uwsgi_size2str(wsgi_req->headers_size+wsgi_req->response_size); return strlen(*buf); } @@ -1320,7 +1319,7 @@ struct uwsgi_logchunk *uwsgi_register_logchunk(char *name, ssize_t (*func)(struc logchunk->func = func; logchunk->free = need_free; logchunk->type = 3; - return logchunk; + return logchunk; } struct uwsgi_logchunk *uwsgi_get_logchunk_by_name(char *name, size_t name_len) { @@ -1444,11 +1443,11 @@ int uwsgi_master_log(void) { ssize_t rlen = read(uwsgi.shared->worker_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize); if (rlen > 0) { -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) uwsgi_alarm_log_check(uwsgi.log_master_buf, rlen); struct uwsgi_regexp_list *url = uwsgi.log_drain_rules; while (url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + if (uwsgi_regexp_match(url->pattern, uwsgi.log_master_buf, rlen) >= 0) { return 0; } url = url->next; @@ -1457,7 +1456,7 @@ int uwsgi_master_log(void) { int show = 0; url = uwsgi.log_filter_rules; while (url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + if (uwsgi_regexp_match(url->pattern, uwsgi.log_master_buf, rlen) >= 0) { show = 1; break; } @@ -1470,7 +1469,7 @@ int uwsgi_master_log(void) { url = uwsgi.log_route; int finish = 0; while (url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + if (uwsgi_regexp_match(url->pattern, uwsgi.log_master_buf, rlen) >= 0) { struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr; if (ul_route) { uwsgi_log_func_do(uwsgi.requested_log_encoders, ul_route, uwsgi.log_master_buf, rlen); @@ -1510,11 +1509,11 @@ int uwsgi_master_req_log(void) { ssize_t rlen = read(uwsgi.shared->worker_req_log_pipe[0], uwsgi.log_master_buf, uwsgi.log_master_bufsize); if (rlen > 0) { -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *url = uwsgi.log_req_route; int finish = 0; while (url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, uwsgi.log_master_buf, rlen) >= 0) { + if (uwsgi_regexp_match(url->pattern, uwsgi.log_master_buf, rlen) >= 0) { struct uwsgi_logger *ul_route = (struct uwsgi_logger *) url->custom_ptr; if (ul_route) { uwsgi_log_func_do(uwsgi.requested_log_req_encoders, ul_route, uwsgi.log_master_buf, rlen); @@ -1665,7 +1664,7 @@ void uwsgi_setup_log_encoders() { exit(1); } struct uwsgi_log_encoder *ule2 = uwsgi_malloc(sizeof(struct uwsgi_log_encoder)); - memcpy(ule2, ule, sizeof(struct uwsgi_log_encoder)); + memcpy(ule2, ule, sizeof(struct uwsgi_log_encoder)); if (use_for) { ule2->use_for = uwsgi_str(use_for+1); *use_for = ':'; @@ -1858,7 +1857,7 @@ void uwsgi_log_encoder_parse_vars(struct uwsgi_log_encoder *ule) { strftime (strftime) */ static char *uwsgi_log_encoder_format(struct uwsgi_log_encoder *ule, char *msg, size_t len, size_t *rlen) { - + if (!ule->configured) { uwsgi_log_encoder_parse_vars(ule); ule->configured = 1; @@ -1886,6 +1885,9 @@ static char *uwsgi_log_encoder_format(struct uwsgi_log_encoder *ule, char *msg, else if (!uwsgi_strncmp(usl->value, usl->len, "micros", 6)) { if (uwsgi_buffer_num64(ub, uwsgi_micros())) goto end; } + else if (!uwsgi_strncmp(usl->value, usl->len, "millis", 6)) { + if (uwsgi_buffer_num64(ub, uwsgi_millis())) goto end; + } else if (!uwsgi_starts_with(usl->value, usl->len, "strftime:", 9)) { char sftime[64]; time_t now = uwsgi_now(); diff --git a/core/loop.c b/core/loop.c index 2c856826e2..c6824edf21 100644 --- a/core/loop.c +++ b/core/loop.c @@ -60,6 +60,9 @@ void *uwsgi_get_loop(char *name) { void simple_loop() { uwsgi_loop_cores_run(simple_loop_run); + // Other threads may still run. Make sure they will stop. + uwsgi.workers[uwsgi.mywid].manage_next_request = 0; + if (uwsgi.workers[uwsgi.mywid].shutdown_sockets) uwsgi_shutdown_all_sockets(); } @@ -78,9 +81,6 @@ void uwsgi_setup_thread_req(long core_id, struct wsgi_request *wsgi_req) { int i; sigset_t smask; - - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &i); - pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &i); pthread_setspecific(uwsgi.tur_key, (void *) wsgi_req); if (core_id > 0) { @@ -127,6 +127,7 @@ void *simple_loop_run(void *arg1) { int main_queue = event_queue_init(); uwsgi_add_sockets_to_queue(main_queue, core_id); + event_queue_add_fd_read(main_queue, uwsgi.loop_stop_pipe[0]); if (uwsgi.signal_socket > -1) { event_queue_add_fd_read(main_queue, uwsgi.signal_socket); diff --git a/core/master.c b/core/master.c index dccd590bd8..44fbc0465d 100644 --- a/core/master.c +++ b/core/master.c @@ -8,16 +8,12 @@ static void master_check_processes() { if (!uwsgi.die_on_no_workers) return; int alive_processes = 0; - int dead_processes = 0; int i; for (i = 1; i <= uwsgi.numproc; i++) { if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) { alive_processes++; } - else { - dead_processes++; - } } if (uwsgi.die_on_no_workers) { @@ -280,7 +276,7 @@ static void master_check_listen_queue() { uint64_t backlog = 0; struct uwsgi_socket *uwsgi_sock = uwsgi.sockets; while(uwsgi_sock) { - if (uwsgi_sock->family == AF_INET) { + if (uwsgi_sock->family == AF_INET || uwsgi_sock->family == AF_INET6) { get_tcp_info(uwsgi_sock); } #ifdef __linux__ diff --git a/core/master_checks.c b/core/master_checks.c index 5b0c1d1b64..c8dc66a7bd 100644 --- a/core/master_checks.c +++ b/core/master_checks.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -180,28 +180,57 @@ void uwsgi_master_check_idle() { } +int uwsgi_master_check_harakiri(int w, int c, time_t harakiri) { + /** + * Triggers a harakiri when the following conditions are met: + * - harakiri timeout > current time + * - listen queue pressure (ie backlog > harakiri_queue_threshold) + * + * The first harakiri attempt on a worker will be graceful if harakiri_graceful_timeout > 0, + * then the worker has harakiri_graceful_timeout seconds to shutdown cleanly, otherwise + * a second harakiri will trigger a SIGKILL + * + */ +#ifdef __linux__ + int backlog = uwsgi.shared->backlog; +#else + int backlog = 0; +#endif + if (harakiri == 0 || harakiri > (time_t) uwsgi.current_time) { + return 0; + } + // no pending harakiri for the worker and no backlog pressure, safe to skip + if (uwsgi.workers[w].pending_harakiri == 0 && backlog < uwsgi.harakiri_queue_threshold) { + uwsgi_log_verbose("HARAKIRI: Skipping harakiri on worker %d. Listen queue is smaller than the threshold (%d < %d)\n", + w, backlog, uwsgi.harakiri_queue_threshold); + return 0; + } + + trigger_harakiri(w); + if (uwsgi.harakiri_graceful_timeout > 0) { + uwsgi.workers[w].cores[c].harakiri = harakiri + uwsgi.harakiri_graceful_timeout; + uwsgi_log_verbose("HARAKIRI: graceful termination attempt on worker %d with signal %d. Next harakiri: %d\n", + w, uwsgi.harakiri_graceful_signal, uwsgi.workers[w].cores[c].harakiri); + } + return 1; +} + int uwsgi_master_check_workers_deadline() { int i,j; int ret = 0; for (i = 1; i <= uwsgi.numproc; i++) { for(j=0;j 0) { - if (uwsgi.workers[i].cores[j].harakiri < (time_t) uwsgi.current_time) { - uwsgi_log_verbose("HARAKIRI triggered by worker %d core %d !!!\n", i, j); - trigger_harakiri(i); - ret = 1; - break; - } + if (uwsgi_master_check_harakiri(i, j, uwsgi.workers[i].cores[j].harakiri)) { + uwsgi_log_verbose("HARAKIRI triggered by worker %d core %d !!!\n", i, j); + ret = 1; + break; } /* then user-defined harakiri */ - if (uwsgi.workers[i].cores[j].user_harakiri > 0) { + if (uwsgi_master_check_harakiri(i, j, uwsgi.workers[i].cores[j].user_harakiri)) { uwsgi_log_verbose("HARAKIRI (user) triggered by worker %d core %d !!!\n", i, j); - if (uwsgi.workers[i].cores[j].user_harakiri < (time_t) uwsgi.current_time) { - trigger_harakiri(i); - ret = 1; - break; - } + ret = 1; + break; } } // then for evil memory checkers @@ -224,7 +253,7 @@ int uwsgi_master_check_workers_deadline() { // check if worker was running longer than allowed lifetime if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cheaped == 0 && uwsgi.max_worker_lifetime > 0) { uint64_t lifetime = uwsgi_now() - uwsgi.workers[i].last_spawn; - if (lifetime > uwsgi.max_worker_lifetime && uwsgi.workers[i].manage_next_request == 1) { + if (lifetime > (uwsgi.max_worker_lifetime + (i-1) * uwsgi.max_worker_lifetime_delta) && uwsgi.workers[i].manage_next_request == 1) { uwsgi_log("worker %d lifetime reached, it was running for %llu second(s)\n", i, (unsigned long long) lifetime); uwsgi.workers[i].manage_next_request = 0; kill(uwsgi.workers[i].pid, SIGWINCH); diff --git a/core/master_events.c b/core/master_events.c index 3466415f3a..a92117413f 100644 --- a/core/master_events.c +++ b/core/master_events.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/master_utils.c b/core/master_utils.c index 7737f1de1a..d147f2d6c4 100644 --- a/core/master_utils.c +++ b/core/master_utils.c @@ -2,7 +2,7 @@ extern struct uwsgi_server uwsgi; -void worker_wakeup() { +void worker_wakeup(int sig) { } uint64_t uwsgi_worker_exceptions(int wid) { @@ -747,9 +747,22 @@ int uwsgi_respawn_worker(int wid) { pthread_mutex_lock(&uwsgi.threaded_logger_lock); } + + for (i = 0; i < 256; i++) { + if (uwsgi.p[i]->pre_uwsgi_fork) { + uwsgi.p[i]->pre_uwsgi_fork(); + } + } + pid_t pid = uwsgi_fork(uwsgi.workers[wid].name); if (pid == 0) { + for (i = 0; i < 256; i++) { + if (uwsgi.p[i]->post_uwsgi_fork) { + uwsgi.p[i]->post_uwsgi_fork(1); + } + } + signal(SIGWINCH, worker_wakeup); signal(SIGTSTP, worker_wakeup); uwsgi.mywid = wid; @@ -812,6 +825,12 @@ int uwsgi_respawn_worker(int wid) { uwsgi_error("fork()"); } else { + for (i = 0; i < 256; i++) { + if (uwsgi.p[i]->post_uwsgi_fork) { + uwsgi.p[i]->post_uwsgi_fork(0); + } + } + // the pid is set only in the master, as the worker should never use it uwsgi.workers[wid].pid = pid; @@ -1605,7 +1624,10 @@ void uwsgi_register_cheaper_algo(char *name, int (*func) (int)) { void trigger_harakiri(int i) { int j; - uwsgi_log_verbose("*** HARAKIRI ON WORKER %d (pid: %d, try: %d) ***\n", i, uwsgi.workers[i].pid, uwsgi.workers[i].pending_harakiri + 1); + uwsgi_log_verbose("*** HARAKIRI ON WORKER %d (pid: %d, try: %d, graceful: %s) ***\n", i, + uwsgi.workers[i].pid, + uwsgi.workers[i].pending_harakiri + 1, + uwsgi.workers[i].pending_harakiri > 0 ? "no": "yes"); if (uwsgi.harakiri_verbose) { #ifdef __linux__ int proc_file; @@ -1654,7 +1676,11 @@ void trigger_harakiri(int i) { } uwsgi_dump_worker(i, "HARAKIRI"); - kill(uwsgi.workers[i].pid, SIGKILL); + if (uwsgi.workers[i].pending_harakiri == 0 && uwsgi.harakiri_graceful_timeout > 0) { + kill(uwsgi.workers[i].pid, uwsgi.harakiri_graceful_signal); + } else { + kill(uwsgi.workers[i].pid, SIGKILL); + } if (!uwsgi.workers[i].pending_harakiri) uwsgi.workers[i].harakiri_count++; uwsgi.workers[i].pending_harakiri++; @@ -1687,7 +1713,8 @@ int uwsgi_cron_task_needs_execution(struct tm *uwsgi_cron_delta, int minute, int uc_hour = hour; uc_day = day; uc_month = month; - uc_week = week; + // support 7 as alias for sunday (0) to match crontab behaviour + uc_week = week == 7 ? 0 : week; // negative values as interval -1 = * , -5 = */5 if (minute < 0) { diff --git a/core/metrics.c b/core/metrics.c index c31b6d501d..733d42084f 100644 --- a/core/metrics.c +++ b/core/metrics.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -703,16 +703,16 @@ int64_t uwsgi_metric_getn(char *name, size_t nlen, char *oid, size_t olen) { int uwsgi_metric_set_max(char *name, char *oid, int64_t value) { um_op; - if (value > *um->value) - *um->value = value; + if (value > *um->value) + *um->value = value; uwsgi_rwunlock(uwsgi.metrics_lock); return 0; } int uwsgi_metric_set_min(char *name, char *oid, int64_t value) { um_op; - if ((value > um->initial_value || 0) && value < *um->value) - *um->value = value; + if ((value > um->initial_value || 0) && value < *um->value) + *um->value = value; uwsgi_rwunlock(uwsgi.metrics_lock); return 0; } diff --git a/core/mount.c b/core/mount.c index 23f5ea4158..add9367db8 100644 --- a/core/mount.c +++ b/core/mount.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* @@ -106,6 +106,7 @@ uint64_t uwsgi_mount_flag(char *mflag) { } int uwsgi_mount(char *fs, char *what, char *where, char *flags, char *data) { +#if defined(__linux__) || defined(__FreeBSD__) || defined(__GNU_kFreeBSD__) #if defined(__FreeBSD__) || defined(__GNU_kFreeBSD__) struct iovec iov[6]; #endif @@ -145,11 +146,13 @@ int uwsgi_mount(char *fs, char *what, char *where, char *flags, char *data) { iov[5].iov_len = strlen(what) + 1; return nmount(iov, 6, (int) mountflags); +#endif #endif return -1; } int uwsgi_umount(char *where, char *flags) { +#if defined(__linux__) || defined(__FreeBSD__) || defined(__GNU_kFreeBSD__) unsigned long mountflags = 0; if (!flags) goto parsed; char *mflags = uwsgi_str(flags); @@ -198,6 +201,7 @@ int uwsgi_umount(char *where, char *flags) { return umount2(where, mountflags); #elif defined(__FreeBSD__) || defined(__GNU_kFreeBSD__) return unmount(where, mountflags); +#endif #endif return -1; } diff --git a/core/notify.c b/core/notify.c index 7c08d68181..2e267dbe6c 100644 --- a/core/notify.c +++ b/core/notify.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/offload.c b/core/offload.c index 09f948bd07..71dbe9ec34 100644 --- a/core/offload.c +++ b/core/offload.c @@ -236,11 +236,11 @@ static void uwsgi_offload_close(struct uwsgi_thread *ut, struct uwsgi_offload_re close(uor->pipe[0]); } - free(uor); - #ifdef UWSGI_DEBUG uwsgi_log("[offload] destroyed session %p\n", uor); #endif + + free(uor); } static void uwsgi_offload_append(struct uwsgi_thread *ut, struct uwsgi_offload_request *uor) { diff --git a/core/plugins_builder.c b/core/plugins_builder.c index f0fd38c909..967f4d2988 100644 --- a/core/plugins_builder.c +++ b/core/plugins_builder.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" #define UWSGI_BUILD_DIR ".uwsgi_plugins_builder" diff --git a/core/progress.c b/core/progress.c index 25bd8c5627..f46f45b620 100644 --- a/core/progress.c +++ b/core/progress.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* diff --git a/core/querystring.c b/core/querystring.c index e77120f068..6f299174fc 100644 --- a/core/querystring.c +++ b/core/querystring.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* diff --git a/core/rb_timers.c b/core/rb_timers.c index 1f2c1f7aa2..9c299ba58f 100644 --- a/core/rb_timers.c +++ b/core/rb_timers.c @@ -10,7 +10,7 @@ the "Introduction to Algorithms" by Cormen, Leiserson and Rivest. */ -#include +#include "uwsgi.h" #define uwsgi_rbt_red(node) ((node)->color = 1) #define uwsgi_rbt_black(node) ((node)->color = 0) diff --git a/core/reader.c b/core/reader.c index fcc1af5cec..bc8e770a80 100644 --- a/core/reader.c +++ b/core/reader.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -283,7 +283,7 @@ char *uwsgi_request_body_readline(struct wsgi_request *wsgi_req, ssize_t hint, s } // no line found, let's return all - *rlen = wsgi_req->post_readline_size - wsgi_req->post_readline_pos; + *rlen = wsgi_req->post_readline_watermark - wsgi_req->post_readline_pos; char *buf = wsgi_req->post_readline_buf + wsgi_req->post_readline_pos; wsgi_req->post_readline_pos = 0; return buf; diff --git a/core/regexp.c b/core/regexp.c index a0569db7bd..7e0c7de1ee 100644 --- a/core/regexp.c +++ b/core/regexp.c @@ -1,4 +1,4 @@ -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) #include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -10,51 +10,119 @@ void uwsgi_opt_pcre_jit(char *opt, char *value, void *foobar) { if (ret != 0 || has_jit != 1) return; uwsgi.pcre_jit = PCRE_STUDY_JIT_COMPILE; +#elif defined(PCRE2_CONFIG_JIT) + int has_jit = 0, ret; + ret = pcre2_config(PCRE2_CONFIG_JIT, &has_jit); + if (ret != 0) + return; + uwsgi.pcre_jit = has_jit; #endif } -int uwsgi_regexp_build(char *re, pcre ** pattern, pcre_extra ** pattern_extra) { +int uwsgi_regexp_build(char *re, uwsgi_pcre ** pattern) { + +#ifdef UWSGI_PCRE2 + int errnbr; + size_t erroff; + *pattern = pcre2_compile((const unsigned char *) re, PCRE2_ZERO_TERMINATED, 0, &errnbr, &erroff, NULL); +#else const char *errstr; int erroff; - *pattern = pcre_compile((const char *) re, 0, &errstr, &erroff, NULL); - if (!*pattern) { + *pattern = uwsgi_malloc(sizeof(uwsgi_pcre)); + (*pattern)->p = pcre_compile((const char *) re, 0, &errstr, &erroff, NULL); +#endif +#ifdef UWSGI_PCRE2 + if (!(*pattern)) { + uwsgi_log("pcre error: code %d at offset %d\n", errnbr, erroff); +#else + if (!((*pattern)->p)) { uwsgi_log("pcre error: %s at offset %d\n", errstr, erroff); +#endif return -1; } +#ifdef UWSGI_PCRE2 + if (uwsgi.pcre_jit) { + errnbr = pcre2_jit_compile(*pattern, PCRE2_JIT_COMPLETE); + if (errnbr) { + pcre2_code_free(*pattern); + uwsgi_log("pcre JIT compile error code %d\n", errnbr); + return -1; + } +#else int opt = uwsgi.pcre_jit; - *pattern_extra = (pcre_extra *) pcre_study((const pcre *) *pattern, opt, &errstr); - if (*pattern_extra == NULL && errstr != NULL) { - pcre_free(*pattern); + (*pattern)->extra = (pcre_extra *) pcre_study((const pcre *) (*pattern)->p, opt, &errstr); + if ((*pattern)->extra == NULL && errstr != NULL) { + pcre_free((*pattern)->p); + free(*pattern); uwsgi_log("pcre (study) error: %s\n", errstr); return -1; +#endif } return 0; } -int uwsgi_regexp_match(pcre * pattern, pcre_extra * pattern_extra, char *subject, int length) { - - return pcre_exec((const pcre *) pattern, (const pcre_extra *) pattern_extra, subject, length, 0, 0, NULL, 0); +int uwsgi_regexp_match(uwsgi_pcre *pattern, const char *subject, int length) { +#ifdef UWSGI_PCRE2 + return uwsgi_regexp_match_ovec(pattern, subject, length, NULL, 0); +#else + return pcre_exec((const pcre *) pattern->p, (const pcre_extra *) pattern->extra, subject, length, 0, 0, NULL, 0); +#endif } -int uwsgi_regexp_match_ovec(pcre * pattern, pcre_extra * pattern_extra, char *subject, int length, int *ovec, int n) { +int uwsgi_regexp_match_ovec(uwsgi_pcre *pattern, const char *subject, int length, int *ovec, int n) { +#ifdef UWSGI_PCRE2 + int rc; + int i; + pcre2_match_data *match_data; + size_t *pcre2_ovec; + + match_data = pcre2_match_data_create_from_pattern(pattern, NULL); + rc = pcre2_match(pattern, (const unsigned char *)subject, length, 0, 0, match_data, NULL); + + /* + * Quoting PCRE{,2} spec, "The first pair of integers, ovector[0] + * and ovector[1], identify the portion of the subject string matched + * by the entire pattern. The next pair is used for the first capturing + * subpattern, and so on." Therefore, the ovector size is the number of + * capturing subpatterns (INFO_CAPTURECOUNT), from uwsgi_regexp_ovector(), + * as matching pairs, plus room for the first pair. + */ if (n > 0) { - return pcre_exec((const pcre *) pattern, (const pcre_extra *) pattern_extra, subject, length, 0, 0, ovec, (n + 1) * 3); + // copy pcre2 output vector to uwsgi output vector + pcre2_ovec = pcre2_get_ovector_pointer(match_data); + for (i=0;i<(n+1)*2;i++) { + ovec[i] = pcre2_ovec[i]; + } +#else + if (n > 0) { + return pcre_exec((const pcre *) pattern->p, (const pcre_extra *) pattern->extra, subject, length, 0, 0, ovec, PCRE_OVECTOR_BYTESIZE(n)); +#endif } - return pcre_exec((const pcre *) pattern, (const pcre_extra *) pattern_extra, subject, length, 0, 0, NULL, 0); + +#ifdef UWSGI_PCRE2 + pcre2_match_data_free(match_data); + + return rc; +#else + return pcre_exec((const pcre *) pattern->p, (const pcre_extra *) pattern->extra, subject, length, 0, 0, NULL, 0); +#endif } -int uwsgi_regexp_ovector(pcre * pattern, pcre_extra * pattern_extra) { +int uwsgi_regexp_ovector(const uwsgi_pcre *pattern) { int n; - - if (pcre_fullinfo((const pcre *) pattern, (const pcre_extra *) pattern_extra, PCRE_INFO_CAPTURECOUNT, &n)) +#ifdef UWSGI_PCRE2 + if (pcre2_pattern_info(pattern, PCRE2_INFO_CAPTURECOUNT, &n)) +#else + if (pcre_fullinfo((const pcre *) pattern->p, (const pcre_extra *) pattern->extra, PCRE_INFO_CAPTURECOUNT, &n)) +#endif return 0; return n; @@ -66,7 +134,7 @@ char *uwsgi_regexp_apply_ovec(char *src, int src_n, char *dst, int dst_n, int *o int dollar = 0; size_t dollars = n; - + for(i=0;i +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -211,7 +211,7 @@ int uwsgi_apply_routes_do(struct uwsgi_route *routes, struct wsgi_request *wsgi_ subject = *subject2 ; subject_len = *subject_len2; } - n = uwsgi_regexp_match_ovec(routes->pattern, routes->pattern_extra, subject, subject_len, routes->ovector[wsgi_req->async_id], routes->ovn[wsgi_req->async_id]); + n = uwsgi_regexp_match_ovec(routes->pattern, subject, subject_len, routes->ovector[wsgi_req->async_id], routes->ovn[wsgi_req->async_id]); } else { int ret = routes->if_func(wsgi_req, routes); @@ -506,15 +506,15 @@ void uwsgi_fixup_routes(struct uwsgi_route *ur) { // fill them if needed... (this is an optimization for route with a static subject) if (ur->subject && ur->subject_len) { - if (uwsgi_regexp_build(ur->orig_route, &ur->pattern, &ur->pattern_extra)) { + if (uwsgi_regexp_build(ur->orig_route, &ur->pattern)) { exit(1); } int i; for(i=0;iovn[i] = uwsgi_regexp_ovector(ur->pattern, ur->pattern_extra); + ur->ovn[i] = uwsgi_regexp_ovector(ur->pattern); if (ur->ovn[i] > 0) { - ur->ovector[i] = uwsgi_calloc(sizeof(int) * (3 * (ur->ovn[i] + 1))); + ur->ovector[i] = uwsgi_calloc(sizeof(int) * PCRE_OVECTOR_BYTESIZE(ur->ovn[i])); } } } @@ -1484,38 +1484,47 @@ static int uwsgi_route_condition_regexp(struct wsgi_request *wsgi_req, struct uw ur->condition_ub[wsgi_req->async_id] = uwsgi_routing_translate(wsgi_req, ur, NULL, 0, ur->subject_str, semicolon - ur->subject_str); if (!ur->condition_ub[wsgi_req->async_id]) return -1; - pcre *pattern; - pcre_extra *pattern_extra; + uwsgi_pcre *pattern; char *re = uwsgi_concat2n(semicolon+1, ur->subject_str_len - ((semicolon+1) - ur->subject_str), "", 0); - if (uwsgi_regexp_build(re, &pattern, &pattern_extra)) { + if (uwsgi_regexp_build(re, &pattern)) { free(re); return -1; } free(re); // a condition has no initialized vectors, let's create them - ur->ovn[wsgi_req->async_id] = uwsgi_regexp_ovector(pattern, pattern_extra); + ur->ovn[wsgi_req->async_id] = uwsgi_regexp_ovector(pattern); if (ur->ovn[wsgi_req->async_id] > 0) { ur->ovector[wsgi_req->async_id] = uwsgi_calloc(sizeof(int) * (3 * (ur->ovn[wsgi_req->async_id] + 1))); } - if (uwsgi_regexp_match_ovec(pattern, pattern_extra, ur->condition_ub[wsgi_req->async_id]->buf, ur->condition_ub[wsgi_req->async_id]->pos, ur->ovector[wsgi_req->async_id], ur->ovn[wsgi_req->async_id] ) >= 0) { - pcre_free(pattern); + if (uwsgi_regexp_match_ovec(pattern, ur->condition_ub[wsgi_req->async_id]->buf, ur->condition_ub[wsgi_req->async_id]->pos, ur->ovector[wsgi_req->async_id], ur->ovn[wsgi_req->async_id] ) >= 0) { +#ifdef UWSGI_PCRE2 + pcre2_code_free(pattern); +#else + pcre_free(pattern->p); #ifdef PCRE_STUDY_JIT_COMPILE - pcre_free_study(pattern_extra); + pcre_free_study(pattern->extra); #else - pcre_free(pattern_extra); + pcre_free(pattern->extra); +#endif + free(pattern); #endif return 1; } - pcre_free(pattern); +#ifdef UWSGI_PCRE2 + pcre2_code_free(pattern); +#else + pcre_free(pattern->p); #ifdef PCRE_STUDY_JIT_COMPILE - pcre_free_study(pattern_extra); + pcre_free_study(pattern->extra); #else - pcre_free(pattern_extra); + pcre_free(pattern->extra); #endif - return 0; + free(pattern); +#endif + return 0; } static int uwsgi_route_condition_empty(struct wsgi_request *wsgi_req, struct uwsgi_route *ur) { diff --git a/core/rpc.c b/core/rpc.c index ed98751812..b6f0baf8b1 100644 --- a/core/rpc.c +++ b/core/rpc.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -17,6 +17,11 @@ int uwsgi_register_rpc(char *name, struct uwsgi_plugin *plugin, uint8_t args, vo return -1; } + if (strlen(name) >= UMAX8) { + uwsgi_log("the supplied RPC name string is too long, max size is %d\n", UMAX8-1); + return -1; + } + uwsgi_lock(uwsgi.rpc_table_lock); // first check if a function is already registered diff --git a/core/sharedarea.c b/core/sharedarea.c index 1628572ff3..a4ca4b47b8 100644 --- a/core/sharedarea.c +++ b/core/sharedarea.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/signal.c b/core/signal.c index a7b4452184..d5d9f8a0ca 100644 --- a/core/signal.c +++ b/core/signal.c @@ -500,7 +500,7 @@ int uwsgi_receive_signal(struct wsgi_request *wsgi_req, int fd, char *name, int destroy: // better to kill the whole worker... - uwsgi_log_verbose("uWSGI %s %d screams: UAAAAAAH my master disconnected: I will kill myself!!!\n", name, id); + uwsgi_log_verbose("uWSGI %s %d error: the master disconnected from this worker. Shutting down the worker.\n", name, id); end_me(0); // never here return 0; diff --git a/core/snmp.c b/core/snmp.c index f0b42c46a9..028cf4a418 100644 --- a/core/snmp.c +++ b/core/snmp.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/ssl.c b/core/ssl.c index c6cea47bea..18b1103642 100644 --- a/core/ssl.c +++ b/core/ssl.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" #include #include #include @@ -145,10 +145,10 @@ static int uwsgi_sni_cb(SSL *ssl, int *ad, void *arg) { if (uwsgi.subscription_dotsplit) goto end; -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *url = uwsgi.sni_regexp; while(url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, (char *)servername, servername_len) >= 0) { + if (uwsgi_regexp_match(url->pattern, (char *)servername, servername_len) >= 0) { SSL_set_SSL_CTX(ssl, url->custom_ptr); return SSL_TLSEXT_ERR_OK; } @@ -628,7 +628,7 @@ void uwsgi_opt_sni(char *opt, char *value, void *foobar) { return; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) if (!strcmp(opt, "sni-regexp")) { struct uwsgi_regexp_list *url = uwsgi_regexp_new_list(&uwsgi.sni_regexp, v); url->custom_ptr = ctx; @@ -637,7 +637,7 @@ void uwsgi_opt_sni(char *opt, char *value, void *foobar) { #endif struct uwsgi_string_list *usl = uwsgi_string_new_list(&uwsgi.sni, v); usl->custom_ptr = ctx; -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) } #endif diff --git a/core/static.c b/core/static.c index 27f225e4bf..9764db291c 100644 --- a/core/static.c +++ b/core/static.c @@ -35,11 +35,11 @@ int uwsgi_static_want_gzip(struct wsgi_request *wsgi_req, char *filename, size_t usl = usl->next; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) // check for regexp struct uwsgi_regexp_list *url = uwsgi.static_gzip; while(url) { - if (uwsgi_regexp_match(url->pattern, url->pattern_extra, filename, *filename_len) >= 0) { + if (uwsgi_regexp_match(url->pattern, filename, *filename_len) >= 0) { goto gzip; } url = url->next; @@ -220,7 +220,7 @@ int uwsgi_add_expires_type(struct wsgi_request *wsgi_req, char *mime_type, int m return 0; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) int uwsgi_add_expires(struct wsgi_request *wsgi_req, char *filename, int filename_len, struct stat *st) { struct uwsgi_dyn_dict *udd = uwsgi.static_expires; @@ -229,7 +229,7 @@ int uwsgi_add_expires(struct wsgi_request *wsgi_req, char *filename, int filenam char expires[31]; while (udd) { - if (uwsgi_regexp_match(udd->pattern, udd->pattern_extra, filename, filename_len) >= 0) { + if (uwsgi_regexp_match(udd->pattern, filename, filename_len) >= 0) { int delta = uwsgi_str_num(udd->value, udd->vallen); int size = uwsgi_http_date(now + delta, expires); if (size > 0) { @@ -242,7 +242,7 @@ int uwsgi_add_expires(struct wsgi_request *wsgi_req, char *filename, int filenam udd = uwsgi.static_expires_mtime; while (udd) { - if (uwsgi_regexp_match(udd->pattern, udd->pattern_extra, filename, filename_len) >= 0) { + if (uwsgi_regexp_match(udd->pattern, filename, filename_len) >= 0) { int delta = uwsgi_str_num(udd->value, udd->vallen); int size = uwsgi_http_date(st->st_mtime + delta, expires); if (size > 0) { @@ -264,7 +264,7 @@ int uwsgi_add_expires_path_info(struct wsgi_request *wsgi_req, struct stat *st) char expires[31]; while (udd) { - if (uwsgi_regexp_match(udd->pattern, udd->pattern_extra, wsgi_req->path_info, wsgi_req->path_info_len) >= 0) { + if (uwsgi_regexp_match(udd->pattern, wsgi_req->path_info, wsgi_req->path_info_len) >= 0) { int delta = uwsgi_str_num(udd->value, udd->vallen); int size = uwsgi_http_date(now + delta, expires); if (size > 0) { @@ -277,7 +277,7 @@ int uwsgi_add_expires_path_info(struct wsgi_request *wsgi_req, struct stat *st) udd = uwsgi.static_expires_path_info_mtime; while (udd) { - if (uwsgi_regexp_match(udd->pattern, udd->pattern_extra, wsgi_req->path_info, wsgi_req->path_info_len) >= 0) { + if (uwsgi_regexp_match(udd->pattern, wsgi_req->path_info, wsgi_req->path_info_len) >= 0) { int delta = uwsgi_str_num(udd->value, udd->vallen); int size = uwsgi_http_date(st->st_mtime + delta, expires); if (size > 0) { @@ -299,7 +299,7 @@ int uwsgi_add_expires_uri(struct wsgi_request *wsgi_req, struct stat *st) { char expires[31]; while (udd) { - if (uwsgi_regexp_match(udd->pattern, udd->pattern_extra, wsgi_req->uri, wsgi_req->uri_len) >= 0) { + if (uwsgi_regexp_match(udd->pattern, wsgi_req->uri, wsgi_req->uri_len) >= 0) { int delta = uwsgi_str_num(udd->value, udd->vallen); int size = uwsgi_http_date(now + delta, expires); if (size > 0) { @@ -312,7 +312,7 @@ int uwsgi_add_expires_uri(struct wsgi_request *wsgi_req, struct stat *st) { udd = uwsgi.static_expires_uri_mtime; while (udd) { - if (uwsgi_regexp_match(udd->pattern, udd->pattern_extra, wsgi_req->uri, wsgi_req->uri_len) >= 0) { + if (uwsgi_regexp_match(udd->pattern, wsgi_req->uri, wsgi_req->uri_len) >= 0) { int delta = uwsgi_str_num(udd->value, udd->vallen); int size = uwsgi_http_date(st->st_mtime + delta, expires); if (size > 0) { @@ -511,7 +511,7 @@ int uwsgi_real_file_serve(struct wsgi_request *wsgi_req, char *real_filename, si if (uwsgi_response_prepare_headers(wsgi_req, "200 OK", 6)) return -1; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) uwsgi_add_expires(wsgi_req, real_filename, real_filename_len, st); uwsgi_add_expires_path_info(wsgi_req, st); uwsgi_add_expires_uri(wsgi_req, st); diff --git a/core/strings.c b/core/strings.c index 7565879dbe..6ba04a73a3 100644 --- a/core/strings.c +++ b/core/strings.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" char *uwsgi_str_split_nget(char *str, size_t len, char what, size_t pos, size_t *rlen) { size_t i; diff --git a/core/timebomb.c b/core/timebomb.c index 0e60507660..d5e63304d5 100644 --- a/core/timebomb.c +++ b/core/timebomb.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* diff --git a/core/transformations.c b/core/transformations.c index c946c79244..2d3a97c32f 100644 --- a/core/transformations.c +++ b/core/transformations.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* diff --git a/core/utils.c b/core/utils.c index 2f6c974284..c944f64781 100644 --- a/core/utils.c +++ b/core/utils.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -580,13 +580,15 @@ void uwsgi_as_root() { } } - if (uwsgi.chroot && !uwsgi.reloads) { + if (uwsgi.chroot && !uwsgi.is_chrooted && !uwsgi.reloads) { if (!uwsgi.master_as_root) uwsgi_log("chroot() to %s\n", uwsgi.chroot); + if (chroot(uwsgi.chroot)) { uwsgi_error("chroot()"); exit(1); } + uwsgi.is_chrooted = 1; #ifdef __linux__ if (uwsgi.logging_options.memory_report) { uwsgi_log("*** Warning, on linux system you have to bind-mount the /proc fs in your chroot to get memory debug/report.\n"); @@ -987,7 +989,7 @@ void uwsgi_as_root() { return; nonroot: - if (uwsgi.chroot && !uwsgi.is_a_reload) { + if (uwsgi.chroot && !uwsgi.is_chrooted && !uwsgi.is_a_reload) { uwsgi_log("cannot chroot() as non-root user\n"); exit(1); } @@ -1032,12 +1034,6 @@ void uwsgi_destroy_request(struct wsgi_request *wsgi_req) { close_and_free_request(wsgi_req); - int foo; - if (uwsgi.threads > 1) { - // now the thread can die... - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &foo); - } - // reset for avoiding following requests to fail on non-uwsgi protocols // thanks Marko Tiikkaja for catching it wsgi_req->uh->_pktsize = 0; @@ -1129,11 +1125,6 @@ void uwsgi_close_request(struct wsgi_request *wsgi_req) { func(wsgi_req); } - if (uwsgi.threads > 1) { - // now the thread can die... - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &tmp_id); - } - // leave harakiri mode if (uwsgi.workers[uwsgi.mywid].cores[wsgi_req->async_id].harakiri > 0) { set_harakiri(wsgi_req, 0); @@ -1581,18 +1572,12 @@ int wsgi_req_accept(int queue, struct wsgi_request *wsgi_req) { } } - // kill the thread after the request completion - if (uwsgi.threads > 1) - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &ret); - if (uwsgi.signal_socket > -1 && (interesting_fd == uwsgi.signal_socket || interesting_fd == uwsgi.my_signal_socket)) { thunder_unlock; uwsgi_receive_signal(wsgi_req, interesting_fd, "worker", uwsgi.mywid); - if (uwsgi.threads > 1) - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &ret); return -1; } @@ -1603,8 +1588,6 @@ int wsgi_req_accept(int queue, struct wsgi_request *wsgi_req) { wsgi_req->fd = wsgi_req->socket->proto_accept(wsgi_req, interesting_fd); thunder_unlock; if (wsgi_req->fd < 0) { - if (uwsgi.threads > 1) - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &ret); return -1; } @@ -1619,8 +1602,6 @@ int wsgi_req_accept(int queue, struct wsgi_request *wsgi_req) { } thunder_unlock; - if (uwsgi.threads > 1) - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &ret); return -1; } @@ -1946,6 +1927,12 @@ char *uwsgi_64bit2str(int64_t num) { return str; } +char *uwsgi_size2str(size_t num) { + char *str = uwsgi_malloc(sizeof(UMAX64_STR) + 1); + snprintf(str, sizeof(UMAX64_STR) + 1, "%llu", (unsigned long long) num); + return str; +} + int uwsgi_num2str2(int num, char *ptr) { return snprintf(ptr, 11, "%d", num); @@ -2332,7 +2319,7 @@ struct uwsgi_string_list *uwsgi_string_new_list(struct uwsgi_string_list **list, return uwsgi_string; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *uwsgi_regexp_custom_new_list(struct uwsgi_regexp_list **list, char *value, char *custom) { struct uwsgi_regexp_list *url = *list, *old_url; @@ -2351,7 +2338,7 @@ struct uwsgi_regexp_list *uwsgi_regexp_custom_new_list(struct uwsgi_regexp_list old_url->next = url; } - if (uwsgi_regexp_build(value, &url->pattern, &url->pattern_extra)) { + if (uwsgi_regexp_build(value, &url->pattern)) { exit(1); } url->next = NULL; @@ -2364,14 +2351,13 @@ struct uwsgi_regexp_list *uwsgi_regexp_custom_new_list(struct uwsgi_regexp_list int uwsgi_regexp_match_pattern(char *pattern, char *str) { - pcre *regexp; - pcre_extra *regexp_extra; + uwsgi_pcre *regexp; - if (uwsgi_regexp_build(pattern, ®exp, ®exp_extra)) + if (uwsgi_regexp_build(pattern, ®exp)) return 1; - return !uwsgi_regexp_match(regexp, regexp_extra, str, strlen(str)); -} + return !uwsgi_regexp_match(regexp, str, strlen(str)); +} #endif @@ -3713,6 +3699,7 @@ char *uwsgi_expand_path(char *dir, int dir_len, char *ptr) { void uwsgi_set_cpu_affinity() { +#if defined(__linux__) || defined(__FreeBSD__) || defined(__GNU_kFreeBSD__) char buf[4096]; int ret; int pos = 0; @@ -3732,7 +3719,6 @@ void uwsgi_set_cpu_affinity() { #elif defined(__FreeBSD__) cpuset_t cpuset; #endif -#if defined(__linux__) || defined(__FreeBSD__) || defined(__GNU_kFreeBSD__) CPU_ZERO(&cpuset); int i; for (i = 0; i < uwsgi.cpu_affinity; i++) { @@ -3747,7 +3733,6 @@ void uwsgi_set_cpu_affinity() { pos += ret; base_cpu++; } -#endif #if defined(__linux__) || defined(__GNU_kFreeBSD__) if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset)) { uwsgi_error("sched_setaffinity()"); @@ -3759,7 +3744,7 @@ void uwsgi_set_cpu_affinity() { #endif uwsgi_log("%s\n", buf); } - +#endif } #ifdef UWSGI_ELF @@ -4090,7 +4075,7 @@ void uwsgi_uuid(char *buf) { uuid_generate(uuid_value); uuid_unparse(uuid_value, buf); #else - int i, r[11]; + unsigned int i, r[11]; if (!uwsgi_file_exists("/dev/urandom")) goto fallback; int fd = open("/dev/urandom", O_RDONLY); @@ -4106,7 +4091,7 @@ void uwsgi_uuid(char *buf) { goto done; fallback: for (i = 0; i < 11; i++) { - r[i] = rand(); + r[i] = (unsigned int) rand(); } done: snprintf(buf, 37, "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10]); diff --git a/core/uwsgi.c b/core/uwsgi.c index c7ff72e967..bbdb37e3c0 100755 --- a/core/uwsgi.c +++ b/core/uwsgi.c @@ -1,27 +1,4 @@ -/* - - *** uWSGI *** - - Copyright (C) 2009-2017 Unbit S.a.s. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -*/ - - -#include +#include "uwsgi.h" struct uwsgi_server uwsgi; pid_t masterpid; @@ -83,6 +60,9 @@ static struct uwsgi_option uwsgi_base_options[] = { {"thunder-lock-watchdog", no_argument, 0, "watchdog for buggy pthread robust mutexes", uwsgi_opt_true, &uwsgi.use_thunder_lock_watchdog, 0}, {"harakiri", required_argument, 't', "set harakiri timeout", uwsgi_opt_set_int, &uwsgi.harakiri_options.workers, 0}, {"harakiri-verbose", no_argument, 0, "enable verbose mode for harakiri", uwsgi_opt_true, &uwsgi.harakiri_verbose, 0}, + {"harakiri-graceful-timeout", required_argument, 0, "interval between graceful harakiri attempt and a sigkill", uwsgi_opt_set_int, &uwsgi.harakiri_graceful_timeout, 0}, + {"harakiri-graceful-signal", required_argument, 0, "use this signal instead of sigterm for graceful harakiri attempts", uwsgi_opt_set_int, &uwsgi.harakiri_graceful_signal, 0}, + {"harakiri-queue-threshold", required_argument, 0, "only trigger harakiri if queue is greater than this threshold", uwsgi_opt_set_int, &uwsgi.harakiri_queue_threshold, 0}, {"harakiri-no-arh", no_argument, 0, "do not enable harakiri during after-request-hook", uwsgi_opt_true, &uwsgi.harakiri_no_arh, 0}, {"no-harakiri-arh", no_argument, 0, "do not enable harakiri during after-request-hook", uwsgi_opt_true, &uwsgi.harakiri_no_arh, 0}, {"no-harakiri-after-req-hook", no_argument, 0, "do not enable harakiri during after-request-hook", uwsgi_opt_true, &uwsgi.harakiri_no_arh, 0}, @@ -129,7 +109,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"if-hostname", required_argument, 0, "(opt logic) check for hostname", uwsgi_opt_logic, (void *) uwsgi_logic_opt_if_hostname, UWSGI_OPT_IMMEDIATE}, {"if-not-hostname", required_argument, 0, "(opt logic) check for hostname", uwsgi_opt_logic, (void *) uwsgi_logic_opt_if_not_hostname, UWSGI_OPT_IMMEDIATE}, -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) {"if-hostname-match", required_argument, 0, "(opt logic) try to match hostname against a regular expression", uwsgi_opt_logic, (void *) uwsgi_logic_opt_if_hostname_match, UWSGI_OPT_IMMEDIATE}, {"if-not-hostname-match", required_argument, 0, "(opt logic) try to match hostname against a regular expression", uwsgi_opt_logic, (void *) uwsgi_logic_opt_if_not_hostname_match, UWSGI_OPT_IMMEDIATE}, #endif @@ -198,7 +178,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"freebind", no_argument, 0, "put socket in freebind mode", uwsgi_opt_true, &uwsgi.freebind, 0}, #endif {"map-socket", required_argument, 0, "map sockets to specific workers", uwsgi_opt_add_string_list, &uwsgi.map_socket, 0}, - {"enable-threads", no_argument, 'T', "enable threads", uwsgi_opt_true, &uwsgi.has_threads, 0}, + {"enable-threads", no_argument, 'T', "enable threads (stub option this is true by default)", uwsgi_opt_true, &uwsgi.has_threads, 0}, {"no-threads-wait", no_argument, 0, "do not wait for threads cancellation on quit/reload", uwsgi_opt_true, &uwsgi.no_threads_wait, 0}, {"auto-procname", no_argument, 0, "automatically set processes name to something meaningful", uwsgi_opt_true, &uwsgi.auto_procname, 0}, @@ -209,7 +189,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"procname-master", required_argument, 0, "set master process name", uwsgi_opt_set_str, &uwsgi.procname_master, UWSGI_OPT_PROCNAME}, {"single-interpreter", no_argument, 'i', "do not use multiple interpreters (where available)", uwsgi_opt_true, &uwsgi.single_interpreter, 0}, - {"need-app", no_argument, 0, "exit if no app can be loaded", uwsgi_opt_true, &uwsgi.need_app, 0}, + {"need-app", optional_argument, 0, "exit if no app can be loaded", uwsgi_opt_true, &uwsgi.need_app, 0}, {"dynamic-apps", no_argument, 0, "allows apps to be dynamically loaded via uwsgi protocol", uwsgi_opt_true, &uwsgi.dynamic_apps, 0}, {"master", no_argument, 'M', "enable master process", uwsgi_opt_true, &uwsgi.master_process, 0}, {"honour-stdin", no_argument, 0, "do not remap stdin to /dev/null", uwsgi_opt_true, &uwsgi.honour_stdin, 0}, @@ -255,6 +235,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"emperor-use-fork-server", required_argument, 0, "connect to the specified fork server instead of using plain fork() for new vassals", uwsgi_opt_set_str, &uwsgi.emperor_use_fork_server, 0}, {"vassal-fork-base", required_argument, 0, "use plain fork() for the specified vassal (instead of a fork-server)", uwsgi_opt_add_string_list, &uwsgi.vassal_fork_base, 0}, {"emperor-subreaper", no_argument, 0, "force the Emperor to be a sub-reaper (if supported)", uwsgi_opt_true, &uwsgi.emperor_subreaper, 0}, + {"emperor-graceful-shutdown", no_argument, 0, "use vassals graceful shutdown during ragnarok", uwsgi_opt_true, &uwsgi.emperor_graceful_shutdown, 0}, #ifdef UWSGI_CAP {"emperor-cap", required_argument, 0, "set vassals capability", uwsgi_opt_set_emperor_cap, NULL, 0}, {"vassals-cap", required_argument, 0, "set vassals capability", uwsgi_opt_set_emperor_cap, NULL, 0}, @@ -297,8 +278,9 @@ static struct uwsgi_option uwsgi_base_options[] = { {"reaper", no_argument, 'r', "call waitpid(-1,...) after each request to get rid of zombies", uwsgi_opt_true, &uwsgi.reaper, 0}, {"max-requests", required_argument, 'R', "reload workers after the specified amount of managed requests", uwsgi_opt_set_64bit, &uwsgi.max_requests, 0}, {"max-requests-delta", required_argument, 0, "add (worker_id * delta) to the max_requests value of each worker", uwsgi_opt_set_64bit, &uwsgi.max_requests_delta, 0}, - {"min-worker-lifetime", required_argument, 0, "number of seconds worker must run before being reloaded (default is 60)", uwsgi_opt_set_64bit, &uwsgi.min_worker_lifetime, 0}, + {"min-worker-lifetime", required_argument, 0, "number of seconds worker must run before being reloaded (default is 10)", uwsgi_opt_set_64bit, &uwsgi.min_worker_lifetime, 0}, {"max-worker-lifetime", required_argument, 0, "reload workers after the specified amount of seconds (default is disabled)", uwsgi_opt_set_64bit, &uwsgi.max_worker_lifetime, 0}, + {"max-worker-lifetime-delta", required_argument, 0, "add (worker_id * delta) seconds to the max_worker_lifetime value of each worker", uwsgi_opt_set_int, &uwsgi.max_worker_lifetime_delta, 0}, {"socket-timeout", required_argument, 'z', "set internal sockets timeout", uwsgi_opt_set_int, &uwsgi.socket_timeout, 0}, {"no-fd-passing", no_argument, 0, "disable file descriptor passing", uwsgi_opt_true, &uwsgi.no_fd_passing, 0}, @@ -583,7 +565,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"ksm", optional_argument, 0, "enable Linux KSM", uwsgi_opt_set_int, &uwsgi.linux_ksm, 0}, #endif #endif -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) {"pcre-jit", no_argument, 0, "enable pcre jit (if available)", uwsgi_opt_pcre_jit, NULL, UWSGI_OPT_IMMEDIATE}, #endif {"never-swap", no_argument, 0, "lock all memory pages avoiding swapping", uwsgi_opt_true, &uwsgi.never_swap, 0}, @@ -667,7 +649,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"legion-freq", required_argument, 0, "set the frequency of legion packets", uwsgi_opt_set_int, &uwsgi.legion_freq, UWSGI_OPT_MASTER}, {"legion-tolerance", required_argument, 0, "set the tolerance of legion subsystem", uwsgi_opt_set_int, &uwsgi.legion_tolerance, UWSGI_OPT_MASTER}, {"legion-death-on-lord-error", required_argument, 0, "declare itself as a dead node for the specified amount of seconds if one of the lord hooks fails", uwsgi_opt_set_int, &uwsgi.legion_death_on_lord_error, UWSGI_OPT_MASTER}, - {"legion-skew-tolerance", required_argument, 0, "set the clock skew tolerance of legion subsystem (default 30 seconds)", uwsgi_opt_set_int, &uwsgi.legion_skew_tolerance, UWSGI_OPT_MASTER}, + {"legion-skew-tolerance", required_argument, 0, "set the clock skew tolerance of legion subsystem (default 60 seconds)", uwsgi_opt_set_int, &uwsgi.legion_skew_tolerance, UWSGI_OPT_MASTER}, {"legion-lord", required_argument, 0, "action to call on Lord election", uwsgi_opt_legion_hook, NULL, UWSGI_OPT_MASTER}, {"legion-unlord", required_argument, 0, "action to call on Lord dismiss", uwsgi_opt_legion_hook, NULL, UWSGI_OPT_MASTER}, {"legion-setup", required_argument, 0, "action to call on legion setup", uwsgi_opt_legion_hook, NULL, UWSGI_OPT_MASTER}, @@ -719,7 +701,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"ssl-enable-sslv3", no_argument, 0, "enable SSLv3 (insecure)", uwsgi_opt_true, &uwsgi.sslv3, 0}, {"ssl-enable-tlsv1", no_argument, 0, "enable TLSv1 (insecure)", uwsgi_opt_true, &uwsgi.tlsv1, 0}, {"ssl-option", required_argument, 0, "set a raw ssl option (numeric value)", uwsgi_opt_add_string_list, &uwsgi.ssl_options, 0}, -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) {"sni-regexp", required_argument, 0, "add an SNI-governed SSL context (the key is a regexp)", uwsgi_opt_sni, NULL, 0}, #endif {"ssl-tmp-dir", required_argument, 0, "store ssl-related temp files in the specified directory", uwsgi_opt_set_str, &uwsgi.ssl_tmp_dir, 0}, @@ -761,7 +743,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"worker-log-req-encoder", required_argument, 0, "add an item in the log req encoder chain", uwsgi_opt_add_string_list, &uwsgi.requested_log_req_encoders, 0}, -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) {"log-drain", required_argument, 0, "drain (do not show) log lines matching the specified regexp", uwsgi_opt_add_regexp_list, &uwsgi.log_drain_rules, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"log-filter", required_argument, 0, "show only log lines matching the specified regexp", uwsgi_opt_add_regexp_list, &uwsgi.log_filter_rules, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"log-route", required_argument, 0, "log to the specified named logger if regexp applied on logline matches", uwsgi_opt_add_regexp_custom_list, &uwsgi.log_route, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, @@ -773,7 +755,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"alarm", required_argument, 0, "create a new alarm, syntax: ", uwsgi_opt_add_string_list, &uwsgi.alarm_list, UWSGI_OPT_MASTER}, {"alarm-cheap", required_argument, 0, "use main alarm thread rather than create dedicated threads for curl-based alarms", uwsgi_opt_true, &uwsgi.alarm_cheap, 0}, {"alarm-freq", required_argument, 0, "tune the anti-loop alarm system (default 3 seconds)", uwsgi_opt_set_int, &uwsgi.alarm_freq, 0}, - {"alarm-fd", required_argument, 0, "raise the specified alarm when an fd is read for read (by default it reads 1 byte, set 8 for eventfd)", uwsgi_opt_add_string_list, &uwsgi.alarm_fd_list, UWSGI_OPT_MASTER}, + {"alarm-fd", required_argument, 0, "raise the specified alarm when an fd is ready for read (by default it reads 1 byte, set 8 for eventfd)", uwsgi_opt_add_string_list, &uwsgi.alarm_fd_list, UWSGI_OPT_MASTER}, {"alarm-segfault", required_argument, 0, "raise the specified alarm when the segmentation fault handler is executed", uwsgi_opt_add_string_list, &uwsgi.alarm_segfault, UWSGI_OPT_MASTER}, {"segfault-alarm", required_argument, 0, "raise the specified alarm when the segmentation fault handler is executed", uwsgi_opt_add_string_list, &uwsgi.alarm_segfault, UWSGI_OPT_MASTER}, {"alarm-backlog", required_argument, 0, "raise the specified alarm when the socket backlog queue is full", uwsgi_opt_add_string_list, &uwsgi.alarm_backlog, UWSGI_OPT_MASTER}, @@ -782,7 +764,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"alarm-lq", required_argument, 0, "raise the specified alarm when the socket backlog queue is full", uwsgi_opt_add_string_list, &uwsgi.alarm_backlog, UWSGI_OPT_MASTER}, {"alarm-listen-queue", required_argument, 0, "raise the specified alarm when the socket backlog queue is full", uwsgi_opt_add_string_list, &uwsgi.alarm_backlog, UWSGI_OPT_MASTER}, {"listen-queue-alarm", required_argument, 0, "raise the specified alarm when the socket backlog queue is full", uwsgi_opt_add_string_list, &uwsgi.alarm_backlog, UWSGI_OPT_MASTER}, -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) {"log-alarm", required_argument, 0, "raise the specified alarm when a log line matches the specified regexp, syntax: [,alarm...] ", uwsgi_opt_add_string_list, &uwsgi.alarm_logs_list, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"alarm-log", required_argument, 0, "raise the specified alarm when a log line matches the specified regexp, syntax: [,alarm...] ", uwsgi_opt_add_string_list, &uwsgi.alarm_logs_list, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, {"not-log-alarm", required_argument, 0, "skip the specified alarm when a log line matches the specified regexp, syntax: [,alarm...] ", uwsgi_opt_add_string_list_custom, &uwsgi.alarm_logs_list, UWSGI_OPT_MASTER | UWSGI_OPT_LOG_MASTER}, @@ -963,7 +945,7 @@ static struct uwsgi_option uwsgi_base_options[] = { {"static-expires-type", required_argument, 0, "set the Expires header based on content type", uwsgi_opt_add_dyn_dict, &uwsgi.static_expires_type, UWSGI_OPT_MIME}, {"static-expires-type-mtime", required_argument, 0, "set the Expires header based on content type and file mtime", uwsgi_opt_add_dyn_dict, &uwsgi.static_expires_type_mtime, UWSGI_OPT_MIME}, -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) {"static-expires", required_argument, 0, "set the Expires header based on filename regexp", uwsgi_opt_add_regexp_dyn_dict, &uwsgi.static_expires, UWSGI_OPT_MIME}, {"static-expires-mtime", required_argument, 0, "set the Expires header based on filename regexp and file mtime", uwsgi_opt_add_regexp_dyn_dict, &uwsgi.static_expires_mtime, UWSGI_OPT_MIME}, @@ -1244,57 +1226,28 @@ void warn_pipe() { } } -// in threading mode we need to use the cancel pthread subsystem -void wait_for_threads() { +// This function is called from signal handler or main thread to wait worker threads. +// `uwsgi.workers[uwsgi.mywid].manage_next_request` should be set to 0 to stop worker threads. +static void wait_for_threads() { int i, ret; - // on some platform thread cancellation is REALLY flaky + // This option was added because we used pthread_cancel(). + // thread cancellation is REALLY flaky if (uwsgi.no_threads_wait) return; - int sudden_death = 0; - - pthread_mutex_lock(&uwsgi.six_feet_under_lock); - for (i = 1; i < uwsgi.threads; i++) { - if (!pthread_equal(uwsgi.workers[uwsgi.mywid].cores[i].thread_id, pthread_self())) { - if (pthread_cancel(uwsgi.workers[uwsgi.mywid].cores[i].thread_id)) { - uwsgi_error("pthread_cancel()\n"); - sudden_death = 1; - } - } - } - - if (sudden_death) - goto end; - // wait for thread termination - for (i = 1; i < uwsgi.threads; i++) { + for (i = 0; i < uwsgi.threads; i++) { if (!pthread_equal(uwsgi.workers[uwsgi.mywid].cores[i].thread_id, pthread_self())) { ret = pthread_join(uwsgi.workers[uwsgi.mywid].cores[i].thread_id, NULL); if (ret) { uwsgi_log("pthread_join() = %d\n", ret); } + else { + // uwsgi_worker_is_busy() should not consider this thread as busy. + uwsgi.workers[uwsgi.mywid].cores[i].in_request = 0; + } } } - - // cancel initial thread last since after pthread_cancel() and - // pthread_join() is called on it, the whole process will appear to be - // a zombie. although it won't eliminate process zombie time, but it - // should minimize it. - if (!pthread_equal(uwsgi.workers[uwsgi.mywid].cores[0].thread_id, pthread_self())) { - if (pthread_cancel(uwsgi.workers[uwsgi.mywid].cores[0].thread_id)) { - uwsgi_error("pthread_cancel() on initial thread\n"); - goto end; - } - - ret = pthread_join(uwsgi.workers[uwsgi.mywid].cores[0].thread_id, NULL); - if (ret) { - uwsgi_log("pthread_join() = %d on initial thread\n", ret); - } - } - -end: - - pthread_mutex_unlock(&uwsgi.six_feet_under_lock); } @@ -1302,16 +1255,17 @@ void gracefully_kill(int signum) { uwsgi_log("Gracefully killing worker %d (pid: %d)...\n", uwsgi.mywid, uwsgi.mypid); uwsgi.workers[uwsgi.mywid].manage_next_request = 0; + if (uwsgi.threads > 1) { - struct wsgi_request *wsgi_req = current_wsgi_req(); - wait_for_threads(); - if (!uwsgi.workers[uwsgi.mywid].cores[wsgi_req->async_id].in_request) { - if (uwsgi.workers[uwsgi.mywid].shutdown_sockets) - uwsgi_shutdown_all_sockets(); - exit(UWSGI_RELOAD_CODE); + // Stop event_queue_wait() in other threads. + // We use loop_stop_pipe only in threaded workers to avoid + // unintensional behavior changes in single threaded workers. + int fd; + if ((fd = uwsgi.loop_stop_pipe[1]) > 0) { + close(fd); + uwsgi.loop_stop_pipe[1] = 0; } return; - // never here } // still not found a way to gracefully reload in async mode @@ -1337,14 +1291,23 @@ void end_me(int signum) { } static void simple_goodbye_cruel_world(const char *reason) { - - if (uwsgi.threads > 1 && !uwsgi_instance_is_dying) { - wait_for_threads(); + int prev = uwsgi.workers[uwsgi.mywid].manage_next_request; + uwsgi.workers[uwsgi.mywid].manage_next_request = 0; + if (prev) { + // Avoid showing same message from all threads. + uwsgi_log("...The work of process %d is done (%s). Seeya!\n", getpid(), (reason != NULL ? reason : "no reason given")); } - uwsgi.workers[uwsgi.mywid].manage_next_request = 0; - uwsgi_log("...The work of process %d is done (%s). Seeya!\n", getpid(), (reason != NULL ? reason : "no reason given")); - exit(0); + if (uwsgi.threads > 1) { + // Stop event_queue_wait() in other threads. + // We use loop_stop_pipe only in threaded workers to avoid + // unintensional behavior changes in single threaded workers. + int fd; + if ((fd = uwsgi.loop_stop_pipe[1]) > 0) { + close(fd); + uwsgi.loop_stop_pipe[1] = 0; + } + } } void goodbye_cruel_world(const char *reason_fmt, ...) { @@ -1373,7 +1336,7 @@ void kill_them_all(int signum) { // unsubscribe if needed uwsgi_unsubscribe_all(); - uwsgi_log("SIGINT/SIGQUIT received...killing workers...\n"); + uwsgi_log("SIGINT/SIGTERM received...killing workers...\n"); int i; for (i = 1; i <= uwsgi.numproc; i++) { @@ -1393,6 +1356,8 @@ void kill_them_all(int signum) { // gracefully destroy void gracefully_kill_them_all(int signum) { + int waitpid_status; + if (uwsgi_instance_is_dying) return; uwsgi.status.gracefully_destroying = 1; @@ -1414,6 +1379,12 @@ void gracefully_kill_them_all(int signum) { } } + for (i = 1; i <= uwsgi.numproc; i++) { + if (uwsgi.workers[i].pid > 0) { + waitpid(uwsgi.workers[i].pid, &waitpid_status, 0); + } + } + uwsgi_destroy_processes(); } @@ -1515,7 +1486,7 @@ void reap_them_all(int signum) { void harakiri() { - uwsgi_log("\nF*CK !!! i must kill myself (pid: %d app_id: %d)...\n", uwsgi.mypid, uwsgi.wsgi_req->app_id); + uwsgi_log("\nKilling the current process (pid: %d app_id: %d)...\n", uwsgi.mypid, uwsgi.wsgi_req->app_id); if (!uwsgi.master_process) { uwsgi_log("*** if you want your workers to be automatically respawned consider enabling the uWSGI master process ***\n"); @@ -1889,7 +1860,7 @@ void uwsgi_plugins_atexit(void) { void uwsgi_backtrace(int depth) { -#if defined(__GLIBC__) || (defined(__APPLE__) && !defined(NO_EXECINFO)) || defined(UWSGI_HAS_EXECINFO) +#if (!defined(__UCLIBC__) && defined(__GLIBC__)) || (defined(__APPLE__) && !defined(NO_EXECINFO)) || defined(UWSGI_HAS_EXECINFO) #include @@ -2518,7 +2489,7 @@ void uwsgi_setup(int argc, char *argv[], char *envp[]) { #endif uwsgi_log_initial("clock source: %s\n", uwsgi.clock->name); -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) if (uwsgi.pcre_jit) { uwsgi_log_initial("pcre jit enabled\n"); } @@ -3192,10 +3163,6 @@ int uwsgi_start(void *v_argv) { if (uwsgi.logformat) { uwsgi_build_log_format(uwsgi.logformat); uwsgi.logit = uwsgi_logit_lf; - // TODO check it - //if (uwsgi.logformat_strftime) { - //uwsgi.logit = uwsgi_logit_lf_strftime; - //} uwsgi.logvectors = uwsgi_malloc(sizeof(struct iovec *) * uwsgi.cores); for (j = 0; j < uwsgi.cores; j++) { uwsgi.logvectors[j] = uwsgi_malloc(sizeof(struct iovec) * uwsgi.logformat_vectors); @@ -3661,7 +3628,6 @@ void uwsgi_worker_run() { if (uwsgi.cores > 1) { uwsgi.workers[uwsgi.mywid].cores[0].thread_id = pthread_self(); - pthread_mutex_init(&uwsgi.six_feet_under_lock, NULL); } uwsgi_ignition(); @@ -3695,6 +3661,10 @@ void uwsgi_ignition() { exit(1); } } + if (pipe(&uwsgi.loop_stop_pipe[0])) { + uwsgi_error("pipe()") + exit(1); + } // mark the worker as "accepting" (this is a mark used by chain reloading) uwsgi.workers[uwsgi.mywid].accepting = 1; @@ -3743,6 +3713,11 @@ void uwsgi_ignition() { } } + // main thread waits other threads. + if (uwsgi.threads > 1) { + wait_for_threads(); + } + // end of the process... end_me(0); } @@ -3986,13 +3961,6 @@ void uwsgi_init_all_apps() { if (uwsgi.need_app) { if (!uwsgi.lazy) uwsgi_log("*** no app loaded. GAME OVER ***\n"); - if (uwsgi.lazy_apps) { - if (uwsgi.master_process) { - if (kill(uwsgi.workers[0].pid, SIGINT)) { - uwsgi_error("kill()"); - } - } - } exit(UWSGI_FAILED_APP_CODE); } else { @@ -4138,7 +4106,11 @@ void uwsgi_opt_safe_fd(char *opt, char *value, void *foobar) { void uwsgi_opt_set_int(char *opt, char *value, void *key) { int *ptr = (int *) key; if (value) { - *ptr = atoi((char *) value); + char *endptr; + *ptr = (int)strtol(value, &endptr, 10); + if (*endptr) { + uwsgi_log("[WARNING] non-numeric value \"%s\" for option \"%s\" - using %d !\n", value, opt, *ptr); + } } else { *ptr = 1; @@ -4320,7 +4292,7 @@ void uwsgi_opt_add_string_list_custom(char *opt, char *value, void *list) { usl->custom = 1; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) void uwsgi_opt_add_regexp_list(char *opt, char *value, void *list) { struct uwsgi_regexp_list **ptr = (struct uwsgi_regexp_list **) list; uwsgi_regexp_new_list(ptr, value); @@ -4586,7 +4558,7 @@ void uwsgi_opt_add_dyn_dict(char *opt, char *value, void *dict) { } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) void uwsgi_opt_add_regexp_dyn_dict(char *opt, char *value, void *dict) { char *space = strchr(value, ' '); @@ -4601,7 +4573,7 @@ void uwsgi_opt_add_regexp_dyn_dict(char *opt, char *value, void *dict) { char *regexp = uwsgi_concat2n(value, space - value, "", 0); - if (uwsgi_regexp_build(regexp, &new_udd->pattern, &new_udd->pattern_extra)) { + if (uwsgi_regexp_build(regexp, &new_udd->pattern)) { exit(1); } diff --git a/core/webdav.c b/core/webdav.c index 811c592654..a0bb1d07ed 100644 --- a/core/webdav.c +++ b/core/webdav.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/core/websockets.c b/core/websockets.c index 60ef20c185..65ebd580d7 100644 --- a/core/websockets.c +++ b/core/websockets.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* @@ -54,6 +54,10 @@ static int uwsgi_websockets_pong(struct wsgi_request *wsgi_req) { return uwsgi_response_write_body_do(wsgi_req, uwsgi.websockets_pong->buf, uwsgi.websockets_pong->pos); } +static int uwsgi_websockets_close(struct wsgi_request *wsgi_req) { + return uwsgi_response_write_body_do(wsgi_req, uwsgi.websockets_close->buf, uwsgi.websockets_close->pos); +} + static int uwsgi_websockets_check_pingpong(struct wsgi_request *wsgi_req) { time_t now = uwsgi_now(); // first round @@ -144,6 +148,7 @@ int uwsgi_websocket_send_binary_from_sharedarea(struct wsgi_request *wsgi_req, i static void uwsgi_websocket_parse_header(struct wsgi_request *wsgi_req) { uint8_t byte1 = wsgi_req->websocket_buf->buf[0]; uint8_t byte2 = wsgi_req->websocket_buf->buf[1]; + wsgi_req->websocket_is_fin = byte1 >> 7; wsgi_req->websocket_opcode = byte1 & 0xf; wsgi_req->websocket_has_mask = byte2 >> 7; wsgi_req->websocket_size = byte2 & 0x7f; @@ -161,14 +166,38 @@ static struct uwsgi_buffer *uwsgi_websockets_parse(struct wsgi_request *wsgi_req } } - struct uwsgi_buffer *ub = uwsgi_buffer_new(wsgi_req->websocket_size); + struct uwsgi_buffer *ub = NULL; + if (wsgi_req->websocket_opcode == 0) { + if (uwsgi.websockets_continuation_buffer == NULL) { + uwsgi_log("Error continuation with empty previous buffer"); + goto error; + } + ub = uwsgi.websockets_continuation_buffer; + } + else { + ub = uwsgi_buffer_new(wsgi_req->websocket_size); + } if (uwsgi_buffer_append(ub, (char *) ptr, wsgi_req->websocket_size)) goto error; if (uwsgi_buffer_decapitate(wsgi_req->websocket_buf, wsgi_req->websocket_pktsize)) goto error; wsgi_req->websocket_phase = 0; wsgi_req->websocket_need = 2; + + if (wsgi_req->websocket_is_fin) { + uwsgi.websockets_continuation_buffer = NULL; + /// Freeing websockets_continuation_buffer is done by the caller + return ub; + } + uwsgi.websockets_continuation_buffer = ub; + /// Message is not complete, send empty dummy buffer to signal waiting for full message + ub = uwsgi_buffer_new(1); + uwsgi_buffer_append(ub, "\0", 1); return ub; error: uwsgi_buffer_destroy(ub); + if (uwsgi.websockets_continuation_buffer != NULL && ub != uwsgi.websockets_continuation_buffer) { + uwsgi_buffer_destroy(uwsgi.websockets_continuation_buffer); + } + uwsgi.websockets_continuation_buffer = NULL; return NULL; } @@ -293,6 +322,7 @@ static struct uwsgi_buffer *uwsgi_websocket_recv_do(struct wsgi_request *wsgi_re return uwsgi_websockets_parse(wsgi_req); // close case 0x8: + uwsgi_websockets_close(wsgi_req); return NULL; // ping case 0x9: @@ -338,12 +368,20 @@ static struct uwsgi_buffer *uwsgi_websocket_recv_do(struct wsgi_request *wsgi_re return NULL; } +static void clear_continuation_buffer() { + if (uwsgi.websockets_continuation_buffer != NULL) { + uwsgi_buffer_destroy(uwsgi.websockets_continuation_buffer); + uwsgi.websockets_continuation_buffer = NULL; + } +} + struct uwsgi_buffer *uwsgi_websocket_recv(struct wsgi_request *wsgi_req) { if (wsgi_req->websocket_closed) { return NULL; } struct uwsgi_buffer *ub = uwsgi_websocket_recv_do(wsgi_req, 0); if (!ub) { + clear_continuation_buffer(); wsgi_req->websocket_closed = 1; } return ub; @@ -355,6 +393,7 @@ struct uwsgi_buffer *uwsgi_websocket_recv_nb(struct wsgi_request *wsgi_req) { } struct uwsgi_buffer *ub = uwsgi_websocket_recv_do(wsgi_req, 1); if (!ub) { + clear_continuation_buffer(); wsgi_req->websocket_closed = 1; } return ub; @@ -429,7 +468,10 @@ void uwsgi_websockets_init() { uwsgi_buffer_append(uwsgi.websockets_pong, "\x8A\0", 2); uwsgi.websockets_ping = uwsgi_buffer_new(2); uwsgi_buffer_append(uwsgi.websockets_ping, "\x89\0", 2); + uwsgi.websockets_close = uwsgi_buffer_new(2); + uwsgi_buffer_append(uwsgi.websockets_close, "\x88\0", 2); uwsgi.websockets_ping_freq = 30; uwsgi.websockets_pong_tolerance = 3; uwsgi.websockets_max_size = 1024; + uwsgi.websockets_continuation_buffer = NULL; } diff --git a/core/yaml.c b/core/yaml.c index e1fd6736d1..e859435660 100644 --- a/core/yaml.c +++ b/core/yaml.c @@ -156,44 +156,49 @@ void uwsgi_yaml_config(char *file, char *magic_table[]) { break; case YAML_FLOW_SEQUENCE_START_TOKEN: case YAML_BLOCK_SEQUENCE_START_TOKEN: - status = 3; + if (in_uwsgi_section) + in_uwsgi_section++; + // fallthrough + case YAML_FLOW_ENTRY_TOKEN: + case YAML_BLOCK_ENTRY_TOKEN: + status = 3; // inside a sequence break; case YAML_BLOCK_MAPPING_START_TOKEN: - if (!in_uwsgi_section) { - if (key) { - if (!strcmp(section_asked, key)) { - in_uwsgi_section = 1; - } + if (in_uwsgi_section) { + in_uwsgi_section++; + break; + } + if (key) { + if (!strcmp(section_asked, key)) { + in_uwsgi_section = 1; } } break; case YAML_BLOCK_END_TOKEN: - if (in_uwsgi_section) { - parsing = 0; - break; - } + case YAML_FLOW_SEQUENCE_END_TOKEN: + if (in_uwsgi_section) + parsing = !!(--in_uwsgi_section); + key = NULL; + status = 0; break; case YAML_SCALAR_TOKEN: - case YAML_FLOW_ENTRY_TOKEN: - case YAML_BLOCK_ENTRY_TOKEN: if (status == 1) { key = (char *) token.data.scalar.value; } - else if (status == 2) { + else if (status == 2 || status == 3) { val = (char *) token.data.scalar.value; if (key && val && in_uwsgi_section) { add_exported_option(key, val, 0); } - status = 0; - } - else if (status == 3) { - val = (char *) token.data.scalar.value; - if (key && val && in_uwsgi_section) { - add_exported_option(key, val, 0); + + // If this was the scalar of a value token, forget the state. + if (status == 2) { + key = NULL; + status = 0; } } else { - uwsgi_log("unsupported YAML token in %s block\n", section_asked); + uwsgi_log("unsupported YAML token %d in %s block\n", token.type, section_asked); parsing = 0; break; } @@ -206,7 +211,6 @@ void uwsgi_yaml_config(char *file, char *magic_table[]) { #else int depth; int current_depth = 0; - int lines = 1; char *yaml_line; char *section = ""; @@ -216,7 +220,6 @@ void uwsgi_yaml_config(char *file, char *magic_table[]) { if (yaml_line == NULL) { break; } - lines++; // skip empty line if (yaml[0] == 0) diff --git a/core/zeus.c b/core/zeus.c index aa20f81581..6e83f3cb51 100644 --- a/core/zeus.c +++ b/core/zeus.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" /* diff --git a/core/zlib.c b/core/zlib.c index 5857980ecd..2ea5d9db52 100644 --- a/core/zlib.c +++ b/core/zlib.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" char gzheader[10] = { 0x1f, 0x8b, Z_DEFLATED, 0, 0, 0, 0, 0, 0, 3 }; diff --git a/examples/bootstrap5.py b/examples/bootstrap5.py index 340e6889eb..f8dc3136b5 100644 --- a/examples/bootstrap5.py +++ b/examples/bootstrap5.py @@ -1,3 +1,3 @@ import uwsgi -print uwsgi.extract("data://0") +print(uwsgi.extract("data://0")) diff --git a/examples/mjpeg_stream.py b/examples/mjpeg_stream.py index 9ed082c175..8897f92677 100644 --- a/examples/mjpeg_stream.py +++ b/examples/mjpeg_stream.py @@ -1,4 +1,5 @@ import os +import subprocess def application(env, start_response): @@ -16,8 +17,8 @@ def application(env, start_response): while 1: yield "Content-Type: image/jpeg\r\n\r\n" - print os.system('screencapture -t jpg -m -T 1 screenshot.jpg') + print(subprocess.call('screencapture -t jpg -m -T 1 screenshot.jpg', shell=True)) f = open('screenshot.jpg') yield env['wsgi.file_wrapper'](f) yield "\r\n--%s\r\n" % boundary - # os.system('./isightcapture -w 640 -h 480 screenshot.jpg') + # subprocess.call('./isightcapture -w 640 -h 480 screenshot.jpg', shell=True) diff --git a/examples/simple_app.py b/examples/simple_app.py index 7a5afbb9bf..f6a565e874 100644 --- a/examples/simple_app.py +++ b/examples/simple_app.py @@ -10,14 +10,14 @@ def ciao(): def ciao2(): print("nuovo uwsgi_server") - print os.getpid() + print(os.getpid()) counter = 0 #if uwsgi.load_plugin(0, 'plugins/example/example_plugin.so', 'ciao'): -# print "example plugin loaded" +# print("example plugin loaded") #else: -# print "unable to load example plugin" +# print("unable to load example plugin") #uwsgi.event_add(uwsgi.EVENT_FILE, "/tmp", ciao) #uwsgi.event_add(uwsgi.EVENT_DNSSD, "_uwsgi._tcp", ciao2) diff --git a/examples/simple_app_wsgi2.py b/examples/simple_app_wsgi2.py index ba176e433e..e18a8dc03e 100644 --- a/examples/simple_app_wsgi2.py +++ b/examples/simple_app_wsgi2.py @@ -1,6 +1,7 @@ +from six.moves import range def mygen(uri): - for i in xrange(1, 100): + for i in range(1, 100): yield "ciao %s
" % uri diff --git a/examples/taskqueue.py b/examples/taskqueue.py index 71131759d3..02f7b26c84 100644 --- a/examples/taskqueue.py +++ b/examples/taskqueue.py @@ -1,7 +1,9 @@ -import Queue from threading import Thread import uwsgi +from six.moves import queue + + CONSUMERS = 4 @@ -15,7 +17,7 @@ def consumer(q): def spawn_consumers(): global q - q = Queue.Queue() + q = queue.Queue() for i in range(CONSUMERS): t = Thread(target=consumer, args=(q,)) t.daemon = True diff --git a/examples/uwsgirouter.py b/examples/uwsgirouter.py index 541238e4f9..630913d367 100644 --- a/examples/uwsgirouter.py +++ b/examples/uwsgirouter.py @@ -11,13 +11,13 @@ def application(env, start_response): # has timed out ? if env['x-wsgiorg.fdevent.timeout']: - print "connection timed out !!!" + print("connection timed out !!!") uwsgi.close(fd) raise StopIteration # connection refused ? if not uwsgi.is_connected(fd): - print "unable to connect" + print("unable to connect") uwsgi.close(fd) raise StopIteration @@ -42,7 +42,7 @@ def application(env, start_response): bufsize = min(cl, 4096) yield uwsgi.wait_fd_read(input, 30) if env['x-wsgiorg.fdevent.timeout']: - print "connection timed out !!!" + print("connection timed out !!!") uwsgi.close(fd) raise StopIteration body = uwsgi.recv(input, bufsize) @@ -57,7 +57,7 @@ def application(env, start_response): # has timed out ? if env['x-wsgiorg.fdevent.timeout']: - print "connection timed out !!!" + print("connection timed out !!!") uwsgi.close(fd) raise StopIteration @@ -68,7 +68,7 @@ def application(env, start_response): # wait for response yield uwsgi.wait_fd_read(fd, 30) if env['x-wsgiorg.fdevent.timeout']: - print "connection timed out !!!" + print("connection timed out !!!") uwsgi.close(fd) raise StopIteration data = uwsgi.recv(fd) diff --git a/examples/uwsgirouter3.py b/examples/uwsgirouter3.py index 37b4b894cb..07c8358248 100644 --- a/examples/uwsgirouter3.py +++ b/examples/uwsgirouter3.py @@ -9,10 +9,10 @@ def application(e, s): global current_node nodes = uwsgi.cluster_nodes() - print nodes + print(nodes) if len(nodes) == 0: - print "no cluster node available" + print("no cluster node available") raise StopIteration if current_node >= len(nodes): diff --git a/examples/uwsgirouter4.py b/examples/uwsgirouter4.py index 7786ff6ebf..e7b5c97d93 100644 --- a/examples/uwsgirouter4.py +++ b/examples/uwsgirouter4.py @@ -5,10 +5,10 @@ def application(e, s): node = uwsgi.cluster_best_node() - print node + print(node) if not node: - print "sorry node unavailable" + print("sorry node unavailable") raise StopIteration for part in uwsgi.send_message(node, 0, 0, e, 0, e['wsgi.input'].fileno(), uwsgi.cl()): diff --git a/examples/uwsgistatus.py b/examples/uwsgistatus.py index 1123b6d909..9581349b26 100644 --- a/examples/uwsgistatus.py +++ b/examples/uwsgistatus.py @@ -6,7 +6,7 @@ def application(env, start_response): - print env + print(env) start_response('200 OK', [('Content-Type', 'text/html')]) yield '

uWSGI %s status

' % uwsgi.version @@ -50,11 +50,11 @@ def application(env, start_response): yield '

workers

' for w in workers: - # print w - # print w['running_time'] + # print(w) + # print(w['running_time']) if w is not None: yield '' + str(w['id']) + '' + str(w['pid']) + '' + str(w['pid']) + '' + str(w['requests']) + '' + str(w['running_time']) + '' + str(w['vsz']) + '' + str(w['rss']) + '' - print w + print(w) yield '' diff --git a/examples/welcome.py b/examples/welcome.py index 5aa4ca88b0..ea1ccbfb8b 100644 --- a/examples/welcome.py +++ b/examples/welcome.py @@ -4,9 +4,13 @@ import sys from uwsgidecorators import rpc, signal, postfork +from os.path import abspath, dirname, join + +logo_png = abspath(join(dirname(__file__), "../logo_uWSGI.png")) + print(sys.version) print(sys.version_info) -if 'set_debug' in gc.__dict__: +if "set_debug" in gc.__dict__: gc.set_debug(gc.DEBUG_SAVEALL) print(os.environ) @@ -14,7 +18,7 @@ print(sys.argv) try: - DEBUG = sys.argv[1] == 'debug' + DEBUG = sys.argv[1] == "debug" except IndexError: DEBUG = False @@ -22,10 +26,11 @@ def after_request_hook(): print("request finished") + uwsgi.after_req_hook = after_request_hook -@rpc('hello') +@rpc(b"hello") def hello_rpc(one, two, three): arg0 = one[::-1] arg1 = two[::-1] @@ -39,56 +44,65 @@ def ciao_mondo(signum): def xsendfile(e, sr): - sr('200 OK', [('Content-Type', 'image/png'), ('X-Sendfile', os.path.abspath('logo_uWSGI.png'))]) - return '' + sr( + "200 OK", [("Content-Type", "image/png"), ("X-Sendfile", logo_png),], + ) + return [] def serve_logo(e, sr): # use raw facilities (status will not be set...) - uwsgi.send("%s 200 OK\r\nContent-Type: image/png\r\n\r\n" % e['SERVER_PROTOCOL']) - uwsgi.sendfile('logo_uWSGI.png') - return '' + uwsgi.send( + b"%s 200 OK\r\nContent-Type: image/png\r\n\r\n" + % e["SERVER_PROTOCOL"].encode("latin1") + ) + uwsgi.sendfile(logo_png) + return [] def serve_config(e, sr): - sr('200 OK', [('Content-Type', 'text/html')]) - for opt in uwsgi.opt.keys(): - yield "%s = %s
" % (opt, uwsgi.opt[opt]) + sr("200 OK", [("Content-Type", "text/html")]) + for key in uwsgi.opt.keys(): + opt = uwsgi.opt[key] + if not isinstance(opt, bytes): + opt = str(opt).encode("utf-8") + yield b"%s = %s
" % (key, opt) + routes = {} -routes['/xsendfile'] = xsendfile -routes['/logo'] = serve_logo -routes['/config'] = serve_config +routes["/xsendfile"] = xsendfile +routes["/logo"] = serve_logo +routes["/config"] = serve_config @postfork def setprocname(): if uwsgi.worker_id() > 0: - uwsgi.setprocname("i am the worker %d" % uwsgi.worker_id()) + uwsgi.setprocname(b"i am the worker %d" % uwsgi.worker_id()) def application(env, start_response): try: - uwsgi.mule_msg(env['REQUEST_URI'], 1) + uwsgi.mule_msg(env["REQUEST_URI"], 1) except Exception: pass - req = uwsgi.workers()[uwsgi.worker_id()-1]['requests'] + req = uwsgi.workers()[uwsgi.worker_id() - 1]["requests"] - uwsgi.setprocname("worker %d managed %d requests" % (uwsgi.worker_id(), req)) + uwsgi.setprocname(b"worker %d managed %d requests" % (uwsgi.worker_id(), req)) try: gc.collect(2) except Exception: pass if DEBUG: - print(env['wsgi.input'].fileno()) + print(env["wsgi.input"].fileno()) - if env['PATH_INFO'] in routes: - return routes[env['PATH_INFO']](env, start_response) + if env["PATH_INFO"] in routes: + return routes[env["PATH_INFO"]](env, start_response) if DEBUG: - print(env['wsgi.input'].fileno()) + print(env["wsgi.input"].fileno()) try: gc.collect(2) @@ -98,24 +112,37 @@ def application(env, start_response): if DEBUG: print(len(gc.get_objects())) - workers = '' + workers = "" for w in uwsgi.workers(): apps = '' - for app in w['apps']: - apps += '' % (app['id'], app['mountpoint'], app['startup_time'], app['requests']) - apps += '
idmountpointstartup timerequests
%d%s%d%d
' + for app in w["apps"]: + apps += "%d%s%d%d" % ( + app["id"], + app["mountpoint"], + app["startup_time"], + app["requests"], + ) + apps += "" workers += """ %d%d%s%d%d%d%s - """ % (w['id'], w['pid'], w['status'], w['running_time']/1000, w['avg_rt']/1000, w['tx'], apps) + """ % ( + w["id"], + w["pid"], + w["status"], + w["running_time"] / 1000, + w["avg_rt"] / 1000, + w["tx"], + apps, + ) output = """ - version %s running on %s (remote user: %s)
+ version %s running on %s (remote user: %s)
Configuration
-
+

Workers and applications
@@ -126,9 +153,15 @@ def application(env, start_response): %s - """ % (uwsgi.version, uwsgi.hostname, env.get('REMOTE_USER', 'None'), workers) + """ % ( + uwsgi.version, + uwsgi.hostname, + env.get("REMOTE_USER", "None"), + workers, + ) - start_response('200 OK', [('Content-Type', 'text/html'), ('Content-Length', str(len(output)))]) + start_response( + "200 OK", [("Content-Type", "text/html"), ("Content-Length", str(len(output)))] + ) - # return bytes(output.encode('latin1')) - return output + return [output.format(script_name=env["SCRIPT_NAME"]).encode("utf-8")] diff --git a/examples/welcome3.py b/examples/welcome3.py index 4d308d17f4..6273ec89ef 100644 --- a/examples/welcome3.py +++ b/examples/welcome3.py @@ -1,35 +1,50 @@ import uwsgi import os +from os.path import abspath, dirname, join + +logo_png = abspath(join(dirname(__file__), "../logo_uWSGI.png")) + def xsendfile(e, sr): - sr('200 OK', [('Content-Type', 'image/png'), ('X-Sendfile', os.path.abspath('logo_uWSGI.png'))]) - return b'' + sr( + "200 OK", [("Content-Type", "image/png"), ("X-Sendfile", logo_png),], + ) + return b"" def serve_logo(e, sr): - sr('200 OK', [('Content-Type', 'image/png')]) - return uwsgi.sendfile('logo_uWSGI.png') + sr("200 OK", [("Content-Type", "image/png")]) + return uwsgi.sendfile(logo_png) def serve_config(e, sr): - sr('200 OK', [('Content-Type', 'text/html')]) + sr("200 OK", [("Content-Type", "text/html")]) for opt in uwsgi.opt.keys(): - body = "{opt} = {optvalue}
".format(opt=opt, optvalue=uwsgi.opt[opt].decode('ascii')) - yield bytes(body.encode('ascii')) + + def decode_if_bytes(val): + if isinstance(val, bytes): + return val.decode("ascii") + return val + + body = "{opt} = {optvalue}
".format( + opt=opt.decode("ascii"), optvalue=decode_if_bytes(uwsgi.opt[opt]) + ) + yield bytes(body.encode("ascii")) + routes = {} -routes['/xsendfile'] = xsendfile -routes['/logo'] = serve_logo -routes['/config'] = serve_config +routes["/xsendfile"] = xsendfile +routes["/logo"] = serve_logo +routes["/config"] = serve_config def application(env, start_response): - if env['PATH_INFO'] in routes: - return routes[env['PATH_INFO']](env, start_response) + if env["PATH_INFO"] in routes: + return routes[env["PATH_INFO"]](env, start_response) - start_response('200 OK', [('Content-Type', 'text/html')]) + start_response("200 OK", [("Content-Type", "text/html")]) body = """ version {version}
@@ -40,6 +55,8 @@ def application(env, start_response):
- """.format(version=uwsgi.version.decode('ascii')) + """.format( + version=uwsgi.version + ) - return bytes(body.encode('ascii')) + return [bytes(body.encode("ascii"))] diff --git a/plugins/asyncio/uwsgiplugin.py b/plugins/asyncio/uwsgiplugin.py index 7cb17f54eb..56e56e8150 100644 --- a/plugins/asyncio/uwsgiplugin.py +++ b/plugins/asyncio/uwsgiplugin.py @@ -1,11 +1,19 @@ -from distutils import sysconfig +try: + from distutils import sysconfig + paths = [ + sysconfig.get_python_inc(), + sysconfig.get_python_inc(plat_specific=True), + ] +except ImportError: + import sysconfig + paths = [ + sysconfig.get_path('include'), + sysconfig.get_path('platinclude'), + ] NAME = 'asyncio' -CFLAGS = [ - '-I' + sysconfig.get_python_inc(), - '-I' + sysconfig.get_python_inc(plat_specific=True) -] +CFLAGS = ['-I' + path for path in paths] LDFLAGS = [] LIBS = [] GCC_LIST = ['asyncio'] diff --git a/plugins/cgi/cgi_plugin.c b/plugins/cgi/cgi_plugin.c index d032db17c3..89d36bed09 100644 --- a/plugins/cgi/cgi_plugin.c +++ b/plugins/cgi/cgi_plugin.c @@ -22,6 +22,7 @@ struct uwsgi_cgi { int do_not_kill_on_error; int async_max_attempts; int close_stdin_on_eof; + int dontresolve; } uc ; static void uwsgi_opt_add_cgi(char *opt, char *value, void *foobar) { @@ -75,6 +76,8 @@ struct uwsgi_option uwsgi_cgi_options[] = { {"cgi-safe", required_argument, 0, "skip security checks if the cgi file is under the specified path", uwsgi_opt_add_string_list, &uc.cgi_safe, 0}, + {"cgi-dontresolve", no_argument, 0 , "call symbolic link directly instead of the real path", uwsgi_opt_true,&uc.dontresolve, 0}, + {0, 0, 0, 0, 0, 0, 0}, }; @@ -475,6 +478,7 @@ static int uwsgi_cgi_request(struct wsgi_request *wsgi_req) { char full_path[PATH_MAX]; char tmp_path[PATH_MAX]; + char symbolic_path[PATH_MAX]; struct stat cgi_stat; int need_free = 0; int is_a_file = 0; @@ -533,6 +537,10 @@ static int uwsgi_cgi_request(struct wsgi_request *wsgi_req) { uwsgi_404(wsgi_req); return UWSGI_OK; } + if (uc.dontresolve) { + full_path_len = strlen(full_path); + memcpy(symbolic_path, full_path, full_path_len+1); + } full_path_len = strlen(tmp_path); // add +1 to copy the null byte @@ -639,6 +647,11 @@ static int uwsgi_cgi_request(struct wsgi_request *wsgi_req) { } } + if (uc.dontresolve) { + full_path_len = strlen(symbolic_path); + memcpy(full_path, symbolic_path, full_path_len+1); + } + int ret = uwsgi_cgi_run(wsgi_req, docroot, docroot_len, full_path, helper, path_info, script_name, is_a_file, discard_base); if (need_free) free(docroot); return ret; @@ -691,23 +704,50 @@ static int uwsgi_cgi_run(struct wsgi_request *wsgi_req, char *docroot, size_t do uwsgi_socket_nb(cgi_pipe[0]); uwsgi_socket_nb(post_pipe[1]); - // ok start sending post data... - size_t remains = wsgi_req->post_cl; - while(remains > 0) { - ssize_t rlen = 0; - char *buf = uwsgi_request_body_read(wsgi_req, 8192, &rlen); - if (!buf) { - goto clear2; - } - if (buf == uwsgi.empty) break; - // write data to the node - if (uwsgi_write_true_nb(post_pipe[1], buf, rlen, uc.timeout)) { - goto clear2; - } - remains -= rlen; - } - - if (uc.close_stdin_on_eof) { + // Start sending request body + if (wsgi_req->body_is_chunked) { + // Write through to process chunk by chunk + while (1) { + struct uwsgi_buffer *ubuf = uwsgi_chunked_read_smart(wsgi_req, 8192, uwsgi.socket_timeout); + if (!ubuf) { + uwsgi_log("error reading chunk from CGI request !!!\n"); + kill_on_error + goto clear2; + } + if (!ubuf->pos) { + // Last chunk received, go and close process's stdin + uwsgi_buffer_destroy(ubuf); + break; + } + // Write chunk to process's stdin + int err = uwsgi_write_true_nb(post_pipe[1], ubuf->buf, ubuf->pos, uc.timeout); + uwsgi_buffer_destroy(ubuf); + if (err) { + uwsgi_log("error writing chunk to CGI process !!!\n"); + kill_on_error + goto clear2; + } + } + } else { + // Normal request with content length set + size_t remains = wsgi_req->post_cl; + while(remains > 0) { + ssize_t rlen = 0; + char *buf = uwsgi_request_body_read(wsgi_req, 8192, &rlen); + if (!buf) { + goto clear2; + } + if (buf == uwsgi.empty) break; + // write data to the node + if (uwsgi_write_true_nb(post_pipe[1], buf, rlen, uc.timeout)) { + goto clear2; + } + remains -= rlen; + } + } + + // For chunked requests, close stdin to tell the script body has ended + if (uc.close_stdin_on_eof || wsgi_req->body_is_chunked) { close(post_pipe[1]); stdin_closed = 1; } @@ -780,8 +820,21 @@ static int uwsgi_cgi_run(struct wsgi_request *wsgi_req, char *docroot, size_t do close(cgi_pipe[1]); // close all the fd > 2 - for(i=3;i<(int)uwsgi.max_fd;i++) { - close(i); + DIR *dirp = opendir("/proc/self/fd"); + if (dirp == NULL) + dirp = opendir("/dev/fd"); + if (dirp != NULL) { + struct dirent *dent; + while ((dent = readdir(dirp)) != NULL) { + int fd = atoi(dent->d_name); + if ((fd > 2) && fd != dirfd(dirp)) + close(fd); + } + closedir(dirp); + } else { + for(i=3;i<(int)uwsgi.max_fd;i++) { + close(i); + } } // fill cgi env diff --git a/plugins/corerouter/cr_common.c b/plugins/corerouter/cr_common.c index e66a162d52..55746d3eb1 100644 --- a/plugins/corerouter/cr_common.c +++ b/plugins/corerouter/cr_common.c @@ -62,7 +62,7 @@ void uwsgi_corerouter_setup_sockets(struct uwsgi_corerouter *ucr) { // fix SERVER_PORT if (!ugs->port || !ugs->port_len) { - ugs->port = strchr(ugs->name, ':'); + ugs->port = strrchr(ugs->name, ':'); if (ugs->port) { ugs->port++; ugs->port_len = strlen(ugs->port); diff --git a/plugins/coroae/uwsgiplugin.py b/plugins/coroae/uwsgiplugin.py index 83c2b25e7c..16da85bdb6 100644 --- a/plugins/coroae/uwsgiplugin.py +++ b/plugins/coroae/uwsgiplugin.py @@ -11,7 +11,7 @@ coroapi = p if not coroapi: - print "unable to find the Coro perl module !!!" + print("unable to find the Coro perl module !!!") sys.exit(1) CFLAGS = os.popen('perl -MExtUtils::Embed -e ccopts').read().rstrip().split() diff --git a/plugins/fiber/fiber.c b/plugins/fiber/fiber.c index 30603ae099..d68f7ccac7 100644 --- a/plugins/fiber/fiber.c +++ b/plugins/fiber/fiber.c @@ -15,12 +15,12 @@ struct uwsgi_option fiber_options[] = { }; -VALUE uwsgi_fiber_request() { +VALUE uwsgi_fiber_request(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, callback_arg)) { async_schedule_to_req_green(); return Qnil; } -VALUE rb_fiber_schedule_to_req() { +VALUE rb_fiber_schedule_to_req(VALUE v) { int id = uwsgi.wsgi_req->async_id; if (!uwsgi.wsgi_req->suspended) { diff --git a/plugins/gccgo/uwsgiplugin.py b/plugins/gccgo/uwsgiplugin.py index fee9364767..dbb2052c44 100644 --- a/plugins/gccgo/uwsgiplugin.py +++ b/plugins/gccgo/uwsgiplugin.py @@ -1,4 +1,5 @@ import os +import subprocess NAME = 'gccgo' @@ -10,6 +11,6 @@ def post_build(config): if os.path.exists('plugins/gccgo/uwsgi.go.o'): - if os.system("objcopy -j .go_export plugins/gccgo/uwsgi.go.o plugins/gccgo/uwsgi.gox") != 0: + if subprocess.call("objcopy -j .go_export plugins/gccgo/uwsgi.go.o plugins/gccgo/uwsgi.gox", shell=True) != 0: os._exit(1) print("*** uwsgi.gox available in %s/plugins/gccgo ***" % os.getcwd()) diff --git a/plugins/gevent/gevent.c b/plugins/gevent/gevent.c index 5ddf100ddf..b0d0c36160 100644 --- a/plugins/gevent/gevent.c +++ b/plugins/gevent/gevent.c @@ -33,7 +33,7 @@ PyObject *py_uwsgi_gevent_ctrl_gl(PyObject *self, PyObject *args) { for(;;) { PyObject *gevent_sleep_args = PyTuple_New(1); PyTuple_SetItem(gevent_sleep_args, 0, PyInt_FromLong(60)); - PyObject *gswitch = PyEval_CallObject(ugevent.greenlet_switch, gevent_sleep_args); + PyObject *gswitch = PyObject_CallObject(ugevent.greenlet_switch, gevent_sleep_args); // could be NULL on exception if (!gswitch) { // just for being paranoid @@ -389,7 +389,7 @@ static void gil_gevent_get() { } static void gil_gevent_release() { - PyGILState_Release((PyGILState_STATE) pthread_getspecific(up.upt_gil_key)); + PyGILState_Release((PyGILState_STATE)(long) pthread_getspecific(up.upt_gil_key)); } static void monkey_patch() { @@ -451,8 +451,13 @@ static void gevent_loop() { ugevent.spawn = PyDict_GetItemString(gevent_dict, "spawn"); if (!ugevent.spawn) uwsgi_pyexit; - ugevent.signal = PyDict_GetItemString(gevent_dict, "signal"); - if (!ugevent.signal) uwsgi_pyexit; + ugevent.signal = PyDict_GetItemString(gevent_dict, "signal_handler"); + if (!ugevent.signal) { + // gevent.signal_handler appears in gevent 1.3. + // On older gevent, fall back to the deprecated gevent.signal. + ugevent.signal = PyDict_GetItemString(gevent_dict, "signal"); + if (!ugevent.signal) uwsgi_pyexit; + } ugevent.greenlet_switch = PyDict_GetItemString(gevent_dict, "sleep"); if (!ugevent.greenlet_switch) uwsgi_pyexit; diff --git a/plugins/gevent/gevent.h b/plugins/gevent/gevent.h index c39034ec59..07f5cb1787 100644 --- a/plugins/gevent/gevent.h +++ b/plugins/gevent/gevent.h @@ -4,7 +4,7 @@ int uwsgi_gevent_wait_write_hook(int, int); int uwsgi_gevent_wait_read_hook(int, int); int uwsgi_gevent_wait_milliseconds_hook(int); -#define GEVENT_SWITCH PyObject *gswitch = python_call(ugevent.greenlet_switch, ugevent.greenlet_switch_args, 0, NULL); Py_DECREF(gswitch) +#define GEVENT_SWITCH PyObject *gswitch = python_call(ugevent.greenlet_switch, ugevent.greenlet_switch_args, 0, NULL); if (gswitch) { Py_DECREF(gswitch); } #define GET_CURRENT_GREENLET python_call(ugevent.get_current, ugevent.get_current_args, 0, NULL) #define free_req_queue uwsgi.async_queue_unused_ptr++; uwsgi.async_queue_unused[uwsgi.async_queue_unused_ptr] = wsgi_req #define stop_the_watchers if (timer) { ret = PyObject_CallMethod(timer, "stop", NULL);\ diff --git a/plugins/gevent/uwsgiplugin.py b/plugins/gevent/uwsgiplugin.py index 4ff550a9f9..422a5ffc19 100644 --- a/plugins/gevent/uwsgiplugin.py +++ b/plugins/gevent/uwsgiplugin.py @@ -1,11 +1,19 @@ -from distutils import sysconfig +try: + from distutils import sysconfig + paths = [ + sysconfig.get_python_inc(), + sysconfig.get_python_inc(plat_specific=True), + ] +except ImportError: + import sysconfig + paths = [ + sysconfig.get_path('include'), + sysconfig.get_path('platinclude'), + ] NAME = 'gevent' -CFLAGS = [ - '-I' + sysconfig.get_python_inc(), - '-I' + sysconfig.get_python_inc(plat_specific=True) -] +CFLAGS = ['-I' + path for path in paths] LDFLAGS = [] LIBS = [] diff --git a/plugins/glusterfs/glusterfs.c b/plugins/glusterfs/glusterfs.c index 83428faf3f..c0063f4e58 100644 --- a/plugins/glusterfs/glusterfs.c +++ b/plugins/glusterfs/glusterfs.c @@ -46,7 +46,7 @@ struct uwsgi_glusterfs_async_io { ssize_t rlen; }; -static void uwsgi_glusterfs_read_async_cb(glfs_fd_t *fd, ssize_t rlen, void *data) { +static void uwsgi_glusterfs_read_async_cb(glfs_fd_t *fd, ssize_t rlen, struct glfs_stat *prestat, struct glfs_stat *poststat, void *data) { struct uwsgi_glusterfs_async_io *aio = (struct uwsgi_glusterfs_async_io *) data; #ifdef UWSGI_DEBUG uwsgi_log("[glusterfs-cb] rlen = %d\n", rlen); diff --git a/plugins/greenlet/uwsgiplugin.py b/plugins/greenlet/uwsgiplugin.py index 27b1da1235..c712b53699 100644 --- a/plugins/greenlet/uwsgiplugin.py +++ b/plugins/greenlet/uwsgiplugin.py @@ -1,11 +1,19 @@ -from distutils import sysconfig +try: + from distutils import sysconfig + paths = [ + sysconfig.get_python_inc(), + sysconfig.get_python_inc(plat_specific=True), + ] +except ImportError: + import sysconfig + paths = [ + sysconfig.get_path('include'), + sysconfig.get_path('platinclude'), + ] NAME = 'greenlet' -CFLAGS = [ - '-I' + sysconfig.get_python_inc(), - '-I' + sysconfig.get_python_inc(plat_specific=True) -] +CFLAGS = ['-I' + path for path in paths] LDFLAGS = [] LIBS = [] diff --git a/plugins/http/http.c b/plugins/http/http.c index 05e3832ea9..ab14d7df47 100644 --- a/plugins/http/http.c +++ b/plugins/http/http.c @@ -32,7 +32,7 @@ struct uwsgi_option http_options[] = { {"http-events", required_argument, 0, "set the number of concurrent http async events", uwsgi_opt_set_int, &uhttp.cr.nevents, 0}, {"http-subscription-server", required_argument, 0, "enable the subscription server", uwsgi_opt_corerouter_ss, &uhttp, 0}, {"http-subscription-fallback-key", required_argument, 0, "key to use for fallback http handler", uwsgi_opt_corerouter_fallback_key, &uhttp.cr, 0}, - {"http-timeout", required_argument, 0, "set internal http socket timeout", uwsgi_opt_set_int, &uhttp.cr.socket_timeout, 0}, + {"http-timeout", required_argument, 0, "set internal http socket timeout (default: 60 seconds)", uwsgi_opt_set_int, &uhttp.cr.socket_timeout, 0}, {"http-manage-expect", optional_argument, 0, "manage the Expect HTTP request header (optionally checking for Content-Length)", uwsgi_opt_set_64bit, &uhttp.manage_expect, 0}, {"http-keepalive", optional_argument, 0, "HTTP 1.1 keepalive support (non-pipelined) requests", uwsgi_opt_set_int, &uhttp.keepalive, 0}, {"http-auto-chunked", no_argument, 0, "automatically transform output to chunked encoding during HTTP 1.1 keepalive (if needed)", uwsgi_opt_true, &uhttp.auto_chunked, 0}, @@ -1224,8 +1224,9 @@ ssize_t http_parse(struct corerouter_peer *main_peer) { } if (hr->remains > 0) { - if (hr->content_length < hr->remains) { - hr->remains = hr->content_length; + if (hr->content_length < hr->remains) { + if (hr->content_length > 0 || !hr->raw_body) + hr->remains = hr->content_length; hr->content_length = 0; // we need to avoid problems with pipelined requests hr->session.can_keepalive = 0; diff --git a/plugins/jvm/uwsgiplugin.py b/plugins/jvm/uwsgiplugin.py index bd03cea3d8..f3c369abf6 100644 --- a/plugins/jvm/uwsgiplugin.py +++ b/plugins/jvm/uwsgiplugin.py @@ -1,5 +1,6 @@ import os import shutil +import subprocess NAME = 'jvm' @@ -31,11 +32,14 @@ JVM_INCPATH = ['-I"/cygdrive/c/Program Files/Java/jdk1.7.0_17/include"', '-I"/cygdrive/c/Program Files/Java/jdk1.7.0_17/include/win32"'] JVM_LIBPATH = ['-L"/cygdrive/c/Program Files/Java/jdk1.7.0_17/jre/bin/server"'] else: - known_jvms = ('/usr/lib/jvm/java-7-openjdk', '/usr/local/openjdk7', '/usr/lib/jvm/java-6-openjdk', '/usr/local/openjdk', '/usr/java', '/usr/lib/jvm/java/', '/usr/lib/jvm/java-8-openjdk-%s' % arch) + known_jvms = ('/usr/lib/jvm/java-7-openjdk', '/usr/local/openjdk7', '/usr/lib/jvm/java-6-openjdk', '/usr/local/openjdk', '/usr/java', '/usr/lib/jvm/java/', '/usr/lib/jvm/java-8-openjdk-%s' % arch, '/usr/lib/jvm/java-11-openjdk-%s' % arch) for jvm in known_jvms: if os.path.exists(jvm + '/include'): JVM_INCPATH = ["-I%s/include/" % jvm, "-I%s/include/%s" % (jvm, operating_system)] - JVM_LIBPATH = ["-L%s/jre/lib/%s/server" % (jvm, arch)] + if os.path.exists("%s/jre" % jvm): + JVM_LIBPATH = ["-L%s/jre/lib/%s/server" % (jvm, arch)] + else: + JVM_LIBPATH = ["-L%s/lib/server" % (jvm,)] break if os.path.exists("%s-%s/include" % (jvm, arch)): jvm = "%s-%s" % (jvm, arch) @@ -72,9 +76,9 @@ def post_build(config): - if os.system("javac %s/plugins/jvm/uwsgi.java" % os.getcwd()) != 0: + if subprocess.call("javac %s/plugins/jvm/uwsgi.java" % os.getcwd(), shell=True) != 0: os._exit(1) - if os.system("cd %s/plugins/jvm ; jar cvf uwsgi.jar *.class" % os.getcwd()) != 0: + if subprocess.call("cd %s/plugins/jvm ; jar cvf uwsgi.jar *.class" % os.getcwd(), shell=True) != 0: os._exit(1) print("*** uwsgi.jar available in %s/plugins/jvm/uwsgi.jar ***" % os.getcwd()) diff --git a/plugins/logsocket/logsocket_plugin.c b/plugins/logsocket/logsocket_plugin.c index a603b3adc8..9ac8f456b6 100644 --- a/plugins/logsocket/logsocket_plugin.c +++ b/plugins/logsocket/logsocket_plugin.c @@ -43,6 +43,7 @@ ssize_t uwsgi_socket_logger(struct uwsgi_logger *ul, char *message, size_t len) else { ul->msg.msg_iov = uwsgi_malloc(sizeof(struct iovec)); ul->msg.msg_iovlen = 1; + ul->count = 0; } if (comma) { diff --git a/plugins/mono/uwsgiplugin.py b/plugins/mono/uwsgiplugin.py index 2232f88eb3..129619116a 100644 --- a/plugins/mono/uwsgiplugin.py +++ b/plugins/mono/uwsgiplugin.py @@ -1,4 +1,5 @@ import os +import subprocess NAME = 'mono' @@ -12,8 +13,8 @@ def post_build(config): - if os.system("sn -k plugins/mono/uwsgi.key") != 0: + if subprocess.call("sn -k plugins/mono/uwsgi.key", shell=True) != 0: os._exit(1) - if os.system("mcs /target:library /r:System.Configuration.dll /r:System.Web.dll /keyfile:plugins/mono/uwsgi.key plugins/mono/uwsgi.cs") != 0: + if subprocess.call("mcs /target:library /r:System.Configuration.dll /r:System.Web.dll /keyfile:plugins/mono/uwsgi.key plugins/mono/uwsgi.cs", shell=True) != 0: os._exit(1) print("*** uwsgi.dll available in %s/plugins/mono/uwsgi.dll ***" % os.getcwd()) diff --git a/plugins/php/common.h b/plugins/php/common.h index 9bf1c0690b..c5bb98dd29 100644 --- a/plugins/php/common.h +++ b/plugins/php/common.h @@ -3,14 +3,11 @@ #include "php_main.h" #include "php_variables.h" -#if (PHP_MAJOR_VERSION < 7) -#include "ext/standard/php_smart_str.h" -#else -#define UWSGI_PHP7 -#endif #include "ext/standard/info.h" #include "ext/session/php_session.h" +#include "ext/standard/head.h" + #include diff --git a/plugins/php/php_plugin.c b/plugins/php/php_plugin.c index 72c3902234..a888e524bd 100644 --- a/plugins/php/php_plugin.c +++ b/plugins/php/php_plugin.c @@ -6,6 +6,9 @@ static sapi_module_struct uwsgi_sapi_module; static int uwsgi_php_init(void); +typedef size_t php_strlen_size; +typedef zend_long php_long_size; + struct uwsgi_php { struct uwsgi_string_list *allowed_docroot; struct uwsgi_string_list *allowed_ext; @@ -13,7 +16,7 @@ struct uwsgi_php { struct uwsgi_string_list *index; struct uwsgi_string_list *set; struct uwsgi_string_list *append_config; -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *app_bypass; #endif struct uwsgi_string_list *vars; @@ -24,6 +27,7 @@ struct uwsgi_php { char *fallback; char *fallback2; char *fallback_qs; + char *ini_entries; size_t ini_size; int dump_config; char *server_software; @@ -66,7 +70,7 @@ struct uwsgi_option uwsgi_php_options[] = { {"php-fallback", required_argument, 0, "run the specified php script when the requested one does not exist", uwsgi_opt_set_str, &uphp.fallback, 0}, {"php-fallback2", required_argument, 0, "run the specified php script relative to the document root when the requested one does not exist", uwsgi_opt_set_str, &uphp.fallback2, 0}, {"php-fallback-qs", required_argument, 0, "php-fallback with QUERY_STRING set", uwsgi_opt_set_str, &uphp.fallback_qs, 0}, -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) {"php-app-bypass", required_argument, 0, "if the regexp matches the uri the --php-app is bypassed", uwsgi_opt_add_regexp_list, &uphp.app_bypass, 0}, #endif {"php-var", required_argument, 0, "add/overwrite a CGI variable at each request", uwsgi_opt_add_string_list, &uphp.vars, 0}, @@ -85,11 +89,7 @@ struct uwsgi_option uwsgi_php_options[] = { }; -#ifdef UWSGI_PHP7 -static size_t sapi_uwsgi_ub_write(const char *str, size_t str_length TSRMLS_DC) -#else -static int sapi_uwsgi_ub_write(const char *str, uint str_length TSRMLS_DC) -#endif +static size_t sapi_uwsgi_ub_write(const char *str, size_t str_length) { struct wsgi_request *wsgi_req = (struct wsgi_request *) SG(server_context); @@ -101,7 +101,7 @@ static int sapi_uwsgi_ub_write(const char *str, uint str_length TSRMLS_DC) return str_length; } -static int sapi_uwsgi_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC) +static int sapi_uwsgi_send_headers(sapi_headers_struct *sapi_headers) { sapi_header_struct *h; zend_llist_position pos; @@ -135,11 +135,7 @@ static int sapi_uwsgi_send_headers(sapi_headers_struct *sapi_headers TSRMLS_DC) return SAPI_HEADER_SENT_SUCCESSFULLY; } -#ifdef UWSGI_PHP7 -static size_t sapi_uwsgi_read_post(char *buffer, size_t count_bytes TSRMLS_DC) -#else -static int sapi_uwsgi_read_post(char *buffer, uint count_bytes TSRMLS_DC) -#endif +static size_t sapi_uwsgi_read_post(char *buffer, size_t count_bytes) { uint read_bytes = 0; @@ -163,7 +159,7 @@ static int sapi_uwsgi_read_post(char *buffer, uint count_bytes TSRMLS_DC) } -static char *sapi_uwsgi_read_cookies(TSRMLS_D) +static char *sapi_uwsgi_read_cookies() { uint16_t len = 0; struct wsgi_request *wsgi_req = (struct wsgi_request *) SG(server_context); @@ -176,55 +172,55 @@ static char *sapi_uwsgi_read_cookies(TSRMLS_D) return NULL; } -static void sapi_uwsgi_register_variables(zval *track_vars_array TSRMLS_DC) +static void sapi_uwsgi_register_variables(zval *track_vars_array) { int i; struct wsgi_request *wsgi_req = (struct wsgi_request *) SG(server_context); - php_import_environment_variables(track_vars_array TSRMLS_CC); + php_import_environment_variables(track_vars_array); if (uphp.server_software) { if (!uphp.server_software_len) uphp.server_software_len = strlen(uphp.server_software); - php_register_variable_safe("SERVER_SOFTWARE", uphp.server_software, uphp.server_software_len, track_vars_array TSRMLS_CC); + php_register_variable_safe("SERVER_SOFTWARE", uphp.server_software, uphp.server_software_len, track_vars_array); } else { - php_register_variable_safe("SERVER_SOFTWARE", "uWSGI", 5, track_vars_array TSRMLS_CC); + php_register_variable_safe("SERVER_SOFTWARE", "uWSGI", 5, track_vars_array); } for (i = 0; i < wsgi_req->var_cnt; i += 2) { php_register_variable_safe( estrndup(wsgi_req->hvec[i].iov_base, wsgi_req->hvec[i].iov_len), wsgi_req->hvec[i + 1].iov_base, wsgi_req->hvec[i + 1].iov_len, - track_vars_array TSRMLS_CC); + track_vars_array); } - php_register_variable_safe("PATH_INFO", wsgi_req->path_info, wsgi_req->path_info_len, track_vars_array TSRMLS_CC); + php_register_variable_safe("PATH_INFO", wsgi_req->path_info, wsgi_req->path_info_len, track_vars_array); if (wsgi_req->query_string_len > 0) { - php_register_variable_safe("QUERY_STRING", wsgi_req->query_string, wsgi_req->query_string_len, track_vars_array TSRMLS_CC); + php_register_variable_safe("QUERY_STRING", wsgi_req->query_string, wsgi_req->query_string_len, track_vars_array); } - php_register_variable_safe("SCRIPT_NAME", wsgi_req->script_name, wsgi_req->script_name_len, track_vars_array TSRMLS_CC); - php_register_variable_safe("SCRIPT_FILENAME", wsgi_req->file, wsgi_req->file_len, track_vars_array TSRMLS_CC); + php_register_variable_safe("SCRIPT_NAME", wsgi_req->script_name, wsgi_req->script_name_len, track_vars_array); + php_register_variable_safe("SCRIPT_FILENAME", wsgi_req->file, wsgi_req->file_len, track_vars_array); - php_register_variable_safe("DOCUMENT_ROOT", wsgi_req->document_root, wsgi_req->document_root_len, track_vars_array TSRMLS_CC); + php_register_variable_safe("DOCUMENT_ROOT", wsgi_req->document_root, wsgi_req->document_root_len, track_vars_array); if (wsgi_req->path_info_len) { char *path_translated = ecalloc(1, wsgi_req->file_len + wsgi_req->path_info_len + 1); memcpy(path_translated, wsgi_req->file, wsgi_req->file_len); memcpy(path_translated + wsgi_req->file_len, wsgi_req->path_info, wsgi_req->path_info_len); - php_register_variable_safe("PATH_TRANSLATED", path_translated, wsgi_req->file_len + wsgi_req->path_info_len , track_vars_array TSRMLS_CC); + php_register_variable_safe("PATH_TRANSLATED", path_translated, wsgi_req->file_len + wsgi_req->path_info_len , track_vars_array); } else { - php_register_variable_safe("PATH_TRANSLATED", "", 0, track_vars_array TSRMLS_CC); + php_register_variable_safe("PATH_TRANSLATED", "", 0, track_vars_array); } - php_register_variable_safe("PHP_SELF", wsgi_req->script_name, wsgi_req->script_name_len, track_vars_array TSRMLS_CC); + php_register_variable_safe("PHP_SELF", wsgi_req->script_name, wsgi_req->script_name_len, track_vars_array); struct uwsgi_string_list *usl = uphp.vars; while(usl) { char *equal = strchr(usl->value, '='); if (equal) { php_register_variable_safe( estrndup(usl->value, equal-usl->value), - equal+1, strlen(equal+1), track_vars_array TSRMLS_CC); + equal+1, strlen(equal+1), track_vars_array); } usl = usl->next; } @@ -237,21 +233,22 @@ static sapi_module_struct uwsgi_sapi_module; void uwsgi_php_append_config(char *filename) { size_t file_size = 0; - char *file_content = uwsgi_open_and_read(filename, &file_size, 1, NULL); - uwsgi_sapi_module.ini_entries = realloc(uwsgi_sapi_module.ini_entries, uphp.ini_size + file_size); - memcpy(uwsgi_sapi_module.ini_entries + uphp.ini_size, file_content, file_size); + char *file_content = uwsgi_open_and_read(filename, &file_size, 1, NULL); + uphp.ini_entries = realloc(uphp.ini_entries, uphp.ini_size + file_size); + memcpy(uphp.ini_entries + uphp.ini_size, file_content, file_size); uphp.ini_size += file_size-1; free(file_content); + uwsgi_sapi_module.ini_entries = uphp.ini_entries; } void uwsgi_php_set(char *opt) { - uwsgi_sapi_module.ini_entries = realloc(uwsgi_sapi_module.ini_entries, uphp.ini_size + strlen(opt)+2); - memcpy(uwsgi_sapi_module.ini_entries + uphp.ini_size, opt, strlen(opt)); - + uphp.ini_entries = realloc(uphp.ini_entries, uphp.ini_size + strlen(opt)+2); + memcpy(uphp.ini_entries + uphp.ini_size, opt, strlen(opt)); uphp.ini_size += strlen(opt)+1; - uwsgi_sapi_module.ini_entries[uphp.ini_size-1] = '\n'; - uwsgi_sapi_module.ini_entries[uphp.ini_size] = 0; + uphp.ini_entries[uphp.ini_size-1] = '\n'; + uphp.ini_entries[uphp.ini_size] = 0; + uwsgi_sapi_module.ini_entries = uphp.ini_entries; } extern ps_module ps_mod_uwsgi; @@ -265,10 +262,7 @@ PHP_MINIT_FUNCTION(uwsgi_php_minit) { char *name = usl->value; char *strval = equal + 1; equal = NULL; -#ifndef UWSGI_PHP7 - name_len = name_len + 1; -#endif - zend_register_string_constant(name, name_len, strval, CONST_CS | CONST_PERSISTENT, module_number TSRMLS_CC); + zend_register_string_constant(name, name_len, strval, CONST_CS | CONST_PERSISTENT, module_number); } usl = usl->next; } @@ -276,11 +270,7 @@ PHP_MINIT_FUNCTION(uwsgi_php_minit) { } PHP_FUNCTION(uwsgi_version) { -#ifdef UWSGI_PHP7 RETURN_STRING(UWSGI_VERSION); -#else - RETURN_STRING(UWSGI_VERSION, 1); -#endif } PHP_FUNCTION(uwsgi_worker_id) { @@ -297,11 +287,11 @@ PHP_FUNCTION(uwsgi_masterpid) { PHP_FUNCTION(uwsgi_cache_exists) { char *key = NULL; - int keylen = 0; + php_strlen_size keylen = 0; char *cache = NULL; - int cachelen = 0; + php_strlen_size cachelen = 0; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|s", &key, &keylen, &cache, &cachelen) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "s|s", &key, &keylen, &cache, &cachelen) == FAILURE) { RETURN_NULL(); } @@ -315,9 +305,9 @@ PHP_FUNCTION(uwsgi_cache_exists) { PHP_FUNCTION(uwsgi_cache_clear) { char *cache = NULL; - int cachelen = 0; + php_strlen_size cachelen = 0; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "|s", &cache, &cachelen) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "|s", &cache, &cachelen) == FAILURE) { RETURN_NULL(); } @@ -330,13 +320,13 @@ PHP_FUNCTION(uwsgi_cache_clear) { PHP_FUNCTION(uwsgi_cache_del) { - + char *key = NULL; - int keylen = 0; + php_strlen_size keylen = 0; char *cache = NULL; - int cachelen = 0; + php_strlen_size cachelen = 0; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|s", &key, &keylen, &cache, &cachelen) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "s|s", &key, &keylen, &cache, &cachelen) == FAILURE) { RETURN_NULL(); } @@ -350,15 +340,15 @@ PHP_FUNCTION(uwsgi_cache_del) { PHP_FUNCTION(uwsgi_cache_get) { char *key = NULL; - int keylen = 0; + php_strlen_size keylen = 0; char *cache = NULL; - int cachelen = 0; + php_strlen_size cachelen = 0; uint64_t valsize; if (!uwsgi.caches) RETURN_NULL(); - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|s", &key, &keylen, &cache, &cachelen) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "s|s", &key, &keylen, &cache, &cachelen) == FAILURE) { RETURN_NULL(); } @@ -366,28 +356,24 @@ PHP_FUNCTION(uwsgi_cache_get) { if (value) { char *ret = estrndup(value, valsize); free(value); -#ifdef UWSGI_PHP7 RETURN_STRING(ret); -#else - RETURN_STRING(ret, 0); -#endif } RETURN_NULL(); } PHP_FUNCTION(uwsgi_cache_set) { char *key = NULL; - int keylen; + php_strlen_size keylen = 0; char *value = NULL; - int vallen; - uint64_t expires = 0; + php_strlen_size vallen = 0; + php_long_size expires = 0; char *cache = NULL; - int cachelen = 0; + php_strlen_size cachelen = 0; if (!uwsgi.caches) RETURN_NULL(); - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|ls", &key, &keylen, &value, &vallen, &expires, &cache, &cachelen) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "ss|ls", &key, &keylen, &value, &vallen, &expires, &cache, &cachelen) == FAILURE) { RETURN_NULL(); } @@ -400,17 +386,17 @@ PHP_FUNCTION(uwsgi_cache_set) { PHP_FUNCTION(uwsgi_cache_update) { char *key = NULL; - int keylen; + php_strlen_size keylen = 0; char *value = NULL; - int vallen; - uint64_t expires = 0; + php_strlen_size vallen = 0; + php_long_size expires = 0; char *cache = NULL; - int cachelen = 0; + php_strlen_size cachelen = 0; if (!uwsgi.caches) RETURN_NULL(); - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "ss|ls", &key, &keylen, &value, &vallen, &expires, &cache, &cachelen) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "ss|ls", &key, &keylen, &value, &vallen, &expires, &cache, &cachelen) == FAILURE) { RETURN_NULL(); } @@ -435,7 +421,7 @@ PHP_FUNCTION(uwsgi_rpc) { uint16_t argvs[256]; uint64_t size = 0; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "+", &varargs, &num_args) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "+", &varargs, &num_args) == FAILURE) { RETURN_NULL(); } @@ -474,11 +460,7 @@ PHP_FUNCTION(uwsgi_rpc) { // here we do not free varargs for performance reasons char *ret = estrndup(response, size); free(response); -#ifdef UWSGI_PHP7 RETURN_STRING(ret); -#else - RETURN_STRING(ret, 0); -#endif } clear: @@ -493,7 +475,7 @@ PHP_FUNCTION(uwsgi_setprocname) { char *name; int name_len; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s", &name, &name_len) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "s", &name, &name_len) == FAILURE) { RETURN_NULL(); } @@ -507,7 +489,7 @@ PHP_FUNCTION(uwsgi_signal) { long long_signum; uint8_t signum = 0; - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "l", &long_signum) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "l", &long_signum) == FAILURE) { RETURN_NULL(); } @@ -517,21 +499,40 @@ PHP_FUNCTION(uwsgi_signal) { RETURN_NULL(); } +PHP_FUNCTION(uwsgi_disconnect) { + + struct wsgi_request *wsgi_req = (struct wsgi_request *) SG(server_context); + + php_output_end_all(); + php_header(); + + uwsgi_disconnect(wsgi_req); + + php_output_set_status(PHP_OUTPUT_DISABLED); + + RETURN_NULL(); +} + + +ZEND_BEGIN_ARG_INFO_EX(arginfo_void, 0, 0, 0) +ZEND_END_ARG_INFO() + zend_function_entry uwsgi_php_functions[] = { - PHP_FE(uwsgi_version, NULL) - PHP_FE(uwsgi_setprocname, NULL) - PHP_FE(uwsgi_worker_id, NULL) - PHP_FE(uwsgi_masterpid, NULL) - PHP_FE(uwsgi_signal, NULL) - - PHP_FE(uwsgi_rpc, NULL) - - PHP_FE(uwsgi_cache_get, NULL) - PHP_FE(uwsgi_cache_set, NULL) - PHP_FE(uwsgi_cache_update, NULL) - PHP_FE(uwsgi_cache_del, NULL) - PHP_FE(uwsgi_cache_clear, NULL) - PHP_FE(uwsgi_cache_exists, NULL) + PHP_FE(uwsgi_version, arginfo_void) + PHP_FE(uwsgi_setprocname, arginfo_void) + PHP_FE(uwsgi_worker_id, arginfo_void) + PHP_FE(uwsgi_masterpid, arginfo_void) + PHP_FE(uwsgi_signal, arginfo_void) + + PHP_FE(uwsgi_rpc, arginfo_void) + + PHP_FE(uwsgi_cache_get, arginfo_void) + PHP_FE(uwsgi_cache_set, arginfo_void) + PHP_FE(uwsgi_cache_update, arginfo_void) + PHP_FE(uwsgi_cache_del, arginfo_void) + PHP_FE(uwsgi_cache_clear, arginfo_void) + PHP_FE(uwsgi_cache_exists, arginfo_void) + PHP_FE(uwsgi_disconnect, arginfo_void) { NULL, NULL, NULL}, }; @@ -634,17 +635,23 @@ static void activate_user_config(const char *filename, const char *doc_root, siz static int php_uwsgi_startup(sapi_module_struct *sapi_module) { +#if ((PHP_MAJOR_VERSION >= 8) && (PHP_MINOR_VERSION >= 2)) + if (php_module_startup(&uwsgi_sapi_module, &uwsgi_module_entry)==FAILURE) { +#else if (php_module_startup(&uwsgi_sapi_module, &uwsgi_module_entry, 1)==FAILURE) { +#endif return FAILURE; } else { return SUCCESS; } } -#if ((PHP_MAJOR_VERSION >= 7) && (PHP_MINOR_VERSION >= 1)) +#if (PHP_MAJOR_VERSION >= 8) +static void sapi_uwsgi_log_message(const char *message, int syslog_type_int) { +#elif ((PHP_MAJOR_VERSION == 7) && (PHP_MINOR_VERSION >= 1)) static void sapi_uwsgi_log_message(char *message, int syslog_type_int) { #else -static void sapi_uwsgi_log_message(char *message TSRMLS_DC) { +static void sapi_uwsgi_log_message(char *message) { #endif uwsgi_log("%s\n", message); } @@ -844,10 +851,6 @@ int uwsgi_php_request(struct wsgi_request *wsgi_req) { zend_file_handle file_handle; -#ifdef ZTS - TSRMLS_FETCH(); -#endif - SG(server_context) = (void *) wsgi_req; if (uwsgi_parse_vars(wsgi_req)) { @@ -887,10 +890,14 @@ int uwsgi_php_request(struct wsgi_request *wsgi_req) { wsgi_req->document_root_len = strlen(wsgi_req->document_root); if (uphp.app) { -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *bypass = uphp.app_bypass; while (bypass) { +#ifdef UWSGI_PCRE2 + if (uwsgi_regexp_match(bypass->pattern, wsgi_req->uri, wsgi_req->uri_len) >= 0) { +#else if (uwsgi_regexp_match(bypass->pattern, bypass->pattern_extra, wsgi_req->uri, wsgi_req->uri_len) >= 0) { +#endif goto oldstyle; } bypass = bypass->next; @@ -899,6 +906,7 @@ int uwsgi_php_request(struct wsgi_request *wsgi_req) { strncpy(real_filename, uphp.app, PATH_MAX); real_filename[PATH_MAX-1] = '\0'; + real_filename_len = strlen(real_filename); if (wsgi_req->path_info_len == 1 && wsgi_req->path_info[0] == '/') { goto appready; } @@ -926,7 +934,7 @@ int uwsgi_php_request(struct wsgi_request *wsgi_req) { goto secure2; } -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) oldstyle: #endif @@ -1115,26 +1123,30 @@ int uwsgi_php_request(struct wsgi_request *wsgi_req) { SG(request_info).path_translated = wsgi_req->file; - file_handle.type = ZEND_HANDLE_FILENAME; - file_handle.filename = real_filename; - file_handle.free_filename = 0; - file_handle.opened_path = NULL; +#if PHP_VERSION_ID >= 80100 + zend_string *handle_filename = zend_string_init(real_filename, real_filename_len, 0); +#else + const char *handle_filename = real_filename; +#endif + memset(&file_handle, 0, sizeof(zend_file_handle)); + file_handle.type = ZEND_HANDLE_FILENAME; + file_handle.filename = handle_filename; - if (php_request_startup(TSRMLS_C) == FAILURE) { + if (php_request_startup() == FAILURE) { uwsgi_500(wsgi_req); - return -1; - } + return -1; + } struct uwsgi_string_list *usl=NULL; uwsgi_foreach(usl, uphp.exec_before) { - if (zend_eval_string_ex(usl->value, NULL, "uWSGI php exec before", 1 TSRMLS_CC) == FAILURE) goto end; + if (zend_eval_string_ex(usl->value, NULL, "uWSGI php exec before", 1) == FAILURE) goto end; } - php_execute_script(&file_handle TSRMLS_CC); + php_execute_script(&file_handle); uwsgi_foreach(usl, uphp.exec_after) { - if (zend_eval_string_ex(usl->value, NULL, "uWSGI php exec after", 1 TSRMLS_CC) == FAILURE) goto end; + if (zend_eval_string_ex(usl->value, NULL, "uWSGI php exec after", 1) == FAILURE) goto end; } end: diff --git a/plugins/php/session.c b/plugins/php/session.c index ddc375797f..cce0698559 100644 --- a/plugins/php/session.c +++ b/plugins/php/session.c @@ -12,37 +12,20 @@ PS_CLOSE_FUNC(uwsgi) { PS_READ_FUNC(uwsgi) { char *cache = PS_GET_MOD_DATA(); uint64_t valsize = 0; -#ifdef UWSGI_PHP7 char *value = uwsgi_cache_magic_get(key->val, key->len , &valsize, NULL, cache); -#else - char *value = uwsgi_cache_magic_get((char *)key, strlen((char *)key), &valsize, NULL, cache); -#endif if (!value) { *val = STR_EMPTY_ALLOC(); return SUCCESS; } -#ifdef UWSGI_PHP7 *val = zend_string_init(value, valsize, 0); -#else - char *new_val = emalloc(valsize); - memcpy(new_val, value, valsize); - free(value); - *val = new_val; - *vallen = valsize; -#endif return SUCCESS; } PS_WRITE_FUNC(uwsgi) { char *cache = PS_GET_MOD_DATA(); -#ifdef UWSGI_PHP7 if (val->len == 0) return SUCCESS; if (!uwsgi_cache_magic_set(key->val, key->len, val->val, val->len, 0, UWSGI_CACHE_FLAG_UPDATE, cache)) { -#else - if (vallen == 0) return SUCCESS; - if (!uwsgi_cache_magic_set((char *)key, strlen(key), (char *)val, vallen, 0, UWSGI_CACHE_FLAG_UPDATE, cache)) { -#endif return SUCCESS; } return FAILURE; @@ -50,14 +33,10 @@ PS_WRITE_FUNC(uwsgi) { PS_DESTROY_FUNC(uwsgi) { char *cache = PS_GET_MOD_DATA(); -#ifdef UWSGI_PHP7 if (!uwsgi_cache_magic_exists(key->val, key->len, cache)) return SUCCESS; if (!uwsgi_cache_magic_del(key->val, key->len, cache)) { -#else - if (!uwsgi_cache_magic_del((char *)key, strlen(key), cache)) { -#endif return SUCCESS; } return FAILURE; diff --git a/plugins/php/uwsgiplugin.py b/plugins/php/uwsgiplugin.py index 8657eb704d..8a3e71b5e7 100644 --- a/plugins/php/uwsgiplugin.py +++ b/plugins/php/uwsgiplugin.py @@ -21,7 +21,12 @@ LDFLAGS.append('-L%s' % ld_run_path) os.environ['LD_RUN_PATH'] = ld_run_path -LIBS = [os.popen(PHPPATH + ' --libs').read().rstrip(), '-lphp' + php_version] +# PHP8 and above does not add the version to the library +# name +if int(php_version) < 8: + LIBS = [os.popen(PHPPATH + ' --libs').read().rstrip(), '-lphp' + php_version] +else: + LIBS = [os.popen(PHPPATH + ' --libs').read().rstrip(), '-lphp'] phplibdir = os.environ.get('UWSGICONFIG_PHPLIBDIR') if phplibdir: diff --git a/plugins/psgi/psgi.h b/plugins/psgi/psgi.h index 220eab334e..c96d8da681 100644 --- a/plugins/psgi/psgi.h +++ b/plugins/psgi/psgi.h @@ -93,3 +93,5 @@ void uwsgi_perl_check_auto_reload(void); void uwsgi_psgi_preinit_apps(void); int uwsgi_perl_add_app(struct wsgi_request *, char *, PerlInterpreter **, SV **, time_t); + +extern struct uwsgi_perl uperl; diff --git a/plugins/psgi/psgi_loader.c b/plugins/psgi/psgi_loader.c index cdcecc1d13..b28bbf2e9c 100644 --- a/plugins/psgi/psgi_loader.c +++ b/plugins/psgi/psgi_loader.c @@ -1,7 +1,6 @@ #include "psgi.h" extern struct uwsgi_server uwsgi; -struct uwsgi_perl uperl; extern struct uwsgi_plugin psgi_plugin; diff --git a/plugins/psgi/psgi_plugin.c b/plugins/psgi/psgi_plugin.c index 0b1ce190ed..d2703aacb2 100644 --- a/plugins/psgi/psgi_plugin.c +++ b/plugins/psgi/psgi_plugin.c @@ -3,11 +3,7 @@ extern char **environ; extern struct uwsgi_server uwsgi; -#ifdef __APPLE__ -extern struct uwsgi_perl uperl; -#else struct uwsgi_perl uperl; -#endif struct uwsgi_plugin psgi_plugin; diff --git a/plugins/psgi/uwsgi_plmodule.c b/plugins/psgi/uwsgi_plmodule.c index e875fbec04..4a6d998033 100644 --- a/plugins/psgi/uwsgi_plmodule.c +++ b/plugins/psgi/uwsgi_plmodule.c @@ -659,8 +659,13 @@ XS(XS_add_rb_timer) { uint8_t uwsgi_signal = SvIV(ST(0)); int seconds = SvIV(ST(1)); + int iterations = 0; - if (uwsgi_signal_add_rb_timer(uwsgi_signal, seconds, 0)) { + if (items > 2) { + iterations = SvIV(ST(2)); + } + + if (uwsgi_signal_add_rb_timer(uwsgi_signal, seconds, iterations)) { croak("unable to register rb timer"); XSRETURN_UNDEF; } @@ -885,7 +890,7 @@ XS(XS_chunked_read) { XSRETURN_UNDEF; } - ST(0) = newSVpv(chunk, len); + ST(0) = newSVpvn(chunk, len); sv_2mortal(ST(0)); XSRETURN(1); } @@ -904,7 +909,7 @@ XS(XS_chunked_read_nb) { XSRETURN_UNDEF; } - ST(0) = newSVpv(chunk, len); + ST(0) = newSVpvn(chunk, len); sv_2mortal(ST(0)); XSRETURN(1); } diff --git a/plugins/pypy/pypy_plugin.c b/plugins/pypy/pypy_plugin.c index 62d177dd3f..f674537004 100644 --- a/plugins/pypy/pypy_plugin.c +++ b/plugins/pypy/pypy_plugin.c @@ -303,7 +303,7 @@ static void uwsgi_pypy_enable_threads() { } } -static void uwsgi_pypy_init_thread() { +static void uwsgi_pypy_init_thread(int sig) { if (u_pypy_thread_attach) { pthread_mutex_lock(&upypy.attach_thread_lock); u_pypy_thread_attach(); diff --git a/plugins/pypy/pypy_setup.py b/plugins/pypy/pypy_setup.py index badf54e446..148533d6bd 100644 --- a/plugins/pypy/pypy_setup.py +++ b/plugins/pypy/pypy_setup.py @@ -48,8 +48,8 @@ uwsgi_defines = [] uwsgi_cflags = ffi.string(lib0.uwsgi_get_cflags()).split() for cflag in uwsgi_cflags: - if cflag.startswith('-D'): - line = cflag[2:] + if cflag.startswith(b'-D'): + line = cflag[2:].decode() if '=' in line: (key, value) = line.split('=', 1) uwsgi_cdef.append('#define %s ...' % key) @@ -166,22 +166,22 @@ struct uwsgi_plugin *p[]; ...; }; -struct uwsgi_server uwsgi; +extern struct uwsgi_server uwsgi; -struct uwsgi_plugin pypy_plugin; +extern struct uwsgi_plugin pypy_plugin; -const char *uwsgi_pypy_version; +extern const char *uwsgi_pypy_version; char *uwsgi_binary_path(); void *uwsgi_malloc(size_t); struct uwsgi_logvar { - char key[256]; - uint8_t keylen; - char val[256]; - uint8_t vallen; - struct uwsgi_logvar *next; + char key[256]; + uint8_t keylen; + char val[256]; + uint8_t vallen; + struct uwsgi_logvar *next; }; struct uwsgi_logvar *uwsgi_logvar_get(struct wsgi_request *, char *, uint8_t); @@ -271,7 +271,7 @@ extern struct uwsgi_server uwsgi; extern struct uwsgi_plugin pypy_plugin; %s -''' % ('\n'.join(uwsgi_defines), uwsgi_dot_h, hooks) +''' % ('\n'.join(uwsgi_defines), uwsgi_dot_h.decode(), hooks) ffi.cdef(cdefines) lib = ffi.verify(cverify) @@ -286,7 +286,7 @@ # fix argv if needed if len(sys.argv) == 0: - sys.argv.insert(0, ffi.string(lib.uwsgi_binary_path())) + sys.argv.insert(0, ffi.string(lib.uwsgi_binary_path()).decode()) @ffi.callback("void(char *)") @@ -305,7 +305,7 @@ def uwsgi_pypy_loader(module): load a wsgi module """ global wsgi_application - m = ffi.string(module) + m = ffi.string(module).decode() c = 'application' if ':' in m: m, c = m.split(':') @@ -324,7 +324,7 @@ def uwsgi_pypy_file_loader(filename): global wsgi_application w = ffi.string(filename) c = 'application' - mod = imp.load_source('uwsgi_file_wsgi', w) + mod = imp.load_source('uwsgi_file_wsgi', w.decode()) wsgi_application = getattr(mod, c) @@ -334,16 +334,16 @@ def uwsgi_pypy_paste_loader(config): load a .ini paste app """ global wsgi_application - c = ffi.string(config) + c = ffi.string(config).decode() if c.startswith('config:'): c = c[7:] if c[0] != '/': c = os.getcwd() + '/' + c try: - from paste.script.util.logging_config import fileConfig + from logging.config import fileConfig fileConfig(c) except ImportError: - print("PyPy WARNING: unable to load paste.script.util.logging_config") + print("PyPy WARNING: unable to load logging.config") from paste.deploy import loadapp wsgi_application = loadapp('config:%s' % c) @@ -363,7 +363,7 @@ def uwsgi_pypy_pythonpath(item): """ add an item to the pythonpath """ - path = ffi.string(item) + path = ffi.string(item).decode() sys.path.append(path) print("added %s to pythonpath" % path) @@ -470,15 +470,17 @@ def writer(data): def start_response(status, headers, exc_info=None): if exc_info: traceback.print_exception(*exc_info) + status = status.encode() lib.uwsgi_response_prepare_headers(wsgi_req, ffi.new("char[]", status), len(status)) for hh in headers: + hh = (hh[0].encode(), hh[1].encode()) lib.uwsgi_response_add_header(wsgi_req, ffi.new("char[]", hh[0]), len(hh[0]), ffi.new("char[]", hh[1]), len(hh[1])) return writer environ = {} iov = wsgi_req.hvec for i in range(0, wsgi_req.var_cnt, 2): - environ[ffi.string(ffi.cast("char*", iov[i].iov_base), iov[i].iov_len)] = ffi.string(ffi.cast("char*", iov[i+1].iov_base), iov[i+1].iov_len) + environ[ffi.string(ffi.cast("char*", iov[i].iov_base), iov[i].iov_len).decode()] = ffi.string(ffi.cast("char*", iov[i+1].iov_base), iov[i+1].iov_len).decode() environ['wsgi.version'] = (1, 0) scheme = 'http' @@ -598,8 +600,8 @@ def uwsgi_pypy_rpc(node, func, *args): def uwsgi_pypy_call(func, *args): node = None - if '@' in func: - (func, node) = func.split('@') + if b'@' in func: + (func, node) = func.split(b'@') return uwsgi_pypy_rpc(node, func, *args) uwsgi.call = uwsgi_pypy_call @@ -1067,7 +1069,7 @@ def uwsgi_pypy_continulet_switch(wsgi_req): def uwsgi_pypy_setup_continulets(): - if lib.uwsgi.async < 1: + if lib.uwsgi["async"] < 1: raise Exception("pypy continulets require async mode !!!") lib.uwsgi.schedule_to_main = uwsgi_pypy_continulet_switch lib.uwsgi.schedule_to_req = uwsgi_pypy_continulet_schedule diff --git a/plugins/python/profiler.c b/plugins/python/profiler.c index 92bc3da787..2a7e1cc231 100644 --- a/plugins/python/profiler.c +++ b/plugins/python/profiler.c @@ -13,6 +13,14 @@ int PyFrame_GetLineNumber(PyFrameObject *frame) { } #endif +#if PY_VERSION_HEX < 0x030900B1 +PyCodeObject* PyFrame_GetCode(PyFrameObject *frame) +{ + Py_INCREF(frame->f_code); + return frame->f_code; +} +#endif + #ifdef PYTHREE #undef PyString_AsString static char *PyString_AsString(PyObject *o) { @@ -27,27 +35,32 @@ int uwsgi_python_profiler_call(PyObject *obj, PyFrameObject *frame, int what, Py static uint64_t last_ts = 0; uint64_t now = uwsgi_micros(); uint64_t delta = 0; + PyCodeObject *code; switch(what) { case PyTrace_CALL: if (last_ts == 0) delta = 0; else delta = now - last_ts; last_ts = now; + code = PyFrame_GetCode(frame); uwsgi_log("[uWSGI Python profiler %llu] CALL: %s (line %d) -> %s %d args, stacksize %d\n", (unsigned long long) delta, - PyString_AsString(frame->f_code->co_filename), + PyString_AsString(code->co_filename), PyFrame_GetLineNumber(frame), - PyString_AsString(frame->f_code->co_name), frame->f_code->co_argcount, frame->f_code->co_stacksize); + PyString_AsString(code->co_name), code->co_argcount, code->co_stacksize); + Py_DECREF(code); break; case PyTrace_C_CALL: if (last_ts == 0) delta = 0; else delta = now - last_ts; last_ts = now; + code = PyFrame_GetCode(frame); uwsgi_log("[uWSGI Python profiler %llu] C CALL: %s (line %d) -> %s %d args, stacksize %d\n", (unsigned long long) delta, - PyString_AsString(frame->f_code->co_filename), + PyString_AsString(code->co_filename), PyFrame_GetLineNumber(frame), - PyEval_GetFuncName(arg), frame->f_code->co_argcount, frame->f_code->co_stacksize); + PyEval_GetFuncName(arg), code->co_argcount, code->co_stacksize); + Py_DECREF(code); break; } @@ -68,7 +81,9 @@ int uwsgi_python_tracer(PyObject *obj, PyFrameObject *frame, int what, PyObject delta = now - last_ts; } last_ts = now; - uwsgi_log("[uWSGI Python profiler %llu] file %s line %d: %s argc:%d\n", (unsigned long long)delta, PyString_AsString(frame->f_code->co_filename), PyFrame_GetLineNumber(frame), PyString_AsString(frame->f_code->co_name), frame->f_code->co_argcount); + PyCodeObject *code = PyFrame_GetCode(frame); + uwsgi_log("[uWSGI Python profiler %llu] file %s line %d: %s argc:%d\n", (unsigned long long)delta, PyString_AsString(code->co_filename), PyFrame_GetLineNumber(frame), PyString_AsString(code->co_name), code->co_argcount); + Py_DECREF(code); } return 0; diff --git a/plugins/python/pyloader.c b/plugins/python/pyloader.c index 5afcc8e7c4..b4853939dd 100644 --- a/plugins/python/pyloader.c +++ b/plugins/python/pyloader.c @@ -22,7 +22,7 @@ PyMethodDef uwsgi_eventfd_write_method[] = { {"uwsgi_eventfd_write", py_eventfd_ void set_dyn_pyhome(char *home, uint16_t pyhome_len) { - char venv_version[15]; + char venv_version[30]; PyObject *site_module; PyObject *pysys_dict = get_uwsgi_pydict("sys"); @@ -45,8 +45,8 @@ void set_dyn_pyhome(char *home, uint16_t pyhome_len) { PyDict_SetItemString(pysys_dict, "prefix", venv_path); PyDict_SetItemString(pysys_dict, "exec_prefix", venv_path); - venv_version[14] = 0; - if (snprintf(venv_version, 15, "/lib/python%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION) == -1) { + bzero(venv_version, 30); + if (snprintf(venv_version, 30, "/lib/python%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION) == -1) { return; } @@ -97,7 +97,9 @@ int init_uwsgi_app(int loader, void *arg1, struct wsgi_request *wsgi_req, PyThre memset(wi, 0, sizeof(struct uwsgi_app)); wi->modifier1 = python_plugin.modifier1; wi->mountpoint_len = wsgi_req->appid_len < 0xff ? wsgi_req->appid_len : (0xff-1); - strncpy(wi->mountpoint, wsgi_req->appid, wi->mountpoint_len); + if (wi->mountpoint_len > 0) { + strncpy(wi->mountpoint, wsgi_req->appid, wi->mountpoint_len); + } // dynamic chdir ? if (wsgi_req->chdir_len > 0) { @@ -127,21 +129,33 @@ int init_uwsgi_app(int loader, void *arg1, struct wsgi_request *wsgi_req, PyThre p = strchr(*e, '='); if (p == NULL) continue; +#ifdef PYTHREE + k = PyUnicode_FromStringAndSize(*e, (int)(p-*e)); +#else k = PyString_FromStringAndSize(*e, (int)(p-*e)); +#endif if (k == NULL) { PyErr_Print(); continue; } - env_value = PyString_FromString(p+1); +#ifdef PYTHREE + env_value = PyUnicode_FromString(p+1); +#else + env_value = PyString_FromString(p+1); +#endif if (env_value == NULL) { PyErr_Print(); Py_DECREF(k); continue; } - + #ifdef UWSGI_DEBUG +#ifdef PYTHREE + uwsgi_log("%s = %s\n", PyUnicode_AsUTF8(k), PyUnicode_AsUTF8(env_value)); +#else uwsgi_log("%s = %s\n", PyString_AsString(k), PyString_AsString(env_value)); +#endif #endif if (PyObject_SetItem(py_environ, k, env_value)) { @@ -207,7 +221,7 @@ int init_uwsgi_app(int loader, void *arg1, struct wsgi_request *wsgi_req, PyThre if (multiapp < 1) { uwsgi_log("you have to define at least one app in the applications dictionary\n"); goto doh; - } + } PyObject *app_mnt = PyList_GetItem(app_list, 0); if (!PyString_Check(app_mnt)) { @@ -226,7 +240,7 @@ int init_uwsgi_app(int loader, void *arg1, struct wsgi_request *wsgi_req, PyThre if (PyString_Check((PyObject *) wi->callable)) { PyObject *callables_dict = get_uwsgi_pydict((char *)arg1); if (callables_dict) { - wi->callable = PyDict_GetItem(callables_dict, (PyObject *)wi->callable); + wi->callable = PyDict_GetItem(callables_dict, (PyObject *)wi->callable); if (!wi->callable) { uwsgi_log("skipping broken app %s\n", wsgi_req->appid); goto multiapp; @@ -375,7 +389,7 @@ int init_uwsgi_app(int loader, void *arg1, struct wsgi_request *wsgi_req, PyThre multiapp: if (multiapp > 1) { for(i=1;i code\n"); - exit(UWSGI_FAILED_APP_CODE); - } - - wsgi_compiled_node = (PyObject *) PyNode_Compile(wsgi_eval_node, "uwsgi_eval_config"); - + wsgi_compiled_node = Py_CompileString(code, "uwsgi_eval_config", Py_file_input); if (!wsgi_compiled_node) { PyErr_Print(); uwsgi_log( "failed to compile eval code\n"); exit(UWSGI_FAILED_APP_CODE); } - wsgi_eval_module = PyImport_ExecCodeModule("uwsgi_eval_config", wsgi_compiled_node); if (!wsgi_eval_module) { PyErr_Print(); @@ -796,7 +800,7 @@ PyObject *uwsgi_eval_loader(void *arg1) { wsgi_eval_callable = PyDict_GetItemString(up.loader_dict, up.callable); } else { - + wsgi_eval_callable = PyDict_GetItemString(up.loader_dict, "application"); } diff --git a/plugins/python/python_plugin.c b/plugins/python/python_plugin.c index 25657986f9..289607b868 100644 --- a/plugins/python/python_plugin.c +++ b/plugins/python/python_plugin.c @@ -195,6 +195,7 @@ struct uwsgi_option uwsgi_python_options[] = { #endif {"py-call-osafterfork", no_argument, 0, "enable child processes running cpython to trap OS signals", uwsgi_opt_true, &up.call_osafterfork, 0}, + {"py-call-uwsgi-fork-hooks", no_argument, 0, "call pre and post hooks when uswgi forks to update the internal interpreter state of CPython", uwsgi_opt_true, &up.call_uwsgi_fork_hooks, 0}, {"early-python", no_argument, 0, "load the python VM as soon as possible (useful for the fork server)", uwsgi_early_python, NULL, UWSGI_OPT_IMMEDIATE}, {"early-pyimport", required_argument, 0, "import a python module in the early phase", uwsgi_early_python_import, NULL, UWSGI_OPT_IMMEDIATE}, @@ -207,6 +208,9 @@ struct uwsgi_option uwsgi_python_options[] = { {"py-master-check-signals", no_argument, 0, "enable python signal handlers in master", uwsgi_opt_true, &up.master_check_signals, 0}, + {"py-executable", required_argument, 0, "override sys.executable value", uwsgi_opt_set_str, &up.executable, 0}, + {"py-sys-executable", required_argument, 0, "override sys.executable value", uwsgi_opt_set_str, &up.executable, 0}, + {0, 0, 0, 0, 0, 0, 0}, }; @@ -229,6 +233,21 @@ void uwsgi_python_pthread_child(void) { PyMethodDef uwsgi_spit_method[] = { {"uwsgi_spit", py_uwsgi_spit, METH_VARARGS, ""} }; PyMethodDef uwsgi_write_method[] = { {"uwsgi_write", py_uwsgi_write, METH_VARARGS, ""} }; +PyDoc_STRVAR(uwsgi_py_doc, "uWSGI api module."); + +#ifdef PYTHREE +static PyModuleDef uwsgi_module3 = { + PyModuleDef_HEAD_INIT, + "uwsgi", + uwsgi_py_doc, + -1, + NULL, +}; +PyObject *init_uwsgi3(void) { + return PyModule_Create(&uwsgi_module3); +} +#endif + int uwsgi_python_init() { char *pyversion = strchr(Py_GetVersion(), '\n'); @@ -294,6 +313,9 @@ int uwsgi_python_init() { wchar_t *pname = uwsgi_calloc(sizeof(wchar_t) * (strlen(program_name)+1)); mbstowcs(pname, program_name, strlen(program_name)+1); Py_SetProgramName(pname); +#ifdef UWSGI_PY312 + PyImport_AppendInittab("uwsgi", init_uwsgi3); +#endif #else Py_SetProgramName(program_name); #endif @@ -353,7 +375,7 @@ void uwsgi_python_reset_random_seed() { PyObject *random_args = PyTuple_New(1); // pass no args PyTuple_SetItem(random_args, 0, Py_None); - PyEval_CallObject(random_seed, random_args); + PyObject_CallObject(random_seed, random_args); if (PyErr_Occurred()) { PyErr_Print(); } @@ -425,13 +447,21 @@ void uwsgi_python_atexit() { void uwsgi_python_post_fork() { + // Need to acquire the gil when no master process is used as first worker + // will not have been forked like others + // Necessary if uwsgi fork hooks called to update interpreter state + if (up.call_uwsgi_fork_hooks && !uwsgi.master_process && uwsgi.mywid == 1) { + UWSGI_GET_GIL + } + if (uwsgi.i_am_a_spooler) { UWSGI_GET_GIL } // reset python signal flags so child processes can trap signals - if (up.call_osafterfork) { -#ifdef HAS_NOT_PyOS_AfterFork_Child + // Necessary if uwsgi fork hooks not called to update interpreter state + if (!up.call_uwsgi_fork_hooks && up.call_osafterfork) { +#ifdef HAS_NOT_PYOS_FORK_STABLE_API PyOS_AfterFork(); #else PyOS_AfterFork_Child(); @@ -470,8 +500,7 @@ UWSGI_RELEASE_GIL PyObject *uwsgi_pyimport_by_filename(char *name, char *filename) { - FILE *pyfile; - struct _node *py_file_node = NULL; + char *pycontent; PyObject *py_compiled_node, *py_file_module; int is_a_package = 0; struct stat pystat; @@ -480,7 +509,7 @@ PyObject *uwsgi_pyimport_by_filename(char *name, char *filename) { if (!uwsgi_check_scheme(filename)) { - pyfile = fopen(filename, "r"); + FILE *pyfile = fopen(filename, "r"); if (!pyfile) { uwsgi_log("failed to open python file %s\n", filename); return NULL; @@ -504,37 +533,32 @@ PyObject *uwsgi_pyimport_by_filename(char *name, char *filename) { } } - py_file_node = PyParser_SimpleParseFile(pyfile, real_filename, Py_file_input); - if (!py_file_node) { - PyErr_Print(); - uwsgi_log("failed to parse file %s\n", real_filename); - if (is_a_package) + fclose(pyfile); + pycontent = uwsgi_simple_file_read(real_filename); + + if (!pycontent) { + if (is_a_package) { free(real_filename); - fclose(pyfile); + } + uwsgi_log("no data read from file %s\n", real_filename); return NULL; } - fclose(pyfile); } else { size_t pycontent_size = 0; - char *pycontent = uwsgi_open_and_read(filename, &pycontent_size, 1, NULL); + pycontent = uwsgi_open_and_read(filename, &pycontent_size, 1, NULL); - if (pycontent) { - py_file_node = PyParser_SimpleParseString(pycontent, Py_file_input); - if (!py_file_node) { - PyErr_Print(); - uwsgi_log("failed to parse url %s\n", real_filename); - return NULL; - } + if (!pycontent) { + uwsgi_log("no data read from url %s\n", real_filename); + return NULL; } } - py_compiled_node = (PyObject *) PyNode_Compile(py_file_node, real_filename); - + py_compiled_node = Py_CompileString(pycontent, real_filename, Py_file_input); if (!py_compiled_node) { PyErr_Print(); - uwsgi_log("failed to compile python file %s\n", real_filename); + uwsgi_log("failed to compile %s\n", real_filename); return NULL; } @@ -585,7 +609,7 @@ void init_uwsgi_vars() { #ifdef HAS_NO_ERRORS_IN_PyFile_FromFd PyObject *new_stdprint = PyFile_FromFd(2, NULL, "w", _IOLBF, NULL, NULL, 0); #else - PyObject *new_stdprint = PyFile_FromFd(2, NULL, "w", _IOLBF, NULL, NULL, NULL, 0); + PyObject *new_stdprint = PyFile_FromFd(2, NULL, "w", _IOLBF, NULL, "backslashreplace", NULL, 0); #endif PyDict_SetItemString(pysys_dict, "stdout", new_stdprint); PyDict_SetItemString(pysys_dict, "__stdout__", new_stdprint); @@ -654,21 +678,6 @@ void init_uwsgi_vars() { -PyDoc_STRVAR(uwsgi_py_doc, "uWSGI api module."); - -#ifdef PYTHREE -static PyModuleDef uwsgi_module3 = { - PyModuleDef_HEAD_INIT, - "uwsgi", - uwsgi_py_doc, - -1, - NULL, -}; -PyObject *init_uwsgi3(void) { - return PyModule_Create(&uwsgi_module3); -} -#endif - void init_uwsgi_embedded_module() { PyObject *new_uwsgi_module, *zero; int i; @@ -689,7 +698,9 @@ void init_uwsgi_embedded_module() { #ifdef PYTHREE +#ifndef UWSGI_PY312 PyImport_AppendInittab("uwsgi", init_uwsgi3); +#endif new_uwsgi_module = PyImport_AddModule("uwsgi"); #else new_uwsgi_module = Py_InitModule3("uwsgi", NULL, uwsgi_py_doc); @@ -816,8 +827,13 @@ void init_uwsgi_embedded_module() { PyObject *py_opt_dict = PyDict_New(); for (i = 0; i < uwsgi.exported_opts_cnt; i++) { - if (PyDict_Contains(py_opt_dict, PyString_FromString(uwsgi.exported_opts[i]->key))) { - PyObject *py_opt_item = PyDict_GetItemString(py_opt_dict, uwsgi.exported_opts[i]->key); +#ifdef PYTHREE + PyObject *key = PyUnicode_FromString(uwsgi.exported_opts[i]->key); +#else + PyObject *key = PyString_FromString(uwsgi.exported_opts[i]->key); +#endif + if (PyDict_Contains(py_opt_dict, key)) { + PyObject *py_opt_item = PyDict_GetItem(py_opt_dict, key); if (PyList_Check(py_opt_item)) { if (uwsgi.exported_opts[i]->value == NULL) { PyList_Append(py_opt_item, Py_True); @@ -836,15 +852,15 @@ void init_uwsgi_embedded_module() { PyList_Append(py_opt_list, PyString_FromString(uwsgi.exported_opts[i]->value)); } - PyDict_SetItemString(py_opt_dict, uwsgi.exported_opts[i]->key, py_opt_list); + PyDict_SetItem(py_opt_dict, key, py_opt_list); } } else { if (uwsgi.exported_opts[i]->value == NULL) { - PyDict_SetItemString(py_opt_dict, uwsgi.exported_opts[i]->key, Py_True); + PyDict_SetItem(py_opt_dict, key, Py_True); } else { - PyDict_SetItemString(py_opt_dict, uwsgi.exported_opts[i]->key, PyString_FromString(uwsgi.exported_opts[i]->value)); + PyDict_SetItem(py_opt_dict, key, PyString_FromString(uwsgi.exported_opts[i]->value)); } } } @@ -1127,6 +1143,13 @@ void uwsgi_python_preinit_apps() { up.loaders[LOADER_CALLABLE] = uwsgi_callable_loader; up.loaders[LOADER_STRING_CALLABLE] = uwsgi_string_callable_loader; + // GIL was released in previous initialization steps but init_pyargv expects + // the GIL to be acquired + // Necessary if uwsgi fork hooks called to update interpreter state + if (up.call_uwsgi_fork_hooks) { + UWSGI_GET_GIL + } + init_pyargv(); init_uwsgi_embedded_module(); @@ -1172,19 +1195,33 @@ void uwsgi_python_preinit_apps() { upli = upli->next; } + // Release the GIL before moving on forward with initialization + // Necessary if uwsgi fork hooks called to update interpreter state + if (up.call_uwsgi_fork_hooks) { + UWSGI_RELEASE_GIL + } + } void uwsgi_python_init_apps() { // lazy ? - if (uwsgi.mywid > 0) { + // Also necessary if uwsgi fork hooks called to update interpreter state + if (uwsgi.mywid > 0 || up.call_uwsgi_fork_hooks) { UWSGI_GET_GIL; } // prepare for stack suspend/resume if (uwsgi.async > 0) { +#ifdef UWSGI_PY312 + up.current_c_recursion_remaining = uwsgi_malloc(sizeof(int)*uwsgi.async); + up.current_py_recursion_remaining = uwsgi_malloc(sizeof(int)*uwsgi.async); +#elif defined UWSGI_PY311 + up.current_recursion_remaining = uwsgi_malloc(sizeof(int)*uwsgi.async); +#else up.current_recursion_depth = uwsgi_malloc(sizeof(int)*uwsgi.async); - up.current_frame = uwsgi_malloc(sizeof(struct _frame)*uwsgi.async); +#endif + up.current_frame = uwsgi_malloc(sizeof(up.current_frame[0])*uwsgi.async); } struct uwsgi_string_list *upli = up.import_list; @@ -1289,7 +1326,8 @@ void uwsgi_python_init_apps() { } } // lazy ? - if (uwsgi.mywid > 0) { + // Also necessary if uwsgi fork hooks called to update interpreter state + if (uwsgi.mywid > 0 || up.call_uwsgi_fork_hooks) { UWSGI_RELEASE_GIL; } @@ -1302,6 +1340,10 @@ void uwsgi_python_master_fixup(int step) { if (!uwsgi.master_process) return; + // Skip master fixup if uwsgi fork hooks called to update interpreter state + if (up.call_uwsgi_fork_hooks) + return; + if (uwsgi.has_threads) { if (step == 0) { if (!master_fixed) { @@ -1318,9 +1360,53 @@ void uwsgi_python_master_fixup(int step) { } } +void uwsgi_python_pre_uwsgi_fork() { + if (!up.call_uwsgi_fork_hooks) + return; + + if (uwsgi.has_threads) { + // Acquire the gil and import lock before forking in order to avoid + // deadlocks in workers + UWSGI_GET_GIL +#ifdef HAS_NOT_PYOS_FORK_STABLE_API + _PyImport_AcquireLock(); +#else + PyOS_BeforeFork(); +#endif + } +} + + +void uwsgi_python_post_uwsgi_fork(int step) { + if (!up.call_uwsgi_fork_hooks) + return; + + if (uwsgi.has_threads) { + if (step == 0) { + // Release locks within master process +#ifdef HAS_NOT_PYOS_FORK_STABLE_API + _PyImport_ReleaseLock(); +#else + PyOS_AfterFork_Parent(); +#endif + UWSGI_RELEASE_GIL + } + else { + // Ensure thread state and locks are cleaned up in child process +#ifdef HAS_NOT_PYOS_FORK_STABLE_API + PyOS_AfterFork(); +#else + PyOS_AfterFork_Child(); +#endif + } + } +} + void uwsgi_python_enable_threads() { +#ifdef UWSGI_SHOULD_CALL_PYEVAL_INITTHREADS PyEval_InitThreads(); +#endif if (pthread_key_create(&up.upt_save_key, NULL)) { uwsgi_error("pthread_key_create()"); exit(1); @@ -1345,7 +1431,11 @@ void uwsgi_python_enable_threads() { up.reset_ts = threaded_reset_ts; } - + // Release the newly created gil from call to PyEval_InitThreads above + // Necessary if uwsgi fork hooks called to update interpreter state + if (up.call_uwsgi_fork_hooks) { + UWSGI_RELEASE_GIL + } uwsgi_log("python threads support enabled\n"); @@ -1353,18 +1443,14 @@ void uwsgi_python_enable_threads() { } void uwsgi_python_set_thread_name(int core_id) { - // call threading.currentThread (taken from mod_wsgi, but removes DECREFs as thread in uWSGI are fixed) + // call threading.current_thread (taken from mod_wsgi, but removes DECREFs as thread in uWSGI are fixed) PyObject *threading_module = PyImport_ImportModule("threading"); if (threading_module) { PyObject *threading_module_dict = PyModule_GetDict(threading_module); if (threading_module_dict) { -#ifdef PYTHREE PyObject *threading_current = PyDict_GetItemString(threading_module_dict, "current_thread"); -#else - PyObject *threading_current = PyDict_GetItemString(threading_module_dict, "currentThread"); -#endif if (threading_current) { - PyObject *current_thread = PyEval_CallObject(threading_current, (PyObject *)NULL); + PyObject *current_thread = PyObject_CallObject(threading_current, (PyObject *)NULL); if (!current_thread) { // ignore the error PyErr_Clear(); @@ -1446,13 +1532,9 @@ PyObject *uwsgi_python_setup_thread(char *name, PyInterpreterState *interpreter) if (threading_module) { PyObject *threading_module_dict = PyModule_GetDict(threading_module); if (threading_module_dict) { -#ifdef PYTHREE PyObject *threading_current = PyDict_GetItemString(threading_module_dict, "current_thread"); -#else - PyObject *threading_current = PyDict_GetItemString(threading_module_dict, "currentThread"); -#endif if (threading_current) { - PyObject *current_thread = PyEval_CallObject(threading_current, (PyObject *)NULL); + PyObject *current_thread = PyObject_CallObject(threading_current, (PyObject *)NULL); if (!current_thread) { // ignore the error PyErr_Clear(); @@ -1577,12 +1659,38 @@ void uwsgi_python_suspend(struct wsgi_request *wsgi_req) { PyGILState_Release(pgst); if (wsgi_req) { +#ifdef UWSGI_PY313 + up.current_c_recursion_remaining[wsgi_req->async_id] = tstate->c_recursion_remaining; + up.current_py_recursion_remaining[wsgi_req->async_id] = tstate->py_recursion_remaining; + up.current_frame[wsgi_req->async_id] = tstate->current_frame; +#elif defined UWSGI_PY312 + up.current_c_recursion_remaining[wsgi_req->async_id] = tstate->c_recursion_remaining; + up.current_py_recursion_remaining[wsgi_req->async_id] = tstate->py_recursion_remaining; + up.current_frame[wsgi_req->async_id] = tstate->cframe; +#elif defined UWSGI_PY311 + up.current_recursion_remaining[wsgi_req->async_id] = tstate->recursion_remaining; + up.current_frame[wsgi_req->async_id] = tstate->cframe; +#else up.current_recursion_depth[wsgi_req->async_id] = tstate->recursion_depth; up.current_frame[wsgi_req->async_id] = tstate->frame; +#endif } else { +#ifdef UWSGI_PY313 + up.current_main_c_recursion_remaining = tstate->c_recursion_remaining; + up.current_main_py_recursion_remaining = tstate->py_recursion_remaining; + up.current_main_frame = tstate->current_frame; +#elif defined UWSGI_PY312 + up.current_main_c_recursion_remaining = tstate->c_recursion_remaining; + up.current_main_py_recursion_remaining = tstate->py_recursion_remaining; + up.current_main_frame = tstate->cframe; +#elif defined UWSGI_PY311 + up.current_main_recursion_remaining = tstate->recursion_remaining; + up.current_main_frame = tstate->cframe; +#else up.current_main_recursion_depth = tstate->recursion_depth; up.current_main_frame = tstate->frame; +#endif } } @@ -1676,8 +1784,10 @@ uint64_t uwsgi_python_rpc(void *func, uint8_t argc, char **argv, uint16_t argvs[ PyObject *pyargs = PyTuple_New(argc); PyObject *ret; - if (!pyargs) + if (!pyargs) { + UWSGI_RELEASE_GIL; return 0; + } for (i = 0; i < argc; i++) { PyTuple_SetItem(pyargs, i, PyString_FromStringAndSize(argv[i], argvs[i])); @@ -1810,12 +1920,38 @@ void uwsgi_python_resume(struct wsgi_request *wsgi_req) { PyGILState_Release(pgst); if (wsgi_req) { +#ifdef UWSGI_PY313 + tstate->c_recursion_remaining = up.current_c_recursion_remaining[wsgi_req->async_id]; + tstate->py_recursion_remaining = up.current_py_recursion_remaining[wsgi_req->async_id]; + tstate->current_frame = up.current_frame[wsgi_req->async_id]; +#elif defined UWSGI_PY312 + tstate->c_recursion_remaining = up.current_c_recursion_remaining[wsgi_req->async_id]; + tstate->py_recursion_remaining = up.current_py_recursion_remaining[wsgi_req->async_id]; + tstate->cframe = up.current_frame[wsgi_req->async_id]; +#elif defined UWSGI_PY311 + tstate->recursion_remaining = up.current_recursion_remaining[wsgi_req->async_id]; + tstate->cframe = up.current_frame[wsgi_req->async_id]; +#else tstate->recursion_depth = up.current_recursion_depth[wsgi_req->async_id]; tstate->frame = up.current_frame[wsgi_req->async_id]; +#endif } else { +#ifdef UWSGI_PY313 + tstate->c_recursion_remaining = up.current_main_c_recursion_remaining; + tstate->py_recursion_remaining = up.current_main_py_recursion_remaining; + tstate->current_frame = up.current_main_frame; +#elif defined UWSGI_PY312 + tstate->c_recursion_remaining = up.current_main_c_recursion_remaining; + tstate->py_recursion_remaining = up.current_main_py_recursion_remaining; + tstate->cframe = up.current_main_frame; +#elif defined UWSGI_PY311 + tstate->recursion_remaining = up.current_main_recursion_remaining; + tstate->cframe = up.current_main_frame; +#else tstate->recursion_depth = up.current_main_recursion_depth; tstate->frame = up.current_main_frame; +#endif } } @@ -1897,7 +2033,7 @@ int uwsgi_python_mule(char *opt) { PyObject *arglist = Py_BuildValue("()"); PyObject *callable = up.loaders[LOADER_MOUNT](opt); if (callable) { - result = PyEval_CallObject(callable, arglist); + result = PyObject_CallObject(callable, arglist); } Py_XDECREF(result); Py_XDECREF(arglist); @@ -2010,7 +2146,7 @@ static ssize_t uwsgi_python_logger(struct uwsgi_logger *ul, char *message, size_ py_getLogger_args = PyTuple_New(1); PyTuple_SetItem(py_getLogger_args, 0, UWSGI_PYFROMSTRING(ul->arg)); } - ul->data = (void *) PyEval_CallObject(py_getLogger, py_getLogger_args); + ul->data = (void *) PyObject_CallObject(py_getLogger, py_getLogger_args); if (PyErr_Occurred()) { PyErr_Clear(); } @@ -2040,8 +2176,9 @@ static int uwsgi_python_worker() { return 0; UWSGI_GET_GIL; // ensure signals can be used again from python - if (!up.call_osafterfork) -#ifdef HAS_NOT_PyOS_AfterFork_Child + // Necessary if fork hooks have been not used to update interpreter state + if (!up.call_osafterfork && !up.call_uwsgi_fork_hooks) +#ifdef HAS_NOT_PYOS_FORK_STABLE_API PyOS_AfterFork(); #else PyOS_AfterFork_Child(); @@ -2126,4 +2263,6 @@ struct uwsgi_plugin python_plugin = { .worker = uwsgi_python_worker, + .pre_uwsgi_fork = uwsgi_python_pre_uwsgi_fork, + .post_uwsgi_fork = uwsgi_python_post_uwsgi_fork, }; diff --git a/plugins/python/pyutils.c b/plugins/python/pyutils.c index 4c984ce82f..4adbf763b0 100644 --- a/plugins/python/pyutils.c +++ b/plugins/python/pyutils.c @@ -75,7 +75,7 @@ struct uwsgi_buffer *uwsgi_python_backtrace(struct wsgi_request *wsgi_req) { PyObject *args = PyTuple_New(1); Py_INCREF(traceback); PyTuple_SetItem(args, 0, traceback); - PyObject *result = PyEval_CallObject(extract_tb, args); + PyObject *result = PyObject_CallObject(extract_tb, args); Py_DECREF(args); if (!result) goto end; @@ -83,12 +83,12 @@ struct uwsgi_buffer *uwsgi_python_backtrace(struct wsgi_request *wsgi_req) { ub = uwsgi_buffer_new(4096); Py_ssize_t i; // we have to build a uwsgi array with 5 items (4 are taken from the python tb) - for(i=0;i< PyList_Size(result);i++) { - PyObject *t = PyList_GetItem(result, i); - PyObject *tb_filename = PyTuple_GetItem(t, 0); - PyObject *tb_lineno = PyTuple_GetItem(t, 1); - PyObject *tb_function = PyTuple_GetItem(t, 2); - PyObject *tb_text = PyTuple_GetItem(t, 3); + for(i=0;i< PySequence_Size(result);i++) { + PyObject *t = PySequence_GetItem(result, i); + PyObject *tb_filename = PySequence_GetItem(t, 0); + PyObject *tb_lineno = PySequence_GetItem(t, 1); + PyObject *tb_function = PySequence_GetItem(t, 2); + PyObject *tb_text = PySequence_GetItem(t, 3); int64_t line_no = PyInt_AsLong(tb_lineno); #ifdef PYTHREE @@ -276,7 +276,7 @@ PyObject *python_call(PyObject *callable, PyObject *args, int catch, struct wsgi //uwsgi_log("ready to call %p %p\n", callable, args); - PyObject *pyret = PyEval_CallObject(callable, args); + PyObject *pyret = PyObject_CallObject(callable, args); //uwsgi_log("called\n"); @@ -395,10 +395,12 @@ void init_pyargv() { uwsgi_log("unable to load python sys module !!!\n"); exit(1); } + if (!up.executable) + up.executable = uwsgi.binary_path; #ifdef PYTHREE - PyDict_SetItemString(sys_dict, "executable", PyUnicode_FromString(uwsgi.binary_path)); + PyDict_SetItemString(sys_dict, "executable", PyUnicode_FromString(up.executable)); #else - PyDict_SetItemString(sys_dict, "executable", PyString_FromString(uwsgi.binary_path)); + PyDict_SetItemString(sys_dict, "executable", PyString_FromString(up.executable)); #endif diff --git a/plugins/python/raw.c b/plugins/python/raw.c index 47f66d20bf..3cd50720df 100644 --- a/plugins/python/raw.c +++ b/plugins/python/raw.c @@ -64,7 +64,7 @@ int uwsgi_request_python_raw(struct wsgi_request *wsgi_req) { UWSGI_GET_GIL PyObject * args = PyTuple_New(1); PyTuple_SetItem(args, 0, PyInt_FromLong(wsgi_req->fd)); - wsgi_req->async_result = PyEval_CallObject(up.raw_callable, args); + wsgi_req->async_result = PyObject_CallObject(up.raw_callable, args); Py_DECREF(args); if (wsgi_req->async_result) { for (;;) { diff --git a/plugins/python/tracebacker.c b/plugins/python/tracebacker.c index e19e3e1c44..a91692f059 100644 --- a/plugins/python/tracebacker.c +++ b/plugins/python/tracebacker.c @@ -14,7 +14,7 @@ char *uwsgi_python_get_thread_name(PyObject *thread_id) { PyObject *threading_enumerate = PyDict_GetItemString(threading_dict, "enumerate"); if (!threading_enumerate) return NULL; - PyObject *threads_list = PyEval_CallObject(threading_enumerate, (PyObject *)NULL); + PyObject *threads_list = PyObject_CallObject(threading_enumerate, (PyObject *)NULL); if (!threads_list) return NULL; PyObject *threads_list_iter = PyObject_GetIter(threads_list); @@ -106,13 +106,13 @@ void *uwsgi_python_tracebacker_thread(void *foobar) { } UWSGI_GET_GIL; // here is the core of the tracebacker - PyObject *current_frames = PyEval_CallObject(_current_frames, (PyObject *)NULL); + PyObject *current_frames = PyObject_CallObject(_current_frames, (PyObject *)NULL); if (!current_frames) goto end2; PyObject *current_frames_items = PyObject_GetAttrString(current_frames, "items"); if (!current_frames_items) goto end; - PyObject *frames_ret = PyEval_CallObject(current_frames_items, (PyObject *)NULL); + PyObject *frames_ret = PyObject_CallObject(current_frames_items, (PyObject *)NULL); if (!frames_ret) goto end3; PyObject *frames_iter = PyObject_GetIter(frames_ret); @@ -134,7 +134,7 @@ void *uwsgi_python_tracebacker_thread(void *foobar) { PyObject *arg_tuple = PyTuple_New(1); PyTuple_SetItem(arg_tuple, 0, stack); Py_INCREF(stack); - PyObject *stacktrace = PyEval_CallObject( extract_stack, arg_tuple); + PyObject *stacktrace = PyObject_CallObject( extract_stack, arg_tuple); Py_DECREF(arg_tuple); if (!stacktrace) goto next2; diff --git a/plugins/python/uwsgi_pymodule.c b/plugins/python/uwsgi_pymodule.c index f9a4377796..cc6948746a 100644 --- a/plugins/python/uwsgi_pymodule.c +++ b/plugins/python/uwsgi_pymodule.c @@ -2039,7 +2039,7 @@ PyObject *py_uwsgi_sharedarea_read(PyObject * self, PyObject * args) { } // HACK: we are safe as rlen can only be lower or equal to len - Py_SIZE(ret) = rlen; + Py_SET_SIZE((PyVarObject *) ret, rlen); return ret; } @@ -2384,8 +2384,6 @@ PyObject *py_uwsgi_workers(PyObject * self, PyObject * args) { goto clear; } - apps_tuple = PyDict_GetItemString(worker_dict, "apps"); - PyDict_Clear(worker_dict); zero = PyInt_FromLong(uwsgi.workers[i + 1].id); @@ -2539,9 +2537,10 @@ PyObject *py_uwsgi_workers(PyObject * self, PyObject * args) { PyTuple_SetItem(apps_tuple, j, apps_dict); } - + PyDict_SetItemString(worker_dict, "apps", apps_tuple); + Py_DECREF(apps_tuple); } diff --git a/plugins/python/uwsgi_python.h b/plugins/python/uwsgi_python.h index 1d7d1caa11..080824c301 100644 --- a/plugins/python/uwsgi_python.h +++ b/plugins/python/uwsgi_python.h @@ -1,5 +1,8 @@ #include +/* See https://docs.python.org/3.10/whatsnew/3.10.html#id2 */ +#define PY_SSIZE_T_CLEAN #include +#include #include @@ -14,6 +17,18 @@ #define UWSGI_PYTHON_OLD #endif +#if (PY_VERSION_HEX >= 0x030b0000) +# define UWSGI_PY311 +#endif + +#if (PY_VERSION_HEX >= 0x030c0000) +# define UWSGI_PY312 +#endif + +#if (PY_VERSION_HEX >= 0x030d0000) +# define UWSGI_PY313 +#endif + #if PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION < 7 #define HAS_NOT_PyMemoryView_FromBuffer #endif @@ -30,21 +45,21 @@ #define HAS_NO_ERRORS_IN_PyFile_FromFd #endif -#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 7 -#define HAS_NOT_PyOS_AfterFork_Child -#endif - -#if PY_MAJOR_VERSION < 3 -#define HAS_NOT_PyOS_AfterFork_Child +#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 7) || PY_MAJOR_VERSION < 3 +#define HAS_NOT_PYOS_FORK_STABLE_API #endif #if PY_MAJOR_VERSION > 2 #define PYTHREE #endif -#if (PY_VERSION_HEX < 0x02060000) -#ifndef Py_SIZE -#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 7) || PY_MAJOR_VERSION < 3 +#define UWSGI_SHOULD_CALL_PYEVAL_INITTHREADS +#endif + +#if (PY_VERSION_HEX < 0x03090000) +#ifndef Py_SET_SIZE +#define Py_SET_SIZE(o, size) ((o)->ob_size = (size)) #endif #endif @@ -171,11 +186,35 @@ struct uwsgi_python { char *callable; +#ifdef UWSGI_PY313 + int *current_c_recursion_remaining; + int *current_py_recursion_remaining; + struct _PyInterpreterFrame **current_frame; + + int current_main_c_recursion_remaining; + int current_main_py_recursion_remaining; + struct _PyInterpreterFrame *current_main_frame; +#elif defined UWSGI_PY312 + int *current_c_recursion_remaining; + int *current_py_recursion_remaining; + _PyCFrame **current_frame; + + int current_main_c_recursion_remaining; + int current_main_py_recursion_remaining; + _PyCFrame *current_main_frame; +#elif defined UWSGI_PY311 + int *current_recursion_remaining; + _PyCFrame **current_frame; + + int current_main_recursion_remaining; + _PyCFrame *current_main_frame; +#else int *current_recursion_depth; struct _frame **current_frame; int current_main_recursion_depth; struct _frame *current_main_frame; +#endif void (*swap_ts)(struct wsgi_request *, struct uwsgi_app *); void (*reset_ts)(struct wsgi_request *, struct uwsgi_app *); @@ -232,6 +271,10 @@ struct uwsgi_python { int wsgi_manage_chunked_input; int master_check_signals; + + char *executable; + + int call_uwsgi_fork_hooks; }; diff --git a/plugins/python/uwsgiplugin.py b/plugins/python/uwsgiplugin.py index 1b33972122..47d30dd11b 100644 --- a/plugins/python/uwsgiplugin.py +++ b/plugins/python/uwsgiplugin.py @@ -1,8 +1,17 @@ import os import sys - -from distutils import sysconfig - +try: + from distutils import sysconfig + paths = [ + sysconfig.get_python_inc(), + sysconfig.get_python_inc(plat_specific=True), + ] +except ImportError: + import sysconfig + paths = [ + sysconfig.get_path('include'), + sysconfig.get_path('platinclude'), + ] def get_python_version(): version = sysconfig.get_config_var('VERSION') @@ -30,16 +39,14 @@ def get_python_version(): 'raw' ] -CFLAGS = [ - '-I' + sysconfig.get_python_inc(), - '-I' + sysconfig.get_python_inc(plat_specific=True), -] +CFLAGS = ['-I' + path for path in paths] LDFLAGS = [] if 'UWSGI_PYTHON_NOLIB' not in os.environ: LIBS = sysconfig.get_config_var('LIBS').split() + sysconfig.get_config_var('SYSLIBS').split() # check if it is a non-shared build (but please, add --enable-shared to your python's ./configure script) - if not sysconfig.get_config_var('Py_ENABLE_SHARED'): + use_static_lib = not sysconfig.get_config_var('Py_ENABLE_SHARED') + if use_static_lib: libdir = sysconfig.get_config_var('LIBPL') # libdir does not exists, try to get it from the venv version = get_python_version() @@ -52,6 +59,10 @@ def get_python_version(): # try 3.x style config dir if not os.path.exists(libdir): libdir = '%s/lib/python%s/config-%s' % (sys.prefix, version, get_python_version()) + # try >=3.6 style config dir with arch as suffix + if not os.path.exists(libdir): + multiarch = sysconfig.get_config_var('MULTIARCH') + libdir = '%s/lib/python%s/config-%s-%s' % (sys.prefix, version, get_python_version(), multiarch) # get cpu type uname = os.uname() @@ -65,11 +76,17 @@ def get_python_version(): libpath = '%s/%s' % (libdir, sysconfig.get_config_var('LIBRARY')) if not os.path.exists(libpath): libpath = '%s/libpython%s.a' % (libdir, version) - LIBS.append(libpath) - # hack for messy linkers/compilers - if '-lutil' in LIBS: - LIBS.append('-lutil') - else: + + if os.path.exists(libpath): + LIBS.append(libpath) + # hack for messy linkers/compilers + if '-lutil' in LIBS: + LIBS.append('-lutil') + if '-lrt' in LIBS: + LIBS.append('-lrt') + else: + use_static_lib = False + if not use_static_lib: try: libdir = sysconfig.get_config_var('LIBDIR') except Exception: diff --git a/plugins/python/wsgi_handlers.c b/plugins/python/wsgi_handlers.c index b6a2b24101..20c6dfeab7 100644 --- a/plugins/python/wsgi_handlers.c +++ b/plugins/python/wsgi_handlers.c @@ -65,6 +65,7 @@ static PyObject *uwsgi_Input_read(uwsgi_Input *self, PyObject *args) { char *buf = NULL; if (wsgi_req->body_is_chunked && up.wsgi_manage_chunked_input) { struct uwsgi_buffer *ubuf = uwsgi_chunked_read_smart(wsgi_req, arg_len, uwsgi.socket_timeout); + UWSGI_GET_GIL if (!ubuf) { return PyErr_Format(PyExc_IOError, "error during chunked read(%ld) on wsgi.input", arg_len); } @@ -475,31 +476,32 @@ void uwsgi_after_request_wsgi(struct wsgi_request *wsgi_req) { } PyObject *py_uwsgi_sendfile(PyObject * self, PyObject * args) { - + int chunk_size; + PyObject *filelike; struct wsgi_request *wsgi_req = py_current_wsgi_req(); - if (!PyArg_ParseTuple(args, "O|i:uwsgi_sendfile", &wsgi_req->async_sendfile, &wsgi_req->sendfile_fd_chunk)) { + if (!PyArg_ParseTuple(args, "O|i:uwsgi_sendfile", &filelike, &chunk_size)) { return NULL; } -#ifdef PYTHREE - wsgi_req->sendfile_fd = PyObject_AsFileDescriptor(wsgi_req->async_sendfile); - if (wsgi_req->sendfile_fd >= 0) { - Py_INCREF((PyObject *)wsgi_req->async_sendfile); - } -#else - if (PyFile_Check((PyObject *)wsgi_req->async_sendfile)) { - Py_INCREF((PyObject *)wsgi_req->async_sendfile); - wsgi_req->sendfile_fd = PyObject_AsFileDescriptor(wsgi_req->async_sendfile); + if (!PyObject_HasAttrString(filelike, "read")) { + PyErr_SetString(PyExc_AttributeError, "object has no attribute 'read'"); + return NULL; } -#endif - // PEP 333 hack - wsgi_req->sendfile_obj = wsgi_req->async_sendfile; - //wsgi_req->sendfile_obj = (void *) PyTuple_New(0); + // wsgi.file_wrapper called a second time? Forget the old reference. + if (wsgi_req->async_sendfile) { + Py_DECREF(wsgi_req->async_sendfile); + } - Py_INCREF((PyObject *) wsgi_req->sendfile_obj); - return (PyObject *) wsgi_req->sendfile_obj; + // XXX: Not 100% sure why twice. + // Maybe: We keep one at async_sendfile and transfer + // one to the caller (even though he gave it to us). + Py_INCREF(filelike); + Py_INCREF(filelike); + wsgi_req->async_sendfile = filelike; + wsgi_req->sendfile_fd_chunk = chunk_size; + return filelike; } void threaded_swap_ts(struct wsgi_request *wsgi_req, struct uwsgi_app *wi) { diff --git a/plugins/python/wsgi_subhandler.c b/plugins/python/wsgi_subhandler.c index a8ad492ab8..faa3fd8521 100644 --- a/plugins/python/wsgi_subhandler.c +++ b/plugins/python/wsgi_subhandler.c @@ -76,7 +76,18 @@ int uwsgi_python_send_body(struct wsgi_request *wsgi_req, PyObject *chunk) { return 1; } return 0; -} +} + +/* + * attempt to make a fd from a file-like PyObject - returns -1 on error. + */ +static int uwsgi_python_try_filelike_as_fd(PyObject *filelike) { + int fd; + if ((fd = PyObject_AsFileDescriptor(filelike)) < 0) { + PyErr_Clear(); + } + return fd; +} /* this is a hack for supporting non-file object passed to wsgi.file_wrapper @@ -92,7 +103,7 @@ static void uwsgi_python_consume_file_wrapper_read(struct wsgi_request *wsgi_req read_method_args = PyTuple_New(0); } for(;;) { - PyObject *read_method_output = PyEval_CallObject(read_method, read_method_args); + PyObject *read_method_output = PyObject_CallObject(read_method, read_method_args); if (PyErr_Occurred()) { uwsgi_manage_exception(wsgi_req, 0); break; @@ -244,6 +255,7 @@ void *uwsgi_request_subhandler_wsgi(struct wsgi_request *wsgi_req, struct uwsgi_ // call if (PyTuple_GetItem(wsgi_req->async_args, 0) != wsgi_req->async_environ) { + Py_INCREF(wsgi_req->async_environ); if (PyTuple_SetItem(wsgi_req->async_args, 0, wsgi_req->async_environ)) { uwsgi_log_verbose("unable to set environ to the python application callable, consider using the holy env allocator\n"); return NULL; @@ -262,7 +274,9 @@ int uwsgi_response_subhandler_wsgi(struct wsgi_request *wsgi_req) { if (uwsgi_python_send_body(wsgi_req, (PyObject *)wsgi_req->async_result)) goto clear; } - if (wsgi_req->sendfile_obj == wsgi_req->async_result) { + // Check if wsgi.file_wrapper has been used. + if (wsgi_req->async_sendfile == wsgi_req->async_result) { + wsgi_req->sendfile_fd = uwsgi_python_try_filelike_as_fd((PyObject *)wsgi_req->async_sendfile); if (wsgi_req->sendfile_fd >= 0) { UWSGI_RELEASE_GIL uwsgi_response_sendfile_do(wsgi_req, wsgi_req->sendfile_fd, 0, 0); @@ -308,8 +322,14 @@ int uwsgi_response_subhandler_wsgi(struct wsgi_request *wsgi_req) { Py_DECREF(pychunk); goto clear; } - } - else if (wsgi_req->sendfile_obj == pychunk) { + } else if (wsgi_req->async_sendfile == pychunk) { + // + // XXX: It's not clear whether this can ever be sensibly reached + // based on PEP 333/3333 - it would mean the iterator yielded + // the result of wsgi.file_wrapper. However, the iterator should + // only ever yield `bytes`... + // + wsgi_req->sendfile_fd = uwsgi_python_try_filelike_as_fd((PyObject *)wsgi_req->async_sendfile); if (wsgi_req->sendfile_fd >= 0) { UWSGI_RELEASE_GIL uwsgi_response_sendfile_do(wsgi_req, wsgi_req->sendfile_fd, 0, 0); @@ -319,41 +339,50 @@ int uwsgi_response_subhandler_wsgi(struct wsgi_request *wsgi_req) { else if (PyObject_HasAttrString(pychunk, "read")) { uwsgi_python_consume_file_wrapper_read(wsgi_req, pychunk); } - uwsgi_py_check_write_errors { uwsgi_py_write_exception(wsgi_req); Py_DECREF(pychunk); goto clear; } + } else { + // The iterator returned something that we were not able to handle. + PyObject *pystr = PyObject_Repr(pychunk); +#ifdef PYTHREE + const char *cstr = PyUnicode_AsUTF8(pystr); +#else + const char *cstr = PyString_AsString(pystr); +#endif + uwsgi_log("[ERROR] Unhandled object from iterator: %s (%p)\n", cstr, pychunk); + Py_DECREF(pystr); } - Py_DECREF(pychunk); return UWSGI_AGAIN; clear: + // Release the reference that we took in py_uwsgi_sendfile. + if (wsgi_req->async_sendfile != NULL) { + Py_DECREF((PyObject *) wsgi_req->async_sendfile); + } - if (wsgi_req->sendfile_fd != -1) { - Py_DECREF((PyObject *)wsgi_req->async_sendfile); + if (wsgi_req->async_placeholder != NULL) { + Py_DECREF((PyObject *)wsgi_req->async_placeholder); } - if (wsgi_req->async_placeholder) { - // CALL close() ALWAYS if we are working with an iterator !!! - if (PyObject_HasAttrString((PyObject *)wsgi_req->async_result, "close")) { - PyObject *close_method = PyObject_GetAttrString((PyObject *)wsgi_req->async_result, "close"); - PyObject *close_method_args = PyTuple_New(0); + // Call close() on the response, this is required by the WSGI spec + if (PyObject_HasAttrString((PyObject *)wsgi_req->async_result, "close")) { + PyObject *close_method = PyObject_GetAttrString((PyObject *)wsgi_req->async_result, "close"); + PyObject *close_method_args = PyTuple_New(0); #ifdef UWSGI_DEBUG - uwsgi_log("calling close() for %.*s %p %p\n", wsgi_req->uri_len, wsgi_req->uri, close_method, close_method_args); + uwsgi_log("calling close() for %.*s %p %p\n", wsgi_req->uri_len, wsgi_req->uri, close_method, close_method_args); #endif - PyObject *close_method_output = PyEval_CallObject(close_method, close_method_args); - if (PyErr_Occurred()) { - uwsgi_manage_exception(wsgi_req, 0); - } - Py_DECREF(close_method_args); - Py_XDECREF(close_method_output); - Py_DECREF(close_method); - } - Py_DECREF((PyObject *)wsgi_req->async_placeholder); + PyObject *close_method_output = PyObject_CallObject(close_method, close_method_args); + if (PyErr_Occurred()) { + uwsgi_manage_exception(wsgi_req, 0); + } + Py_DECREF(close_method_args); + Py_XDECREF(close_method_output); + Py_DECREF(close_method); } Py_DECREF((PyObject *)wsgi_req->async_result); diff --git a/plugins/pyuwsgi/pyuwsgi.c b/plugins/pyuwsgi/pyuwsgi.c index 7079d1e24b..14135a6117 100644 --- a/plugins/pyuwsgi/pyuwsgi.c +++ b/plugins/pyuwsgi/pyuwsgi.c @@ -115,12 +115,6 @@ pyuwsgi_setup(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - //TODO: ...??? - // actually do the thing! - PyThreadState *_tstate = PyThreadState_Get(); - uwsgi_setup(orig_argc, orig_argv, environ); - PyThreadState_Swap(_tstate); - Py_INCREF(self); return self; } @@ -133,6 +127,7 @@ pyuwsgi_init(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + uwsgi_setup(orig_argc, orig_argv, environ); int rc = uwsgi_run(); // never(?) here @@ -149,6 +144,7 @@ pyuwsgi_run(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + uwsgi_setup(orig_argc, orig_argv, environ); int rc = uwsgi_run(); // never(?) here diff --git a/plugins/pyuwsgi/uwsgiplugin.py b/plugins/pyuwsgi/uwsgiplugin.py index 4b6bce042a..3bfd66eddf 100644 --- a/plugins/pyuwsgi/uwsgiplugin.py +++ b/plugins/pyuwsgi/uwsgiplugin.py @@ -1,14 +1,23 @@ -from distutils import sysconfig -import os, sys +import os +import sys +try: + from distutils import sysconfig + paths = [ + sysconfig.get_python_inc(), + sysconfig.get_python_inc(plat_specific=True), + ] +except ImportError: + import sysconfig + paths = [ + sysconfig.get_path('include'), + sysconfig.get_path('platinclude'), + ] os.environ['UWSGI_PYTHON_NOLIB'] = '1' NAME = 'pyuwsgi' -CFLAGS = [ - '-I' + sysconfig.get_python_inc(), - '-I' + sysconfig.get_python_inc(plat_specific=True), -] +CFLAGS = ['-I' + path for path in paths] LDFLAGS = [] LIBS = [] diff --git a/plugins/rack/rack_api.c b/plugins/rack/rack_api.c index 6a1396e012..004fb17ada 100644 --- a/plugins/rack/rack_api.c +++ b/plugins/rack/rack_api.c @@ -7,13 +7,13 @@ extern struct uwsgi_plugin rack_plugin; #define uwsgi_rack_api(x, y, z) rb_define_module_function(rb_uwsgi_embedded, x, y, z) -static VALUE rack_uwsgi_metric_get(VALUE *class, VALUE key) { +static VALUE rack_uwsgi_metric_get(VALUE class, VALUE key) { Check_Type(key, T_STRING); int64_t value = uwsgi_metric_get(RSTRING_PTR(key), NULL); return LONG2NUM(value); } -static VALUE rack_uwsgi_metric_set(VALUE *class, VALUE key, VALUE val) { +static VALUE rack_uwsgi_metric_set(VALUE class, VALUE key, VALUE val) { Check_Type(key, T_STRING); Check_Type(val, T_FIXNUM); // should be T_BIGNUM... if (uwsgi_metric_set(RSTRING_PTR(key), NULL, NUM2LONG(val) )) { @@ -22,7 +22,7 @@ static VALUE rack_uwsgi_metric_set(VALUE *class, VALUE key, VALUE val) { return Qtrue; } -static VALUE rack_uwsgi_metric_inc(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_metric_inc(int argc, VALUE *argv, VALUE class) { int64_t value = 1; if (argc == 0) return Qnil; Check_Type(argv[0], T_STRING); @@ -38,7 +38,7 @@ static VALUE rack_uwsgi_metric_inc(int argc, VALUE *argv, VALUE *class) { return Qtrue; } -static VALUE rack_uwsgi_metric_dec(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_metric_dec(int argc, VALUE *argv, VALUE class) { int64_t value = 1; if (argc == 0) return Qnil; Check_Type(argv[0], T_STRING); @@ -54,7 +54,7 @@ static VALUE rack_uwsgi_metric_dec(int argc, VALUE *argv, VALUE *class) { return Qtrue; } -static VALUE rack_uwsgi_metric_mul(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_metric_mul(int argc, VALUE *argv, VALUE class) { int64_t value = 1; if (argc == 0) return Qnil; Check_Type(argv[0], T_STRING); @@ -70,7 +70,7 @@ static VALUE rack_uwsgi_metric_mul(int argc, VALUE *argv, VALUE *class) { return Qtrue; } -static VALUE rack_uwsgi_metric_div(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_metric_div(int argc, VALUE *argv, VALUE class) { int64_t value = 1; if (argc == 0) return Qnil; Check_Type(argv[0], T_STRING); @@ -89,7 +89,7 @@ static VALUE rack_uwsgi_metric_div(int argc, VALUE *argv, VALUE *class) { -static VALUE rack_uwsgi_warning(VALUE *class, VALUE rbmessage) { +static VALUE rack_uwsgi_warning(VALUE class, VALUE rbmessage) { Check_Type(rbmessage, T_STRING); char *message = RSTRING_PTR(rbmessage); @@ -108,7 +108,7 @@ static VALUE rack_uwsgi_warning(VALUE *class, VALUE rbmessage) { return Qnil; } -static VALUE rack_uwsgi_user_harakiri(VALUE *class, VALUE sec) { +static VALUE rack_uwsgi_user_harakiri(VALUE class, VALUE sec) { Check_Type(sec, T_FIXNUM); struct wsgi_request *wsgi_req = current_wsgi_req(); set_user_harakiri(wsgi_req, NUM2INT(sec)); @@ -116,7 +116,7 @@ static VALUE rack_uwsgi_user_harakiri(VALUE *class, VALUE sec) { } -static VALUE rack_uwsgi_log(VALUE *class, VALUE msg) { +static VALUE rack_uwsgi_log(VALUE class, VALUE msg) { Check_Type(msg, T_STRING); @@ -125,7 +125,7 @@ static VALUE rack_uwsgi_log(VALUE *class, VALUE msg) { return Qnil; } -static VALUE rack_uwsgi_i_am_the_spooler(VALUE *class) { +static VALUE rack_uwsgi_i_am_the_spooler(VALUE class) { if (uwsgi.i_am_a_spooler) { return Qtrue; } @@ -133,7 +133,7 @@ static VALUE rack_uwsgi_i_am_the_spooler(VALUE *class) { } #ifdef UWSGI_SSL -static VALUE rack_uwsgi_i_am_the_lord(VALUE *class, VALUE legion_name) { +static VALUE rack_uwsgi_i_am_the_lord(VALUE class, VALUE legion_name) { Check_Type(legion_name, T_STRING); if (uwsgi_legion_i_am_the_lord(RSTRING_PTR(legion_name))) { return Qtrue; @@ -142,14 +142,14 @@ static VALUE rack_uwsgi_i_am_the_lord(VALUE *class, VALUE legion_name) { } #endif -static VALUE rack_uwsgi_connection_fd(VALUE *class) { +static VALUE rack_uwsgi_connection_fd(VALUE class) { struct wsgi_request *wsgi_req = current_wsgi_req(); return INT2NUM(wsgi_req->fd); } -static VALUE rack_uwsgi_setprocname(VALUE *class, VALUE rbname) { +static VALUE rack_uwsgi_setprocname(VALUE class, VALUE rbname) { Check_Type(rbname, T_STRING); char *name = RSTRING_PTR(rbname); @@ -158,7 +158,7 @@ static VALUE rack_uwsgi_setprocname(VALUE *class, VALUE rbname) { return Qnil; } -static VALUE rack_uwsgi_mem(VALUE *class) { +static VALUE rack_uwsgi_mem(VALUE class) { uint64_t rss=0, vsz = 0; VALUE ml = rb_ary_new2(2); @@ -172,23 +172,23 @@ static VALUE rack_uwsgi_mem(VALUE *class) { } -static VALUE rack_uwsgi_request_id(VALUE *class) { +static VALUE rack_uwsgi_request_id(VALUE class) { return ULONG2NUM(uwsgi.workers[uwsgi.mywid].requests); } -static VALUE rack_uwsgi_worker_id(VALUE *class) { +static VALUE rack_uwsgi_worker_id(VALUE class) { return INT2NUM(uwsgi.mywid); } -static VALUE rack_uwsgi_mule_id(VALUE *class) { +static VALUE rack_uwsgi_mule_id(VALUE class) { return INT2NUM(uwsgi.muleid); } -static VALUE rack_uwsgi_logsize(VALUE *class) { +static VALUE rack_uwsgi_logsize(VALUE class) { return ULONG2NUM(uwsgi.shared->logsize); } -static VALUE rack_uwsgi_mule_msg(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_mule_msg(int argc, VALUE *argv, VALUE class) { int fd = -1; int mule_id = -1; @@ -267,7 +267,7 @@ static int uwsgi_ruby_hash_mule_callback(VALUE key, VALUE val, VALUE arg_array) return 0; } -static VALUE rack_uwsgi_mule_get_msg(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_mule_get_msg(int argc, VALUE *argv, VALUE class) { int manage_signals = 1; int manage_farms = 1; @@ -323,7 +323,7 @@ static VALUE rack_uwsgi_mule_get_msg(int argc, VALUE *argv, VALUE *class) { } -static VALUE rack_uwsgi_lock(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_lock(int argc, VALUE *argv, VALUE class) { int lock_num = 0; @@ -341,7 +341,7 @@ static VALUE rack_uwsgi_lock(int argc, VALUE *argv, VALUE *class) { return Qnil; } -static VALUE rack_uwsgi_unlock(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_unlock(int argc, VALUE *argv, VALUE class) { int lock_num = 0; @@ -363,7 +363,7 @@ static VALUE rack_uwsgi_unlock(int argc, VALUE *argv, VALUE *class) { -static VALUE rack_uwsgi_cache_set(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_set(int argc, VALUE *argv, VALUE class) { if (argc < 2) goto error; @@ -399,7 +399,7 @@ static VALUE rack_uwsgi_cache_set(int argc, VALUE *argv, VALUE *class) { } -static VALUE rack_uwsgi_cache_update(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_update(int argc, VALUE *argv, VALUE class) { if (argc < 2) goto error; @@ -436,7 +436,7 @@ static VALUE rack_uwsgi_cache_update(int argc, VALUE *argv, VALUE *class) { } -static VALUE rack_uwsgi_cache_set_exc(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_set_exc(int argc, VALUE *argv, VALUE class) { VALUE ret = rack_uwsgi_cache_set(argc, argv, class); if (ret == Qnil) { rb_raise(rb_eRuntimeError, "unable to set value in uWSGI cache"); @@ -444,7 +444,7 @@ static VALUE rack_uwsgi_cache_set_exc(int argc, VALUE *argv, VALUE *class) { return ret; } -static VALUE rack_uwsgi_cache_update_exc(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_update_exc(int argc, VALUE *argv, VALUE class) { VALUE ret = rack_uwsgi_cache_update(argc, argv, class); if (ret == Qnil) { rb_raise(rb_eRuntimeError, "unable to update value in uWSGI cache"); @@ -452,7 +452,7 @@ static VALUE rack_uwsgi_cache_update_exc(int argc, VALUE *argv, VALUE *class) { return ret; } -static VALUE rack_uwsgi_cache_del(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_del(int argc, VALUE *argv, VALUE class) { if (argc == 0) goto error; @@ -478,7 +478,7 @@ static VALUE rack_uwsgi_cache_del(int argc, VALUE *argv, VALUE *class) { return Qnil; } -static VALUE rack_uwsgi_cache_del_exc(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_del_exc(int argc, VALUE *argv, VALUE class) { VALUE ret = rack_uwsgi_cache_del(argc, argv, class); if (ret == Qnil) { rb_raise(rb_eRuntimeError, "unable to delete object from uWSGI cache"); @@ -488,7 +488,7 @@ static VALUE rack_uwsgi_cache_del_exc(int argc, VALUE *argv, VALUE *class) { -static VALUE rack_uwsgi_cache_exists(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_exists(int argc, VALUE *argv, VALUE class) { if (argc == 0) goto error; @@ -516,7 +516,7 @@ static VALUE rack_uwsgi_cache_exists(int argc, VALUE *argv, VALUE *class) { -static VALUE rack_uwsgi_cache_clear(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_clear(int argc, VALUE *argv, VALUE class) { char *cache = NULL; @@ -532,7 +532,7 @@ static VALUE rack_uwsgi_cache_clear(int argc, VALUE *argv, VALUE *class) { return Qnil; } -static VALUE rack_uwsgi_cache_clear_exc(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_clear_exc(int argc, VALUE *argv, VALUE class) { VALUE ret = rack_uwsgi_cache_clear(argc, argv, class); if (ret == Qnil) { rb_raise(rb_eRuntimeError, "unable to clear the uWSGI cache"); @@ -542,7 +542,7 @@ static VALUE rack_uwsgi_cache_clear_exc(int argc, VALUE *argv, VALUE *class) { -static VALUE rack_uwsgi_cache_get(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_get(int argc, VALUE *argv, VALUE class) { if (argc == 0) goto error; @@ -572,7 +572,7 @@ static VALUE rack_uwsgi_cache_get(int argc, VALUE *argv, VALUE *class) { } -static VALUE rack_uwsgi_cache_get_exc(int argc, VALUE *argv, VALUE *class) { +static VALUE rack_uwsgi_cache_get_exc(int argc, VALUE *argv, VALUE class) { VALUE ret = rack_uwsgi_cache_get(argc, argv, class); if (ret == Qnil) { rb_raise(rb_eRuntimeError, "unable to get value from uWSGI cache"); @@ -583,7 +583,7 @@ static VALUE rack_uwsgi_cache_get_exc(int argc, VALUE *argv, VALUE *class) { -static VALUE rack_uwsgi_add_cron(VALUE *class, VALUE rbsignum, VALUE rbmin, VALUE rbhour, VALUE rbday, VALUE rbmon, VALUE rbweek) { +static VALUE rack_uwsgi_add_cron(VALUE class, VALUE rbsignum, VALUE rbmin, VALUE rbhour, VALUE rbday, VALUE rbmon, VALUE rbweek) { Check_Type(rbsignum, T_FIXNUM); Check_Type(rbmin, T_FIXNUM); @@ -609,7 +609,7 @@ static VALUE rack_uwsgi_add_cron(VALUE *class, VALUE rbsignum, VALUE rbmin, VALU -static VALUE rack_uwsgi_add_timer(VALUE *class, VALUE rbsignum, VALUE secs) { +static VALUE rack_uwsgi_add_timer(VALUE class, VALUE rbsignum, VALUE secs) { Check_Type(rbsignum, T_FIXNUM); Check_Type(secs, T_FIXNUM); @@ -627,7 +627,7 @@ static VALUE rack_uwsgi_add_timer(VALUE *class, VALUE rbsignum, VALUE secs) { -static VALUE rack_uwsgi_add_rb_timer(VALUE *class, VALUE rbsignum, VALUE secs) { +static VALUE rack_uwsgi_add_rb_timer(VALUE class, VALUE rbsignum, VALUE secs) { Check_Type(rbsignum, T_FIXNUM); Check_Type(secs, T_FIXNUM); @@ -645,7 +645,7 @@ static VALUE rack_uwsgi_add_rb_timer(VALUE *class, VALUE rbsignum, VALUE secs) { } -static VALUE rack_uwsgi_alarm(VALUE *class, VALUE alarm, VALUE msg) { +static VALUE rack_uwsgi_alarm(VALUE class, VALUE alarm, VALUE msg) { Check_Type(alarm, T_STRING); Check_Type(msg, T_STRING); @@ -655,7 +655,7 @@ static VALUE rack_uwsgi_alarm(VALUE *class, VALUE alarm, VALUE msg) { return Qnil; } -static VALUE rack_uwsgi_add_file_monitor(VALUE *class, VALUE rbsignum, VALUE rbfilename) { +static VALUE rack_uwsgi_add_file_monitor(VALUE class, VALUE rbsignum, VALUE rbfilename) { Check_Type(rbsignum, T_FIXNUM); Check_Type(rbfilename, T_STRING); @@ -672,7 +672,7 @@ static VALUE rack_uwsgi_add_file_monitor(VALUE *class, VALUE rbsignum, VALUE rbf } -static VALUE uwsgi_ruby_wait_fd_read(VALUE *class, VALUE arg1, VALUE arg2) { +static VALUE uwsgi_ruby_wait_fd_read(VALUE class, VALUE arg1, VALUE arg2) { Check_Type(arg1, T_FIXNUM); Check_Type(arg2, T_FIXNUM); @@ -690,7 +690,7 @@ static VALUE uwsgi_ruby_wait_fd_read(VALUE *class, VALUE arg1, VALUE arg2) { return Qtrue; } -static VALUE uwsgi_ruby_wait_fd_write(VALUE *class, VALUE arg1, VALUE arg2) { +static VALUE uwsgi_ruby_wait_fd_write(VALUE class, VALUE arg1, VALUE arg2) { Check_Type(arg1, T_FIXNUM); Check_Type(arg2, T_FIXNUM); @@ -707,7 +707,7 @@ static VALUE uwsgi_ruby_wait_fd_write(VALUE *class, VALUE arg1, VALUE arg2) { return Qtrue; } -static VALUE uwsgi_ruby_async_connect(VALUE *class, VALUE arg) { +static VALUE uwsgi_ruby_async_connect(VALUE class, VALUE arg) { Check_Type(arg, T_STRING); @@ -717,7 +717,7 @@ static VALUE uwsgi_ruby_async_connect(VALUE *class, VALUE arg) { } -static VALUE uwsgi_ruby_async_sleep(VALUE *class, VALUE arg) { +static VALUE uwsgi_ruby_async_sleep(VALUE class, VALUE arg) { Check_Type(arg, T_FIXNUM); @@ -731,7 +731,7 @@ static VALUE uwsgi_ruby_async_sleep(VALUE *class, VALUE arg) { return Qtrue; } -static VALUE uwsgi_ruby_masterpid(VALUE *class) { +static VALUE uwsgi_ruby_masterpid(VALUE class) { if (uwsgi.master_process) { return INT2NUM(uwsgi.workers[0].pid); @@ -739,7 +739,7 @@ static VALUE uwsgi_ruby_masterpid(VALUE *class) { return INT2NUM(0); } -static VALUE uwsgi_ruby_suspend(VALUE *class) { +static VALUE uwsgi_ruby_suspend(VALUE class) { struct wsgi_request *wsgi_req = current_wsgi_req(); @@ -750,7 +750,7 @@ static VALUE uwsgi_ruby_suspend(VALUE *class) { } -static VALUE uwsgi_ruby_signal_wait(int argc, VALUE *argv, VALUE *class) { +static VALUE uwsgi_ruby_signal_wait(int argc, VALUE *argv, VALUE class) { struct wsgi_request *wsgi_req = current_wsgi_req(); int wait_for_specific_signal = 0; @@ -782,7 +782,7 @@ static VALUE uwsgi_ruby_signal_wait(int argc, VALUE *argv, VALUE *class) { return Qnil; } -static VALUE uwsgi_ruby_signal_received(VALUE *class) { +static VALUE uwsgi_ruby_signal_received(VALUE class) { struct wsgi_request *wsgi_req = current_wsgi_req(); @@ -790,7 +790,7 @@ static VALUE uwsgi_ruby_signal_received(VALUE *class) { } -static VALUE uwsgi_ruby_signal_registered(VALUE *class, VALUE signum) { +static VALUE uwsgi_ruby_signal_registered(VALUE class, VALUE signum) { Check_Type(signum, T_FIXNUM); @@ -803,7 +803,7 @@ static VALUE uwsgi_ruby_signal_registered(VALUE *class, VALUE signum) { return Qfalse; } -static VALUE uwsgi_ruby_do_rpc(int argc, VALUE *rpc_argv, VALUE *class) { +static VALUE uwsgi_ruby_do_rpc(int argc, VALUE *rpc_argv, VALUE class) { char *node = NULL, *func; uint64_t size = 0; @@ -854,7 +854,7 @@ static VALUE uwsgi_ruby_do_rpc(int argc, VALUE *rpc_argv, VALUE *class) { return Qnil; } -static VALUE uwsgi_ruby_register_rpc(int argc, VALUE *argv, VALUE *class) { +static VALUE uwsgi_ruby_register_rpc(int argc, VALUE *argv, VALUE class) { int rb_argc = 0; @@ -880,7 +880,7 @@ static VALUE uwsgi_ruby_register_rpc(int argc, VALUE *argv, VALUE *class) { return Qtrue; } -static VALUE uwsgi_ruby_register_signal(VALUE *class, VALUE signum, VALUE sigkind, VALUE rbhandler) { +static VALUE uwsgi_ruby_register_signal(VALUE class, VALUE signum, VALUE sigkind, VALUE rbhandler) { Check_Type(signum, T_FIXNUM); Check_Type(sigkind, T_STRING); @@ -900,7 +900,7 @@ static VALUE uwsgi_ruby_register_signal(VALUE *class, VALUE signum, VALUE sigkin } -static VALUE uwsgi_ruby_signal(int argc, VALUE *argv, VALUE *class) { +static VALUE uwsgi_ruby_signal(int argc, VALUE *argv, VALUE class) { if (argc < 1) { rb_raise(rb_eRuntimeError, "you have to specify a signum"); @@ -968,7 +968,7 @@ static int rack_uwsgi_build_spool(VALUE rbkey, VALUE rbval, VALUE argv) { } -static VALUE rack_uwsgi_send_spool(VALUE *class, VALUE args) { +static VALUE rack_uwsgi_send_spool(VALUE class, VALUE args) { char *body = NULL; size_t body_len= 0; @@ -1006,7 +1006,7 @@ static VALUE rack_uwsgi_send_spool(VALUE *class, VALUE args) { } -static VALUE uwsgi_ruby_websocket_handshake(int argc, VALUE *argv, VALUE *class) { +static VALUE uwsgi_ruby_websocket_handshake(int argc, VALUE *argv, VALUE class) { struct wsgi_request *wsgi_req = current_wsgi_req(); @@ -1035,7 +1035,7 @@ static VALUE uwsgi_ruby_websocket_handshake(int argc, VALUE *argv, VALUE *class) return Qnil; } -static VALUE uwsgi_ruby_websocket_send(VALUE *class, VALUE *msg) { +static VALUE uwsgi_ruby_websocket_send(VALUE class, VALUE msg) { Check_Type(msg, T_STRING); char *message = RSTRING_PTR(msg); size_t message_len = RSTRING_LEN(msg); @@ -1046,7 +1046,7 @@ static VALUE uwsgi_ruby_websocket_send(VALUE *class, VALUE *msg) { return Qnil; } -static VALUE uwsgi_ruby_websocket_recv(VALUE *class) { +static VALUE uwsgi_ruby_websocket_recv(VALUE class) { struct wsgi_request *wsgi_req = current_wsgi_req(); struct uwsgi_buffer *ub = uwsgi_websocket_recv(wsgi_req); @@ -1060,7 +1060,7 @@ static VALUE uwsgi_ruby_websocket_recv(VALUE *class) { } -static VALUE uwsgi_ruby_websocket_recv_nb(VALUE *class) { +static VALUE uwsgi_ruby_websocket_recv_nb(VALUE class) { struct wsgi_request *wsgi_req = current_wsgi_req(); struct uwsgi_buffer *ub = uwsgi_websocket_recv_nb(wsgi_req); diff --git a/plugins/rack/rack_plugin.c b/plugins/rack/rack_plugin.c index 1ddd2c892f..792cc4dc78 100644 --- a/plugins/rack/rack_plugin.c +++ b/plugins/rack/rack_plugin.c @@ -208,7 +208,7 @@ VALUE rb_uwsgi_io_init(int argc, VALUE *argv, VALUE self) { return self; } -VALUE rb_uwsgi_io_gets(VALUE obj, VALUE args) { +VALUE rb_uwsgi_io_gets(VALUE obj) { struct wsgi_request *wsgi_req; Data_Get_Struct(obj, struct wsgi_request, wsgi_req); @@ -222,14 +222,14 @@ VALUE rb_uwsgi_io_gets(VALUE obj, VALUE args) { return Qnil; } -VALUE rb_uwsgi_io_each(VALUE obj, VALUE args) { +VALUE rb_uwsgi_io_each(VALUE obj) { if (!rb_block_given_p()) rb_raise(rb_eArgError, "Expected block on rack.input 'each' method"); // yield strings chunks for(;;) { - VALUE chunk = rb_uwsgi_io_gets(obj, Qnil); + VALUE chunk = rb_uwsgi_io_gets(obj); if (chunk == Qnil) { return Qnil; } @@ -273,7 +273,7 @@ VALUE rb_uwsgi_io_read(VALUE obj, VALUE args) { return Qnil; } -VALUE rb_uwsgi_io_rewind(VALUE obj, VALUE args) { +VALUE rb_uwsgi_io_rewind(VALUE obj) { struct wsgi_request *wsgi_req; Data_Get_Struct(obj, struct wsgi_request, wsgi_req); @@ -454,9 +454,13 @@ void uwsgi_ruby_gemset(char *gemset) { } -static void rack_hack_dollar_zero(VALUE name, ID id) { +static void rack_hack_dollar_zero(VALUE name, ID id, VALUE *_) { ur.dollar_zero = rb_obj_as_string(name); + // From ruby 2.7 onwards this is a noop, from ruby 3.2 onwards + // this function no longer exists +#if !defined(RUBY27) rb_obj_taint(ur.dollar_zero); +#endif } #ifndef RUBY19 @@ -698,7 +702,7 @@ VALUE call_dispatch(VALUE env) { } -static VALUE send_body(VALUE obj) { +static VALUE send_body(VALUE obj, VALUE data, int argc, const VALUE *argv, VALUE blockarg) { struct wsgi_request *wsgi_req = current_wsgi_req(); @@ -731,7 +735,7 @@ VALUE iterate_body(VALUE body) { #endif } -VALUE send_header(VALUE obj, VALUE headers) { +VALUE send_header(VALUE obj, VALUE headers, int argc, const VALUE *argv, VALUE blockarg) { struct wsgi_request *wsgi_req = current_wsgi_req(); @@ -792,7 +796,7 @@ VALUE send_header(VALUE obj, VALUE headers) { VALUE iterate_headers(VALUE headers) { #ifdef RUBY19 - return rb_block_call(headers, rb_intern("each"), 0, 0, send_header, headers ); + return rb_block_call(headers, rb_intern("each"), 0, 0, send_header, headers); #else return rb_iterate(rb_each, headers, send_header, headers); #endif @@ -1051,18 +1055,22 @@ VALUE init_rack_app( VALUE script ) { } #endif - VALUE rackup = rb_funcall( rb_const_get(rack, rb_intern("Builder")), rb_intern("parse_file"), 1, script); - if (TYPE(rackup) != T_ARRAY) { - uwsgi_log("unable to parse %s file\n", RSTRING_PTR(script)); - return Qnil; - } - - if (RARRAY_LEN(rackup) < 1) { - uwsgi_log("invalid rack config file: %s\n", RSTRING_PTR(script)); + VALUE rackup = rb_funcall( rb_const_get(rack, rb_intern("Builder")), rb_intern("parse_file"), 1, script); + if (TYPE(rackup) == T_OBJECT) { // rack +3 + return rackup; + } + else if (TYPE(rackup) == T_ARRAY) { // rack << 3 + if (RARRAY_LEN(rackup) < 1) { + uwsgi_log("invalid rack config file: %s\n", RSTRING_PTR(script)); + return Qnil; + } + return RARRAY_PTR(rackup)[0] ; + } + else { + uwsgi_log("unable to parse %s file %d\n", RSTRING_PTR(script), TYPE(rackup)); return Qnil; - } + } - return RARRAY_PTR(rackup)[0] ; } int uwsgi_rack_magic(char *mountpoint, char *lazy) { diff --git a/plugins/rack/uwsgiplugin.py b/plugins/rack/uwsgiplugin.py index 8b13f74df9..9ac983114f 100644 --- a/plugins/rack/uwsgiplugin.py +++ b/plugins/rack/uwsgiplugin.py @@ -14,6 +14,8 @@ if (v[0] == '1' and v[1] == '9') or v[0] >= '2': CFLAGS = os.popen(RUBYPATH + " -e \"require 'rbconfig';print RbConfig::CONFIG['CFLAGS']\"").read().rstrip().split() CFLAGS.append('-DRUBY19') + if version >= '2.7': + CFLAGS.append('-DRUBY27') CFLAGS.append('-Wno-unused-parameter') rbconfig = 'RbConfig' else: diff --git a/plugins/redislog/redislog_plugin.c b/plugins/redislog/redislog_plugin.c index 2669368fec..6c19eb75c9 100644 --- a/plugins/redislog/redislog_plugin.c +++ b/plugins/redislog/redislog_plugin.c @@ -159,7 +159,7 @@ ssize_t uwsgi_redis_logger(struct uwsgi_logger *ul, char *message, size_t len) { uredislog->fd = uwsgi_connect(uredislog->address, uwsgi.socket_timeout, 0); if (uredislog->password) { setup_iov.iov_len = snprintf( - setup_buf, sizeof (setup_buf), "*2\r\n$4\r\nauth\r\n$%lu\r\n%*s\r\n", + setup_buf, sizeof (setup_buf), "*2\r\n$4\r\nauth\r\n$%zu\r\n%*s\r\n", strlen(uredislog->password), (int)strlen(uredislog->password), uredislog->password); setup_iov.iov_base = setup_buf; ret = writev(uredislog->fd, &setup_iov, 1); @@ -172,7 +172,7 @@ ssize_t uwsgi_redis_logger(struct uwsgi_logger *ul, char *message, size_t len) { } if (uredislog->id) { setup_iov.iov_len = snprintf( - setup_buf, sizeof (setup_buf), "*2\r\n$6\r\nselect\r\n$%lu\r\n%*s\r\n", + setup_buf, sizeof (setup_buf), "*2\r\n$6\r\nselect\r\n$%zu\r\n%*s\r\n", strlen(uredislog->id), (int)strlen(uredislog->id), uredislog->id); setup_iov.iov_base = setup_buf; ret = writev(uredislog->fd, &setup_iov, 1); diff --git a/plugins/router_basicauth/router_basicauth.c b/plugins/router_basicauth/router_basicauth.c index 429bade659..0b7161ea60 100644 --- a/plugins/router_basicauth/router_basicauth.c +++ b/plugins/router_basicauth/router_basicauth.c @@ -3,7 +3,7 @@ #ifdef UWSGI_ROUTING // TODO: Add more crypt_r supported platfroms here -#if defined(__linux__) && defined(__GLIBC__) +#if defined(__linux__) && defined(__GLIBC__) && !defined(__UCLIBC__) #include #elif defined(__CYGWIN__) #include @@ -67,7 +67,7 @@ static uint16_t htpasswd_check(char *filename, char *auth) { if (clen > 13) cpwd[13] = 0; -#if defined(__linux__) && defined(__GLIBC__) +#if defined(__linux__) && defined(__GLIBC__) && !defined(__UCLIBC__) struct crypt_data cd; memset(&cd, 0, sizeof(struct crypt_data)); /* work around glibc-2.2.5 bug, diff --git a/plugins/router_xmldir/router_xmldir.c b/plugins/router_xmldir/router_xmldir.c index 525146beb2..c2cbff8fca 100644 --- a/plugins/router_xmldir/router_xmldir.c +++ b/plugins/router_xmldir/router_xmldir.c @@ -1,23 +1,3 @@ -/* - * Copyright (C) 2013 Unbit S.a.s. - * Copyright (C) 2013 Guido Berhoerster - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - #include #include #include diff --git a/plugins/ruby19/uwsgiplugin.py b/plugins/ruby19/uwsgiplugin.py index a3bffd4ce5..994db614bc 100644 --- a/plugins/ruby19/uwsgiplugin.py +++ b/plugins/ruby19/uwsgiplugin.py @@ -11,9 +11,11 @@ GCC_LIST = ['../rack/rack_plugin', '../rack/rack_api'] -if v[0] == '1' and v[1] == '9': +if (v[0] == '1' and v[1] == '9') or v[0] >= '2': CFLAGS = os.popen(RUBYPATH + " -e \"require 'rbconfig';print RbConfig::CONFIG['CFLAGS']\"").read().rstrip().split() CFLAGS.append('-DRUBY19') + if version >= '2.7': + CFLAGS.append('-DRUBY27') CFLAGS.append('-Wno-unused-parameter') rbconfig = 'RbConfig' else: diff --git a/plugins/stackless/uwsgiplugin.py b/plugins/stackless/uwsgiplugin.py index d720e2b9bd..a304b97a52 100644 --- a/plugins/stackless/uwsgiplugin.py +++ b/plugins/stackless/uwsgiplugin.py @@ -1,11 +1,19 @@ -from distutils import sysconfig +try: + from distutils import sysconfig + paths = [ + sysconfig.get_python_inc(), + sysconfig.get_python_inc(plat_specific=True), + ] +except ImportError: + import sysconfig + paths = [ + sysconfig.get_path('include'), + sysconfig.get_path('platinclude'), + ] NAME = 'stackless' -CFLAGS = [ - '-I' + sysconfig.get_python_inc(), - '-I' + sysconfig.get_python_inc(plat_specific=True), -] +CFLAGS = ['-I' + path for path in paths] LDFLAGS = [] LIBS = [] diff --git a/plugins/tornado/uwsgiplugin.py b/plugins/tornado/uwsgiplugin.py index c0179b609b..f80b6efbf1 100644 --- a/plugins/tornado/uwsgiplugin.py +++ b/plugins/tornado/uwsgiplugin.py @@ -1,11 +1,19 @@ -from distutils import sysconfig +try: + from distutils import sysconfig + paths = [ + sysconfig.get_python_inc(), + sysconfig.get_python_inc(plat_specific=True), + ] +except ImportError: + import sysconfig + paths = [ + sysconfig.get_path('include'), + sysconfig.get_path('platinclude'), + ] NAME = 'tornado' -CFLAGS = [ - '-I' + sysconfig.get_python_inc(), - '-I' + sysconfig.get_python_inc(plat_specific=True), -] +CFLAGS = ['-I' + path for path in paths] LDFLAGS = [] LIBS = [] diff --git a/proto/BUILD b/proto/BUILD new file mode 100644 index 0000000000..10a5c3ed32 --- /dev/null +++ b/proto/BUILD @@ -0,0 +1 @@ +exports_files(glob(["*.c"])) diff --git a/proto/base.c b/proto/base.c index 2325e7db3a..5bef156a7f 100644 --- a/proto/base.c +++ b/proto/base.c @@ -1,4 +1,4 @@ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/proto/fastcgi.c b/proto/fastcgi.c index d55971d8fc..6e8718015a 100644 --- a/proto/fastcgi.c +++ b/proto/fastcgi.c @@ -1,6 +1,6 @@ /* async fastcgi protocol parser */ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/proto/http.c b/proto/http.c index 981401c06b..53ce424378 100644 --- a/proto/http.c +++ b/proto/http.c @@ -1,6 +1,6 @@ /* async http protocol parser */ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; @@ -311,7 +311,7 @@ static int http_parse(struct wsgi_request *wsgi_req, char *watermark) { } } else { - char *server_port = strchr(wsgi_req->socket->name, ':'); + char *server_port = strrchr(wsgi_req->socket->name, ':'); if (server_port) { wsgi_req->len += proto_base_add_uwsgi_var(wsgi_req, "SERVER_PORT", 11, server_port+1, strlen(server_port+1)); } diff --git a/proto/puwsgi.c b/proto/puwsgi.c index ba08e93bb7..f7bde2c28c 100644 --- a/proto/puwsgi.c +++ b/proto/puwsgi.c @@ -1,6 +1,6 @@ /* async uwsgi protocol parser */ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/proto/scgi.c b/proto/scgi.c index 707fe3e5d0..09921528a6 100644 --- a/proto/scgi.c +++ b/proto/scgi.c @@ -1,6 +1,6 @@ /* async SCGI protocol parser */ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/proto/uwsgi.c b/proto/uwsgi.c index feaa47c85b..451a1b8c82 100644 --- a/proto/uwsgi.c +++ b/proto/uwsgi.c @@ -1,6 +1,6 @@ /* async uwsgi protocol parser */ -#include +#include "uwsgi.h" extern struct uwsgi_server uwsgi; diff --git a/setup.cpyext.py b/setup.cpyext.py index c29b1d3a9a..9a238d7ab0 100644 --- a/setup.cpyext.py +++ b/setup.cpyext.py @@ -12,9 +12,8 @@ import shlex import uwsgiconfig -from setuptools import setup +from setuptools import setup, Extension from setuptools.command.build_ext import build_ext -from distutils.core import Extension class uWSGIBuildExt(build_ext): diff --git a/setup.py b/setup.py index cc1868c7dc..1b1446244e 100644 --- a/setup.py +++ b/setup.py @@ -144,6 +144,12 @@ def get_extra_require(): 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], install_requires=get_extra_require() ) diff --git a/t/cachetest.py b/t/cachetest.py index b717e8d3d6..8dc93946d5 100644 --- a/t/cachetest.py +++ b/t/cachetest.py @@ -13,7 +13,7 @@ def gen_rand_n(max_n): def gen_rand_s(size): return ''.join(random.choice(string.letters) for i in range(size)) -print 'filling cache...' +print('filling cache...') for i in range(0, 1000): kl = gen_rand_n(200) key = gen_rand_s(kl) @@ -22,14 +22,14 @@ def gen_rand_s(size): items[key] = val uwsgi.cache_set(key, val) -print 'checking cache...' +print('checking cache...') count = 0 for key in items.keys(): val = uwsgi.cache_get(key) count += 1 if val != items[key]: - print len(val), val - print len(items[key]), items[key] + print(len(val), val) + print(len(items[key]), items[key]) raise Exception('CACHE TEST FAILED AFTER %d ITERATIONS !!!' % count) -print "TEST PASSED" +print("TEST PASSED") diff --git a/t/cgi/hello.cgi b/t/cgi/hello.cgi new file mode 100755 index 0000000000..b3e56cfcc2 --- /dev/null +++ b/t/cgi/hello.cgi @@ -0,0 +1,6 @@ +#!/bin/sh + +echo "Content-Type: text/plain" +echo +echo "Hello world!" +echo "PATH_INFO=${PATH_INFO}" diff --git a/t/core/readline/__init__.py b/t/core/readline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/t/core/readline/app.py b/t/core/readline/app.py new file mode 100644 index 0000000000..168d6d01dd --- /dev/null +++ b/t/core/readline/app.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +# uwsgi --plugin python,http --http 0.0.0.0:8000 -w app + +from werkzeug.wrappers import Request, Response + + +def application(env, start_response): + request = Request(env) + lines = b"" + while True: + line = request.stream.readline() + if not line: + break; + lines += line + response = Response(lines) + return response(env, start_response) diff --git a/t/core/readline/client.py b/t/core/readline/client.py new file mode 100644 index 0000000000..0a8734d435 --- /dev/null +++ b/t/core/readline/client.py @@ -0,0 +1,8 @@ +import requests + +headers = {'Content-Type': 'application/octet-stream'} +data = '\n'.join(['{:04}'.format(i) for i in range(1001)] + ['final']) + +r = requests.post("http://127.0.0.1:8000", data=data, headers=headers) + +assert r.text == data diff --git a/t/core/readline/requirements.txt b/t/core/readline/requirements.txt new file mode 100644 index 0000000000..82b955f6ef --- /dev/null +++ b/t/core/readline/requirements.txt @@ -0,0 +1,2 @@ +requests +Werkzeug diff --git a/t/php/config.ini b/t/php/config.ini index 35a6dee9f6..b1bea5d48f 100644 --- a/t/php/config.ini +++ b/t/php/config.ini @@ -1,10 +1,13 @@ [uwsgi] http-socket = :8080 http-socket-modifier1 = 14 +# required for php +need-app = false +plugins = php cache2 = name=session,items=1000,store=/tmp/uwsgi-session-cache,bitmap=1 php-set = session.save_handler=uwsgi php-set = session.save_path=session -php-docroot = t/php/ +php-docroot = %d diff --git a/t/pypy/config.ini b/t/pypy/config.ini new file mode 100644 index 0000000000..1e9e71bd6c --- /dev/null +++ b/t/pypy/config.ini @@ -0,0 +1,7 @@ +[uwsgi] +plugin = pypy +need-app = false +pypy-lib = /usr/lib/x86_64-linux-gnu/libpypy3-c.so +pypy-home = /usr/lib/pypy3 +pypy-setup = %d../../plugins/pypy/pypy_setup.py +pypy-wsgi-file = %d../python/helloapp.py diff --git a/t/pypy/t_continulet1.py b/t/pypy/t_continulet1.py index d4ad90c453..668797dfbc 100644 --- a/t/pypy/t_continulet1.py +++ b/t/pypy/t_continulet1.py @@ -14,7 +14,7 @@ def application(e, sr): # call suspend 10 times and yield some value for i in range(0, 10): - print i + print(i) uwsgi.suspend() yield str(i) @@ -36,7 +36,7 @@ def application(e, sr): finally: uwsgi.close(fd) - print "sleeping for 3 seconds..." + print("sleeping for 3 seconds...") uwsgi.async_sleep(3) uwsgi.suspend() yield "done" diff --git a/t/pypy/t_continulet2.py b/t/pypy/t_continulet2.py index 0d7ab6d1e6..df700f2fa9 100644 --- a/t/pypy/t_continulet2.py +++ b/t/pypy/t_continulet2.py @@ -14,7 +14,7 @@ def application(e, sr): # suspend 10 times and yield a value for i in range(1, 10): - print i + print(i) uwsgi.suspend() yield str(i) @@ -43,7 +43,7 @@ def application(e, sr): # always ensure sockets are closed uwsgi.close(fd) - print "sleeping for 3 seconds..." + print("sleeping for 3 seconds...") uwsgi.async_sleep(3) uwsgi.suspend() yield "done" diff --git a/t/python/helloapp.py b/t/python/helloapp.py new file mode 100644 index 0000000000..49f1907cc3 --- /dev/null +++ b/t/python/helloapp.py @@ -0,0 +1,3 @@ +def application(env, start_response): + start_response("200 OK", [("Content-Type", "text/html")]) + return [b"Hello World"] diff --git a/t/python/manage_script_name/manage_script_name_test.ini b/t/python/manage_script_name/manage_script_name_test.ini index f2287768f5..3563486fce 100644 --- a/t/python/manage_script_name/manage_script_name_test.ini +++ b/t/python/manage_script_name/manage_script_name_test.ini @@ -1,8 +1,4 @@ [uwsgi] -http-socket = :8080 - -master = 1 - ; Three apps on three mountpoints wsgi-file = %d/useless_app.py diff --git a/t/python/manage_script_name/test_manage_script_name.py b/t/python/manage_script_name/test_manage_script_name.py deleted file mode 100644 index 0908b0fc3c..0000000000 --- a/t/python/manage_script_name/test_manage_script_name.py +++ /dev/null @@ -1,59 +0,0 @@ -#! /usr/bin/env python3 -# coding = utf-8 -# author = Adriano Di Luzio - -# I require requests! - -""" -First run: - $ ./uwsgi t/python/manage_script_name/manage_script_name_test.ini - -Then run me! -""" - -import unittest -import requests - -HOST = "http://127.0.0.1:8080" - - -class ManageScriptNameTest(unittest.TestCase): - - def test_classic_mountpoints(self): - mps = { - "/foo", - "/foobis/", - "/footris/" - } - - for mp in mps: - # Requests to /foo should kick-in the managed script name. - r = requests.get(HOST + mp) - self.assertEqual(r.text, mp) - - ends = mp.endswith("/") - - # And equally requests to /foo/ - r = requests.get( - HOST + mp + "/") if not ends else requests.get(HOST + mp[:-1]) - self.assertEqual(r.text, mp) - - # Or correct requests (/foo/resource) - r = requests.get( - HOST + mp + "/" + "resource") if not ends else requests.get(HOST + mp + "resource") - self.assertEqual(r.text, mp) - - def test_intriguing_mountpoints(self): - mps = { - "/fooanything", - "/foobisis/", - "/foofighters", - } - - for mp in mps: - r = requests.get(HOST + mp) - self.assertEqual(r.text, "") - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/t/python/manage_script_name/useless_app.py b/t/python/manage_script_name/useless_app.py index b19c08104c..115f4dbc9e 100644 --- a/t/python/manage_script_name/useless_app.py +++ b/t/python/manage_script_name/useless_app.py @@ -1,3 +1,3 @@ def application(env, start_response): start_response('200 OK', [('Content-Type', 'text/html')]) - return env['SCRIPT_NAME'] + return [env['SCRIPT_NAME'].encode('ascii'), ] diff --git a/t/rack/app.ru b/t/rack/app.ru new file mode 100644 index 0000000000..4603375e7d --- /dev/null +++ b/t/rack/app.ru @@ -0,0 +1,9 @@ +class App + + def call(environ) + [200, {"content-type" => "text/plain"}, ['Hello']] + end + +end + +run App.new diff --git a/t/runner b/t/runner new file mode 100755 index 0000000000..b59966b5d4 --- /dev/null +++ b/t/runner @@ -0,0 +1,258 @@ +#!/usr/bin/python3 +# +# This test suite runner runs some integration tests for uwsgi, that is +# each test launches a test server with a specific configuration and +# verifies (usually using a HTTP request) that this test server behaves as +# expected. +# +# buildconf/integration-tests.ini holds the build configuration for this +# to run fine. + + +import os +import requests +import signal +import socket +import subprocess +import sys +import time +import unittest + + +TESTS_DIR = os.path.dirname(__file__) +UWSGI_BINARY = os.getenv("UWSGI_BINARY", os.path.join(TESTS_DIR, "..", "uwsgi")) +UWSGI_PLUGINS = os.getenv("UWSGI_PLUGINS_TEST", "all").split(" ") +UWSGI_ADDR = "127.0.0.1" +UWSGI_PORT = 8000 +UWSGI_HTTP = f"{UWSGI_ADDR}:{UWSGI_PORT}" + + +def plugins_available(plugins): + available = False + if "all" in UWSGI_PLUGINS: + available = True + else: + available = all([plugin in UWSGI_PLUGINS for plugin in plugins]) + return available, f"{plugins} plugins not available but required for this test case" + + +class UwsgiTest(unittest.TestCase): + + def start_server(self, args): + self.testserver = subprocess.Popen( + [UWSGI_BINARY] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + def uwsgi_ready(self): + try: + s = socket.socket() + s.connect( + ( + UWSGI_ADDR, + UWSGI_PORT, + ) + ) + except socket.error: + return False + else: + return True + finally: + s.close() + + def start_listen_server(self, args): + self.start_server(["--http-socket", UWSGI_HTTP] + args) + + # ensure server is ready + retries = 10 + while not self.uwsgi_ready() and retries > 0: + time.sleep(0.1) + retries = retries - 1 + if retries == 0: + raise RuntimeError("uwsgi test server is not available") + + def tearDown(self): + if hasattr(self._outcome, "errors"): + # Python 3.4 - 3.10 (These two methods have no side effects) + result = self.defaultTestResult() + self._feedErrorsToResult(result, self._outcome.errors) + else: + # Python 3.11+ + result = self._outcome.result + ok = not (result.errors + result.failures) + + if hasattr(self, "testserver"): + self.testserver.send_signal(signal.SIGTERM) + if not ok: + print(self.testserver.stdout.read(), file=sys.stderr) + + self.testserver.wait() + self.testserver.stdout.close() + + def assert_GET_body(self, url_path, body_expected): + with requests.get(f"http://{UWSGI_HTTP}{url_path}") as r: + self.assertEqual(r.text, body_expected) + + def test_static_expires(self): + self.start_listen_server( + [ + "--plugin", + "notfound", + os.path.join(TESTS_DIR, "static", "config.ini"), + ] + ) + + with requests.get(f"http://{UWSGI_HTTP}/foobar/config.ini") as r: + self.assertTrue("Expires" in r.headers) + + @unittest.skipUnless(*plugins_available(["python"])) + def test_mountpoints(self): + self.start_listen_server( + [ + "--plugin", + "python", + os.path.join( + TESTS_DIR, + "python", + "manage_script_name", + "manage_script_name_test.ini", + ), + ] + ) + + mps = {"/foo", "/foobis/", "/footris/"} + + for mp in mps: + # Requests to /foo should kick-in the managed script name. + self.assert_GET_body(mp, mp) + + ends = mp.endswith("/") + + # And equally requests to /foo/ + self.assert_GET_body(f"{mp}/" if not ends else f"{mp}"[:-1], mp) + + # Or correct requests (/foo/resource) + self.assert_GET_body(f"{mp}/resource" if not ends else f"{mp}resource", mp) + + mps = { + "/fooanything", + "/foobisis/", + "/foofighters", + } + + for mp in mps: + self.assert_GET_body(mp, "") + + @unittest.skipUnless(*plugins_available(["python"])) + def test_python3_helloworld(self): + self.start_listen_server( + [ + "--plugin", + "python", + "--wsgi-file", + os.path.join(TESTS_DIR, "python", "helloapp.py"), + ] + ) + + self.assert_GET_body("/", "Hello World") + + @unittest.skipUnless(*plugins_available(["pypy"])) + def test_pypy3_helloworld(self): + self.start_listen_server( + [ + os.path.join(TESTS_DIR, "pypy", "config.ini"), + ] + ) + + self.assert_GET_body("/", "Hello World") + + @unittest.skipUnless(*plugins_available(["php"])) + def test_php_session(self): + self.start_listen_server( + [ + os.path.join(TESTS_DIR, "php", "config.ini"), + ] + ) + + self.assert_GET_body("/test.php", "PASS\n") + + @unittest.skipUnless(*plugins_available(["jvm"])) + def test_jvm_hellworld(self): + classpath = ":".join( + [ + "/usr/share/java/uwsgi.jar", + os.path.join(TESTS_DIR, "java"), + os.path.join(TESTS_DIR, "..", "plugins", "jvm"), + ] + ) + + subprocess.call( + [ + "javac", + "-classpath", + classpath, + os.path.join(TESTS_DIR, "java", "rpc.java"), + ] + ) + + self.start_listen_server( + [ + "--need-app=0", + "--plugins", + "0:jvm,jwsgi", + "--jvm-classpath", + classpath, + "--jwsgi", + "rpc:application", + ] + ) + + self.assert_GET_body("/", "

null

") + + @unittest.skipUnless(*plugins_available(["psgi"])) + def test_psgi_helloworld(self): + self.start_listen_server( + [ + "--plugins", + "psgi", + "--psgi", + os.path.join(TESTS_DIR, "perl", "test_hello.psgi"), + ] + ) + + self.assert_GET_body("/", "Hello, world!") + + @unittest.skipUnless(*plugins_available(["cgi"])) + def test_cgi_helloworld(self): + self.start_listen_server( + [ + "--need-app=0", + "--plugins", + "0:cgi", + "--cgi", + os.path.join(TESTS_DIR, "cgi", "hello.cgi"), + ] + ) + + self.assert_GET_body( + "/foobar/say_hello", "Hello world!\nPATH_INFO=/foobar/say_hello\n" + ) + + @unittest.skipUnless(*plugins_available(["rack"])) + def test_rack_helloworld(self): + self.start_listen_server( + [ + "--plugins", + "0:rack", + "--rack", + os.path.join(TESTS_DIR, "rack", "app.ru"), + ] + ) + + self.assert_GET_body("/", "Hello") + + +if __name__ == "__main__": + unittest.main() diff --git a/t/sharedarea/bigranges.py b/t/sharedarea/bigranges.py index 8543221ec9..4ac5fdcc41 100644 --- a/t/sharedarea/bigranges.py +++ b/t/sharedarea/bigranges.py @@ -4,12 +4,12 @@ class SharedareaTest(unittest.TestCase): def test_32(self): - pos = 2L * (1024L ** 3) + pos = 2 * (1024 ** 3) uwsgi.sharedarea_write32(0, pos, 17) self.assertEqual(uwsgi.sharedarea_read32(0, pos), 17) def test_64(self): - pos = 2L * (1024L ** 3) + pos = 2 * (1024 ** 3) uwsgi.sharedarea_write64(0, pos, 30) self.assertEqual(uwsgi.sharedarea_read64(0, pos), 30) diff --git a/t/static/config.ini b/t/static/config.ini new file mode 100644 index 0000000000..b6bedc0ed5 --- /dev/null +++ b/t/static/config.ini @@ -0,0 +1,4 @@ +[uwsgi] +need-app = False +static-map = /foobar=%d +static-expires-uri = ^/foobar/ 315360000 diff --git a/tests/cpubound_stackless.py b/tests/cpubound_stackless.py index a4c397246a..5934a02d43 100644 --- a/tests/cpubound_stackless.py +++ b/tests/cpubound_stackless.py @@ -11,4 +11,4 @@ def application(env, start_response): if i % 2 == 0: stackless.schedule() - print "DONE AT %d" % i + print("DONE AT %d" % i) diff --git a/tests/deadlocks/main.py b/tests/deadlocks/main.py new file mode 100644 index 0000000000..66e3b37ddf --- /dev/null +++ b/tests/deadlocks/main.py @@ -0,0 +1,4 @@ +def application(env, start_response): + start_response("200 OK", [("Content-Type", "text/plain")]) + message = "Hello World" + return [message.encode("utf-8")] diff --git a/tests/deadlocks/master-nothreads.ini b/tests/deadlocks/master-nothreads.ini new file mode 100644 index 0000000000..15f79da446 --- /dev/null +++ b/tests/deadlocks/master-nothreads.ini @@ -0,0 +1,6 @@ +[uwsgi] +show-config = true +master = true +enable-threads = false +workers = 1 +py-call-uwsgi-fork-hooks = true diff --git a/tests/deadlocks/master-singleinterpreter-threads-10workers.ini b/tests/deadlocks/master-singleinterpreter-threads-10workers.ini new file mode 100644 index 0000000000..47812651b9 --- /dev/null +++ b/tests/deadlocks/master-singleinterpreter-threads-10workers.ini @@ -0,0 +1,7 @@ +[uwsgi] +show-config = true +master = true +enable-threads = true +workers = 10 +single-interpreter = true +py-call-uwsgi-fork-hooks = true diff --git a/tests/deadlocks/master-singleinterpreter-threads-1worker.ini b/tests/deadlocks/master-singleinterpreter-threads-1worker.ini new file mode 100644 index 0000000000..61a943b039 --- /dev/null +++ b/tests/deadlocks/master-singleinterpreter-threads-1worker.ini @@ -0,0 +1,7 @@ +[uwsgi] +show-config = true +master = true +enable-threads = true +workers = 1 +single-interpreter = true +py-call-uwsgi-fork-hooks = true diff --git a/tests/deadlocks/master-threads-10workers.ini b/tests/deadlocks/master-threads-10workers.ini new file mode 100644 index 0000000000..3dc7de510b --- /dev/null +++ b/tests/deadlocks/master-threads-10workers.ini @@ -0,0 +1,6 @@ +[uwsgi] +show-config = true +master = true +enable-threads = true +workers = 10 +py-call-uwsgi-fork-hooks = true diff --git a/tests/deadlocks/master-threads-1worker.ini b/tests/deadlocks/master-threads-1worker.ini new file mode 100644 index 0000000000..f9d2382c4f --- /dev/null +++ b/tests/deadlocks/master-threads-1worker.ini @@ -0,0 +1,6 @@ +[uwsgi] +show-config = true +master = true +enable-threads = true +workers = 1 +py-call-uwsgi-fork-hooks = true diff --git a/tests/deadlocks/nomaster-threads-10workers.ini b/tests/deadlocks/nomaster-threads-10workers.ini new file mode 100644 index 0000000000..7f6ff91b8a --- /dev/null +++ b/tests/deadlocks/nomaster-threads-10workers.ini @@ -0,0 +1,6 @@ +[uwsgi] +show-config = true +master = false +enable-threads = true +workers = 10 +py-call-uwsgi-fork-hooks = true diff --git a/tests/deadlocks/nomaster-threads-1worker.ini b/tests/deadlocks/nomaster-threads-1worker.ini new file mode 100644 index 0000000000..8252820817 --- /dev/null +++ b/tests/deadlocks/nomaster-threads-1worker.ini @@ -0,0 +1,6 @@ +[uwsgi] +show-config = true +master = false +enable-threads = true +workers = 1 +py-call-uwsgi-fork-hooks = true diff --git a/tests/deadlocks/sitecustomize.py b/tests/deadlocks/sitecustomize.py new file mode 100644 index 0000000000..0aff259f7b --- /dev/null +++ b/tests/deadlocks/sitecustomize.py @@ -0,0 +1,14 @@ +import threading +import time + + +def run(): + print("[DEADLOCKS] started run") + st = time.time() + while time.time() < st + 5: + pass + print("[DEADLOCKS] finished run") + +t = threading.Thread(target=run) +t.daemon = True +t.start() diff --git a/tests/decoratortest.py b/tests/decoratortest.py index 8a2f41b95d..8117fc3572 100644 --- a/tests/decoratortest.py +++ b/tests/decoratortest.py @@ -8,6 +8,9 @@ import time +from six.moves import range + + # register rpc function helloworld @rpc("helloworld") def hello2(): @@ -47,7 +50,7 @@ def tmpmodified(num): # spool a long running task @spool def a_long_task(args): - for i in xrange(1, 10): + for i in range(1, 10): print("%s = %d" % (str(args), i)) print(uwsgi.call('helloworld')) time.sleep(1) @@ -56,7 +59,7 @@ def a_long_task(args): # continuously spool a long running task @spoolforever def an_infinite_task(args): - for i in xrange(1, 4): + for i in range(1, 4): print("infinite: %d %s" % (i, str(args))) print(uwsgi.call('helloworld')) uwsgi.signal(100) @@ -127,7 +130,7 @@ def fork_happened2(): @lock def locked_func(): print("starting locked function on worker %d" % uwsgi.worker_id()) - for i in xrange(1, 5): + for i in range(1, 5): time.sleep(1) print("[locked %d] waiting..." % uwsgi.worker_id()) print("done with locked function on worker %d" % uwsgi.worker_id()) diff --git a/tests/gevent_spool.py b/tests/gevent_spool.py index bdf90363c9..2742466f0c 100644 --- a/tests/gevent_spool.py +++ b/tests/gevent_spool.py @@ -4,7 +4,7 @@ @spool def longtask(*args): - print args + print(args) return uwsgi.SPOOL_OK diff --git a/tests/gh-deadlocks.sh b/tests/gh-deadlocks.sh new file mode 100755 index 0000000000..328f338c15 --- /dev/null +++ b/tests/gh-deadlocks.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -u + +PYTHON_VERSION="$1" +. "./tests/gh-shared.sh" + + +for INI_FILE in tests/deadlocks/*.ini ; do + test_python_deadlocks "${PYTHON_VERSION}" "$INI_FILE" +done + +results diff --git a/tests/gh-python.sh b/tests/gh-python.sh new file mode 100755 index 0000000000..903758b3ea --- /dev/null +++ b/tests/gh-python.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -u + +PYTHON_VERSION="$1" +. "./tests/gh-shared.sh" + + +for WSGI_FILE in tests/staticfile.py tests/testworkers.py tests/testrpc.py ; do + test_python "${PYTHON_VERSION}" "${WSGI_FILE}" +done + +results diff --git a/tests/gh-rack.sh b/tests/gh-rack.sh new file mode 100755 index 0000000000..1cdaa599ff --- /dev/null +++ b/tests/gh-rack.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -u + +RACK_VERSION="$1" +. "./tests/gh-shared.sh" + + +for RACK in examples/config2.ru ; do + test_rack "${RACK_VERSION}" "${RACK}" +done + +results diff --git a/tests/gh-shared.sh b/tests/gh-shared.sh new file mode 100755 index 0000000000..bad6dd043b --- /dev/null +++ b/tests/gh-shared.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +txtund=$(tput sgr 0 1) # underline +txtbld=$(tput bold) # bold +bldred=${txtbld}$(tput setaf 1) # red +bldgre=${txtbld}$(tput setaf 2) # green +bldyel=${txtbld}$(tput setaf 3) # yellow +bldblu=${txtbld}$(tput setaf 4) # blue +txtcya=$(tput setaf 6) # cyan +bldwht=${txtbld}$(tput setaf 7) # white +txtrst=$(tput sgr0) # reset + + +ERROR=0 +SUCCESS=0 + + +die() { + date > reload.txt + sleep 3 + pidof uwsgi && killall uwsgi + sleep 1 + pidof uwsgi && killall -9 uwsgi + echo -e "$@" + if [ -e uwsgi.log ]; then + echo -e "${bldyel}>>> uwsgi.log:${txtrst}" + echo -e "${txtcya}" + cat uwsgi.log + echo -e "${txtrst}" + fi +} + + +http_test() { + URL=$1 + UPID=`pidof uwsgi` + if [ "$UPID" != "" ]; then + echo -e "${bldgre}>>> Spawned PID $UPID, running tests${txtrst}" + sleep 5 + curl -fI $URL + RET=$? + if [ $RET != 0 ]; then + die "${bldred}>>> Error during curl run${txtrst}" + ERROR=$((ERROR+1)) + else + SUCCESS=$((SUCCESS+1)) + fi + die "${bldyel}>>> SUCCESS: Done${txtrst}" + else + die "${bldred}>>> ERROR: uWSGI did not start${txtrst}" + ERROR=$((ERROR+1)) + fi +} + + +test_python() { + date > reload.txt + rm -f uwsgi.log + echo -e "${bldyel}================== TESTING $1 $2 =====================${txtrst}" + echo -e "${bldyel}>>> Spawning uWSGI python app${txtrst}" + echo -en "${bldred}" + ./uwsgi --master --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --wsgi-file $2 --daemonize uwsgi.log + sleep 1 + echo -en "${txtrst}" + http_test "http://localhost:8080/" + echo -e "${bldyel}===================== DONE $1 $2 =====================${txtrst}\n\n" +} + + +test_python_deadlocks() { + date > reload.txt + rm -f uwsgi.log + echo -e "${bldyel}================== TESTING DEADLOCKS $1 $2 =====================${txtrst}" + echo -e "${bldyel}>>> Starting python app${txtrst}" + echo -en "${bldred}" + # initialize with tests/deadlocks/sitecustomize.py + PYTHONPATH=tests/deadlocks ./uwsgi --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --wsgi-file tests/deadlocks/main.py --ini $2 --daemonize uwsgi.log + sleep 1 + echo -en "${txtrst}" + http_test "http://localhost:8080/" + echo -e "${bldyel}===================== DONE $1 $2 =====================${txtrst}\n\n" +} + + +test_rack() { + date > reload.txt + rm -f uwsgi.log + # the code assumes that ruby environment is activated by `rvm use` + echo -e "${bldyel}================== TESTING $1 $2 =====================${txtrst}" + echo -e "${bldyel}>>> Installing sinatra gem using gem${txtrst}" + sudo gem install sinatra -v 2.2.2 || die + echo -e "${bldyel}>>> Spawning uWSGI rack app${txtrst}" + echo -en "${bldred}" + ./uwsgi --master --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --rack $2 --daemonize uwsgi.log + echo -en "${txtrst}" + http_test "http://localhost:8080/hi" + echo -e "${bldyel}===================== DONE $1 $2 =====================${txtrst}\n\n" +} + +results() { + echo "${bldgre}>>> $SUCCESS SUCCESSFUL TEST(S)${txtrst}" + if [ $ERROR -ge 1 ]; then + echo "${bldred}>>> $ERROR FAILED TEST(S)${txtrst}" + exit 1 + fi + + exit 0 +} diff --git a/tests/grunter.py b/tests/grunter.py index d3af2c77ce..1efbb12485 100644 --- a/tests/grunter.py +++ b/tests/grunter.py @@ -2,6 +2,9 @@ import time +from six.moves import range + + def application(env, start_response): start_response('200 Ok', [('Content-Type', 'text/html')]) @@ -11,11 +14,11 @@ def application(env, start_response): grunt = uwsgi.grunt() if grunt is None: - print "worker %d detached" % uwsgi.worker_id() + print("worker %d detached" % uwsgi.worker_id()) else: yield "And now i am the grunt with a fix worker id of %d
" % uwsgi.worker_id() time.sleep(2) yield "Now, i will start a very slow task...
" - for i in xrange(1, 10): + for i in range(1, 10): yield "waiting for %d seconds
" % i time.sleep(i) diff --git a/tests/harakiri.py b/tests/harakiri.py new file mode 100644 index 0000000000..88c24afa4d --- /dev/null +++ b/tests/harakiri.py @@ -0,0 +1,25 @@ +# ./uwsgi --master --http :8080 --harakiri 1 --wsgi-file tests/harakiri.py --harakiri-graceful-timeout 1 --py-call-osafterfork --lazy-apps --enable-threads --threads 2 --harakiri-graceful-signal 31 +import time +import uwsgi +import signal +import sys +import atexit + +def sig_handler(n, fp): + print("[Python App] attempting graceful shutdown triggered by harakiri (signal %d)" % n) + exit(1) + +def application(e, s): + print("[Python App] sleeping") + time.sleep(3) + s('200 OK', [('Content-Type', 'text/html')]) + return [b"OK"] + + +def exit_handler(): + time.sleep(3) + # Should not reach this line (graceful harakiri deadline expired) + print("[Python App] exiting now") + +atexit.register(exit_handler) +signal.signal(signal.SIGSYS, sig_handler) diff --git a/tests/iobound_async_unix.py b/tests/iobound_async_unix.py index 8fb23e7855..81051bc810 100644 --- a/tests/iobound_async_unix.py +++ b/tests/iobound_async_unix.py @@ -44,7 +44,7 @@ def application(env, start_response): for r in send_request(env, s): yield r else: - print c + print(c) start_response('500 Internal Server Error', [('Content-Type', 'text/plain')]) yield "Internal Server Error" diff --git a/tests/mulefunc.py b/tests/mulefunc.py index a05bebb2b9..24ae60239b 100644 --- a/tests/mulefunc.py +++ b/tests/mulefunc.py @@ -3,22 +3,22 @@ @timer(3, target='mule1') def hello_timer(signum): - print "3 seconds elapsed" + print("3 seconds elapsed") @mulefunc def conto_fino_a_dieci(uno, due, tre): - print "MULE ID %d: conto_fino_a_dieci" % uwsgi.mule_id(), uno, due, tre + print("MULE ID %d: conto_fino_a_dieci" % uwsgi.mule_id(), uno, due, tre) @mulefunc(2) def conto_fino_a_venti(uno, due, tre): - print "MULE ID %d: conto_fino_a_venti" % uwsgi.mule_id(), uno, due, tre + print("MULE ID %d: conto_fino_a_venti" % uwsgi.mule_id(), uno, due, tre) @mulefunc('topogigio') def conto_fino_a_trenta(uno, due, tre): - print "MULE ID %d: conto_fino_a_trenta" % uwsgi.mule_id(), uno, due, tre + print("MULE ID %d: conto_fino_a_trenta" % uwsgi.mule_id(), uno, due, tre) def application(e, sr): diff --git a/tests/myadmin.py b/tests/myadmin.py index f05df27ce0..3d5ced0b2e 100644 --- a/tests/myadmin.py +++ b/tests/myadmin.py @@ -2,7 +2,7 @@ import struct import sys -print sys.argv +print(sys.argv) if len(sys.argv) == 3: chunks = uwsgi.send_message(sys.argv[1], 10, int(sys.argv[2]), '') @@ -11,6 +11,6 @@ for chunk in chunks: pkt += chunk - print "%d = %d" % (int(sys.argv[2]), struct.unpack("I", pkt)[0]) + print("%d = %d" % (int(sys.argv[2]), struct.unpack("I", pkt)[0])) elif len(sys.argv) == 4: uwsgi.send_message(sys.argv[1], 10, int(sys.argv[2]), struct.pack("I", int(sys.argv[3]))) diff --git a/tests/pgbound_async.py b/tests/pgbound_async.py index b7d6101cb8..b4728cc75f 100644 --- a/tests/pgbound_async.py +++ b/tests/pgbound_async.py @@ -23,7 +23,7 @@ def application(env, start_response): for i in pg_wait(connection, env, 3): yield i - print "connected" + print("connected") cursor = connection.cursor() cursor.execute("SELECT * FROM foo") @@ -31,7 +31,7 @@ def application(env, start_response): for i in pg_wait(cursor.connection, env, 3): yield i - print "query result available" + print("query result available") for record in cursor: yield str(record) diff --git a/tests/picazzo.py b/tests/picazzo.py index 87e197434d..39a9bb2105 100644 --- a/tests/picazzo.py +++ b/tests/picazzo.py @@ -12,7 +12,7 @@ def login(req): def login_post(req): - print req + print(req) req["session"]["user"] = "James" return redirect("/") diff --git a/tests/refcount.py b/tests/refcount.py index c83085f4d0..97dc4387cc 100644 --- a/tests/refcount.py +++ b/tests/refcount.py @@ -3,5 +3,5 @@ def application(e, sr): sr('200 OK', [('Content-Type', 'text/html')]) - print sys.gettotalrefcount() + print(sys.gettotalrefcount()) yield '%s' % sys.gettotalrefcount() diff --git a/tests/runningthread.py b/tests/runningthread.py index dfed522cfb..b222fe7e13 100644 --- a/tests/runningthread.py +++ b/tests/runningthread.py @@ -5,7 +5,7 @@ def mess(): while True: - for i in xrange(0, 100): + for i in range(0, 100): if uwsgi.ready(): uwsgi.signal(17) print(i) diff --git a/tests/sendchunked.py b/tests/sendchunked.py index 99b36657f5..84f4fdfe66 100644 --- a/tests/sendchunked.py +++ b/tests/sendchunked.py @@ -1,6 +1,9 @@ import socket import sys +from six.moves import input + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) (addr, port) = sys.argv[1].split(':') @@ -10,5 +13,5 @@ s.send("Transfer-Encoding: chunked\r\n\r\n") while True: - msg = raw_input("msg >> ") + msg = input("msg >> ") s.send("%X\r\n%s\r\n" % (len(msg), msg)) diff --git a/tests/signals.py b/tests/signals.py index 4a2e8592e0..49640bcd99 100644 --- a/tests/signals.py +++ b/tests/signals.py @@ -2,19 +2,19 @@ def hello_signal(num, payload): - print "i am the signal %d" % num + print("i am the signal %d" % num) def hello_signal2(num, payload): - print "i am the signal %d with payload: %s" % (num, payload) + print("i am the signal %d with payload: %s" % (num, payload)) def hello_file(num, filename): - print "file %s has been modified !!!" % filename + print("file %s has been modified !!!" % filename) def hello_timer(num, secs): - print "%s seconds elapsed" % secs + print("%s seconds elapsed" % secs) # uwsgi.register_signal(30, uwsgi.SIGNAL_KIND_WORKER, hello_signal) uwsgi.register_signal(30, "workers", hello_signal) diff --git a/tests/sleeping_async.py b/tests/sleeping_async.py index 7a5d536ae5..3a2af66d32 100644 --- a/tests/sleeping_async.py +++ b/tests/sleeping_async.py @@ -6,5 +6,5 @@ def application(env, start_response): start_response('200 Ok', [('Content-type', 'text/html')]) yield uwsgi.async_sleep(sleepvalue) - # print "TIMEOUT: ", env['x-wsgiorg.fdevent.timeout'] + # print("TIMEOUT: ", env['x-wsgiorg.fdevent.timeout']) yield "

Hello World after %d seconds

" % sleepvalue diff --git a/tests/sleeping_green.py b/tests/sleeping_green.py index df9ee8a5dd..355ac1aa2c 100644 --- a/tests/sleeping_green.py +++ b/tests/sleeping_green.py @@ -10,5 +10,5 @@ def application(env, start_response): start_response('200 Ok', [('Content-type', 'text/html')]) start_at = time.time() uwsgi.green_sleep(sleepvalue) - # print "TIMEOUT: ", env['x-wsgiorg.fdevent.timeout'] + # print("TIMEOUT: ", env['x-wsgiorg.fdevent.timeout']) yield "

Hello World after %s seconds

" % str(time.time() - start_at) diff --git a/tests/sleepthreadasync.py b/tests/sleepthreadasync.py index 5795099f84..ef6d9da2e3 100644 --- a/tests/sleepthreadasync.py +++ b/tests/sleepthreadasync.py @@ -6,7 +6,7 @@ def foo(): while True: time.sleep(1) - print "ciao, sono un thread" + print("ciao, sono un thread") t = threading.Thread(target=foo) t.daemon = True diff --git a/tests/slow.py b/tests/slow.py index 61ca5d5b05..f82e328b2c 100644 --- a/tests/slow.py +++ b/tests/slow.py @@ -3,11 +3,11 @@ def application(e, s): - print "locking" + print("locking") uwsgi.lock() - print "locked" + print("locked") time.sleep(3) uwsgi.unlock() - print "UN-locked" + print("UN-locked") s('200 OK', [('Content-Type', 'text/html')]) return "slow" diff --git a/tests/static/test.txt b/tests/static/test.txt new file mode 100644 index 0000000000..871bb87ecb --- /dev/null +++ b/tests/static/test.txt @@ -0,0 +1 @@ +cookie diff --git a/tests/static/test2.txt b/tests/static/test2.txt new file mode 100644 index 0000000000..8504e7eaeb --- /dev/null +++ b/tests/static/test2.txt @@ -0,0 +1 @@ +cookie2 diff --git a/tests/testapp.py b/tests/testapp.py index 99ec401877..fb3e1386a1 100644 --- a/tests/testapp.py +++ b/tests/testapp.py @@ -1,5 +1,6 @@ import uwsgi +from six.moves import reload_module import time import sys import os @@ -19,7 +20,7 @@ class testthread(Thread): def run(self): while 1: time.sleep(2) - print "i am a terrible python thread of the uWSGI master process", uwsgi.applications + print("i am a terrible python thread of the uWSGI master process", uwsgi.applications) tthread = testthread() @@ -29,24 +30,24 @@ def run(self): p = "serena" #while 1: -#print "MARSHALLED OUT: ",uwsgi.send_uwsgi_message("127.0.0.1", 3033, 33, 17, {'prodotto':p, 'tempo': time.time(), 'pippo':'pluto', 'topolino':'paperino', 'callable':4+1, 'nullo': None, 'embedded': {'a':1} }, 17) +#print("MARSHALLED OUT: ",uwsgi.send_uwsgi_message("127.0.0.1", 3033, 33, 17, {'prodotto':p, 'tempo': time.time(), 'pippo':'pluto', 'topolino':'paperino', 'callable':4+1, 'nullo': None, 'embedded': {'a':1} }, 17)) def mako(filename, vars): return uwsgi.send_uwsgi_message("127.0.0.1", 3033, 33, 17, (filename, vars), 17) -#print uwsgi.send_uwsgi_message("127.0.0.1", 3033, 33, 17, ('makotest.txt', {'whattimeisit':time.time(), 'roberta':'serena'}), 17) +#print(uwsgi.send_uwsgi_message("127.0.0.1", 3033, 33, 17, ('makotest.txt', {'whattimeisit':time.time(), 'roberta':'serena'}), 17)) def myspooler(env): - print env + print(env) for i in range(1, 100): uwsgi.sharedarea_inclong(100) # time.sleep(1) uwsgi.spooler = myspooler -#print "SPOOLER: ", uwsgi.send_to_spooler({'TESTKEY':'TESTVALUE', 'APPNAME':'uWSGI'}) +#print("SPOOLER: ", uwsgi.send_to_spooler({'TESTKEY':'TESTVALUE', 'APPNAME':'uWSGI'})) def helloworld(): @@ -62,7 +63,7 @@ def force_harakiri(): def application(env, start_response): - print env + print(env) start_response('200 OK', [('Content-Type', 'text/plain')]) yield { '/': helloworld, @@ -71,7 +72,7 @@ def application(env, start_response): '/uwsgi/': helloworld }[env['PATH_INFO']]() - print env + print(env) def gomako(): @@ -94,9 +95,9 @@ def djangohomepage(): uwsgi.start_response('200 OK', [('Content-Type', 'text/html')]) t = Template("My name is {{ my_name }}.") c = Context({"my_name": "Serena"}) - print t, c + print(t, c) a = t.render(c) - print "ciao", a + print("ciao", a) yield str(a) @@ -109,9 +110,9 @@ def reload(env, start_response): #uwsgi.reload() -# print str(uwsgi.masterpid()) + "\n" +# print(str(uwsgi.masterpid()) + "\n") -# print "i am python" +# print("i am python") #yo() # yield "python" @@ -120,7 +121,7 @@ def reload(env, start_response): # yield str(uwsgi.masterpid()) - #print uwsgi.pippo + #print(uwsgi.pippo) #print 4/0 # try: @@ -130,7 +131,7 @@ def reload(env, start_response): # except Exception: # print "bah" -# print "ok" +# print("ok") # yield 4/0 @@ -153,16 +154,16 @@ def reload(env, start_response): yield '

workers

' for w in workers: - #print w - #print w['running_time'] + #print(w) + #print(w['running_time']) if w is not None: yield '' + str(w['id']) + '' + str(w['pid']) + '' + str(w['pid']) + '' + str(w['requests']) + '' + str(w['running_time']) + '' + str(w['vsz']) + '' + str(w['rss']) + '' - print w + print(w) yield '' #yield out - #print "FATTOfattoFATTO" + #print("FATTOfattoFATTO") def remotemako(env, start_response): @@ -174,9 +175,9 @@ def remotemako(env, start_response): ('192.168.173.5', 3434, [9001, 12000]), ('192.168.173.5', 3435, [12001, 15000]) ) - print clusters + print(clusters) all_values = uwsgi.send_multi_uwsgi_message(clusters, 33, 17, 40) - print all_values + print(all_values) return mako('makotest.txt', { 'whattimeisit': time.time(), 'roberta': 'serena', @@ -196,5 +197,5 @@ def remotemako(env, start_response): '/pippo': reload } -print uwsgi.applications -print uwsgi.applist +print(uwsgi.applications) +print(uwsgi.applist) diff --git a/tests/testfilewrapper.py b/tests/testfilewrapper.py new file mode 100644 index 0000000000..2575c1aed9 --- /dev/null +++ b/tests/testfilewrapper.py @@ -0,0 +1,122 @@ +from __future__ import print_function +import gc +import io +import os.path +import time + +import flask +import flask.helpers + + +application = flask.Flask(__name__) + +FILENAME = os.path.join(os.path.dirname(__file__), "static", "test.txt") +FILENAME2 = os.path.join(os.path.dirname(__file__), "static", "test2.txt") + +@application.after_request +def _after(response): + gc.collect() + fds = os.listdir("/proc/self/fd") + print("PY: objects:", len(gc.get_objects()), "fds:", len(fds)) + return response + +@application.route("/") +def index(): + return "HELLO\n" + +@application.route("/1") +def send_file_1(): + fp = open(FILENAME, "rb") + return flask.send_file(fp, attachment_filename="test.txt") + + +@application.route("/2") +def send_file_2(): + bio = io.BytesIO(b"cookie\n") + return flask.send_file(bio, attachment_filename="test.txt") + + +@application.route("/3") +def send_file_3(): + """ + What happens if we call the wsgi.file_wrapper twice? + + This should respond with cookie2 + """ + fp = open(FILENAME, "rb") + flask.send_file(fp, attachment_filename="test.txt") + fp = open(FILENAME2, "rb") + return flask.send_file(fp, attachment_filename="test.txt") + + +@application.route("/4") +def send_file_4(): + """ + Non-filelike object to send_file/wrap_file/wsgi.file_wrapper. + + AttributeError on the call to wsgi.file_wrapper. + """ + return flask.send_file(object(), attachment_filename="test.txt") + + +@application.route("/stream1") +def stream1(): + """ + Unrelated to wsgi.file_wrapper, just ensuring the iterator stuff still works. + """ + def _yield(): + start = time.time() + for i in range(3): + time.sleep(0.1) + yield " {:.2f} cookie".format(time.time() - start).encode("utf-8") + yield b"\n" + return flask.Response(_yield(), mimetype="text/plain") + + +@application.route("/stream2") +def stream2(): + """ + Yielding the result of a wrap_file call with a file object. + + gunicorn / werkzeug do not support this as it's not required. + """ + fp = open(FILENAME, "rb") + resp = flask.helpers.wrap_file(flask.request.environ, fp) + print("PY: resp after return", hex(id(resp))) + + def _yield(): + print("PY: _yield() run", hex(id(resp)), repr(resp)) + yield resp + return flask.Response(_yield()) + + +@application.route("/stream3") +def stream3(): + """ + Yielding the result of a wrap_file call with a BytesIO object. + + gunicorn / werkzeug do not support this as it's not required. + """ + bio = io.BytesIO(b"cookie\n") + resp = flask.helpers.wrap_file(flask.request.environ, bio) + + def _yield(): + yield resp + return flask.Response(_yield()) + + +@application.route("/stream4") +def stream4(): + """ + werkzeug logs: AssertionError: applications must write bytes + gunicorn logs: TypeError: is not a byte + uwsgi didn't log, should now.. + """ + fp = open(FILENAME, "rb") + resp = flask.send_file(fp, attachment_filename="test.txt") + print("PY: resp after return", hex(id(resp))) + + def _yield(): + print("PY: _yield() run", hex(id(resp)), repr(resp)) + yield resp + return flask.Response(_yield(), direct_passthrough=True) diff --git a/tests/testgevent.py b/tests/testgevent.py index 53849abff3..c0995d8aaf 100644 --- a/tests/testgevent.py +++ b/tests/testgevent.py @@ -6,21 +6,21 @@ def microtask(wid): - print "i am a gevent task" + print("i am a gevent task") gevent.sleep(10) - print "10 seconds elapsed in worker id %d" % wid + print("10 seconds elapsed in worker id %d" % wid) def athread(): while True: time.sleep(1) - print "i am the thread 1" + print("i am the thread 1") def athread2(): while True: time.sleep(1) - print "i am the thread 2" + print("i am the thread 2") t1 = Thread(target=athread) t1.daemon = True diff --git a/tests/testrpc.py b/tests/testrpc.py new file mode 100644 index 0000000000..c3363d6ade --- /dev/null +++ b/tests/testrpc.py @@ -0,0 +1,13 @@ +import uwsgi + +def hello(): + pass + +def application(env, start_response): + try: + uwsgi.register_rpc("A"*300, hello) + start_response('500 Buffer Overflow', [('Content-Type', 'text/plain')]) + except ValueError: + start_response('200 OK', [('Content-Type', 'text/plain')]) + + return () diff --git a/tests/testworkers.py b/tests/testworkers.py new file mode 100644 index 0000000000..ff84428341 --- /dev/null +++ b/tests/testworkers.py @@ -0,0 +1,29 @@ +""" +Regression test for #2056 - uwsgi.workers() leaking objects. +""" +import uwsgi +import gc + + +def application(env, start_response): + gc.collect() + start_objs = len(gc.get_objects()) + + for i in range(200): + workers = uwsgi.workers() + assert workers, "none/empty uwsgi.workers() - " + repr(workers) + for w in workers: + assert w["apps"], "none/empty apps in worker dict: " + repr(w) + + gc.collect() + end_objs = len(gc.get_objects()) + diff_objs = end_objs - start_objs + + # Sometimes there is a spurious diff of 4 objects or so. + if diff_objs > 10: + start_response('500 Leaking', [('Content-Type', 'text/plain')]) + yield "Leaking objects...\n".encode("utf-8") + else: + start_response('200 OK', [('Content-Type', 'text/plain')]) + + yield "{} {} {}\n".format(start_objs, end_objs, diff_objs).encode("utf-8") diff --git a/tests/threads.py b/tests/threads.py index be3dcffdf6..241a001d78 100644 --- a/tests/threads.py +++ b/tests/threads.py @@ -4,6 +4,9 @@ import sys +from six.moves import reload_module + + def monitor1(): while 1: time.sleep(1) @@ -21,7 +24,7 @@ def monitor3(): while 1: time.sleep(5) print("5 seconds elapsed") - # reload(fake) + # reload_module(fake) def spawn_my_magic_threads(): diff --git a/tests/threads_atexit.py b/tests/threads_atexit.py new file mode 100644 index 0000000000..efb216040a --- /dev/null +++ b/tests/threads_atexit.py @@ -0,0 +1,44 @@ +# https://github.com/unbit/uwsgi/pull/2615 +# atexit should be called when reached max-requests. +# +# Start this app: +# +# $ ./uwsgi --http-socket :8000 --master -L --wsgi-file=tests/threads_atexit.py \ +# --workers 1 --threads 32 --max-requests 40 --min-worker-lifetime 6 --lazy-apps +# +# Access to this app with hey[1]: +# +# # Do http access for 5 minutes with 32 concurrency +# $ ./hey -c 32 -z 5m 'http://127.0.0.1:8000/' +# +# Search how many stamp files: +# +# $ ls uwsgi_worker*.txt | wc -l +# 39 # should be 0 +# +# [1] https://github.com/rakyll/hey + +import atexit +import os +import sys +import time + + +pid = os.getpid() +stamp_file = f"./uwsgi_worker{pid}.txt" + + +with open(stamp_file, "w") as f: + print(time.time(), file=f) + + +@atexit.register +def on_finish_worker(): + print(f"removing {stamp_file}", file=sys.stderr) + os.remove(stamp_file) + + +def application(env, start_response): + time.sleep(1) + start_response('200 OK', [('Content-Type', 'text/html')]) + return [b"Hello World"] diff --git a/tests/threads_heavy.py b/tests/threads_heavy.py new file mode 100644 index 0000000000..632cc68e76 --- /dev/null +++ b/tests/threads_heavy.py @@ -0,0 +1,29 @@ +# https://github.com/unbit/uwsgi/pull/2615 +# CPU heavy application in multi threaded uWSGI doesn't shutdown gracefully. +# +# $ ./uwsgi \ +# --wsgi-file=threads_heavy.py --master --http-socket=:8000 \ +# --workers=4 --threads=8 --max-requests=20 --min-worker-lifetime=6 -L \ +# --worker-reload-mercy=20 2>&1 | tee uwsgi.log +# +# $ hey -c 16 -z 3m 'http://127.0.0.1:8000/' +# +# $ grep MERCY uwsgi.log +# Tue Mar 19 14:01:59 2024 - worker 1 (pid: 62113) is taking too much time to die...NO MERCY !!! +# Tue Mar 19 14:02:23 2024 - worker 2 (pid: 62218) is taking too much time to die...NO MERCY !!! +# ... +# +# This was caused by pthread_cancel() is called from non-main thread. + +def fibonacci(n): + if n <= 1: + return n + return fibonacci(n - 1) + fibonacci(n - 2) + + +def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/html')]) + n = 24 + r = fibonacci(n) + s = f"F({n}) = {r}" + return [s.encode()] diff --git a/tests/travis.sh b/tests/travis.sh index 993b1a2534..67ea2f5dbd 100755 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -1,6 +1,8 @@ #!/bin/bash +set -u +CI_CONFIG="$1" txtund=$(tput sgr 0 1) # underline txtbld=$(tput bold) # bold bldred=${txtbld}$(tput setaf 1) # red @@ -57,13 +59,29 @@ http_test() { test_python() { date > reload.txt rm -f uwsgi.log - echo -e "${bldyel}================== TESTING $1 =====================${txtrst}" + echo -e "${bldyel}================== TESTING $1 $2 =====================${txtrst}" echo -e "${bldyel}>>> Spawning uWSGI python app${txtrst}" echo -en "${bldred}" - ./uwsgi --master --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --wsgi-file tests/staticfile.py --daemonize uwsgi.log + ./uwsgi --master --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --wsgi-file $2 --daemonize uwsgi.log + sleep 1 echo -en "${txtrst}" http_test "http://localhost:8080/" - echo -e "${bldyel}===================== DONE $1 =====================${txtrst}\n\n" + echo -e "${bldyel}===================== DONE $1 $2 =====================${txtrst}\n\n" +} + + +test_python_deadlocks() { + date > reload.txt + rm -f uwsgi.log + echo -e "${bldyel}================== TESTING DEADLOCKS $1 $2 =====================${txtrst}" + echo -e "${bldyel}>>> Starting python app${txtrst}" + echo -en "${bldred}" + # initialize with tests/deadlocks/sitecustomize.py + PYTHONPATH=tests/deadlocks ./uwsgi --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --wsgi-file tests/deadlocks/main.py --ini $2 --daemonize uwsgi.log + sleep 1 + echo -en "${txtrst}" + http_test "http://localhost:8080/" + echo -e "${bldyel}===================== DONE $1 $2 =====================${txtrst}\n\n" } @@ -71,31 +89,38 @@ test_rack() { date > reload.txt rm -f uwsgi.log # the code assumes that ruby environment is activated by `rvm use` - echo -e "${bldyel}================== TESTING $1 =====================${txtrst}" + echo -e "${bldyel}================== TESTING $1 $2 =====================${txtrst}" echo -e "${bldyel}>>> Installing sinatra gem using gem${txtrst}" - gem install sinatra || die + sudo gem install sinatra -v 2.2.2 || die echo -e "${bldyel}>>> Spawning uWSGI rack app${txtrst}" echo -en "${bldred}" - ./uwsgi --master --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --rack examples/config2.ru --daemonize uwsgi.log + ./uwsgi --master --plugin 0:$1 --http :8080 --exit-on-reload --touch-reload reload.txt --rack $2 --daemonize uwsgi.log echo -en "${txtrst}" http_test "http://localhost:8080/hi" - echo -e "${bldyel}===================== DONE $1 =====================${txtrst}\n\n" + echo -e "${bldyel}===================== DONE $1 $2 =====================${txtrst}\n\n" } while read PV ; do - test_python $PV -done < <(cat .travis.yml | grep "plugins/python base" | sed s_".*plugins/python base "_""_g) - - + for WSGI_FILE in tests/staticfile.py tests/testworkers.py tests/testrpc.py ; do + test_python $PV $WSGI_FILE + done +done < <(cat "$CI_CONFIG" | grep "plugins/python base" | sed s_".*plugins/python base "_""_g) +while read PV ; do + for INI_FILE in tests/deadlocks/*.ini ; do + test_python_deadlocks $PV $INI_FILE + done +done < <(cat "$CI_CONFIG" | grep "plugins/python base" | sed s_".*plugins/python base "_""_g) while read RV ; do - test_rack $RV -done < <(cat .travis.yml | grep "plugins/rack base" | sed s_".*plugins/rack base "_""_g) + for RACK in examples/config2.ru ; do + test_rack $RV $RACK + done +done < <(cat "$CI_CONFIG" | grep "plugins/rack base" | sed s_".*plugins/rack base "_""_g) -echo "${bldgre}>>> $SUCCESS SUCCESSFUL PLUGIN(S)${txtrst}" +echo "${bldgre}>>> $SUCCESS SUCCESSFUL TEST(S)${txtrst}" if [ $ERROR -ge 1 ]; then - echo "${bldred}>>> $ERROR FAILED PLUGIN(S)${txtrst}" + echo "${bldred}>>> $ERROR FAILED TEST(S)${txtrst}" exit 1 fi diff --git a/tests/ugevent.py b/tests/ugevent.py index bb03a1b2d2..7b847d77fd 100644 --- a/tests/ugevent.py +++ b/tests/ugevent.py @@ -12,28 +12,28 @@ @signal(17) def hello(signum): - print "hello i am signal %d, i am here because the background job is finished" % signum + print("hello i am signal %d, i am here because the background job is finished" % signum) if REFCNT: - print sys.gettotalrefcount() + print(sys.gettotalrefcount()) @timer(10) def ten_seconds(signum): - print "10 seconds elapsed, signal %d raised" % signum + print("10 seconds elapsed, signal %d raised" % signum) if REFCNT: - print sys.gettotalrefcount() + print(sys.gettotalrefcount()) @filemon('/tmp') def tmp_modified(signum): - print "/tmp has been touched, i am the greenlet %s running on worker %d" % (gevent.getcurrent(), uwsgi.worker_id()) + print("/tmp has been touched, i am the greenlet %s running on worker %d" % (gevent.getcurrent(), uwsgi.worker_id())) if REFCNT: - print sys.gettotalrefcount() + print(sys.gettotalrefcount()) def bg_task(): for i in range(1, 10): - print "background task", i + print("background task", i) gevent.sleep(1) # task ended raise a signal !!! @@ -42,7 +42,7 @@ def bg_task(): def long_task(): for i in range(1, 10): - print i + print(i) gevent.sleep() @@ -70,7 +70,7 @@ def application(e, sr): yield "ip = %s
" % j.value if REFCNT: - print sys.gettotalrefcount() + print(sys.gettotalrefcount()) yield "%d" % sys.gettotalrefcount() # this task will goes on after request end diff --git a/tests/websockets.py b/tests/websockets.py index a7a3b38a0d..106e350860 100644 --- a/tests/websockets.py +++ b/tests/websockets.py @@ -55,7 +55,7 @@ def application(env, sr): """ % (ws_scheme, env['HTTP_HOST']) elif env['PATH_INFO'] == '/foobar/': uwsgi.websocket_handshake(env['HTTP_SEC_WEBSOCKET_KEY'], env.get('HTTP_ORIGIN', '')) - print "websockets..." + print("websockets...") while True: msg = uwsgi.websocket_recv_nb() if msg: diff --git a/tests/websockets_chat.py b/tests/websockets_chat.py index de91abf8f5..34b354ff1d 100644 --- a/tests/websockets_chat.py +++ b/tests/websockets_chat.py @@ -55,7 +55,7 @@ def application(env, sr): return "" elif env['PATH_INFO'] == '/foobar/': uwsgi.websocket_handshake(env['HTTP_SEC_WEBSOCKET_KEY'], env.get('HTTP_ORIGIN', '')) - print "websockets..." + print("websockets...") r = redis.StrictRedis(host='localhost', port=6379, db=0) channel = r.pubsub() channel.subscribe('foobar') diff --git a/tests/websockets_echo.py b/tests/websockets_echo.py index 5db57cea55..83184afd0b 100644 --- a/tests/websockets_echo.py +++ b/tests/websockets_echo.py @@ -51,7 +51,7 @@ def application(env, sr): """ % (ws_scheme, env['HTTP_HOST']) elif env['PATH_INFO'] == '/foobar/': uwsgi.websocket_handshake(env['HTTP_SEC_WEBSOCKET_KEY'], env.get('HTTP_ORIGIN', '')) - print "websockets..." + print("websockets...") while True: msg = uwsgi.websocket_recv() uwsgi.websocket_send("[%s] %s" % (time.time(), msg)) diff --git a/tests/werkzeug.py b/tests/werkzeug_app.py similarity index 100% rename from tests/werkzeug.py rename to tests/werkzeug_app.py diff --git a/check/Makefile b/unittest/Makefile similarity index 55% rename from check/Makefile rename to unittest/Makefile index 0f0b447865..c28b8d70c0 100644 --- a/check/Makefile +++ b/unittest/Makefile @@ -1,20 +1,27 @@ CFLAGS = $(shell pkg-config --cflags check) +CFLAGS += -DUWSGI_PCRE2 LDFLAGS = $(shell pkg-config --libs check) LDFLAGS += -ldl -lz LDFLAGS += $(shell xml2-config --libs) LDFLAGS += $(shell pkg-config --libs openssl) -LDFLAGS += $(shell pcre-config --libs) +LDFLAGS += $(shell pcre2-config --libs8) +LDFLAGS += $(shell pkg-config --libs jansson) +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + LDFLAGS += -lcap +endif -objects = check_core + +objects = check_core check_regexp all: $(objects) -$(objects): %: %.c +$(objects): %: %.c ../libuwsgi.a $(CC) $(CFLAGS) -o $@ $< ../libuwsgi.a $(LDFLAGS) -test: +test: all @for file in $(objects); do ./$$file; done clean: diff --git a/unittest/check_core.c b/unittest/check_core.c new file mode 100644 index 0000000000..6aaee620dc --- /dev/null +++ b/unittest/check_core.c @@ -0,0 +1,108 @@ +#include +#include "../uwsgi.h" + + +START_TEST(test_uwsgi_strncmp) +{ + int result; + result = uwsgi_strncmp("test", 4, "test", 4); + ck_assert(result == 0); + + result = uwsgi_strncmp("test", 4, "tes", 3); + ck_assert(result == 1); + + result = uwsgi_strncmp("tes", 3, "test", 4); + ck_assert(result == 1); + + result = uwsgi_strncmp("aaa", 3, "bbb", 3); + ck_assert_msg(result < 0, "result: %d", result); + + result = uwsgi_strncmp("bbb", 3, "aaa", 3); + ck_assert_msg(result > 0, "result: %d", result); +} +END_TEST + +Suite *check_core_strings(void) +{ + Suite *s = suite_create("uwsgi strings"); + TCase *tc = tcase_create("strings"); + + suite_add_tcase(s, tc); + tcase_add_test(tc, test_uwsgi_strncmp); + return s; +} + +START_TEST(test_uwsgi_opt_set_int) +{ + int result; + uwsgi_opt_set_int("", "true", &result); + ck_assert(result == 0); + + uwsgi_opt_set_int("", "false", &result); + ck_assert(result == 0); + + uwsgi_opt_set_int("", "0", &result); + ck_assert(result == 0); + + uwsgi_opt_set_int("", "60", &result); + ck_assert(result == 60); + + // When used with "optional_argument", value will be passed as NULL + uwsgi_opt_set_int("", NULL, &result); + ck_assert(result == 1); +} +END_TEST + +Suite *check_core_opt_parsing(void) +{ + Suite *s = suite_create("uwsgi opt parsing"); + TCase *tc = tcase_create("opt_parsing"); + + suite_add_tcase(s, tc); + tcase_add_test(tc, test_uwsgi_opt_set_int); + return s; +} + +START_TEST(test_uwsgi_cron_task_needs_execution_handles_weekday_7_as_sunday) +{ + int result; + struct tm *t; + time_t now; + + now = time(NULL); + t = localtime(&now); + t->tm_wday= 0; + + result = uwsgi_cron_task_needs_execution(t, -1, -1, -1, -1, 0); + ck_assert(result == 1); + + result = uwsgi_cron_task_needs_execution(t, -1, -1, -1, -1, 7); + ck_assert(result == 1); + + result = uwsgi_cron_task_needs_execution(t, -1, -1, -1, -1, 1); + ck_assert(result == 0); +} +END_TEST + +Suite *check_core_cron(void) +{ + Suite *s = suite_create("uwsgi cron"); + TCase *tc = tcase_create("cron"); + + suite_add_tcase(s, tc); + tcase_add_test(tc, test_uwsgi_cron_task_needs_execution_handles_weekday_7_as_sunday); + return s; +} + +int main(void) +{ + int nf; + SRunner *r = srunner_create(check_core_strings()); + srunner_add_suite(r, check_core_opt_parsing()); + srunner_add_suite(r, check_core_cron()); + srunner_run_all(r, CK_NORMAL); + nf = srunner_ntests_failed(r); + srunner_free(r); + return (nf == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} + diff --git a/unittest/check_regexp.c b/unittest/check_regexp.c new file mode 100644 index 0000000000..b2025bfb2f --- /dev/null +++ b/unittest/check_regexp.c @@ -0,0 +1,86 @@ +#include +#include "../uwsgi.h" + + +START_TEST(test_uwsgi_regexp_match) +{ + int result; + uwsgi_pcre *pattern_all; + uwsgi_pcre *pattern; + + result = uwsgi_regexp_build(".*", &pattern_all); + ck_assert(result == 0); + + result = uwsgi_regexp_match(pattern_all, "/fooba", 6); + ck_assert(result >= 0); + + result = uwsgi_regexp_build("/foobar/.*", &pattern); + ck_assert(result == 0); + + result = uwsgi_regexp_match(pattern, "/fooba", 6); + ck_assert(result < 0); + + result = uwsgi_regexp_match(pattern, "/foobar/baz", 11); + ck_assert(result >= 0); + + pcre2_code_free(pattern_all); + pcre2_code_free(pattern); +} +END_TEST + +START_TEST(test_uwsgi_regexp_match_ovec) +{ + int result; + uwsgi_pcre *pattern; + int *ovec = calloc((2+1)*2, sizeof(int)); + char buf[20], sub[20]; + + result = uwsgi_regexp_build("^/foo/(.*)\\.jpg\\?([0-9]{2})", &pattern); + ck_assert(result == 0); + result = uwsgi_regexp_ovector(pattern); + ck_assert(result == 2); + + result = uwsgi_regexp_match_ovec(pattern, "/fooba", 6, ovec, 2); + ck_assert(result < 0); + + strcpy(buf, "/foo/bar.jpg?422"); + result = uwsgi_regexp_match_ovec(pattern, buf, strlen(buf), ovec, 2); + ck_assert(result >= 0); + strncpy(sub, buf+ovec[0], ovec[1]-ovec[0]); + sub[ovec[1]-ovec[0]] = '\0'; + ck_assert_str_eq(sub, "/foo/bar.jpg?42"); + strncpy(sub, buf+ovec[2], ovec[3]-ovec[2]); + sub[ovec[3]-ovec[2]] = '\0'; + ck_assert_str_eq(sub, "bar"); + strncpy(sub, buf+ovec[4], ovec[5]-ovec[4]); + sub[ovec[5]-ovec[4]] = '\0'; + ck_assert_str_eq(sub, "42"); + + strcpy(sub, uwsgi_regexp_apply_ovec(buf, strlen(buf), "key=$1.$2.jpg", 13, ovec, 2)); + ck_assert_str_eq(sub, "key=bar.42.jpg"); + + pcre2_code_free(pattern); + free(ovec); +} +END_TEST + +Suite *check_regexp(void) +{ + Suite *s = suite_create("uwsgi regexp"); + TCase *tc = tcase_create("regexp"); + + suite_add_tcase(s, tc); + tcase_add_test(tc, test_uwsgi_regexp_match); + tcase_add_test(tc, test_uwsgi_regexp_match_ovec); + return s; +} + +int main(void) +{ + int nf; + SRunner *r = srunner_create(check_regexp()); + srunner_run_all(r, CK_NORMAL); + nf = srunner_ntests_failed(r); + srunner_free(r); + return (nf == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/uwsgi.h b/uwsgi.h index ba3986e794..748bc856a7 100755 --- a/uwsgi.h +++ b/uwsgi.h @@ -449,8 +449,26 @@ struct uwsgi_lock_ops { #define uwsgi_wait_read_req(x) uwsgi.wait_read_hook(x->fd, uwsgi.socket_timeout) ; x->switches++ #define uwsgi_wait_write_req(x) uwsgi.wait_write_hook(x->fd, uwsgi.socket_timeout) ; x->switches++ -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) +#ifdef UWSGI_PCRE2 + +#define PCRE2_CODE_UNIT_WIDTH 8 +#include +#define PCRE_OVECTOR_BYTESIZE(n) (n+1)*2 + +typedef pcre2_code uwsgi_pcre; + +#else + #include +#define PCRE_OVECTOR_BYTESIZE(n) (n+1)*3 + +typedef struct { + pcre *p; + pcre_extra *extra; +} uwsgi_pcre; + +#endif #endif struct uwsgi_dyn_dict { @@ -466,9 +484,8 @@ struct uwsgi_dyn_dict { struct uwsgi_dyn_dict *prev; struct uwsgi_dyn_dict *next; -#ifdef UWSGI_PCRE - pcre *pattern; - pcre_extra *pattern_extra; +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) + uwsgi_pcre *pattern; #endif }; @@ -479,11 +496,10 @@ struct uwsgi_hook { struct uwsgi_hook *next; }; -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list { - pcre *pattern; - pcre_extra *pattern_extra; + uwsgi_pcre *pattern; uint64_t custom; char *custom_str; @@ -625,6 +641,8 @@ struct uwsgi_daemon { char *chdir; int max_throttle; + + int notifypid; }; struct uwsgi_logger { @@ -1093,13 +1111,16 @@ struct uwsgi_plugin { int (*worker)(void); void (*early_post_jail) (void); + + void (*pre_uwsgi_fork) (void); + void (*post_uwsgi_fork) (int); }; -#ifdef UWSGI_PCRE -int uwsgi_regexp_build(char *, pcre **, pcre_extra **); -int uwsgi_regexp_match(pcre *, pcre_extra *, char *, int); -int uwsgi_regexp_match_ovec(pcre *, pcre_extra *, char *, int, int *, int); -int uwsgi_regexp_ovector(pcre *, pcre_extra *); +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) +int uwsgi_regexp_build(char *, uwsgi_pcre **); +int uwsgi_regexp_match(uwsgi_pcre *, const char *, int); +int uwsgi_regexp_match_ovec(uwsgi_pcre *, const char *, int, int *, int); +int uwsgi_regexp_ovector(const uwsgi_pcre *); char *uwsgi_regexp_apply_ovec(char *, int, char *, int, int *, int); int uwsgi_regexp_match_pattern(char *pattern, char *str); @@ -1190,8 +1211,7 @@ struct uwsgi_spooler { struct uwsgi_route { - pcre *pattern; - pcre_extra *pattern_extra; + uwsgi_pcre *pattern; char *orig_route; @@ -1300,15 +1320,14 @@ struct uwsgi_alarm_fd { struct uwsgi_alarm_fd *uwsgi_add_alarm_fd(int, char *, size_t, char *, size_t); -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_alarm_ll { struct uwsgi_alarm_instance *alarm; struct uwsgi_alarm_ll *next; }; struct uwsgi_alarm_log { - pcre *pattern; - pcre_extra *pattern_extra; + uwsgi_pcre *pattern; int negate; struct uwsgi_alarm_ll *alarms; struct uwsgi_alarm_log *next; @@ -1368,6 +1387,9 @@ enum uwsgi_range { UWSGI_RANGE_INVALID, }; +// avoid name clashes on solaris +#undef sun + struct wsgi_request { int fd; struct uwsgi_header *uh; @@ -1648,6 +1670,8 @@ struct wsgi_request { char * if_range; uint16_t if_range_len; + + uint8_t websocket_is_fin; }; @@ -2248,7 +2272,7 @@ struct uwsgi_server { struct uwsgi_string_list *requested_log_encoders; struct uwsgi_string_list *requested_log_req_encoders; -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) int pcre_jit; struct uwsgi_regexp_list *log_drain_rules; struct uwsgi_regexp_list *log_filter_rules; @@ -2330,7 +2354,7 @@ struct uwsgi_server { int static_gzip_all; struct uwsgi_string_list *static_gzip_dir; struct uwsgi_string_list *static_gzip_ext; -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *static_gzip; #endif @@ -2455,7 +2479,7 @@ struct uwsgi_server { int master_queue; int master_interval; - // mainly iseful for broodlord mode + // mainly useful for broodlord mode int vassal_sos_backlog; int no_defer_accept; @@ -2476,6 +2500,9 @@ struct uwsgi_server { int backtrace_depth; int harakiri_verbose; + int harakiri_graceful_timeout; + int harakiri_graceful_signal; + int harakiri_queue_threshold; int harakiri_no_arh; int magic_table_first_round; @@ -2499,7 +2526,6 @@ struct uwsgi_server { // avoid thundering herd in threaded modes pthread_mutex_t thunder_mutex; - pthread_mutex_t six_feet_under_lock; pthread_mutex_t lock_static; int use_thunder_lock; @@ -2732,7 +2758,7 @@ struct uwsgi_server { int ssl_sessions_timeout; struct uwsgi_cache *ssl_sessions_cache; char *ssl_tmp_dir; -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *sni_regexp; #endif struct uwsgi_string_list *sni; @@ -2765,6 +2791,7 @@ struct uwsgi_server { struct uwsgi_buffer *websockets_ping; struct uwsgi_buffer *websockets_pong; + struct uwsgi_buffer *websockets_close; int websockets_ping_freq; int websockets_pong_tolerance; uint64_t websockets_max_size; @@ -2854,6 +2881,10 @@ struct uwsgi_server { int tlsv1; #endif + // uWSGI 2.0.19 + int emperor_graceful_shutdown; + int is_chrooted; + size_t response_header_limit; // uWSGI 2.1 @@ -2918,6 +2949,12 @@ struct uwsgi_server { size_t environ_len; int dynamic_apps; + + struct uwsgi_buffer *websockets_continuation_buffer; + + uint64_t max_worker_lifetime_delta; + // This pipe is used to stop event_queue_wait() in threaded workers. + int loop_stop_pipe[2]; }; struct uwsgi_rpc { @@ -3226,7 +3263,6 @@ void uwsgi_405(struct wsgi_request *); void uwsgi_redirect_to_slash(struct wsgi_request *); void manage_snmp(int, uint8_t *, int, struct sockaddr_in *); -void snmp_init(void); void uwsgi_master_manage_snmp(int); @@ -3259,21 +3295,12 @@ uint32_t uwsgi_swap32(uint32_t); uint64_t uwsgi_swap64(uint64_t); #endif -int uwsgi_parse_request(int, struct wsgi_request *, int); int uwsgi_parse_vars(struct wsgi_request *); -int uwsgi_enqueue_message(char *, int, uint8_t, uint8_t, char *, int, int); - -void manage_opt(int, char *); - -int uwsgi_ping_node(int, struct wsgi_request *); - void uwsgi_async_init(void); void async_loop(); struct wsgi_request *find_first_available_wsgi_req(void); -struct wsgi_request *find_first_accepting_wsgi_req(void); struct wsgi_request *find_wsgi_req_by_fd(int); -struct wsgi_request *find_wsgi_req_by_id(int); void async_schedule_to_req_green(void); void async_schedule_to_req(void); @@ -3281,9 +3308,6 @@ int async_add_fd_write(struct wsgi_request *, int, int); int async_add_fd_read(struct wsgi_request *, int, int); void async_reset_request(struct wsgi_request *); -struct wsgi_request *next_wsgi_req(struct wsgi_request *); - - void async_add_timeout(struct wsgi_request *, int); void uwsgi_as_root(void); @@ -3365,29 +3389,18 @@ void *uwsgi_get_loop(char *); void add_exported_option(char *, char *, int); void add_exported_option_do(char *, char *, int, int); -ssize_t uwsgi_send_empty_pkt(int, char *, uint8_t, uint8_t); - int uwsgi_waitfd_event(int, int, int); #define uwsgi_waitfd(a, b) uwsgi_waitfd_event(a, b, POLLIN) #define uwsgi_waitfd_write(a, b) uwsgi_waitfd_event(a, b, POLLOUT) -int uwsgi_hooked_parse_dict_dgram(int, char *, size_t, uint8_t, uint8_t, void (*)(char *, uint16_t, char *, uint16_t, void *), void *); int uwsgi_hooked_parse(char *, size_t, void (*)(char *, uint16_t, char *, uint16_t, void *), void *); int uwsgi_hooked_parse_array(char *, size_t, void (*) (uint16_t, char *, uint16_t, void *), void *); -int uwsgi_get_dgram(int, struct wsgi_request *); - -int uwsgi_string_sendto(int, uint8_t, uint8_t, struct sockaddr *, socklen_t, char *, size_t); - -void uwsgi_stdin_sendto(char *, uint8_t, uint8_t); - char *generate_socket_name(char *); #define UMIN(a,b) ((a)>(b)?(b):(a)) #define UMAX(a,b) ((a)<(b)?(b):(a)) -ssize_t uwsgi_send_message(int, uint8_t, uint8_t, char *, uint16_t, int, ssize_t, int); - int uwsgi_cache_set2(struct uwsgi_cache *, char *, uint16_t, char *, uint64_t, uint64_t, uint64_t); int uwsgi_cache_del2(struct uwsgi_cache *, char *, uint16_t, uint64_t, uint16_t); char *uwsgi_cache_get2(struct uwsgi_cache *, char *, uint16_t, uint64_t *); @@ -3474,6 +3487,7 @@ void emperor_loop(void); char *uwsgi_num2str(int); char *uwsgi_float2str(float); char *uwsgi_64bit2str(int64_t); +char *uwsgi_size2str(size_t); char *magic_sub(char *, size_t, size_t *, char *[]); void init_magic_table(char *[]); @@ -3611,7 +3625,6 @@ int uwsgi_proto_base_writev(struct wsgi_request *, struct iovec *, size_t *); #ifdef UWSGI_SSL int uwsgi_proto_ssl_write(struct wsgi_request *, char *, size_t); #endif -int uwsgi_proto_base_write_header(struct wsgi_request *, char *, size_t); ssize_t uwsgi_proto_base_read_body(struct wsgi_request *, char *, size_t); ssize_t uwsgi_proto_noop_read_body(struct wsgi_request *, char *, size_t); #ifdef UWSGI_SSL @@ -3669,7 +3682,7 @@ void uwsgi_close_all_unshared_sockets(void); void uwsgi_shutdown_all_sockets(void); struct uwsgi_string_list *uwsgi_string_new_list(struct uwsgi_string_list **, char *); -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) struct uwsgi_regexp_list *uwsgi_regexp_custom_new_list(struct uwsgi_regexp_list **, char *, char *); #define uwsgi_regexp_new_list(x, y) uwsgi_regexp_custom_new_list(x, y, NULL); #endif @@ -3943,7 +3956,7 @@ void uwsgi_opt_add_addr_list(char *, char *, void *); void uwsgi_opt_add_string_list_custom(char *, char *, void *); void uwsgi_opt_add_dyn_dict(char *, char *, void *); void uwsgi_opt_binary_append_data(char *, char *, void *); -#ifdef UWSGI_PCRE +#if defined(UWSGI_PCRE) || defined(UWSGI_PCRE2) void uwsgi_opt_pcre_jit(char *, char *, void *); void uwsgi_opt_add_regexp_dyn_dict(char *, char *, void *); void uwsgi_opt_add_regexp_list(char *, char *, void *); @@ -3956,7 +3969,6 @@ void uwsgi_opt_set_rawint(char *, char *, void *); void uwsgi_opt_set_16bit(char *, char *, void *); void uwsgi_opt_set_64bit(char *, char *, void *); void uwsgi_opt_set_megabytes(char *, char *, void *); -void uwsgi_opt_set_dyn(char *, char *, void *); void uwsgi_opt_set_placeholder(char *, char *, void *); void uwsgi_opt_add_shared_socket(char *, char *, void *); void uwsgi_opt_add_socket(char *, char *, void *); @@ -4100,7 +4112,6 @@ int uwsgi_calc_cheaper(void); int uwsgi_cheaper_algo_spare(int); int uwsgi_cheaper_algo_spare2(int); int uwsgi_cheaper_algo_backlog(int); -int uwsgi_cheaper_algo_backlog2(int); int uwsgi_cheaper_algo_manual(int); int uwsgi_master_log(void); @@ -4271,7 +4282,6 @@ struct uwsgi_logchunk *uwsgi_register_logchunk(char *, ssize_t (*)(struct wsgi_r void uwsgi_logit_simple(struct wsgi_request *); void uwsgi_logit_lf(struct wsgi_request *); -void uwsgi_logit_lf_strftime(struct wsgi_request *); struct uwsgi_logvar *uwsgi_logvar_get(struct wsgi_request *, char *, uint8_t); void uwsgi_logvar_add(struct wsgi_request *, char *, uint8_t, char *, uint8_t); @@ -4570,7 +4580,6 @@ void uwsgi_loop_cores_run(void *(*)(void *)); int uwsgi_kvlist_parse(char *, size_t, char, int, ...); int uwsgi_send_http_stats(int); -ssize_t uwsgi_simple_request_read(struct wsgi_request *, char *, size_t); int uwsgi_plugin_modifier1(char *); void *cache_udp_server_loop(void *); @@ -4645,7 +4654,6 @@ int uwsgi_response_prepare_headers(struct wsgi_request *, char *, uint16_t); int uwsgi_response_prepare_headers_int(struct wsgi_request *, int); int uwsgi_response_add_header(struct wsgi_request *, char *, uint16_t, char *, uint16_t); int uwsgi_response_add_header_force(struct wsgi_request *, char *, uint16_t, char *, uint16_t); -int uwsgi_response_commit_headers(struct wsgi_request *); int uwsgi_response_sendfile_do(struct wsgi_request *, int, size_t, size_t); int uwsgi_response_sendfile_do_can_close(struct wsgi_request *, int, size_t, size_t, int); @@ -5123,7 +5131,11 @@ char *uwsgi_subscription_algo_name(void *); int uwsgi_wait_for_fs(char *, int); int uwsgi_wait_for_mountpoint(char *); int uwsgi_wait_for_socket(char *); + +#if defined(__linux__) && !defined(OBSOLETE_LINUX_KERNEL) void uwsgi_hooks_setns_run(struct uwsgi_string_list *, pid_t, uid_t, gid_t); +#endif + char *vassal_attr_get(struct uwsgi_instance *, char *); int vassal_attr_get_multi(struct uwsgi_instance *, char *, int (*)(struct uwsgi_instance *, char *, void *), void *); diff --git a/uwsgiconfig.py b/uwsgiconfig.py index 51348320e0..4f6c8126d6 100644 --- a/uwsgiconfig.py +++ b/uwsgiconfig.py @@ -12,6 +12,7 @@ import sys import subprocess +import sysconfig from threading import Thread, Lock from optparse import OptionParser @@ -20,13 +21,15 @@ except ImportError: from Queue import Queue -from distutils import sysconfig - try: import ConfigParser except ImportError: import configparser as ConfigParser +try: + from shlex import quote +except ImportError: + from pipes import quote PY3 = sys.version_info[0] == 3 @@ -114,7 +117,7 @@ def thread_compiler(num): print_lock.acquire() print_compilation_output("[thread %d][%s] %s" % (num, GCC, objfile), "[thread %d] %s" % (num, cmdline)) print_lock.release() - ret = os.system(cmdline) + ret = subprocess.call(cmdline, shell=True) if ret != 0: os._exit(1) elif cmdline: @@ -160,7 +163,7 @@ def uniq_warnings(elements): if uwsgi_version.endswith('-dev') and os.path.exists('%s/.git' % os.path.dirname(os.path.abspath(__file__))): try: - uwsgi_version += '-%s' % spcall('git rev-parse --short HEAD') + uwsgi_version += '+%s' % spcall('git rev-parse --short HEAD') except Exception: pass @@ -216,11 +219,12 @@ def has_usable_ucontext(): def spcall3(cmd): p = subprocess.Popen(cmd, shell=True, stdin=open('/dev/null'), stderr=subprocess.PIPE, stdout=subprocess.PIPE) + (out, err) = p.communicate() - if p.wait() == 0: + if p.returncode == 0: if sys.version_info[0] > 2: - return p.stderr.read().rstrip().decode() - return p.stderr.read().rstrip() + return err.rstrip().decode() + return err.rstrip() else: return None @@ -244,14 +248,14 @@ def push_print(msg): def push_command(objfile, cmdline): if not compile_queue: print_compilation_output("[%s] %s" % (GCC, objfile), cmdline) - ret = os.system(cmdline) + ret = subprocess.call(cmdline, shell=True) if ret != 0: sys.exit(1) else: compile_queue.put((objfile, cmdline)) -def compile(cflags, last_cflags_ts, objfile, srcfile): +def uwsgi_compile(cflags, last_cflags_ts, objfile, srcfile): source_stat = os.stat(srcfile) header_stat = os.stat('uwsgi.h') try: @@ -418,13 +422,13 @@ def build_uwsgi(uc, print_only=False, gcll=None): if objfile.endswith('.c') or objfile.endswith('.cc') or objfile.endswith('.m') or objfile.endswith('.go'): if objfile.endswith('.go'): cflags.append('-Wno-error') - compile(' '.join(cflags), last_cflags_ts, objfile + '.o', file) + uwsgi_compile(' '.join(cflags), last_cflags_ts, objfile + '.o', file) if objfile.endswith('.go'): cflags.pop() else: if objfile == 'core/dot_h': cflags.append('-g') - compile(' '.join(cflags), last_cflags_ts, objfile + '.o', file + '.c') + uwsgi_compile(' '.join(cflags), last_cflags_ts, objfile + '.o', file + '.c') if objfile == 'core/dot_h': cflags.pop() @@ -497,25 +501,25 @@ def build_uwsgi(uc, print_only=False, gcll=None): elif cfile.endswith('.o'): gcc_list.append('%s/%s' % (path, cfile)) elif not cfile.endswith('.c') and not cfile.endswith('.cc') and not cfile.endswith('.go') and not cfile.endswith('.m'): - compile(' '.join(uniq_warnings(p_cflags)), last_cflags_ts, + uwsgi_compile(' '.join(uniq_warnings(p_cflags)), last_cflags_ts, path + '/' + cfile + '.o', path + '/' + cfile + '.c') gcc_list.append('%s/%s' % (path, cfile)) else: if cfile.endswith('.go'): p_cflags.append('-Wno-error') - compile(' '.join(uniq_warnings(p_cflags)), last_cflags_ts, + uwsgi_compile(' '.join(uniq_warnings(p_cflags)), last_cflags_ts, path + '/' + cfile + '.o', path + '/' + cfile) gcc_list.append('%s/%s' % (path, cfile)) for bfile in up.get('BINARY_LIST', []): try: - binary_link_cmd = "ld -r -b binary -o %s/%s.o %s/%s" % (path, bfile[1], path, bfile[1]) + binary_link_cmd = "ld -z noexecstack -r -b binary -o %s/%s.o %s/%s" % (path, bfile[1], path, bfile[1]) print(binary_link_cmd) - if os.system(binary_link_cmd) != 0: + if subprocess.call(binary_link_cmd, shell=True) != 0: raise Exception('unable to link binary file') for kind in ('start', 'end'): objcopy_cmd = "objcopy --redefine-sym _binary_%s_%s=%s_%s %s/%s.o" % (binarize('%s/%s' % (path, bfile[1])), kind, bfile[0], kind, path, bfile[1]) print(objcopy_cmd) - if os.system(objcopy_cmd) != 0: + if subprocess.call(objcopy_cmd, shell=True) != 0: raise Exception('unable to link binary file') gcc_list.append('%s/%s.o' % (path, bfile[1])) except Exception: @@ -566,19 +570,19 @@ def build_uwsgi(uc, print_only=False, gcll=None): print("*** uWSGI linking ***") if '--static' in ldflags: ldline = 'ar cru %s %s' % ( - bin_name, + quote(bin_name), ' '.join(map(add_o, gcc_list)) ) else: ldline = "%s -o %s %s %s %s" % ( GCC, - bin_name, + quote(bin_name), ' '.join(uniq_warnings(ldflags)), ' '.join(map(add_o, gcc_list)), ' '.join(uniq_warnings(libs)) ) print(ldline) - ret = os.system(ldline) + ret = subprocess.call(ldline, shell=True) if ret != 0: print("*** error linking uWSGI ***") sys.exit(1) @@ -675,14 +679,16 @@ def __init__(self, filename, mute=False): if 'UWSGI_INCLUDES' in os.environ: self.include_path += os.environ['UWSGI_INCLUDES'].split(',') - self.cflags = [ + cflags = [ '-O2', '-I.', '-Wall', '-Werror', + '-Wno-error=deprecated-declarations', '-D_LARGEFILE_SOURCE', '-D_FILE_OFFSET_BITS=64' - ] + os.environ.get("CFLAGS", "").split() + self.get('cflags', '').split() + ] + self.cflags = cflags + os.environ.get("CFLAGS", "").split() + self.get('cflags', '').split() python_venv_include = os.path.join(sys.prefix, 'include', 'site', 'python{0}.{1}'.format(*sys.version_info)) @@ -706,13 +712,13 @@ def __init__(self, filename, mute=False): if uwsgi_os == 'GNU': self.cflags.append('-D__HURD__') - gcc_version = spcall("%s -dumpversion" % GCC) + gcc_version = spcall("%s -dumpfullversion -dumpversion" % GCC) if not gcc_version and GCC.startswith('gcc'): if uwsgi_os == 'Darwin': GCC = 'llvm-' + GCC else: GCC = 'gcc' - gcc_version = spcall("%s -dumpversion" % GCC) + gcc_version = spcall("%s -dumpfullversion -dumpversion" % GCC) try: add_it = False @@ -762,6 +768,8 @@ def __init__(self, filename, mute=False): self.cflags += ['-Wextra', '-Wno-unused-parameter', '-Wno-missing-field-initializers'] if gcc_major == 4 and gcc_minor < 9: self.cflags.append('-Wno-format -Wno-format-security') + if "gcc" in GCC and gcc_major >= 5: + self.cflags.append('-Wformat-signedness') self.ldflags = os.environ.get("LDFLAGS", "").split() self.libs = ['-lpthread', '-lm', '-rdynamic'] @@ -1091,30 +1099,30 @@ def get_gcll(self): has_pcre = False - # re-enable after pcre fix - if self.get('pcre'): - if self.get('pcre') == 'auto': - pcreconf = spcall('pcre-config --libs') - if pcreconf: - self.libs.append(pcreconf) - pcreconf = spcall("pcre-config --cflags") - self.cflags.append(pcreconf) - self.gcc_list.append('core/regexp') - self.cflags.append("-DUWSGI_PCRE") - has_pcre = True - + required_pcre = self.get('pcre') + if required_pcre: + pcre_libs = spcall('pcre2-config --libs8') + if pcre_libs: + pcre_cflags = spcall("pcre2-config --cflags") + pcre_define = "-DUWSGI_PCRE2" else: - pcreconf = spcall('pcre-config --libs') - if pcreconf is None: - print("*** libpcre headers unavailable. uWSGI build is interrupted. You have to install pcre development package or disable pcre") - sys.exit(1) - else: - self.libs.append(pcreconf) - pcreconf = spcall("pcre-config --cflags") - self.cflags.append(pcreconf) - self.gcc_list.append('core/regexp') - self.cflags.append("-DUWSGI_PCRE") - has_pcre = True + pcre_libs = spcall('pcre-config --libs') + pcre_cflags = spcall("pcre-config --cflags") + pcre_define = "-DUWSGI_PCRE" + else: + pcre_libs = None + + if required_pcre: + if required_pcre != 'auto' and pcre_libs is None: + print("*** libpcre headers unavailable. uWSGI build is interrupted. You have to install pcre development package or disable pcre") + sys.exit(1) + + if pcre_libs: + self.libs.append(pcre_libs) + self.cflags.append(pcre_cflags) + self.gcc_list.append('core/regexp') + self.cflags.append(pcre_define) + has_pcre = True if has_pcre: report['pcre'] = True @@ -1135,7 +1143,7 @@ def get_gcll(self): self.libs.append('-lcap') report['capabilities'] = True - if self.has_include('uuid/uuid.h'): + if self.has_include('uuid/uuid.h') and not self.get('check'): self.cflags.append("-DUWSGI_UUID") if uwsgi_os in ('Linux', 'GNU', 'GNU/kFreeBSD') or uwsgi_os.startswith('CYGWIN') or os.path.exists('/usr/lib/libuuid.so') or os.path.exists('/usr/local/lib/libuuid.so') or os.path.exists('/usr/lib64/libuuid.so') or os.path.exists('/usr/local/lib64/libuuid.so'): self.libs.append('-luuid') @@ -1156,9 +1164,9 @@ def get_gcll(self): if not self.embed_config: self.embed_config = self.get('embed_config') if self.embed_config: - binary_link_cmd = "ld -r -b binary -o %s.o %s" % (binarize(self.embed_config), self.embed_config) + binary_link_cmd = "ld -z noexecstack -r -b binary -o %s.o %s" % (binarize(self.embed_config), self.embed_config) print(binary_link_cmd) - os.system(binary_link_cmd) + subprocess.call(binary_link_cmd, shell=True) self.cflags.append("-DUWSGI_EMBED_CONFIG=_binary_%s_start" % binarize(self.embed_config)) self.cflags.append("-DUWSGI_EMBED_CONFIG_END=_binary_%s_end" % binarize(self.embed_config)) embed_files = os.environ.get('UWSGI_EMBED_FILES') @@ -1175,25 +1183,25 @@ def get_gcll(self): for directory, directories, files in os.walk(ef): for f in files: fname = "%s/%s" % (directory, f) - binary_link_cmd = "ld -r -b binary -o %s.o %s" % (binarize(fname), fname) + binary_link_cmd = "ld -z noexecstack -r -b binary -o %s.o %s" % (binarize(fname), fname) print(binary_link_cmd) - os.system(binary_link_cmd) + subprocess.call(binary_link_cmd, shell=True) if symbase: for kind in ('start', 'end'): objcopy_cmd = "objcopy --redefine-sym _binary_%s_%s=_binary_%s%s_%s build/%s.o" % (binarize(fname), kind, binarize(symbase), binarize(fname[len(ef):]), kind, binarize(fname)) print(objcopy_cmd) - os.system(objcopy_cmd) + subprocess.call(objcopy_cmd, shell=True) binary_list.append(binarize(fname)) else: - binary_link_cmd = "ld -r -b binary -o %s.o %s" % (binarize(ef), ef) + binary_link_cmd = "ld -z noexecstack -r -b binary -o %s.o %s" % (binarize(ef), ef) print(binary_link_cmd) - os.system(binary_link_cmd) + subprocess.call(binary_link_cmd, shell=True) binary_list.append(binarize(ef)) if symbase: for kind in ('start', 'end'): objcopy_cmd = "objcopy --redefine-sym _binary_%s_%s=_binary_%s_%s build/%s.o" % (binarize(ef), kind, binarize(symbase), kind, binarize(ef)) print(objcopy_cmd) - os.system(objcopy_cmd) + subprocess.call(objcopy_cmd, shell=True) self.cflags.append('-DUWSGI_VERSION="\\"' + uwsgi_version + '\\""') @@ -1378,14 +1386,23 @@ def get_remote_plugin(path): if git_dir.endswith('.git'): git_dir = git_dir[:-4] if not os.path.isdir(git_dir): - if os.system('git clone %s' % path) != 0: + if subprocess.call(['git', 'clone', path]) != 0: sys.exit(1) else: - if os.system('cd %s ; git pull' % git_dir) != 0: + if subprocess.call(['git', 'pull'], cwd=git_dir) != 0: sys.exit(1) return git_dir +try: + execfile +except NameError: + def execfile(path, up): + with open(path) as py: + code = compile(py.read(), path, 'exec') + exec(code, up) + + def get_plugin_up(path): up = {} if os.path.isfile(path): @@ -1397,12 +1414,7 @@ def get_plugin_up(path): if not path: path = '.' elif os.path.isdir(path): - try: - execfile('%s/uwsgiplugin.py' % path, up) - except Exception: - f = open('%s/uwsgiplugin.py' % path) - exec(f.read(), up) - f.close() + execfile('%s/uwsgiplugin.py' % path, up) else: print("Error: unable to find directory '%s'" % path) sys.exit(1) @@ -1487,7 +1499,7 @@ def build_plugin(path, uc, cflags, ldflags, libs, name=None): try: binary_link_cmd = "ld -r -b binary -o %s/%s.o %s/%s" % (path, bfile[1], path, bfile[1]) print(binary_link_cmd) - if os.system(binary_link_cmd) != 0: + if subprocess.call(binary_link_cmd, shell=True) != 0: raise Exception('unable to link binary file') for kind in ('start', 'end'): objcopy_cmd = "objcopy --redefine-sym _binary_%s_%s=%s_%s %s/%s.o" % ( @@ -1499,7 +1511,7 @@ def build_plugin(path, uc, cflags, ldflags, libs, name=None): bfile[1] ) print(objcopy_cmd) - if os.system(objcopy_cmd) != 0: + if subprocess.call(objcopy_cmd, shell=True) != 0: raise Exception('unable to link binary file') gcc_list.append('%s/%s.o' % (path, bfile[1])) except Exception: @@ -1558,7 +1570,7 @@ def build_plugin(path, uc, cflags, ldflags, libs, name=None): ) print_compilation_output("[%s] %s.so" % (GCC, plugin_dest), gccline) - ret = os.system(gccline) + ret = subprocess.call(gccline, shell=True) if ret != 0: print("*** unable to build %s plugin ***" % name) sys.exit(1) @@ -1571,7 +1583,7 @@ def build_plugin(path, uc, cflags, ldflags, libs, name=None): f.close() objline = "objcopy %s.so --add-section uwsgi=.uwsgi_plugin_section %s.so" % (plugin_dest, plugin_dest) print_compilation_output(None, objline) - os.system(objline) + subprocess.call(objline, shell=True) os.unlink('.uwsgi_plugin_section') except Exception: pass @@ -1683,15 +1695,15 @@ def vararg_callback(option, opt_str, value, parser): pass build_plugin(options.extra_plugin[0], None, cflags, ldflags, None, name) elif options.clean: - os.system("rm -f core/*.o") - os.system("rm -f proto/*.o") - os.system("rm -f lib/*.o") - os.system("rm -f plugins/*/*.o") - os.system("rm -f build/*.o") - os.system("rm -f core/dot_h.c") - os.system("rm -f core/config_py.c") + subprocess.call("rm -f core/*.o", shell=True) + subprocess.call("rm -f proto/*.o", shell=True) + subprocess.call("rm -f lib/*.o", shell=True) + subprocess.call("rm -f plugins/*/*.o", shell=True) + subprocess.call("rm -f build/*.o", shell=True) + subprocess.call("rm -f core/dot_h.c", shell=True) + subprocess.call("rm -f core/config_py.c", shell=True) elif options.check: - os.system("cppcheck --max-configs=1000 --enable=all -q core/ plugins/ proto/ lib/ apache2/") + subprocess.call("cppcheck --max-configs=1000 --enable=all -q core/ plugins/ proto/ lib/ apache2/", shell=True) else: parser.print_help() sys.exit(1) diff --git a/valgrind/valgrind-generate-sups.sh b/valgrind/valgrind-generate-sups.sh index 37bfe3f883..3e28cb37bb 100755 --- a/valgrind/valgrind-generate-sups.sh +++ b/valgrind/valgrind-generate-sups.sh @@ -1,5 +1,7 @@ #!/bin/bash +pydir=${1:-/usr} + gensup() { for SUP in Cond Free Leak Overlap Addr1 Addr2 Addr4 Addr8 Addr16 Value1 Value2 Value4 Value8 Value16 ; do @@ -16,10 +18,10 @@ gensup() { while read SO ; do gensup libpython "$SO" -done < <(find /usr/lib*/ -type f -name libpython*) +done < <(find ${pydir}/lib*/ -type f -name libpython*) while read SO ; do gensup python "$SO" -done < <(find /usr/lib*/python*/ -type f -name \*.so) +done < <(find ${pydir}/lib*/python*/ -type f -name \*.so)