diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6fe6f35 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.t linguist-language=Text diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml new file mode 100644 index 0000000..595a4c1 --- /dev/null +++ b/.github/workflows/coverity.yml @@ -0,0 +1,68 @@ +name: Coverity + +on: + schedule: + - cron: "0 0 * * *" + +jobs: + scan: + runs-on: ubuntu-18.04 + if: ${{ github.repository_owner == 'openresty' }} + env: + COVERITY_SCAN_PROJECT_NAME: 'rds-json-nginx-module' + COVERITY_SCAN_BRANCH_PATTERN: '*' + COVERITY_SCAN_NOTIFICATION_EMAIL: 'chipitsine@gmail.com' + LUAJIT_PREFIX: '/opt/luajit21' + LUAJIT_LIB: '/opt/luajit21/lib' + LUAJIT_INC: '/opt/luajit21/include/luajit-2.1' + LUA_INCLUDE_DIR: '/opt/luajit21/include/luajit-2.1' + LUA_CMODULE_DIR: '/lib' + JOBS: 3 + NGX_BUILD_JOBS: 3 + NGINX_VERSION: 1.19.9 + CC: gcc + steps: + - uses: actions/checkout@v3 + - name: Install apt dependencies + run: | + sudo apt-get update + sudo apt-get install -y axel libgd-dev + - name: clone OpenResty satellites + run: | + git clone https://github.com/openresty/nginx-devel-utils.git + git clone https://github.com/openresty/openresty.git ../openresty + git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx + git clone https://github.com/simpl/ngx_devel_kit.git ../ndk-nginx-module + git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module + git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core + git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache + git clone https://github.com/openresty/nginx-eval-module.git ../eval-nginx-module + git clone https://github.com/openresty/xss-nginx-module.git ../xss-nginx-module + git clone https://github.com/openresty/set-misc-nginx-module.git ../set-misc-nginx-module + git clone https://github.com/openresty/headers-more-nginx-module.git ../headers-more-nginx-module + git clone https://github.com/openresty/drizzle-nginx-module.git ../drizzle-nginx-module + git clone https://github.com/calio/form-input-nginx-module.git ../form-input-nginx-module + git clone https://github.com/openresty/ngx_postgres.git ../postgres-nginx-module + git clone https://github.com/openresty/openresty.git ../ngx_openresty + git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module + git clone https://github.com/openresty/array-var-nginx-module.git ../array-var-nginx-module + - name: Install libdrizzle + run: | + wget http://openresty.org/download/drizzle7-2011.07.21.tar.gz + tar xzf drizzle7-2011.07.21.tar.gz && cd drizzle7-2011.07.21 + ./configure --prefix=/usr --without-server + sudo PATH=$PATH make libdrizzle-1.0 install-libdrizzle-1.0 + - name: Install luajit2 + run: | + git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git + cd luajit2 + make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' + sudo make install PREFIX=$LUAJIT_PREFIX + - name: Run Coverity Scan + env: + COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + run: | + export COVERITY_SCAN_BUILD_COMMAND="sh util/build.sh $NGINX_VERSION" + export PATH=$PWD/work/nginx/sbin:$PWD/nginx-devel-utils:$PATH + export NGX_BUILD_CC=gcc + curl -fsSL "https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh" | bash || true diff --git a/.gitignore b/.gitignore index abb8e85..8d4b393 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,9 @@ restart misc/ t/servroot/ all -build1[01] +build1[0-9] buildroot/ *.html +*.plist +Makefile +*.patch diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..7b6efa7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,92 @@ +sudo: required +dist: focal + +os: linux + +branches: + only: + - "master" +language: c + +addons: + apt: + packages: + - axel + - cpanminus + - libtest-base-perl + - libtext-diff-perl + - liburi-perl + - libwww-perl + - libtest-longstring-perl + - liblist-moreutils-perl + - libgd-dev + postgresql: "13" + +cache: + apt: true + directories: + - download-cache + +compiler: + - gcc + +env: + global: + - LUAJIT_PREFIX=/opt/luajit21 + - LUAJIT_LIB=$LUAJIT_PREFIX/lib + - LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH + - LUAJIT_INC=$LUAJIT_PREFIX/include/luajit-2.1 + - LUA_INCLUDE_DIR=$LUAJIT_INC + - LUA_CMODULE_DIR=/lib + - JOBS=3 + - DRIZZLE_VER=2011.07.21 + - NGX_BUILD_JOBS=$JOBS + - TEST_NGINX_SLEEP=0.006 + matrix: + - NGINX_VERSION=1.27.1 NGX_EXTRA_OPT=--without-pcre2 + +services: + - mysql + - postgresql + +install: + - if [ ! -f download-cache/drizzle7-$DRIZZLE_VER.tar.gz ]; then wget -P download-cache https://github.com/openresty/openresty-deps-prebuild/releases/download/v20230902/drizzle7-$DRIZZLE_VER.tar.gz; fi + - git clone https://github.com/openresty/openresty-devel-utils.git + - git clone https://github.com/openresty/openresty.git ../openresty + - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx + - git clone https://github.com/simpl/ngx_devel_kit.git ../ndk-nginx-module + - git clone https://github.com/openresty/test-nginx.git + - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git + - git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module + - git clone https://github.com/openresty/lua-resty-core.git ../lua-resty-core + - git clone https://github.com/openresty/lua-resty-lrucache.git ../lua-resty-lrucache + - git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module + - git clone https://github.com/openresty/nginx-eval-module.git ../eval-nginx-module + - git clone https://github.com/openresty/xss-nginx-module.git ../xss-nginx-module + - git clone https://github.com/openresty/set-misc-nginx-module.git ../set-misc-nginx-module + - git clone https://github.com/openresty/array-var-nginx-module.git ../array-var-nginx-module + - git clone https://github.com/openresty/drizzle-nginx-module.git ../drizzle-nginx-module + - git clone https://github.com/calio/form-input-nginx-module.git ../form-input-nginx-module + - git clone https://github.com/openresty/ngx_postgres.git ../postgres-nginx-module + - git clone https://github.com/openresty/openresty.git ../ngx_openresty + +before_script: + - mysql -uroot -e "create database ngx_test; CREATE USER 'ngx_test'@'%' IDENTIFIED WITH mysql_native_password BY 'ngx_test'; grant all on ngx_test.* to 'ngx_test'@'%'; flush privileges;" + - mysql -uroot -e 'alter database ngx_test character set utf8mb4 collate utf8mb4_unicode_ci;' + - psql -c "create database ngx_test;" -U postgres + - psql -c "create user ngx_test with password 'ngx_test';" -U postgres + - psql -c "grant all privileges on database ngx_test to ngx_test;" -U postgres + +script: + - tar xzf download-cache/drizzle7-2011.07.21.tar.gz && cd drizzle7-2011.07.21 + - ./configure --prefix=/usr --without-server > build.log 2>&1 || (cat build.log && exit 1) + - sudo PATH=$PATH make libdrizzle-1.0 install-libdrizzle-1.0 > build.log 2>&1 || (cat build.log && exit 1) + - cd ../luajit2 + - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' > build.log 2>&1 || (cat build.log && exit 1) + - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) + - cd ../test-nginx && sudo cpanm . && cd .. + - export PATH=$PWD/work/nginx/sbin:$PWD/openresty-devel-utils:$PATH + - export NGX_BUILD_CC=$CC + - sh util/build.sh $NGINX_VERSION > build.log 2>&1 || (cat build.log && exit 1) + - nginx -V + - prove -I. -r t diff --git a/README b/README deleted file mode 100644 index 401a0ce..0000000 --- a/README +++ /dev/null @@ -1,223 +0,0 @@ -Name - ngx_rds_json - an output filter that formats Resty - DBD Streams generated by ngx_drizzle and others to JSON - -Status - This module is considered production ready. - - We need your help! If you find this module useful and/or - interesting, please consider joining the development! - Commit bit can be freely delivered at your request ;) - -Synopsis - - server { - location /mysql { - drizzle_query 'select * from cats'; - drizzle_pass my_mysql_upstream; - - rds_json on; - } - - location /foo { - if ($arg_limit !~ '^\d{2}$') { - rds_json_ret 400 'Bad "limit" argument'; - } - - drizzle_query "select * from dogs limit $arg_limit"; - drizzle_pass my_mysql_upstream; - - rds_json on; - } - ... - } - -Description - This module provides an output filter that can format the RDS outputs - generated by ngx_drizzle and ngx_postgres modules to JSON. - -Directives - rds_json - syntax: rds_json on|off - default: rds_json off - - Enables or disables the output filter of this module. - - rds_json_buffer_size - syntax: rds_json_buffer_size - default: rds_json_buffer_size - - Controls the buffer size used by this module. default to the page size (4k/8k). - The bigger the buffer size, the less streammy the conversion - will be. But usually increasing the buffer size - does help reduce CPU time. - - rds_json_format - syntax: rds_json_format normal|compact - default: rds_json_format normal - - Controls the output JSON format. A sample of the default "normal" format - looks like this - - [{"id":1,"name":"marry"},{"id":2,"name":"bob"}] - - while it looks like below when in the "compact" format - - [["id","name"],[1,"marry"],[2,"bob"]] - - that is, the first row holds the column name list. - - rds_json_root - syntax: rds_json_root - default: no - - Specify the "root" key for data rows (if any). - - For example, - - rds_json on; - rds_json_root rows; - - will return JSON output like this: - - {"rows":[{"id":2,"name":null},{"id":3,"name":"bob"}]} - - if "rds_json_format compact" is also specified, then the - output will look like this: - - {"rows":[["id","name"],[2,null],[3,"bob"]]} - - Nginx variables are not supported in the argument of - this directive. - - If this directive is not defined, neither are rds_json_success_property, - nor rds_json_extra_property, the JSON output for select queries will - just be an array at the - top level. - - When either rds_json_success_property or rds_json_extra_property are specified, - this directive takes a default argument of "data". - - rds_json_success_property - syntax: rds_json_success_property - default: no - - Specify the top-level object property name used in the JSON output - for indicating success or false of the query. - - rds_json_user_property - syntax: rds_json_user_property - default: no - - Specify additonal user properties for the top-level object - of the JSON output. - - Multiple instances of this directives are allowed in a single scope. - - Nginx variables are supported in the argument. - - Both of the and arguments will be automatically - quoted according to JSON strings' notation. - - rds_json_ret - syntax: rds_json_ret - default: no - - This directive enables a content handler that simply emits - an response body like this: - - {"errcode":,"errstr":""} - - while the string will be properly quoted as - a JSON string. - - rds_json_content_type - syntax: rds_json_content_type - default: rds_json_content_type application/json - - Controls the Content-Type header of the response generated by - this module's output filter. - -Installation - Grab the nginx source code from nginx.net (), for - example, the version 1.0.5 (see nginx compatibility), and then build - the source with this module: - - $ wget 'http://sysoev.ru/nginx/nginx-1.0.5.tar.gz' - $ tar -xzvf nginx-1.0.5.tar.gz - $ cd nginx-1.0.5/ - - # Here we assume you would install you nginx under /opt/nginx/. - $ ./configure --prefix=/opt/nginx \ - --add-module=/path/to/drizzle-nginx-module - --add-module=/path/to/rds-json-nginx-module - - $ make -j2 - $ make install - - Download the latest version of the release tarball of this module from - rds-json-nginx-module file list - (). - -Compatibility - The following versions of Nginx should work with this module: - - * 1.1.x (last tested: 1.1.5) - - * 1.0.x (last tested: 1.0.8) - - * 0.9.x (last tested: 0.9.4) - - * 0.8.x (last tested: 0.8.55) - - * 0.7.x >= 0.7.46 (last tested: 0.7.68) - - Earlier versions of Nginx like 0.6.x and 0.5.x will *not* work. - - If you find that any particular version of Nginx above 0.7.44 does not - work with this module, please consider reporting a bug. - -TODO - -Known Issues - -See Also - - * ngx_drizzle ( http://github.com/chaoslawful/drizzle-nginx-module ) - - * ngx_postgres ( http://github.com/FRiCKLE/ngx_postgres/ ) - -Author - Zhang "agentzh" Yichun (章亦春) - -Copyright & License - This module is licenced under the BSD license. - - Copyright (c) 2009, 2010, 2011, Taobao Inc., Alibaba Group ( http://www.taobao.com ). - - Copyright (C) 2009, 2010, 2011, Zhang "agentzh" Yichun (章亦春) . - - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ba6455 --- /dev/null +++ b/README.md @@ -0,0 +1,373 @@ +Name +==== + +ngx_rds_json - an output filter that formats Resty DBD Streams generated by ngx_drizzle and others to JSON + +Table of Contents +================= + +* [Name](#name) +* [Status](#status) +* [Synopsis](#synopsis) +* [Description](#description) +* [Directives](#directives) + * [rds_json](#rds_json) + * [rds_json_buffer_size](#rds_json_buffer_size) + * [rds_json_format](#rds_json_format) + * [rds_json_root](#rds_json_root) + * [rds_json_success_property](#rds_json_success_property) + * [rds_json_user_property](#rds_json_user_property) + * [rds_json_errcode_key](#rds_json_errcode_key) + * [rds_json_errstr_key](#rds_json_errstr_key) + * [rds_json_ret](#rds_json_ret) + * [rds_json_content_type](#rds_json_content_type) +* [Installation](#installation) +* [Compatibility](#compatibility) +* [Author](#author) +* [Copyright & License](#copyright--license) +* [See Also](#see-also) + +Status +====== + +This module is considered production ready. + +We need your help! If you find this module useful and/or interesting, please consider joining the development! +Commit bit can be freely delivered at your request ;) + +Synopsis +======== + +```nginx +server { + location /mysql { + drizzle_query 'select * from cats'; + drizzle_pass my_mysql_upstream; + + rds_json on; + } + + location /foo { + if ($arg_limit !~ '^\d{2}$') { + rds_json_ret 400 'Bad "limit" argument'; + } + + drizzle_query "select * from dogs limit $arg_limit"; + drizzle_pass my_mysql_upstream; + + rds_json on; + } + ... +} +``` + +Description +=========== + +This module provides an output filter that can format the RDS outputs +generated by [ngx_drizzle](https://github.com/openresty/drizzle-nginx-module) +and [ngx_postgres](https://github.com/FRiCKLE/ngx_postgres/) modules to JSON. + +[Back to TOC](#table-of-contents) + +Directives +========== + +[Back to TOC](#table-of-contents) + +rds_json +-------- +**syntax:** *rds_json on|off* + +**default:** *rds_json off* + +**context:** *http, server, location, if location* + +Enables or disables the output filter of this module. + +[Back to TOC](#table-of-contents) + +rds_json_buffer_size +-------------------- +**syntax:** *rds_json_buffer_size <bytes>* + +**default:** *rds_json_buffer_size <page-size>* + +**context:** *http, server, location, if location* + +Controls the buffer size used by this module. default to the page size (4k/8k). +The bigger the buffer size, the less streammy the conversion +will be. But usually increasing the buffer size +does help reduce CPU time. + +[Back to TOC](#table-of-contents) + +rds_json_format +--------------- +**syntax:** *rds_json_format normal|compact* + +**default:** *rds_json_format normal* + +**context:** *http, server, location, if location* + +Controls the output JSON format. A sample of the default "normal" format +looks like this + +```json + [{"id":1,"name":"marry"},{"id":2,"name":"bob"}] +``` + +while it looks like below when in the "compact" format + +```json + [["id","name"],[1,"marry"],[2,"bob"]] +``` + +that is, the first row holds the column name list. + +[Back to TOC](#table-of-contents) + +rds_json_root +------------- +**syntax:** *rds_json_root <key>* + +**default:** *no* + +**context:** *http, server, location, if location* + +Specifies the "root" key for data rows (if any). For example, + +```nginx + rds_json on; + rds_json_root rows; +``` + +will return JSON output like this: + +```json + {"rows":[{"id":2,"name":null},{"id":3,"name":"bob"}]} +``` + +if `rds_json_format compact` is also specified, then the +output will look like this: + +```json + {"rows":[["id","name"],[2,null],[3,"bob"]]} +``` + +Nginx variables are not supported in the argument of +this directive. + +If this directive is not defined, neither are [rds_json_success_property](#rds_json_success_property), +nor [rds_json_user_property](#rds_json_user_property), the JSON output for select queries will +just be an array at the top level. + +When either [rds_json_success_property](#rds_json_success_property) or [rds_json_user_property](#rds_json_user_property) are specified, +this directive takes a default argument of "data". + +[Back to TOC](#table-of-contents) + +rds_json_success_property +------------------------- +**syntax:** *rds_json_success_property <key>* + +**default:** *no* + +**context:** *http, server, location, if location* + +Specifies the top-level object property name used in the JSON output +for indicating success or false of the query. + +[Back to TOC](#table-of-contents) + +rds_json_user_property +----------------------- +**syntax:** *rds_json_user_property <key> <value>* + +**default:** *no* + +**context:** *http, server, location, if location* + +Specifies additonal user properties for the top-level object +of the JSON output. + +Multiple instances of this directives are allowed in a single scope. + +Nginx variables are supported in the `value` argument. + +Both of the `key` and `value` arguments will be automatically +quoted according to JSON strings' notation. + +[Back to TOC](#table-of-contents) + +rds_json_errcode_key +--------------------- +**syntax:** *rds_json_errcode_key <key>* + +**default:** *rds_json_errcode_key errcode* + +**context:** *http, server, location, if location* + +Specifies the errcode key name used in the JSON output. + +[Back to TOC](#table-of-contents) + +rds_json_errstr_key +------------------- +**syntax:** *rds_json_errstr_key <key>* + +**default:** *rds_json_errstr_key errstr* + +**context:** *http, server, location, if location* + +Specifies the errstr key name used in the JSON output. + +[Back to TOC](#table-of-contents) + +rds_json_ret +------------ +**syntax:** *rds_json_ret <error-code> <descrption>* + +**default:** *no* + +**context:** *location, if location* + +This directive enables a content handler that simply emits +an response body like this: + +```json + {"errcode":,"errstr":""} +``` + +while the `` string will be properly quoted as +a JSON string. + +[Back to TOC](#table-of-contents) + +rds_json_content_type +--------------------- +**syntax:** *rds_json_content_type <mime-type>* + +**default:** *rds_json_content_type application/json* + +**context:** *http, server, location, if location* + +Controls the `Content-Type` header of the response generated by +this module's output filter. + +[Back to TOC](#table-of-contents) + +Installation +============ + +You're recommended to install this module (as well as the Nginx core and many other goodies) via the [OpenResty bundle](http://openresty.org). See [the detailed instructions](http://openresty.org/#Installation) for downloading and installing OpenResty into your system. This is the easiest and most safe way to set things up. + +Alternatively, you can install this module manually with the Nginx source: + +Grab the nginx source code from [nginx.org](http://nginx.org/), for example, +the version 1.13.6 (see [nginx compatibility](#compatibility)), and then build the source with this module: + +```bash + + $ wget 'http://nginx.org/download/nginx-1.13.6.tar.gz' + $ tar -xzvf nginx-1.13.6.tar.gz + $ cd nginx-1.13.6/ + + # Here we assume you would install you nginx under /opt/nginx/. + $ ./configure --prefix=/opt/nginx \ + --add-module=/path/to/rds-json-nginx-module + + $ make -j2 + $ make install +``` + +Download the latest version of the release tarball of this module from [rds-json-nginx-module file list](https://github.com/openresty/rds-json-nginx-module/tags). + +Starting from NGINX 1.9.11, you can also compile this module as a dynamic module, by using the `--add-dynamic-module=PATH` option +instead of `--add-module=PATH` on the `./configure` command line above. +And then you can explicitly load the module in your `nginx.conf` via the [load_module](http://nginx.org/en/docs/ngx_core_module.html#load_module) directive, for example, + +```nginx +load_module /path/to/modules/ngx_http_rds_json_filter_module.so; +``` + +Also, this module is included and enabled by default in the [OpenResty bundle](http://openresty.org). + +[Back to TOC](#table-of-contents) + +Compatibility +============= +The following versions of Nginx should work with this module: + +* **1.13.x** (last tested: 1.13.6) +* **1.12.x** +* **1.11.x** (last tested: 1.11.2) +* **1.10.x** +* **1.9.x** (last tested: 1.9.7) +* **1.8.x** +* **1.7.x** (last tested: 1.7.10) +* **1.6.x** +* **1.5.x** (last tested: 1.5.12) +* **1.4.x** (last tested: 1.4.4) +* **1.2.x** (last tested: 1.2.9) +* **1.1.x** (last tested: 1.1.5) +* **1.0.x** (last tested: 1.0.9) +* **0.9.x** (last tested: 0.9.4) +* **0.8.x** (last tested: 0.8.55) +* **0.7.x >= 0.7.46** (last tested: 0.7.68) + +Earlier versions of Nginx like 0.6.x and 0.5.x will *not* work. + +If you find that any particular version of Nginx above 0.7.44 does not +work with this module, please consider reporting a bug. + +[Back to TOC](#table-of-contents) + +Author +====== +Yichun "agentzh" Zhang (章亦春) *<agentzh@gmail.com>*, OpenResty Inc. + +[Back to TOC](#table-of-contents) + +Copyright & License +=================== + +This module is licenced under the BSD license. + +Copyright (C) 2009-2017, Yichun Zhang (agentzh) <agentzh@gmail.com>, OpenResty Inc. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +[Back to TOC](#table-of-contents) + +See Also +======== + +* [ngx_drizzle](https://github.com/openresty/drizzle-nginx-module) +* [ngx_postgres](https://github.com/FRiCKLE/ngx_postgres/) +* [ngx_rds_csv](https://github.com/openresty/rds-csv-nginx-module) + +[Back to TOC](#table-of-contents) + diff --git a/config b/config index f9f65d5..5ebf3b2 100644 --- a/config +++ b/config @@ -1,5 +1,30 @@ ngx_addon_name=ngx_http_rds_json_filter_module -HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES ngx_http_rds_json_filter_module" -NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/src/ngx_http_rds_json_filter_module.c $ngx_addon_dir/src/ngx_http_rds_json_processor.c $ngx_addon_dir/src/ngx_http_rds_json_util.c $ngx_addon_dir/src/ngx_http_rds_json_output.c $ngx_addon_dir/src/ngx_http_rds_json_handler.c" -NGX_ADDON_DEPS="$NGX_ADDON_DEPS $ngx_addon_dir/src/ddebug.h $ngx_addon_dir/src/resty_dbd_stream.h $ngx_addon_dir/src/ngx_http_rds_json_filter_module.h $ngx_addon_dir/src/ngx_http_rds_json_processor.h $ngx_addon_dir/src/ngx_http_rds_json_util.h $ngx_addon_dir/src/ngx_http_rds.h $ngx_addon_dir/src/resty_dbd_stream.h $ngx_addon_dir/src/ngx_http_rds_json_output.h $ngx_addon_dir/src/ngx_http_rds_utils.h $ngx_addon_dir/src/ngx_http_rds_json_handler.h" +RDS_JSON_FILTER_SRCS=" \ + $ngx_addon_dir/src/ngx_http_rds_json_filter_module.c \ + $ngx_addon_dir/src/ngx_http_rds_json_processor.c \ + $ngx_addon_dir/src/ngx_http_rds_json_util.c \ + $ngx_addon_dir/src/ngx_http_rds_json_output.c \ + $ngx_addon_dir/src/ngx_http_rds_json_handler.c \ + " + +RDS_JSON_FILTER_DEPS=" \ + $ngx_addon_dir/src/ddebug.h \ + $ngx_addon_dir/src/resty_dbd_stream.h \ + $ngx_addon_dir/src/ngx_http_rds_json_filter_module.h \ + $ngx_addon_dir/src/ngx_http_rds_json_processor.h \ + $ngx_addon_dir/src/ngx_http_rds_json_util.h \ + $ngx_addon_dir/src/ngx_http_rds.h \ + $ngx_addon_dir/src/resty_dbd_stream.h \ + $ngx_addon_dir/src/ngx_http_rds_json_output.h \ + $ngx_addon_dir/src/ngx_http_rds_utils.h \ + $ngx_addon_dir/src/ngx_http_rds_json_handler.h \ + " + + +ngx_module_type=HTTP_AUX_FILTER +ngx_module_name=$ngx_addon_name +ngx_module_srcs="$RDS_JSON_FILTER_SRCS" +ngx_module_deps="$RDS_JSON_FILTER_DEPS" + +. auto/module diff --git a/src/ddebug.h b/src/ddebug.h index 8f57f86..0bfa9b2 100644 --- a/src/ddebug.h +++ b/src/ddebug.h @@ -1,6 +1,7 @@ #ifndef DDEBUG_H #define DDEBUG_H +#include #include #if defined(DDEBUG) && (DDEBUG) @@ -28,7 +29,8 @@ #include -static void dd(const char * fmt, ...) { +static ngx_inline void +dd(const char * fmt, ...) { } # endif @@ -45,7 +47,8 @@ static void dd(const char * fmt, ...) { #include -static void dd(const char * fmt, ...) { +static ngx_inline void +dd(const char * fmt, ...) { } # endif diff --git a/src/ngx_http_rds.h b/src/ngx_http_rds.h index 2e3d878..838f838 100644 --- a/src/ngx_http_rds.h +++ b/src/ngx_http_rds.h @@ -11,6 +11,7 @@ #include #include #include +#include typedef struct { diff --git a/src/ngx_http_rds_json_filter_module.c b/src/ngx_http_rds_json_filter_module.c index 7bcee34..8954abc 100644 --- a/src/ngx_http_rds_json_filter_module.c +++ b/src/ngx_http_rds_json_filter_module.c @@ -1,10 +1,12 @@ /* - * Copyright (C) agentzh + * Copyright (C) Yichun Zhang (agentzh) */ +#ifndef DDEBUG #define DDEBUG 0 +#endif #include "ddebug.h" #include "ngx_http_rds_json_filter_module.h" @@ -18,8 +20,12 @@ #define ngx_http_rds_json_content_type "application/json" +#define ngx_http_rds_json_errcode_default_key "\"errcode\"" +#define ngx_http_rds_json_errstr_default_key "\"errstr\"" + + +static volatile ngx_cycle_t *ngx_http_rds_json_prev_cycle = NULL; -static unsigned ngx_http_rds_json_filter_used = 0; static ngx_conf_enum_t ngx_http_rds_json_formats[] = { { ngx_string("normal"), json_format_normal }, @@ -33,20 +39,24 @@ ngx_http_output_header_filter_pt ngx_http_rds_json_next_header_filter; ngx_http_output_body_filter_pt ngx_http_rds_json_next_body_filter; +static void *ngx_http_rds_json_create_main_conf(ngx_conf_t *cf); static char *ngx_http_rds_json_ret(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); -static void *ngx_http_rds_json_create_conf(ngx_conf_t *cf); -static char *ngx_http_rds_json_merge_conf(ngx_conf_t *cf, void *parent, + void *conf); +static void *ngx_http_rds_json_create_loc_conf(ngx_conf_t *cf); +static char *ngx_http_rds_json_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child); static ngx_int_t ngx_http_rds_json_filter_init(ngx_conf_t *cf); -static char * ngx_http_rds_json_root(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); -static char * ngx_http_rds_json_success_property(ngx_conf_t *cf, - ngx_command_t *cmd, void *conf); -static char * ngx_http_rds_json_user_property(ngx_conf_t *cf, - ngx_command_t *cmd, void *conf); -static char * ngx_http_rds_json(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); -static ngx_int_t ngx_http_rds_json_pre_config(ngx_conf_t *cf); +static char *ngx_http_rds_json_root(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static char *ngx_http_rds_json_success_property(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); +static char *ngx_http_rds_json_user_property(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); +static char *ngx_http_rds_json_errcode_key(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static char *ngx_http_rds_json_errstr_key(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static char *ngx_http_rds_json(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static ngx_command_t ngx_http_rds_json_commands[] = { @@ -57,7 +67,7 @@ static ngx_command_t ngx_http_rds_json_commands[] = { |NGX_CONF_FLAG, ngx_http_rds_json, NGX_HTTP_LOC_CONF_OFFSET, - offsetof(ngx_http_rds_json_conf_t, enabled), + offsetof(ngx_http_rds_json_loc_conf_t, enabled), NULL }, { ngx_string("rds_json_root"), @@ -87,13 +97,31 @@ static ngx_command_t ngx_http_rds_json_commands[] = { 0, NULL }, + { ngx_string("rds_json_errcode_key"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF + |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF + |NGX_CONF_TAKE1, + ngx_http_rds_json_errcode_key, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("rds_json_errstr_key"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF + |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF + |NGX_CONF_TAKE1, + ngx_http_rds_json_errstr_key, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL }, + { ngx_string("rds_json_format"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF |NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF |NGX_CONF_TAKE1, ngx_conf_set_enum_slot, NGX_HTTP_LOC_CONF_OFFSET, - offsetof(ngx_http_rds_json_conf_t, format), + offsetof(ngx_http_rds_json_loc_conf_t, format), &ngx_http_rds_json_formats }, { ngx_string("rds_json_content_type"), @@ -102,7 +130,7 @@ static ngx_command_t ngx_http_rds_json_commands[] = { |NGX_CONF_TAKE1, ngx_conf_set_str_slot, NGX_HTTP_LOC_CONF_OFFSET, - offsetof(ngx_http_rds_json_conf_t, content_type), + offsetof(ngx_http_rds_json_loc_conf_t, content_type), NULL }, { ngx_string("rds_json_ret"), @@ -117,7 +145,7 @@ static ngx_command_t ngx_http_rds_json_commands[] = { |NGX_CONF_TAKE1, ngx_conf_set_size_slot, NGX_HTTP_LOC_CONF_OFFSET, - offsetof(ngx_http_rds_json_conf_t, buf_size), + offsetof(ngx_http_rds_json_loc_conf_t, buf_size), NULL }, ngx_null_command @@ -125,17 +153,17 @@ static ngx_command_t ngx_http_rds_json_commands[] = { static ngx_http_module_t ngx_http_rds_json_filter_module_ctx = { - ngx_http_rds_json_pre_config, /* preconfiguration */ + NULL, /* preconfiguration */ ngx_http_rds_json_filter_init, /* postconfiguration */ - NULL, /* create main configuration */ + ngx_http_rds_json_create_main_conf, /* create main configuration */ NULL, /* init main configuration */ NULL, /* create server configuration */ NULL, /* merge server configuration */ - ngx_http_rds_json_create_conf, /* create location configuration */ - ngx_http_rds_json_merge_conf /* merge location configuration */ + ngx_http_rds_json_create_loc_conf, /* create location configuration */ + ngx_http_rds_json_merge_loc_conf /* merge location configuration */ }; @@ -158,12 +186,16 @@ ngx_module_t ngx_http_rds_json_filter_module = { static ngx_int_t ngx_http_rds_json_header_filter(ngx_http_request_t *r) { - ngx_http_rds_json_ctx_t *ctx; - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_ctx_t *ctx; + ngx_http_rds_json_loc_conf_t *conf; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json header filter, \"%V\"", &r->uri); /* XXX maybe we can generate stub JSON strings like * {"errcode":403,"error":"Permission denied"} * for HTTP error pages? */ + if ((r->headers_out.status < NGX_HTTP_OK) || (r->headers_out.status >= NGX_HTTP_SPECIAL_RESPONSE) || (r->headers_out.status == NGX_HTTP_NO_CONTENT) @@ -171,7 +203,9 @@ ngx_http_rds_json_header_filter(ngx_http_request_t *r) { ngx_http_set_ctx(r, NULL, ngx_http_rds_json_filter_module); - dd("status is not OK: %d, skipping", (int) r->headers_out.status); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json: skipped due to bad status: %ui", + r->headers_out.status); return ngx_http_rds_json_next_header_filter(r); } @@ -180,11 +214,20 @@ ngx_http_rds_json_header_filter(ngx_http_request_t *r) conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_json_filter_module); - if ( ! conf->enabled) { + if (!conf->enabled) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json: skipped because not enabled in the current " + "location"); + return ngx_http_rds_json_next_header_filter(r); } if (ngx_http_rds_json_test_content_type(r) != NGX_OK) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json: skipped due to unmatched Content-Type " + "response header \"%V\"", + &r->headers_out.content_type); + return ngx_http_rds_json_next_header_filter(r); } @@ -226,6 +269,9 @@ ngx_http_rds_json_header_filter(ngx_http_request_t *r) r->filter_need_in_memory = 1; + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json header filter postponed header sending"); + /* we do postpone the header sending to the body filter */ return NGX_OK; } @@ -237,6 +283,9 @@ ngx_http_rds_json_body_filter(ngx_http_request_t *r, ngx_chain_t *in) ngx_http_rds_json_ctx_t *ctx; ngx_int_t rc; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json body filter, \"%V\"", &r->uri); + if (in == NULL || r->header_only) { return ngx_http_rds_json_next_body_filter(r, in); } @@ -247,6 +296,9 @@ ngx_http_rds_json_body_filter(ngx_http_request_t *r, ngx_chain_t *in) return ngx_http_rds_json_next_body_filter(r, in); } + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json body filter postponed header sending"); + switch (ctx->state) { case state_expect_header: rc = ngx_http_rds_json_process_header(r, in, ctx); @@ -270,9 +322,11 @@ ngx_http_rds_json_body_filter(ngx_http_request_t *r, ngx_chain_t *in) case state_done: - /* mark the remaining bufs as consumed */ + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json body filter discarding unexpected trailing " + "buffers"); - dd("discarding bufs"); + /* mark the remaining bufs as consumed */ ngx_http_rds_json_discard_bufs(r->pool, in); @@ -280,8 +334,8 @@ ngx_http_rds_json_body_filter(ngx_http_request_t *r, ngx_chain_t *in) default: ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: invalid internal state: %d", - ctx->state); + "rds_json: invalid internal state: %d", + ctx->state); rc = NGX_ERROR; @@ -293,7 +347,7 @@ ngx_http_rds_json_body_filter(ngx_http_request_t *r, ngx_chain_t *in) if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { ctx->state = state_done; - if (! ctx->header_sent) { + if (!ctx->header_sent) { ctx->header_sent = 1; if (rc == NGX_ERROR) { @@ -302,7 +356,8 @@ ngx_http_rds_json_body_filter(ngx_http_request_t *r, ngx_chain_t *in) r->headers_out.status = rc; - dd("sending ERROR headers"); + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "rds json body filter sending error page headers"); ngx_http_rds_json_next_header_filter(r); ngx_http_send_special(r, NGX_HTTP_LAST); @@ -322,9 +377,25 @@ ngx_http_rds_json_body_filter(ngx_http_request_t *r, ngx_chain_t *in) static ngx_int_t ngx_http_rds_json_filter_init(ngx_conf_t *cf) { - dd("setting next filter"); + int multi_http_blocks; + ngx_http_rds_json_main_conf_t *rmcf; + + rmcf = ngx_http_conf_get_module_main_conf(cf, + ngx_http_rds_json_filter_module); + + if (ngx_http_rds_json_prev_cycle != ngx_cycle) { + ngx_http_rds_json_prev_cycle = ngx_cycle; + multi_http_blocks = 0; + + } else { + multi_http_blocks = 1; + } + + dd("setting next filter: %d", (int) rmcf->requires_filters); + + if (multi_http_blocks || rmcf->requires_filters) { + dd("register filters"); - if (ngx_http_rds_json_filter_used) { ngx_http_rds_json_next_header_filter = ngx_http_top_header_filter; ngx_http_top_header_filter = ngx_http_rds_json_header_filter; @@ -337,11 +408,11 @@ ngx_http_rds_json_filter_init(ngx_conf_t *cf) static void * -ngx_http_rds_json_create_conf(ngx_conf_t *cf) +ngx_http_rds_json_create_loc_conf(ngx_conf_t *cf) { - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_loc_conf_t *conf; - conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_rds_json_conf_t)); + conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_rds_json_loc_conf_t)); if (conf == NULL) { return NULL; } @@ -371,20 +442,18 @@ ngx_http_rds_json_create_conf(ngx_conf_t *cf) static char * -ngx_http_rds_json_merge_conf(ngx_conf_t *cf, void *parent, void *child) +ngx_http_rds_json_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) { - ngx_http_rds_json_conf_t *prev = parent; - ngx_http_rds_json_conf_t *conf = child; + ngx_http_rds_json_loc_conf_t *prev = parent; + ngx_http_rds_json_loc_conf_t *conf = child; ngx_conf_merge_value(conf->enabled, prev->enabled, 0); ngx_conf_merge_uint_value(conf->format, prev->format, json_format_normal); - ngx_conf_merge_str_value(conf->root, prev->root, - ""); + ngx_conf_merge_str_value(conf->root, prev->root, ""); - ngx_conf_merge_str_value(conf->success, prev->success, - ""); + ngx_conf_merge_str_value(conf->success, prev->success, ""); if (conf->user_props == NULL) { conf->user_props = prev->user_props; @@ -394,21 +463,26 @@ ngx_http_rds_json_merge_conf(ngx_conf_t *cf, void *parent, void *child) ngx_str_set(&conf->root, "\"data\""); } + ngx_conf_merge_str_value(conf->errcode_key, prev->errcode_key, + "\"errcode\""); + + ngx_conf_merge_str_value(conf->errstr_key, prev->errstr_key, + "\"errstr\""); + ngx_conf_merge_str_value(conf->content_type, prev->content_type, - ngx_http_rds_json_content_type); + ngx_http_rds_json_content_type); ngx_conf_merge_size_value(conf->buf_size, prev->buf_size, - (size_t) ngx_pagesize); + (size_t) ngx_pagesize); return NGX_CONF_OK; } static char * -ngx_http_rds_json_ret(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf) +ngx_http_rds_json_ret(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { - ngx_http_rds_json_conf_t *jlcf = conf; + ngx_http_rds_json_loc_conf_t *jlcf = conf; ngx_http_core_loc_conf_t *clcf; ngx_str_t *value; ngx_http_compile_complex_value_t ccv; @@ -421,7 +495,7 @@ ngx_http_rds_json_ret(ngx_conf_t *cf, ngx_command_t *cmd, if (jlcf->errcode.len == 0) { ngx_log_error(NGX_LOG_ERR, cf->log, 0, - "rds_json: rds_json_ret: the errcode argument is empty"); + "rds_json: rds_json_ret: the errcode argument is empty"); return NGX_CONF_ERROR; } @@ -432,8 +506,8 @@ ngx_http_rds_json_ret(ngx_conf_t *cf, ngx_command_t *cmd, if (c < '0' || c > '9') { ngx_log_error(NGX_LOG_ERR, cf->log, 0, - "rds_json: rds_json_ret: invalid errcode argument: " - "\"%V\"", &jlcf->errcode); + "rds_json: rds_json_ret: invalid errcode argument: " + "\"%V\"", &jlcf->errcode); return NGX_CONF_ERROR; } @@ -461,8 +535,7 @@ ngx_http_rds_json_ret(ngx_conf_t *cf, ngx_command_t *cmd, done: - clcf = ngx_http_conf_get_module_loc_conf(cf, - ngx_http_core_module); + clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module); if (clcf == NULL) { return NGX_CONF_ERROR; } @@ -474,10 +547,9 @@ ngx_http_rds_json_ret(ngx_conf_t *cf, ngx_command_t *cmd, static char * -ngx_http_rds_json_root(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf) +ngx_http_rds_json_root(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { - ngx_http_rds_json_conf_t *jlcf = conf; + ngx_http_rds_json_loc_conf_t *jlcf = conf; ngx_str_t *value; uintptr_t escape; u_char *p; @@ -493,7 +565,7 @@ ngx_http_rds_json_root(ngx_conf_t *cf, ngx_command_t *cmd, } escape = ngx_http_rds_json_escape_json_str(NULL, value[1].data, - value[1].len); + value[1].len); jlcf->root.len = value[1].len + escape + sizeof("\"\"") - 1; @@ -511,7 +583,7 @@ ngx_http_rds_json_root(ngx_conf_t *cf, ngx_command_t *cmd, } else { p = (u_char *) ngx_http_rds_json_escape_json_str(p, value[1].data, - value[1].len); + value[1].len); } *p++ = '"'; @@ -526,9 +598,9 @@ ngx_http_rds_json_root(ngx_conf_t *cf, ngx_command_t *cmd, static char * ngx_http_rds_json_success_property(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf) + void *conf) { - ngx_http_rds_json_conf_t *jlcf = conf; + ngx_http_rds_json_loc_conf_t *jlcf = conf; ngx_str_t *value; uintptr_t escape; u_char *p; @@ -544,7 +616,7 @@ ngx_http_rds_json_success_property(ngx_conf_t *cf, ngx_command_t *cmd, } escape = ngx_http_rds_json_escape_json_str(NULL, value[1].data, - value[1].len); + value[1].len); jlcf->success.len = value[1].len + escape + sizeof("\"\"") - 1; @@ -562,7 +634,7 @@ ngx_http_rds_json_success_property(ngx_conf_t *cf, ngx_command_t *cmd, } else { p = (u_char *) ngx_http_rds_json_escape_json_str(p, value[1].data, - value[1].len); + value[1].len); } *p++ = '"'; @@ -577,15 +649,14 @@ ngx_http_rds_json_success_property(ngx_conf_t *cf, ngx_command_t *cmd, static char * ngx_http_rds_json_user_property(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf) + void *conf) { - ngx_http_rds_json_conf_t *jlcf = conf; + u_char *p; ngx_str_t *value; - ngx_http_rds_json_property_t *prop; uintptr_t escape; - u_char *p; - - ngx_http_compile_complex_value_t ccv; + ngx_http_rds_json_property_t *prop; + ngx_http_rds_json_loc_conf_t *jlcf = conf; + ngx_http_compile_complex_value_t ccv; value = cf->args->elts; @@ -599,7 +670,7 @@ ngx_http_rds_json_user_property(ngx_conf_t *cf, ngx_command_t *cmd, if (jlcf->user_props == NULL) { jlcf->user_props = ngx_array_create(cf->pool, 4, - sizeof(ngx_http_rds_json_property_t)); + sizeof(ngx_http_rds_json_property_t)); if (jlcf->user_props == NULL) { return NGX_CONF_ERROR; @@ -607,11 +678,14 @@ ngx_http_rds_json_user_property(ngx_conf_t *cf, ngx_command_t *cmd, } prop = ngx_array_push(jlcf->user_props); + if (prop == NULL) { + return NGX_CONF_ERROR; + } /* process the user property key */ escape = ngx_http_rds_json_escape_json_str(NULL, value[1].data, - value[1].len); + value[1].len); prop->key.len = value[1].len + escape + sizeof("\"\"") - 1; @@ -629,7 +703,7 @@ ngx_http_rds_json_user_property(ngx_conf_t *cf, ngx_command_t *cmd, } else { p = (u_char *) ngx_http_rds_json_escape_json_str(p, value[1].data, - value[1].len); + value[1].len); } *p++ = '"'; @@ -653,19 +727,135 @@ ngx_http_rds_json_user_property(ngx_conf_t *cf, ngx_command_t *cmd, } +static char * +ngx_http_rds_json_errcode_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_rds_json_loc_conf_t *jlcf = conf; + u_char *p; + ngx_str_t *value; + uintptr_t escape; + + value = cf->args->elts; + + if (jlcf->errcode_key.len) { + return "is duplicate"; + } + + if (value[1].len == 0) { + return "takes an empty value"; + } + + escape = ngx_http_rds_json_escape_json_str(NULL, value[1].data, + value[1].len); + + jlcf->errcode_key.len = value[1].len + escape + sizeof("\"\"") - 1; + + p = ngx_palloc(cf->pool, jlcf->errcode_key.len); + if (p == NULL) { + return NGX_CONF_ERROR; + } + + jlcf->errcode_key.data = p; + + *p++ = '"'; + + if (escape == 0) { + p = ngx_copy(p, value[1].data, value[1].len); + + } else { + p = (u_char *) ngx_http_rds_json_escape_json_str(p, value[1].data, + value[1].len); + } + + *p++ = '"'; + + if (p - jlcf->errcode_key.data != (ssize_t) jlcf->errcode_key.len) { + return "sees buffer error"; + } + + return NGX_CONF_OK; +} + + +static char * +ngx_http_rds_json_errstr_key(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_rds_json_loc_conf_t *jlcf = conf; + ngx_str_t *value; + uintptr_t escape; + u_char *p; + + value = cf->args->elts; + + if (jlcf->errstr_key.len) { + return "is duplicate"; + } + + if (value[1].len == 0) { + return "takes an empty value"; + } + + escape = ngx_http_rds_json_escape_json_str(NULL, value[1].data, + value[1].len); + + jlcf->errstr_key.len = value[1].len + escape + sizeof("\"\"") - 1; + + p = ngx_palloc(cf->pool, jlcf->errstr_key.len); + if (p == NULL) { + return NGX_CONF_ERROR; + } + + jlcf->errstr_key.data = p; + + *p++ = '"'; + + if (escape == 0) { + p = ngx_copy(p, value[1].data, value[1].len); + + } else { + p = (u_char *) ngx_http_rds_json_escape_json_str(p, value[1].data, + value[1].len); + } + + *p++ = '"'; + + if (p - jlcf->errstr_key.data != (ssize_t) jlcf->errstr_key.len) { + return "sees buffer error"; + } + + return NGX_CONF_OK; +} + + static char * ngx_http_rds_json(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { - ngx_http_rds_json_filter_used = 1; + ngx_http_rds_json_main_conf_t *rmcf; + + rmcf = ngx_http_conf_get_module_main_conf(cf, + ngx_http_rds_json_filter_module); + + dd("set filter used to 1"); + + rmcf->requires_filters = 1; return ngx_conf_set_flag_slot(cf, cmd, conf); } -static ngx_int_t ngx_http_rds_json_pre_config(ngx_conf_t *cf) +static void * +ngx_http_rds_json_create_main_conf(ngx_conf_t *cf) { - ngx_http_rds_json_filter_used = 0; + ngx_http_rds_json_main_conf_t *rmcf; - return NGX_OK; -} + rmcf = ngx_pcalloc(cf->pool, sizeof(ngx_http_rds_json_main_conf_t)); + if (rmcf == NULL) { + return NULL; + } + + /* set by ngx_pcalloc: + * rmcf->requires_filters = 0; + */ + return rmcf; +} diff --git a/src/ngx_http_rds_json_filter_module.h b/src/ngx_http_rds_json_filter_module.h index 275c6fd..9ad044d 100644 --- a/src/ngx_http_rds_json_filter_module.h +++ b/src/ngx_http_rds_json_filter_module.h @@ -11,6 +11,7 @@ #include #include #include +#include #ifndef NGX_HTTP_RESET_CONTENT @@ -39,6 +40,12 @@ typedef struct { } ngx_http_rds_json_property_t; +typedef struct { + unsigned requires_filters; /* :1 */ + +} ngx_http_rds_json_main_conf_t; + + typedef struct { ngx_flag_t enabled; ngx_uint_t format; @@ -48,12 +55,16 @@ typedef struct { * key */ ngx_array_t *user_props; /* rds_json_user_property */ + ngx_str_t errcode_key; + ngx_str_t errstr_key; + size_t buf_size; /* for rds_json_ret */ ngx_str_t errcode; ngx_http_complex_value_t *errstr; -} ngx_http_rds_json_conf_t; + +} ngx_http_rds_json_loc_conf_t; typedef enum { @@ -95,9 +106,9 @@ typedef struct { uint32_t field_data_rest; - ngx_flag_t header_sent:1; - ngx_flag_t seen_stream_end:1; - ngx_flag_t generated_col_names:1; + ngx_uint_t header_sent:1; + ngx_uint_t seen_stream_end:1; + ngx_uint_t generated_col_names:1; } ngx_http_rds_json_ctx_t; diff --git a/src/ngx_http_rds_json_handler.c b/src/ngx_http_rds_json_handler.c index 01bdbc5..28f53d4 100644 --- a/src/ngx_http_rds_json_handler.c +++ b/src/ngx_http_rds_json_handler.c @@ -1,4 +1,6 @@ +#ifndef DDEBUG #define DDEBUG 0 +#endif #include "ddebug.h" #include "ngx_http_rds_json_handler.h" @@ -11,7 +13,6 @@ ngx_http_rds_json_ret_handler(ngx_http_request_t *r) ngx_chain_t *cl; ngx_buf_t *b; size_t len; - ngx_http_rds_json_conf_t *conf; ngx_str_t errstr; ngx_int_t rc; uintptr_t escape = 0; @@ -19,11 +20,11 @@ ngx_http_rds_json_ret_handler(ngx_http_request_t *r) uintptr_t *escapes = NULL; ngx_uint_t i; ngx_http_rds_json_property_t *prop = NULL; + ngx_http_rds_json_loc_conf_t *conf; dd("entered ret handler"); - conf = ngx_http_get_module_loc_conf(r, - ngx_http_rds_json_filter_module); + conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_json_filter_module); /* evaluate the final value of conf->errstr */ @@ -33,23 +34,30 @@ ngx_http_rds_json_ret_handler(ngx_http_request_t *r) /* calculate the buffer size */ - len = sizeof("{\"errcode\":") - 1 + len = sizeof("{") - 1 + + conf->errcode_key.len + + sizeof(":") - 1 + conf->errcode.len + sizeof("}") - 1 ; if (errstr.len) { - escape = ngx_http_rds_json_escape_json_str(NULL, - errstr.data, errstr.len); - - len += sizeof("\"errstr\":\"") - 1 - + errstr.len + escape - + sizeof("\",") - 1 + escape = ngx_http_rds_json_escape_json_str(NULL, errstr.data, + errstr.len); + + len += sizeof(",") - 1 + + conf->errstr_key.len + + sizeof(":") - 1 + + sizeof("\"") - 1 + + errstr.len + + escape + + sizeof("\"") - 1 ; } if (conf->success.len) { len += conf->success.len + sizeof(":,") - 1; + if (ngx_atoi(conf->errcode.data, conf->errcode.len) == 0) { len += sizeof("true") - 1; @@ -60,15 +68,15 @@ ngx_http_rds_json_ret_handler(ngx_http_request_t *r) if (conf->user_props) { values = ngx_pnalloc(r->pool, - conf->user_props->nelts * (sizeof(ngx_str_t) + - sizeof(uintptr_t))); + conf->user_props->nelts + * (sizeof(ngx_str_t) + sizeof(uintptr_t))); if (values == NULL) { return NGX_ERROR; } escapes = (uintptr_t *) ((u_char *) values + - conf->user_props->nelts * sizeof(ngx_str_t)); + conf->user_props->nelts * sizeof(ngx_str_t)); prop = conf->user_props->elts; for (i = 0; i < conf->user_props->nelts; i++) { @@ -79,7 +87,7 @@ ngx_http_rds_json_ret_handler(ngx_http_request_t *r) } escapes[i] = ngx_http_rds_json_escape_json_str(NULL, values[i].data, - values[i].len); + values[i].len); len += sizeof(":\"\",") - 1 + prop[i].key.len + values[i].len + escapes[i]; @@ -110,11 +118,16 @@ ngx_http_rds_json_ret_handler(ngx_http_request_t *r) *b->last++ = '{'; - b->last = ngx_copy_literal(b->last, "\"errcode\":"); + b->last = ngx_copy(b->last, conf->errcode_key.data, conf->errcode_key.len); + *b->last++ = ':'; b->last = ngx_copy(b->last, conf->errcode.data, conf->errcode.len); if (errstr.len) { - b->last = ngx_copy_literal(b->last, ",\"errstr\":\""); + *b->last++ = ','; + b->last = ngx_copy(b->last, conf->errstr_key.data, + conf->errstr_key.len); + *b->last++ = ':'; + *b->last++ = '"'; if (escape == 0) { b->last = ngx_copy(b->last, errstr.data, errstr.len); @@ -161,7 +174,7 @@ ngx_http_rds_json_ret_handler(ngx_http_request_t *r) if (b->last != b->end) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: rds_json_ret: buffer error"); + "rds_json: rds_json_ret: buffer error"); return NGX_ERROR; } diff --git a/src/ngx_http_rds_json_output.c b/src/ngx_http_rds_json_output.c index 6e299be..6f85904 100644 --- a/src/ngx_http_rds_json_output.c +++ b/src/ngx_http_rds_json_output.c @@ -4,9 +4,12 @@ */ +#ifndef DDEBUG #define DDEBUG 0 +#endif #include "ddebug.h" + #include "ngx_http_rds_json_filter_module.h" #include "ngx_http_rds_json_output.h" #include "ngx_http_rds_json_util.h" @@ -14,25 +17,22 @@ #include -static u_char * ngx_http_rds_json_request_mem(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, size_t len); - +static u_char *ngx_http_rds_json_request_mem(ngx_http_request_t *r, + ngx_http_rds_json_ctx_t *ctx, size_t len); static ngx_int_t ngx_http_rds_json_get_buf(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx); - -static u_char * ngx_http_rds_json_get_postponed(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, size_t len); - + ngx_http_rds_json_ctx_t *ctx); +static u_char *ngx_http_rds_json_get_postponed(ngx_http_request_t *r, + ngx_http_rds_json_ctx_t *ctx, size_t len); static ngx_int_t ngx_http_rds_json_submit_mem(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, size_t len, unsigned last_buf); + ngx_http_rds_json_ctx_t *ctx, size_t len, unsigned last_buf); + static size_t ngx_get_num_size(uint64_t i); ngx_int_t ngx_http_rds_json_output_literal(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, - u_char *data, size_t len, ngx_flag_t last_buf) + ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len, int last_buf) { u_char *pos; @@ -59,7 +59,7 @@ ngx_http_rds_json_output_literal(ngx_http_request_t *r, ngx_int_t ngx_http_rds_json_output_bufs(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx) + ngx_http_rds_json_ctx_t *ctx) { ngx_int_t rc; ngx_chain_t *cl; @@ -101,24 +101,23 @@ ngx_http_rds_json_output_bufs(ngx_http_request_t *r, } #if defined(nginx_version) && nginx_version >= 1001004 - ngx_chain_update_chains(r->pool, &ctx->free_bufs, &ctx->busy_bufs, &ctx->out, - ctx->tag); + ngx_chain_update_chains(r->pool, &ctx->free_bufs, &ctx->busy_bufs, + &ctx->out, ctx->tag); #else ngx_chain_update_chains(&ctx->free_bufs, &ctx->busy_bufs, &ctx->out, - ctx->tag); + ctx->tag); #endif ctx->last_out = &ctx->out; } /* impossible to reach here */ - return NGX_ERROR; } ngx_int_t ngx_http_rds_json_output_header(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_header_t *header) + ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_header_t *header) { u_char *pos, *last; size_t size; @@ -129,16 +128,17 @@ ngx_http_rds_json_output_header(ngx_http_request_t *r, uintptr_t *escapes = NULL; ngx_http_rds_json_property_t *prop = NULL; - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_loc_conf_t *conf; conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_json_filter_module); /* calculate the buffer size */ - size = sizeof("{\"errcode\":") - 1 - + ngx_get_num_size(header->std_errcode) - + sizeof("}") - 1 - ; + size = sizeof("{") - 1 + + conf->errcode_key.len + + sizeof(":") - 1 + + ngx_get_num_size(header->std_errcode) + + sizeof("}") - 1; if (conf->success.len) { size += conf->success.len + sizeof(":,") - 1; @@ -152,15 +152,15 @@ ngx_http_rds_json_output_header(ngx_http_request_t *r, if (conf->user_props) { values = ngx_pnalloc(r->pool, - conf->user_props->nelts * (sizeof(ngx_str_t) + - sizeof(uintptr_t))); + conf->user_props->nelts + * (sizeof(ngx_str_t) + sizeof(uintptr_t))); if (values == NULL) { return NGX_ERROR; } - escapes = (uintptr_t *) ((u_char *) values + - conf->user_props->nelts * sizeof(ngx_str_t)); + escapes = (uintptr_t *) ((u_char *) values + + conf->user_props->nelts * sizeof(ngx_str_t)); prop = conf->user_props->elts; for (i = 0; i < conf->user_props->nelts; i++) { @@ -170,36 +170,40 @@ ngx_http_rds_json_output_header(ngx_http_request_t *r, return NGX_ERROR; } - escapes[i] = ngx_http_rds_json_escape_json_str(NULL, values[i].data, - values[i].len); + escapes[i] = ngx_http_rds_json_escape_json_str(NULL, + values[i].data, + values[i].len); size += sizeof(":\"\",") - 1 + prop[i].key.len + values[i].len - + escapes[i]; + + escapes[i]; } } + if (header->errstr.len) { escape = ngx_http_rds_json_escape_json_str(NULL, header->errstr.data, - header->errstr.len); + header->errstr.len); + + size += sizeof(",") - 1 + + conf->errstr_key.len + + sizeof(":") - 1 + + sizeof("\"") - 1 + + header->errstr.len + + escape + + sizeof("\"") - 1; - size += sizeof(",\"errstr\":\"") - 1 - + header->errstr.len + escape - + sizeof("\"") - 1 - ; } else { escape = (uintptr_t) 0; } if (header->insert_id) { size += sizeof(",\"insert_id\":") - 1 - + ngx_get_num_size(header->insert_id) - ; + + ngx_get_num_size(header->insert_id); } if (header->affected_rows) { size += sizeof(",\"affected_rows\":") - 1 - + ngx_get_num_size(header->affected_rows) - ; + + ngx_get_num_size(header->affected_rows); } /* create the buffer */ @@ -236,8 +240,10 @@ ngx_http_rds_json_output_header(ngx_http_request_t *r, last = ngx_copy(last, values[i].data, values[i].len); } else { - last = (u_char *) ngx_http_rds_json_escape_json_str(last, - values[i].data, values[i].len); + last = (u_char *) + ngx_http_rds_json_escape_json_str(last, + values[i].data, + values[i].len); } *last++ = '"'; @@ -245,21 +251,26 @@ ngx_http_rds_json_output_header(ngx_http_request_t *r, } } - last = ngx_copy_literal(last, "\"errcode\":"); + last = ngx_copy(last, conf->errcode_key.data, conf->errcode_key.len); + *last++ = ':'; last = ngx_snprintf(last, NGX_UINT16_LEN, "%uD", - (uint32_t) header->std_errcode); + (uint32_t) header->std_errcode); if (header->errstr.len) { - last = ngx_copy_literal(last, ",\"errstr\":\""); + *last++ = ','; + last = ngx_copy(last, conf->errstr_key.data, conf->errstr_key.len); + *last++ = ':'; + *last++ = '"'; if (escape == 0) { - last = ngx_copy(last, header->errstr.data, - header->errstr.len); + last = ngx_copy(last, header->errstr.data, header->errstr.len); } else { - last = (u_char *) ngx_http_rds_json_escape_json_str(last, - header->errstr.data, header->errstr.len); + last = (u_char *) + ngx_http_rds_json_escape_json_str(last, + header->errstr.data, + header->errstr.len); } *last++ = '"'; @@ -267,22 +278,21 @@ ngx_http_rds_json_output_header(ngx_http_request_t *r, if (header->insert_id) { last = ngx_copy_literal(last, ",\"insert_id\":"); - last = ngx_snprintf(last, NGX_UINT64_LEN, "%uL", - header->insert_id); + last = ngx_snprintf(last, NGX_UINT64_LEN, "%uL", header->insert_id); } if (header->affected_rows) { last = ngx_copy_literal(last, ",\"affected_rows\":"); last = ngx_snprintf(last, NGX_UINT64_LEN, "%uL", - header->affected_rows); + header->affected_rows); } *last++ = '}'; if ((size_t) (last - pos) != size) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: output header buffer error: %O != %uz", - (off_t) (last - pos), size); + "rds_json: output header buffer error: %O != %uz", + (off_t) (last - pos), size); return NGX_ERROR; } @@ -299,7 +309,7 @@ ngx_http_rds_json_output_header(ngx_http_request_t *r, ngx_int_t ngx_http_rds_json_output_props(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_json_conf_t *conf) + ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_json_loc_conf_t *conf) { size_t size; u_char *pos, *last; @@ -317,15 +327,16 @@ ngx_http_rds_json_output_props(ngx_http_request_t *r, if (conf->user_props) { values = ngx_pnalloc(r->pool, - conf->user_props->nelts * (sizeof(ngx_str_t) + - sizeof(uintptr_t))); + conf->user_props->nelts + * (sizeof(ngx_str_t) + sizeof(uintptr_t))); if (values == NULL) { return NGX_ERROR; } - escapes = (uintptr_t *) ((u_char *) values + - conf->user_props->nelts * sizeof(ngx_str_t)); + escapes = (uintptr_t *) ((u_char *) values + + conf->user_props->nelts + * sizeof(ngx_str_t)); prop = conf->user_props->elts; for (i = 0; i < conf->user_props->nelts; i++) { @@ -335,11 +346,12 @@ ngx_http_rds_json_output_props(ngx_http_request_t *r, return NGX_ERROR; } - escapes[i] = ngx_http_rds_json_escape_json_str(NULL, values[i].data, - values[i].len); + escapes[i] = ngx_http_rds_json_escape_json_str(NULL, + values[i].data, + values[i].len); size += sizeof(":\"\",") - 1 + prop[i].key.len + values[i].len - + escapes[i]; + + escapes[i]; } } @@ -367,8 +379,10 @@ ngx_http_rds_json_output_props(ngx_http_request_t *r, last = ngx_copy(last, values[i].data, values[i].len); } else { - last = (u_char *) ngx_http_rds_json_escape_json_str(last, - values[i].data, values[i].len); + last = (u_char *) + ngx_http_rds_json_escape_json_str(last, + values[i].data, + values[i].len); } *last++ = '"'; @@ -381,8 +395,8 @@ ngx_http_rds_json_output_props(ngx_http_request_t *r, if (last - pos != (ssize_t) size) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: output props begin: buffer error: %O != %uz", - (off_t) (last - pos), size); + "rds_json: output props begin: buffer error: %O != %uz", + (off_t) (last - pos), size); return NGX_ERROR; } @@ -393,7 +407,7 @@ ngx_http_rds_json_output_props(ngx_http_request_t *r, ngx_int_t ngx_http_rds_json_output_cols(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx) + ngx_http_rds_json_ctx_t *ctx) { ngx_uint_t i; ngx_http_rds_column_t *col; @@ -405,14 +419,12 @@ ngx_http_rds_json_output_cols(ngx_http_request_t *r, for (i = 0; i < ctx->col_count; i++) { col = &ctx->cols[i]; - key_escape = ngx_http_rds_json_escape_json_str(NULL, - col->name.data, col->name.len); + key_escape = ngx_http_rds_json_escape_json_str(NULL, col->name.data, + col->name.len); dd("key_escape: %d", (int) key_escape); - size += col->name.len + key_escape - + sizeof("\"\"") - 1 - ; + size += col->name.len + key_escape + sizeof("\"\"") - 1; if (i != ctx->col_count - 1) { /* not the last column */ @@ -437,7 +449,8 @@ ngx_http_rds_json_output_cols(ngx_http_request_t *r, *last++ = '"'; last = (u_char *) ngx_http_rds_json_escape_json_str(last, - col->name.data, col->name.len); + col->name.data, + col->name.len); *last++ = '"'; @@ -454,8 +467,7 @@ ngx_http_rds_json_output_cols(ngx_http_request_t *r, ngx_int_t ngx_http_rds_json_output_field(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len, - ngx_flag_t is_null) + ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len, int is_null) { u_char *pos, *last; ngx_http_rds_column_t *col; @@ -465,7 +477,7 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, uintptr_t val_escape = 0; u_char *p; ngx_uint_t i; - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_loc_conf_t *conf; ngx_uint_t format; conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_json_filter_module); @@ -473,8 +485,8 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, format = conf->format; dd("reading row %llu, col %d, len %d", - (unsigned long long) ctx->row, - (int) ctx->cur_col, (int) len); + (unsigned long long) ctx->row, + (int) ctx->cur_col, (int) len); /* calculate the buffer size */ @@ -484,9 +496,11 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, if (ctx->row == 1) { dd("first column, first row"); size = sizeof("{\"") - 1; + } else { size = sizeof(",{\"") - 1; } + } else { size = sizeof(",\"") - 1; } @@ -495,9 +509,11 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, if (ctx->cur_col == 0) { dd("first column"); size = sizeof(",[") - 1; + } else { size = sizeof(",") - 1; } + } else { return NGX_ERROR; } @@ -506,24 +522,23 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, if (format == json_format_normal) { key_escape = ngx_http_rds_json_escape_json_str(NULL, col->name.data, - col->name.len); + col->name.len); dd("key_escape: %d", (int) key_escape); - size += col->name.len + key_escape - + sizeof("\":") - 1 - ; + size += col->name.len + key_escape + sizeof("\":") - 1; } else if (format == json_format_compact) { /* do nothing */ + } else { return NGX_ERROR; } if (len == 0 && ctx->field_data_rest > 0) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: at least one octet should go with the field size " - "in one buf"); + "rds_json: at least one octet should go with the " + "field size in one buf"); return NGX_ERROR; } @@ -554,8 +569,8 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, if (*p < '0' || *p > '9') { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: invalid integral field value: \"%*s\"", - len, data); + "rds_json: invalid integral field value: " + "\"%*s\"", len, data); return NGX_ERROR; } } @@ -567,28 +582,31 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, dd("bool field found"); if (*data == '0' || *data == 'f' || *data == 'F' - || *data == '1' || *data == 't' || *data == 'T' ) + || *data == '1' || *data == 't' || *data == 'T' ) { if (len != 1 || ctx->field_data_rest) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: invalid boolean field value leading " - "by \"%*s\"", len, data); + "rds_json: invalid boolean field value " + "leading by \"%*s\"", len, data); return NGX_ERROR; } if (*data == '0' || *data == 'f' || *data == 'F') { bool_val = 0; size += sizeof("false") - 1; + } else { bool_val = 1; size += sizeof("true") - 1; } + } else { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: output field: invalid boolean value " - "leading by \"%*s\"", len, data); + "rds_json: output field: invalid boolean value " + "leading by \"%*s\"", len, data); return NGX_ERROR; } + break; default: @@ -598,9 +616,7 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, val_escape = ngx_http_rds_json_escape_json_str(NULL, data, len); - size += sizeof("\"") - 1 - + len + val_escape - ; + size += sizeof("\"") - 1 + len + val_escape; if (ctx->field_data_rest == 0) { size += sizeof("\"") - 1; @@ -609,7 +625,7 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, } if (ctx->field_data_rest == 0 - && ctx->cur_col == ctx->col_count - 1) + && ctx->cur_col == ctx->col_count - 1) { /* last column in the row */ if (format == json_format_normal) { @@ -617,6 +633,7 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, } else if (format == json_format_compact) { size += sizeof("]") - 1; + } else { return NGX_ERROR; } @@ -639,9 +656,11 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, if (ctx->row == 1) { dd("first column, first row"); *last++ = '{'; + } else { *last++ = ','; *last++ = '{'; } + } else { *last++ = ','; } @@ -652,16 +671,20 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, last = ngx_copy(last, col->name.data, col->name.len); } else { - last = (u_char *) ngx_http_rds_json_escape_json_str(last, - col->name.data, col->name.len); + last = (u_char *) + ngx_http_rds_json_escape_json_str(last, col->name.data, + col->name.len); } *last++ = '"'; *last++ = ':'; } else if (format == json_format_compact) { + if (ctx->cur_col == 0) { dd("first column"); - *last++ = ','; *last++ = '['; + *last++ = ','; + *last++ = '['; + } else { *last++ = ','; } @@ -677,6 +700,7 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, } else if (len == 0) { dd("copy emtpy string over"); last = ngx_copy_literal(last, "\"\""); + } else { switch (col->std_type & 0xc000) { case rds_rough_col_type_int: @@ -690,9 +714,11 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, case rds_rough_col_type_bool: if (bool_val) { last = ngx_copy_literal(last, "true"); + } else { last = ngx_copy_literal(last, "false"); } + break; default: @@ -701,22 +727,25 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, if (val_escape == 0) { last = ngx_copy(last, data, len); + } else { dd("field: string value escape non-zero: %d", - (int) val_escape); + (int) val_escape); #if DDEBUG p = last; #endif - last = (u_char *) ngx_http_rds_json_escape_json_str(last, - data, len); + last = (u_char *) + ngx_http_rds_json_escape_json_str(last, data, len); +#if DDEBUG dd("escaped value \"%.*s\" (len %d, escape %d, escape2 %d)", - (int) (len + val_escape), - p, (int) (len + val_escape), - (int) val_escape, - (int) ((last - p) - len)); + (int) (len + val_escape), + p, (int) (len + val_escape), + (int) val_escape, + (int) ((last - p) - len)); +#endif } if (ctx->field_data_rest == 0) { @@ -727,14 +756,14 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, } } - if (ctx->field_data_rest == 0 - && ctx->cur_col == ctx->col_count - 1) - { + if (ctx->field_data_rest == 0 && ctx->cur_col == ctx->col_count - 1) { dd("last column in the row"); if (format == json_format_normal) { *last++ = '}'; + } else if (format == json_format_compact) { *last++ = ']'; + } else { return NGX_ERROR; } @@ -742,8 +771,8 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, if ((size_t) (last - pos) != size) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: output field: buffer error (%d left)", - (int) size - (last - pos)); + "rds_json: output field: buffer error (%d left)", + (int) size - (last - pos)); return NGX_ERROR; } @@ -754,7 +783,7 @@ ngx_http_rds_json_output_field(ngx_http_request_t *r, ngx_int_t ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len) + ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len) { u_char *pos, *last; size_t size = 0; @@ -762,7 +791,7 @@ ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, uintptr_t escape = 0; u_char *p; ngx_uint_t i; - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_loc_conf_t *conf; conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_json_filter_module); @@ -775,8 +804,8 @@ ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, for (p = data, i = 0; i < len; i++, p++) { if (*p < '0' || *p > '9') { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: invalid integral field value: \"%*s\"", - len, data); + "rds_json: invalid integral field value: \"%*s\"", + len, data); return NGX_ERROR; } } @@ -804,14 +833,14 @@ ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, break; } - if (ctx->field_data_rest == 0 - && ctx->cur_col == ctx->col_count - 1) - { + if (ctx->field_data_rest == 0 && ctx->cur_col == ctx->col_count - 1) { /* last column in the row */ if (conf->format == json_format_normal) { size += sizeof("}") - 1; + } else if (conf->format == json_format_compact) { size += sizeof("]") - 1; + } else { return NGX_ERROR; } @@ -845,21 +874,22 @@ ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, } else { dd("more field data: string value escape non-zero: %d", - (int) escape); + (int) escape); #if DDEBUG p = last; #endif last = (u_char *) ngx_http_rds_json_escape_json_str(last, - data, len); + data, len); +#if DDEBUG dd("escaped value \"%.*s\" (len %d, escape %d, escape2 %d)", - (int) (len + escape), - p, (int) (len + escape), - (int) escape, - (int) ((last - p) - len)); - + (int) (len + escape), + p, (int) (len + escape), + (int) escape, + (int) ((last - p) - len)); +#endif } if (ctx->field_data_rest == 0) { @@ -869,14 +899,14 @@ ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, break; } /* switch */ - if (ctx->field_data_rest == 0 && - ctx->cur_col == ctx->col_count - 1) - { + if (ctx->field_data_rest == 0 && ctx->cur_col == ctx->col_count - 1) { /* last column in the row */ if (conf->format == json_format_normal) { *last++ = '}'; + } else if (conf->format == json_format_compact) { *last++ = ']'; + } else { return NGX_ERROR; } @@ -884,9 +914,8 @@ ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, if ((size_t) (last - pos) != size) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: output more field data: buffer error " - "(%d left)", (int) (size - (last - pos))); - + "rds_json: output more field data: buffer error " + "(%d left)", (int) (size - (last - pos))); return NGX_ERROR; } @@ -896,7 +925,7 @@ ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, static u_char * ngx_http_rds_json_request_mem(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, size_t len) + ngx_http_rds_json_ctx_t *ctx, size_t len) { ngx_int_t rc; u_char *p; @@ -925,9 +954,9 @@ ngx_http_rds_json_request_mem(ngx_http_request_t *r, static ngx_int_t ngx_http_rds_json_get_buf(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx) + ngx_http_rds_json_ctx_t *ctx) { - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_loc_conf_t *conf; dd("MEM enter"); @@ -945,8 +974,7 @@ ngx_http_rds_json_get_buf(ngx_http_request_t *r, } else { dd("MEM creating temp buf with size: %d", (int) conf->buf_size); - ctx->out_buf = ngx_create_temp_buf(r->pool, - conf->buf_size); + ctx->out_buf = ngx_create_temp_buf(r->pool, conf->buf_size); if (ctx->out_buf == NULL) { return NGX_ERROR; @@ -964,7 +992,7 @@ ngx_http_rds_json_get_buf(ngx_http_request_t *r, static u_char * ngx_http_rds_json_get_postponed(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, size_t len) + ngx_http_rds_json_ctx_t *ctx, size_t len) { u_char *p; @@ -982,21 +1010,22 @@ ngx_http_rds_json_get_postponed(ngx_http_request_t *r, return ctx->cached.start; alloc: - p = ngx_palloc(r->pool, len); - if (p == NULL) { - return NULL; - } - ctx->cached.start = p; - ctx->cached.end = p + len; + p = ngx_palloc(r->pool, len); + if (p == NULL) { + return NULL; + } - return p; + ctx->cached.start = p; + ctx->cached.end = p + len; + + return p; } static ngx_int_t ngx_http_rds_json_submit_mem(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, size_t len, unsigned last_buf) + ngx_http_rds_json_ctx_t *ctx, size_t len, unsigned last_buf) { ngx_chain_t *cl; ngx_int_t rc; @@ -1011,7 +1040,7 @@ ngx_http_rds_json_submit_mem(ngx_http_request_t *r, } ctx->out_buf->last = ngx_copy(ctx->out_buf->last, - ctx->postponed.pos, len); + ctx->postponed.pos, len); ctx->avail_out -= len; @@ -1087,4 +1116,3 @@ ngx_get_num_size(uint64_t i) return n; } - diff --git a/src/ngx_http_rds_json_output.h b/src/ngx_http_rds_json_output.h index d8d42e2..6575f92 100644 --- a/src/ngx_http_rds_json_output.h +++ b/src/ngx_http_rds_json_output.h @@ -3,6 +3,7 @@ * Copyright (C) agentzh */ + #ifndef NGX_HTTP_RDS_JSON_OUTPUT_H #define NGX_HTTP_RDS_JSON_OUTPUT_H @@ -16,22 +17,19 @@ ngx_int_t ngx_http_rds_json_output_header(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_header_t *header); + ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_header_t *header); ngx_int_t ngx_http_rds_json_output_cols(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx); + ngx_http_rds_json_ctx_t *ctx); ngx_int_t ngx_http_rds_json_output_literal(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len, - ngx_flag_t last_buf); + ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len, int last_buf); ngx_int_t ngx_http_rds_json_output_bufs(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx); + ngx_http_rds_json_ctx_t *ctx); ngx_int_t ngx_http_rds_json_output_field(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len, - ngx_flag_t is_null); + ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len, int is_null); ngx_int_t ngx_http_rds_json_output_more_field_data(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len); + ngx_http_rds_json_ctx_t *ctx, u_char *data, size_t len); ngx_int_t ngx_http_rds_json_output_props(ngx_http_request_t *r, - ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_json_conf_t *conf); + ngx_http_rds_json_ctx_t *ctx, ngx_http_rds_json_loc_conf_t *conf); #endif /* NGX_HTTP_RDS_JSON_OUTPUT_H */ - diff --git a/src/ngx_http_rds_json_processor.c b/src/ngx_http_rds_json_processor.c index f83b883..9a242f3 100644 --- a/src/ngx_http_rds_json_processor.c +++ b/src/ngx_http_rds_json_processor.c @@ -1,13 +1,17 @@ + /* * Copyright (C) agentzh */ +#ifndef DDEBUG #define DDEBUG 0 +#endif #include "ddebug.h" -#include "ngx_http_rds_json_processor.h" + #include "ngx_http_rds_json_util.h" +#include "ngx_http_rds_json_processor.h" #include "ngx_http_rds_json_output.h" #include "ngx_http_rds.h" #include "ngx_http_rds_utils.h" @@ -15,9 +19,10 @@ #include #include + ngx_int_t ngx_http_rds_json_process_header(ngx_http_request_t *r, - ngx_chain_t *in, ngx_http_rds_json_ctx_t *ctx) + ngx_chain_t *in, ngx_http_rds_json_ctx_t *ctx) { ngx_buf_t *b; ngx_http_rds_header_t header; @@ -29,11 +34,11 @@ ngx_http_rds_json_process_header(ngx_http_request_t *r, b = in->buf; - if ( ! ngx_buf_in_memory(b) ) { - if ( ! ngx_buf_special(b) ) { + if (!ngx_buf_in_memory(b)) { + if (!ngx_buf_special(b)) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: process header: buf from " - "upstream not in memory"); + "rds_json: process header: buf from " + "upstream not in memory"); goto invalid; } @@ -62,16 +67,15 @@ ngx_http_rds_json_process_header(ngx_http_request_t *r, if (b->pos != b->last) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: header: there's unexpected remaining data " - "in the buf"); - + "rds_json: header: there's unexpected remaining data " + "in the buf"); goto invalid; } ctx->state = state_done; /* now we send the postponed response header */ - if (! ctx->header_sent) { + if (!ctx->header_sent) { ctx->header_sent = 1; rc = ngx_http_rds_json_next_header_filter(r); @@ -93,7 +97,7 @@ ngx_http_rds_json_process_header(ngx_http_request_t *r, } ctx->cols = ngx_palloc(r->pool, - header.col_count * sizeof(ngx_http_rds_column_t)); + header.col_count * sizeof(ngx_http_rds_column_t)); if (ctx->cols == NULL) { goto invalid; @@ -104,7 +108,7 @@ ngx_http_rds_json_process_header(ngx_http_request_t *r, ctx->col_count = header.col_count; /* now we send the postponed response header */ - if (! ctx->header_sent) { + if (!ctx->header_sent) { ctx->header_sent = 1; rc = ngx_http_rds_json_next_header_filter(r); @@ -113,13 +117,13 @@ ngx_http_rds_json_process_header(ngx_http_request_t *r, } } - return ngx_http_rds_json_process_col(r, - b->pos == b->last ? in->next : in, ctx); + return ngx_http_rds_json_process_col(r, b->pos == b->last ? in->next : in, + ctx); invalid: dd("return 500"); - if (! ctx->header_sent) { + if (!ctx->header_sent) { ctx->header_sent = 1; r->headers_out.status = NGX_HTTP_INTERNAL_SERVER_ERROR; @@ -134,12 +138,12 @@ ngx_http_rds_json_process_header(ngx_http_request_t *r, ngx_int_t -ngx_http_rds_json_process_col(ngx_http_request_t *r, - ngx_chain_t *in, ngx_http_rds_json_ctx_t *ctx) +ngx_http_rds_json_process_col(ngx_http_request_t *r, ngx_chain_t *in, + ngx_http_rds_json_ctx_t *ctx) { ngx_buf_t *b; ngx_int_t rc; - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_loc_conf_t *conf; if (in == NULL) { return NGX_OK; @@ -147,11 +151,11 @@ ngx_http_rds_json_process_col(ngx_http_request_t *r, b = in->buf; - if ( ! ngx_buf_in_memory(b) ) { - if ( ! ngx_buf_special(b) ) { + if (!ngx_buf_in_memory(b)) { + if (!ngx_buf_special(b)) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: process col: buf from " - "upstream not in memory"); + "rds_json: process col: buf from " + "upstream not in memory"); return NGX_ERROR; } @@ -188,7 +192,6 @@ ngx_http_rds_json_process_col(ngx_http_request_t *r, ctx->row = 0; dd("output \"[\""); - dd("before output literal"); conf = ngx_http_get_module_loc_conf(r, ngx_http_rds_json_filter_module); @@ -204,7 +207,8 @@ ngx_http_rds_json_process_col(ngx_http_request_t *r, } rc = ngx_http_rds_json_output_literal(r, ctx, - (u_char *)"[", sizeof("[") - 1, 0 /* last buf */); + (u_char *)"[", sizeof("[") - 1, + 0 /* last buf */); dd("after output literal"); @@ -226,7 +230,6 @@ ngx_http_rds_json_process_col(ngx_http_request_t *r, return rc; } - dd("process col is entering process row..."); return ngx_http_rds_json_process_row(r, in, ctx); } @@ -236,13 +239,13 @@ ngx_http_rds_json_process_col(ngx_http_request_t *r, ngx_int_t -ngx_http_rds_json_process_row(ngx_http_request_t *r, - ngx_chain_t *in, ngx_http_rds_json_ctx_t *ctx) +ngx_http_rds_json_process_row(ngx_http_request_t *r, ngx_chain_t *in, + ngx_http_rds_json_ctx_t *ctx) { ngx_buf_t *b; ngx_int_t rc; - ngx_http_rds_json_conf_t *conf; + ngx_http_rds_json_loc_conf_t *conf; if (in == NULL) { return NGX_OK; @@ -252,11 +255,11 @@ ngx_http_rds_json_process_row(ngx_http_request_t *r, b = in->buf; - if ( ! ngx_buf_in_memory(b) ) { - if ( ! ngx_buf_special(b) ) { + if (!ngx_buf_in_memory(b)) { + if (!ngx_buf_special(b)) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: process row: buf from " - "upstream not in memory"); + "rds_json: process row: buf from " + "upstream not in memory"); return NGX_ERROR; } @@ -271,13 +274,13 @@ ngx_http_rds_json_process_row(ngx_http_request_t *r, if (b->last - b->pos < (ssize_t) sizeof(uint8_t)) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: row flag is incomplete in the buf"); + "rds_json: row flag is incomplete in the buf"); return NGX_ERROR; } dd("row flag: %d (offset %d)", - (char) *b->pos, - (int) (b->pos - b->start)); + (char) *b->pos, + (int) (b->pos - b->start)); if (*b->pos++ == 0) { /* end of row list */ @@ -285,8 +288,8 @@ ngx_http_rds_json_process_row(ngx_http_request_t *r, if (b->pos != b->last) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: row: there's unexpected remaining data " - "in the buf"); + "rds_json: row: there's unexpected remaining data " + "in the buf"); return NGX_ERROR; } @@ -294,11 +297,15 @@ ngx_http_rds_json_process_row(ngx_http_request_t *r, if (conf->root.len) { rc = ngx_http_rds_json_output_literal(r, ctx, - (u_char *)"]}", sizeof("]}") - 1, 1 /* last buf*/); + (u_char *)"]}", + sizeof("]}") - 1, + 1 /* last buf*/); } else { rc = ngx_http_rds_json_output_literal(r, ctx, - (u_char *)"]", sizeof("]") - 1, 1 /* last buf*/); + (u_char *)"]", + sizeof("]") - 1, + 1 /* last buf*/); } if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -324,8 +331,8 @@ ngx_http_rds_json_process_row(ngx_http_request_t *r, ngx_int_t -ngx_http_rds_json_process_field(ngx_http_request_t *r, - ngx_chain_t *in, ngx_http_rds_json_ctx_t *ctx) +ngx_http_rds_json_process_field(ngx_http_request_t *r, ngx_chain_t *in, + ngx_http_rds_json_ctx_t *ctx) { size_t total, len; ngx_buf_t *b; @@ -338,13 +345,13 @@ ngx_http_rds_json_process_field(ngx_http_request_t *r, b = in->buf; - if ( ! ngx_buf_in_memory(b) ) { + if (!ngx_buf_in_memory(b)) { dd("buf not in memory"); - if ( ! ngx_buf_special(b) ) { + if (!ngx_buf_special(b)) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: process field: buf from " - "upstream not in memory"); + "rds_json: process field: buf from " + "upstream not in memory"); return NGX_ERROR; } @@ -361,9 +368,9 @@ ngx_http_rds_json_process_field(ngx_http_request_t *r, if (b->last - b->pos < (ssize_t) sizeof(uint32_t)) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: field size is incomplete in the buf: %*s " - "(len: %d)", b->last - b->pos, b->pos, - (int) (b->last - b->pos)); + "rds_json: field size is incomplete in the buf: %*s " + "(len: %d)", b->last - b->pos, b->pos, + (int) (b->last - b->pos)); return NGX_ERROR; } @@ -381,7 +388,7 @@ ngx_http_rds_json_process_field(ngx_http_request_t *r, ctx->field_data_rest = 0; rc = ngx_http_rds_json_output_field(r, ctx, b->pos, len, - 1 /* is null */); + 1 /* is null */); } else { len = (uint32_t) (b->last - b->pos); @@ -393,7 +400,7 @@ ngx_http_rds_json_process_field(ngx_http_request_t *r, ctx->field_data_rest = total - len; rc = ngx_http_rds_json_output_field(r, ctx, b->pos, len, - 0 /* not null */); + 0 /* not null */); } if (rc == NGX_ERROR || rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -428,14 +435,12 @@ ngx_http_rds_json_process_field(ngx_http_request_t *r, } /* impossible to reach here */ - - return NGX_ERROR; } ngx_int_t ngx_http_rds_json_process_more_field_data(ngx_http_request_t *r, - ngx_chain_t *in, ngx_http_rds_json_ctx_t *ctx) + ngx_chain_t *in, ngx_http_rds_json_ctx_t *ctx) { ngx_int_t rc; ngx_buf_t *b; @@ -448,9 +453,9 @@ ngx_http_rds_json_process_more_field_data(ngx_http_request_t *r, b = in->buf; - if ( ! ngx_buf_in_memory(b)) { + if (!ngx_buf_in_memory(b)) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: buf from upstream not in memory"); + "rds_json: buf from upstream not in memory"); return NGX_ERROR; } @@ -501,7 +506,4 @@ ngx_http_rds_json_process_more_field_data(ngx_http_request_t *r, } /* impossible to reach here */ - - return NGX_ERROR; } - diff --git a/src/ngx_http_rds_json_util.c b/src/ngx_http_rds_json_util.c index 707f0fd..d0080e6 100644 --- a/src/ngx_http_rds_json_util.c +++ b/src/ngx_http_rds_json_util.c @@ -1,6 +1,15 @@ + +/* + * Copyright (C) Yichun Zhang (agentzh) + */ + + +#ifndef DDEBUG #define DDEBUG 0 +#endif #include "ddebug.h" + #include "resty_dbd_stream.h" #include "ngx_http_rds_json_util.h" @@ -117,9 +126,9 @@ ngx_http_rds_json_test_content_type(ngx_http_request_t *r) ngx_str_t *type; type = &r->headers_out.content_type; - if (type->len != rds_content_type_len || - ngx_strncmp(type->data, rds_content_type, - rds_content_type_len) != 0) + if (type->len != rds_content_type_len + || ngx_strncmp(type->data, rds_content_type, rds_content_type_len) + != 0) { return NGX_DECLINED; } @@ -136,7 +145,8 @@ ngx_http_rds_json_discard_bufs(ngx_pool_t *pool, ngx_chain_t *in) for (cl = in; cl; cl = cl->next) { #if 0 if (cl->buf->temporary - && ngx_buf_size(cl->buf) > 0) { + && ngx_buf_size(cl->buf) > 0) + { ngx_pfree(pool, cl->buf->start); } #endif @@ -144,4 +154,3 @@ ngx_http_rds_json_discard_bufs(ngx_pool_t *pool, ngx_chain_t *in) cl->buf->pos = cl->buf->last; } } - diff --git a/src/ngx_http_rds_json_util.h b/src/ngx_http_rds_json_util.h index 8002dba..bce303c 100644 --- a/src/ngx_http_rds_json_util.h +++ b/src/ngx_http_rds_json_util.h @@ -1,8 +1,18 @@ + +/* + * Copyright (C) Yichun Zhang (agentzh) + */ + + #ifndef NGX_HTTP_RDS_JSON_UTIL_H #define NGX_HTTP_RDS_JSON_UTIL_H + #include #include +#include +#include + #ifndef NGX_UINT64_LEN #define NGX_UINT64_LEN (sizeof("18446744073709551615") - 1) @@ -17,9 +27,8 @@ #endif - uintptr_t ngx_http_rds_json_escape_json_str(u_char *dst, u_char *src, - size_t size); + size_t size); ngx_int_t ngx_http_rds_json_test_content_type(ngx_http_request_t *r); @@ -27,4 +36,3 @@ void ngx_http_rds_json_discard_bufs(ngx_pool_t *pool, ngx_chain_t *in); #endif /* NGX_HTTP_RDS_JSON_UTIL_H */ - diff --git a/src/ngx_http_rds_utils.h b/src/ngx_http_rds_utils.h index 67e6f79..25d44d2 100644 --- a/src/ngx_http_rds_utils.h +++ b/src/ngx_http_rds_utils.h @@ -3,39 +3,40 @@ * Copyright (C) agentzh */ + #ifndef NGX_HTTP_RDS_UTILS_H #define NGX_HTTP_RDS_UTILS_H static ngx_inline ngx_int_t ngx_http_rds_parse_header(ngx_http_request_t *r, ngx_buf_t *b, - ngx_http_rds_header_t *header) + ngx_http_rds_header_t *header) { ssize_t rest; - rest = sizeof(uint8_t) /* endian type */ - + sizeof(uint32_t) /* format version */ - + sizeof(uint8_t) /* result type */ + rest = sizeof(uint8_t) /* endian type */ + + sizeof(uint32_t) /* format version */ + + sizeof(uint8_t) /* result type */ - + sizeof(uint16_t) /* standard error code */ - + sizeof(uint16_t) /* driver-specific error code */ + + sizeof(uint16_t) /* standard error code */ + + sizeof(uint16_t) /* driver-specific error code */ - + sizeof(uint16_t) /* driver-specific errstr len */ - + 0 /* driver-specific errstr data */ - + sizeof(uint64_t) /* affected rows */ - + sizeof(uint64_t) /* insert id */ - + sizeof(uint16_t) /* column count */ - ; + + sizeof(uint16_t) /* driver-specific errstr len */ + + 0 /* driver-specific errstr data */ + + sizeof(uint64_t) /* affected rows */ + + sizeof(uint64_t) /* insert id */ + + sizeof(uint16_t) /* column count */ + ; if (b->last - b->pos < rest) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds: header is incomplete in the buf"); + "rds: header is incomplete in the buf"); return NGX_ERROR; } /* check endian type */ - if ( *(uint8_t *) b->pos != + if (*(uint8_t *) b->pos != #if (NGX_HAVE_LITTLE_ENDIAN) 0 #else /* big endian */ @@ -44,7 +45,7 @@ ngx_http_rds_parse_header(ngx_http_request_t *r, ngx_buf_t *b, ) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds: endian type in the header differ"); + "rds: endian type in the header differ"); return NGX_ERROR; } @@ -52,9 +53,9 @@ ngx_http_rds_parse_header(ngx_http_request_t *r, ngx_buf_t *b, /* check RDS format version number */ - if ( *(uint32_t *) b->pos != (uint32_t) resty_dbd_stream_version) { + if (*(uint32_t *) b->pos != (uint32_t) resty_dbd_stream_version) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds: RDS format version differ"); + "rds: RDS format version differ"); return NGX_ERROR; } @@ -66,7 +67,7 @@ ngx_http_rds_parse_header(ngx_http_request_t *r, ngx_buf_t *b, if (*b->pos != 0) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds: RDS result type must be 0 for now"); + "rds: RDS result type must be 0 for now"); return NGX_ERROR; } @@ -95,14 +96,14 @@ ngx_http_rds_parse_header(ngx_http_request_t *r, ngx_buf_t *b, /* check the rest data's size */ rest = header->errstr.len - + sizeof(uint64_t) /* affected rows */ - + sizeof(uint64_t) /* insert id */ - + sizeof(uint16_t) /* column count */ - ; + + sizeof(uint64_t) /* affected rows */ + + sizeof(uint64_t) /* insert id */ + + sizeof(uint16_t) /* column count */ + ; if (b->last - b->pos < rest) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds: header is incomplete in the buf"); + "rds: header is incomplete in the buf"); return NGX_ERROR; } @@ -138,18 +139,18 @@ ngx_http_rds_parse_header(ngx_http_request_t *r, ngx_buf_t *b, static ngx_inline ngx_int_t ngx_http_rds_parse_col(ngx_http_request_t *r, ngx_buf_t *b, - ngx_http_rds_column_t *col) + ngx_http_rds_column_t *col) { ssize_t rest; rest = sizeof(uint16_t) /* std col type */ - + sizeof(uint16_t) /* driver col type */ - + sizeof(uint16_t) /* col name str len */ - ; + + sizeof(uint16_t) /* driver col type */ + + sizeof(uint16_t) /* col name str len */ + ; if (b->last - b->pos < rest) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds: column spec is incomplete in the buf"); + "rds: column spec is incomplete in the buf"); return NGX_ERROR; } @@ -168,7 +169,7 @@ ngx_http_rds_parse_col(ngx_http_request_t *r, ngx_buf_t *b, if (col->name.len == 0) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds_json: column name empty"); + "rds_json: column name empty"); return NGX_ERROR; } @@ -176,7 +177,7 @@ ngx_http_rds_parse_col(ngx_http_request_t *r, ngx_buf_t *b, if (b->last - b->pos < rest) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "rds: column name string is incomplete in the buf"); + "rds: column name string is incomplete in the buf"); return NGX_ERROR; } @@ -191,12 +192,11 @@ ngx_http_rds_parse_col(ngx_http_request_t *r, ngx_buf_t *b, b->pos += col->name.len; dd("saved column name \"%.*s\" (len %d, offset %d)", - (int) col->name.len, col->name.data, - (int) col->name.len, (int) (b->pos - b->start)); + (int) col->name.len, col->name.data, + (int) col->name.len, (int) (b->pos - b->start)); return NGX_OK; } #endif /* NGX_HTTP_RDS_UTILS_H */ - diff --git a/t/000_init.t b/t/000_init.t index 43150f1..e57b498 100644 --- a/t/000_init.t +++ b/t/000_init.t @@ -7,15 +7,27 @@ repeat_each(1); plan tests => repeat_each() * blocks(); +$ENV{TEST_NGINX_POSTGRESQL_PORT} ||= 5432; +$ENV{TEST_NGINX_POSTGRESQL_HOST} ||= '127.0.0.1'; + +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream database { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ +our $http_config2 = <<'_EOC_'; + upstream database { + postgres_server $TEST_NGINX_POSTGRESQL_HOST:$TEST_NGINX_POSTGRESQL_PORT + dbname=ngx_test user=ngx_test password=ngx_test; + } +_EOC_ + + worker_connections(128); no_shuffle(); run_tests(); @@ -128,3 +140,62 @@ GET /init --- error_code: 200 --- timeout: 10 + + +=== TEST 8: cats - drop table - pg +--- http_config eval: $::http_config2 +--- config + location = /init { + postgres_pass database; + postgres_query "DROP TABLE cats"; + error_page 500 = /ignore; + } + + location /ignore { echo "ignore"; } +--- request +GET /init +--- error_code: 200 +--- timeout: 10 + + + +=== TEST 9: cats - create table - pg +--- http_config eval: $::http_config2 +--- config + location = /init { + postgres_pass database; + postgres_query "CREATE TABLE cats (id integer, name text)"; + } +--- request +GET /init +--- error_code: 200 +--- timeout: 10 + + + +=== TEST 10: cats - insert value - pg +--- http_config eval: $::http_config2 +--- config + location = /init { + postgres_pass database; + postgres_query "INSERT INTO cats (id) VALUES (2)"; + } +--- request +GET /init +--- error_code: 200 +--- timeout: 10 + + + +=== TEST 11: cats - insert value - pg +--- http_config eval: $::http_config2 +--- config + location = /init { + postgres_pass database; + postgres_query "INSERT INTO cats (id, name) VALUES (3, 'bob')"; + } +--- request +GET /init +--- error_code: 200 +--- timeout: 10 + diff --git a/t/buf.t b/t/buf.t index 9e2ed9e..5118803 100644 --- a/t/buf.t +++ b/t/buf.t @@ -3,7 +3,7 @@ use lib 'lib'; use Test::Nginx::Socket; -repeat_each(1); +repeat_each(2); plan tests => repeat_each() * 2 * blocks(); diff --git a/t/compact/buf.t b/t/compact/buf.t index 2782b8d..f467942 100644 --- a/t/compact/buf.t +++ b/t/compact/buf.t @@ -1,9 +1,9 @@ -# vi:filetype=perl +# vi:filetype= use lib 'lib'; use Test::Nginx::Socket; -repeat_each(1); +repeat_each(2); plan tests => repeat_each() * 2 * blocks(); diff --git a/t/compact/openresty.t b/t/compact/openresty.t index 0d50e85..da8d2c1 100644 --- a/t/compact/openresty.t +++ b/t/compact/openresty.t @@ -8,16 +8,17 @@ repeat_each(2); worker_connections(128); workers(1); -master_on; +#master_on; log_level('warn'); plan tests => repeat_each() * 3 * blocks(); +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ diff --git a/t/compact/sanity-stream.t b/t/compact/sanity-stream.t index bddd3a9..66cf7fd 100644 --- a/t/compact/sanity-stream.t +++ b/t/compact/sanity-stream.t @@ -6,15 +6,16 @@ use Test::Nginx::Socket; #repeat_each(10); no_shuffle(); -repeat_each(1); +repeat_each(2); plan tests => repeat_each() * 2 * blocks() + 2 * repeat_each() * 3; +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ diff --git a/t/compact/sanity.t b/t/compact/sanity.t index f8143c1..91b57ef 100644 --- a/t/compact/sanity.t +++ b/t/compact/sanity.t @@ -1,4 +1,4 @@ -# vi:filetype=perl +# vi:filetype= use lib 'lib'; use Test::Nginx::Socket; @@ -6,15 +6,16 @@ use Test::Nginx::Socket; #repeat_each(10); no_shuffle(); -repeat_each(1); +repeat_each(2); plan tests => repeat_each() * 2 * blocks() + 2 * repeat_each() * 3; +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ diff --git a/t/escape.t b/t/escape.t index 72d3df4..44c6c57 100644 --- a/t/escape.t +++ b/t/escape.t @@ -7,11 +7,12 @@ repeat_each(2); plan tests => repeat_each() * 2 * blocks(); +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ diff --git a/t/form.t b/t/form.t index 99d446b..7fac2b0 100644 --- a/t/form.t +++ b/t/form.t @@ -7,11 +7,12 @@ repeat_each(2); plan tests => repeat_each() * 2 * blocks(); +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ diff --git a/t/openresty.t b/t/openresty.t index f1a22d3..b843a76 100644 --- a/t/openresty.t +++ b/t/openresty.t @@ -8,16 +8,17 @@ repeat_each(2); worker_connections(128); workers(1); -master_on; +#master_on; log_level('warn'); plan tests => repeat_each() * 3 * blocks(); +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ diff --git a/t/pg.t b/t/pg.t index 4606bdd..d7be1fd 100644 --- a/t/pg.t +++ b/t/pg.t @@ -5,8 +5,9 @@ use Test::Nginx::Socket; repeat_each(2); -plan tests => repeat_each() * 2 * blocks(); +plan tests => repeat_each() * (2 * blocks() + 1); +$ENV{TEST_NGINX_POSTGRESQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_POSTGRESQL_PORT} ||= 5432; no_long_string(); @@ -18,7 +19,7 @@ __DATA__ === TEST 1: bool blob field (keepalive off) --- http_config upstream backend { - postgres_server 127.0.0.1:$TEST_NGINX_POSTGRESQL_PORT + postgres_server $TEST_NGINX_POSTGRESQL_HOST:$TEST_NGINX_POSTGRESQL_PORT dbname=ngx_test user=ngx_test password=ngx_test; postgres_keepalive off; } @@ -62,7 +63,7 @@ GET /test === TEST 2: bool blob field (keepalive on) --- http_config upstream backend { - postgres_server 127.0.0.1:$TEST_NGINX_POSTGRESQL_PORT + postgres_server $TEST_NGINX_POSTGRESQL_HOST:$TEST_NGINX_POSTGRESQL_PORT dbname=ngx_test user=ngx_test password=ngx_test; } --- config @@ -100,3 +101,28 @@ GET /test [{"id":1,"flag":true},{"id":2,"flag":false}] --- skip_nginx: 2: < 0.7.46 + + +=== TEST 3: sanity (github issue #2) +--- http_config + upstream backend { + postgres_server $TEST_NGINX_POSTGRESQL_HOST:$TEST_NGINX_POSTGRESQL_PORT + dbname=ngx_test user=ngx_test password=ngx_test; + } +--- config + location /pg { + rds_json on; + rds_json_root url; + + postgres_pass backend; + postgres_query GET "select * from cats order by id"; + postgres_output rds; + } +--- request +GET /pg +--- response_headers +Content-Type: application/json +--- response_body chop +{"url":[{"id":2,"name":null},{"id":3,"name":"bob"}]} +--- skip_nginx: 2: < 0.7.46 + diff --git a/t/property.t b/t/property.t index 403e555..05f9fb0 100644 --- a/t/property.t +++ b/t/property.t @@ -7,11 +7,12 @@ repeat_each(2); plan tests => repeat_each() * 2 * blocks(); +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ @@ -386,3 +387,175 @@ GET /ret --- response_body chomp {"errcode":400,"errstr":"Non zero ret","city":"beijing","\"hi\"":"\"hello\n\""} + + +=== TEST 21: location internal rewrite +--- config + location @err403 { + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } + + location /foo { + error_page 403 = @err403; + return 403; + } +--- request + GET /foo +--- response_body chop +{"errcode":403,"errstr":"Forbidden","ret":false} + + + +=== TEST 22: rds_json_errcode_key +--- config + location /foo { + rds_json_errcode_key "ecode"; + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } +--- request + GET /foo +--- response_body chop +{"ecode":403,"errstr":"Forbidden","ret":false} + + + +=== TEST 23: rds_json_errcode_key +--- config + rds_json_errcode_key "ecoderoot"; + + location /foo { + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } +--- request + GET /foo +--- response_body chop +{"ecoderoot":403,"errstr":"Forbidden","ret":false} + + + +=== TEST 24: rds_json_errcode_key +--- config + rds_json_errcode_key "ecoderoot"; + + location /foo { + rds_json_errcode_key "ecode"; + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } +--- request + GET /foo +--- response_body chop +{"ecode":403,"errstr":"Forbidden","ret":false} + + + +=== TEST 25: rds_json_errstr_key +--- config + rds_json_errstr_key "msg"; + + location /foo { + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } +--- request + GET /foo +--- response_body chop +{"errcode":403,"msg":"Forbidden","ret":false} + + + +=== TEST 26: rds_json_errstr_key +--- config + rds_json_errstr_key "msg"; + + location /foo { + rds_json_errstr_key "msg2"; + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } +--- request + GET /foo +--- response_body chop +{"errcode":403,"msg2":"Forbidden","ret":false} + + + +=== TEST 27: rds_json_errstr_key & rds_json_root +--- config + rds_json_errstr_key "msg2"; + rds_json_root rows; + + location /foo { + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } +--- request + GET /foo +--- response_body chop +{"errcode":403,"msg2":"Forbidden","ret":false} + + + +=== TEST 28: rds_json_errcode_key & rds_json_root +--- config + rds_json_root rows; + rds_json_errcode_key "code"; + + location /foo { + rds_json_success_property ret; + rds_json_ret 403 "Forbidden"; + } +--- request + GET /foo +--- response_body chop +{"code":403,"errstr":"Forbidden","ret":false} + + + +=== TEST 29: update - custom errstr_key +--- http_config eval: $::http_config +--- config + location /mysql { + drizzle_pass backend; + #drizzle_dbname $dbname; + drizzle_query "update cats set name='bob' where name='bob'"; + rds_json on; + rds_json_errcode_key "\"code\""; + rds_json_errstr_key "\"str\""; + + set $name 'Jimmy"'; + set $age 32; + rds_json_user_property name $name; + rds_json_user_property age $age; + } +--- request +GET /mysql +--- response_body chop +{"name":"Jimmy\"","age":"32","\"code\"":0,"\"str\"":"Rows matched: 1 Changed: 0 Warnings: 0"} + + + +=== TEST 30: select - custom errstr_key +--- http_config eval: $::http_config +--- config + location /mysql { + drizzle_pass backend; + drizzle_query "select 'aaa' as a, 'bbb' as b"; + rds_json on; + rds_json_errcode_key "\"code\""; + rds_json_errstr_key "\"str\""; + + rds_json_root data; + + set $name 'Jimmy"'; + set $age 32; + rds_json_user_property name $name; + rds_json_user_property age $age; + } +--- request +GET /mysql +--- response_body chop +{"name":"Jimmy\"","age":"32","data":[{"a":"aaa","b":"bbb"}]} diff --git a/t/ret.t b/t/ret.t index 5cf154a..0d77b28 100644 --- a/t/ret.t +++ b/t/ret.t @@ -4,6 +4,7 @@ use lib 'lib'; use Test::Nginx::Socket; repeat_each(3); +#repeat_each(1); plan tests => repeat_each() * 3 * blocks(); diff --git a/t/sanity-stream.t b/t/sanity-stream.t index 12c00ce..116b987 100644 --- a/t/sanity-stream.t +++ b/t/sanity-stream.t @@ -6,15 +6,16 @@ use Test::Nginx::Socket; #repeat_each(10); no_shuffle(); -repeat_each(1); +repeat_each(2); plan tests => repeat_each() * 2 * blocks() + 2 * repeat_each() * 3; +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ diff --git a/t/sanity.t b/t/sanity.t index 8279b11..94af304 100644 --- a/t/sanity.t +++ b/t/sanity.t @@ -6,20 +6,22 @@ use Test::Nginx::Socket; #repeat_each(10); no_shuffle(); -repeat_each(1); +repeat_each(2); -plan tests => repeat_each() * 2 * blocks() + 2 * repeat_each() * 3; +plan tests => repeat_each() * (2 * blocks() + 7); +$ENV{TEST_NGINX_MYSQL_HOST} ||= '127.0.0.1'; $ENV{TEST_NGINX_MYSQL_PORT} ||= 3306; our $http_config = <<'_EOC_'; upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test; } _EOC_ #no_long_string(); +#master_on(); run_tests(); @@ -33,7 +35,7 @@ __DATA__ location /mysql { drizzle_pass backend; #drizzle_dbname $dbname; - drizzle_query 'select * from cats'; + drizzle_query 'select * from cats order by id'; rds_json on; } --- request @@ -328,9 +330,9 @@ GET /test === TEST 13: strings need to be escaped (forcing utf8) --- http_config upstream backend { - drizzle_server 127.0.0.1:$TEST_NGINX_MYSQL_PORT protocol=mysql + drizzle_server $TEST_NGINX_MYSQL_HOST:$TEST_NGINX_MYSQL_PORT protocol=mysql dbname=ngx_test user=ngx_test password=ngx_test - charset=utf8; + charset=utf8mb4; } --- config @@ -356,12 +358,13 @@ GET /test {"errcode":0} {"errcode":0} {"errcode":0,"insert_id":1,"affected_rows":1} -[{"id":1,"body":"a\r\nb\b??\u001a"}] +[{"id":1,"body":"a\r\nb\b你好\u001a"}] --- timeout: 5 === TEST 14: strings need to be escaped +--- SKIP --- http_config eval: $::http_config --- config location /test { @@ -438,3 +441,21 @@ GET /mysql {"errcode":0,"errstr":"Rows matched: 1 Changed: 0 Warnings: 0"} --- SKIP + + +=== TEST 17: bad MIME type +--- http_config eval: $::http_config +--- config + location /mysql { + default_type "text/css"; + echo hello; + rds_json on; + } +--- request +GET /mysql +--- response_headers +Content-Type: text/css +--- response_body +hello +--- timeout: 15 + diff --git a/t/unused.t b/t/unused.t new file mode 100644 index 0000000..7b3fe76 --- /dev/null +++ b/t/unused.t @@ -0,0 +1,94 @@ +# vi:filetype= + +use lib 'lib'; +use Test::Nginx::Socket; + +repeat_each(2); +#repeat_each(1); + +plan tests => repeat_each() * (5 * blocks()); + +log_level('debug'); + +no_long_string(); + +run_tests(); + +__DATA__ + +=== TEST 1: not using this module's directives at all +--- config + location /foo { + echo Hello; + } +--- request + GET /foo +--- response_headers +Content-Type: text/plain +--- response_body +Hello +--- no_error_log +rds json header filter, "/foo" +rds json body filter, "/foo" + + + +=== TEST 2: using rds_json_ret, but not rds_json +--- config + location /foo { + rds_json_ret 400 "Bad request"; + } +--- request + GET /foo +--- response_headers +Content-Type: application/json +--- response_body chop +{"errcode":400,"errstr":"Bad request"} +--- no_error_log +rds json header filter, "/foo" +rds json body filter, "/foo" + + + +=== TEST 3: using rds_json +--- config + location /foo { + echo Hello; + rds_json on; + } +--- request + GET /foo +--- response_headers +Content-Type: text/plain +--- response_body +Hello +--- error_log +rds json header filter, "/foo" +rds json body filter, "/foo" + + + +=== TEST 4: multiple http {} blocks +This test case won't run with nginx 1.9.3+ since duplicate http {} blocks +have been prohibited since then. +--- SKIP +--- config + location /foo { + echo Hello; + rds_json on; + } + +--- post_main_config + http { + } + +--- request + GET /foo +--- response_headers +Content-Type: text/plain +--- response_body +Hello +--- error_log +rds json header filter, "/foo" +rds json body filter, "/foo" + diff --git a/util/build.sh b/util/build.sh index db59847..630a3eb 100755 --- a/util/build.sh +++ b/util/build.sh @@ -3,50 +3,25 @@ # this file is mostly meant to be used by the author himself. root=`pwd` -version=$1 home=~ +version=$1 +force=$2 #opts=$2 -target=$root/work/nginx - -if [ ! -d ./buildroot ]; then - mkdir ./buildroot || exit 1 -fi - -cd buildroot || exit 1 - -if [ ! -s "nginx-$version.tar.gz" ]; then - if [ -f ~/work/nginx-$version.tar.gz ]; then - cp ~/work/nginx-$version.tar.gz ./ || exit 1 - else - wget "http://sysoev.ru/nginx/nginx-$version.tar.gz" -O nginx-$version.tar.gz || exit 1 - cp nginx-$version.tar.gz ~/work/ || exit 1 - fi - tar -xzvf nginx-$version.tar.gz || exit 1 - cp $root/../no-pool-nginx/nginx-$version-no_pool.patch ./ || exit 1 - patch -p0 < nginx-$version-no_pool.patch || exit 1 -fi - -#tar -xzvf nginx-$version.tar.gz || exit 1 -#cp $root/../no-pool-nginx/nginx-$version-no_pool.patch ./ || exit 1 -#patch -p0 < nginx-$version-no_pool.patch || exit 1 - -cd nginx-$version/ || exit 1 - -if [[ "$BUILD_CLEAN" -eq 1 || ! -f Makefile || "$root/config" -nt Makefile || "$root/util/build.sh" -nt Makefile ]]; then - ./configure --prefix=$target \ +ngx-build $force $version \ --with-cc-opt="-O1" \ - --with-ld-opt="-Wl,-rpath,/opt/drizzle/lib" \ - --without-mail_pop3_module \ - --without-mail_imap_module \ - --without-mail_smtp_module \ - --without-http_upstream_ip_hash_module \ - --without-http_empty_gif_module \ - --without-http_memcached_module \ - --without-http_referer_module \ - --without-http_autoindex_module \ - --without-http_auth_basic_module \ - --without-http_userid_module \ + $NGX_EXTRA_OPT \ + --with-ld-opt="-Wl,-rpath,/opt/drizzle/lib:/opt/pg9/lib" \ + --without-mail_pop3_module \ + --without-mail_imap_module \ + --without-mail_smtp_module \ + --without-http_upstream_ip_hash_module \ + --without-http_empty_gif_module \ + --without-http_memcached_module \ + --without-http_referer_module \ + --without-http_autoindex_module \ + --without-http_auth_basic_module \ + --without-http_userid_module \ --add-module=$root/../eval-nginx-module \ --add-module=$root/../echo-nginx-module \ --add-module=$root/../xss-nginx-module \ @@ -57,26 +32,11 @@ if [[ "$BUILD_CLEAN" -eq 1 || ! -f Makefile || "$root/config" -nt Makefile || "$ --add-module=$root/../drizzle-nginx-module \ --add-module=$root/../form-input-nginx-module \ --add-module=$root/../postgres-nginx-module \ - --with-debug \ - || exit 1 + --with-debug #--add-module=$root/../lua-nginx-module \ #--add-module=$home/work/ngx_http_auth_request-0.1 #\ #--with-rtsig_module #--with-cc-opt="-g3 -O0" #--add-module=$root/../echo-nginx-module \ #--without-http_ssi_module # we cannot disable ssi because echo_location_async depends on it (i dunno why?!) - if [ $? -ne 0 ]; then - echo "Failed to configure" - exit 1 - fi -fi - -if [ -f $target/sbin/nginx ]; then - rm -f $target/sbin/nginx -fi -if [ -f $target/logs/nginx.pid ]; then - kill `cat $target/logs/nginx.pid` -fi -make -j3 -make install diff --git a/valgrind.suppress b/valgrind.suppress index a74d487..8632e25 100644 --- a/valgrind.suppress +++ b/valgrind.suppress @@ -1,3 +1,24 @@ +{ + + Memcheck:Leak + fun:malloc + fun:ngx_alloc + obj:* +} +{ + + Memcheck:Param + epoll_ctl(event) + fun:epoll_ctl +} +{ + + Memcheck:Leak + fun:malloc + fun:ngx_alloc + fun:ngx_event_process_init + fun:ngx_single_process_cycle +} { nginx-core-process-init Memcheck:Leak @@ -78,16 +99,6 @@ fun:ngx_create_pool fun:main } -{ - sendmsg_uninit - Memcheck:Param - socketcall.sendmsg(msg.msg_iov[i]) - fun:__sendmsg_nocancel - fun:ngx_write_channel - fun:ngx_signal_worker_processes - fun:ngx_master_process_cycle - fun:main -} { spawn_process_alloc Memcheck:Leak @@ -117,7 +128,6 @@ fun:ngx_init_cycle fun:main } - { Memcheck:Leak @@ -236,19 +246,6 @@ fun:ngx_init_cycle fun:main } -{ - - Memcheck:Addr8 - fun:ngx_http_upstream_drizzle_send_query - fun:ngx_http_drizzle_process_events - fun:ngx_http_drizzle_wev_handler - fun:ngx_http_upstream_dbd_handler - fun:ngx_epoll_process_events - fun:ngx_process_events_and_timers - fun:ngx_single_process_cycle - fun:main -} - { Memcheck:Leak @@ -297,4 +294,33 @@ fun:ngx_http_core_server fun:ngx_conf_parse } - +{ + + Memcheck:Cond + fun:index + fun:expand_dynamic_string_token + fun:_dl_map_object + fun:map_doit + fun:_dl_catch_error + fun:do_preload + fun:dl_main +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:ngx_alloc + fun:ngx_set_environment + fun:ngx_single_process_cycle +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:ngx_alloc + fun:ngx_set_environment + fun:ngx_worker_process_init + fun:ngx_worker_process_cycle +}