From 0939833b0cca2a50054bf1db437db62295c1e299 Mon Sep 17 00:00:00 2001 From: Sami Jaktholm Date: Mon, 12 Jun 2017 18:08:40 +0300 Subject: [PATCH 001/130] add http proxy support to request_uri() These changes introduce support for making HTTP requests through a forward proxy when using the high-level request_uri() API. The new set_proxy_options() function can be used to configure HTTP proxy settings. This method takes a table as a parameter that can include the following fields: * http_proxy - an URI to a proxy that should be used for http:// requests * https_proxy - an URI to a proxy that should be used for https:// requests * no_proxy - a comma separated list of hosts / IPs that should not go through the configured proxy When request_uri() is called with an http:// URI and http_proxy has been configured for the client instance, the connection will be established to the proxy host. Once the connection has been established, the HTTP request is sent to the proxy host as if the peer on the other end of the connection was the remote server. The only difference to the non-proxy case is that the path sent to the proxy is actually a full URI instead of a relative path. When request_uri() is called with an https:// URI and https_proxy has been configured for the client instance, the connection will be established to the proxy host. Once the connection has been established, we go ahead and perform a CONNECT request to open a TCP tunnel to the remote server. If the proxy gives a success response, to the CONNECT request, the client continues to perform the TLS handshake with the remote server and making the request as if there was no proxy configured. Some tests have been included which verify that the proxy options are interpreted correctly in different cases and that http_proxy works as real proxies expect. This is about as much there is to test without actually including a real forward proxy in the test harness. Manual testing has been performed against squid and tinyproxy. Fixes #63 --- README.md | 12 +++ lib/resty/http.lua | 103 ++++++++++++++++++- t/14-host-header.t | 27 +++++ t/16-http-proxy.t | 247 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 t/16-http-proxy.t diff --git a/README.md b/README.md index aee6028d..5f21ff2c 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,18 @@ Note that calling this instead of `close` is "safe" in that it will conditionall In case of success, returns `1`. In case of errors, returns `nil, err`. In the case where the conneciton is conditionally closed as described above, returns `2` and the error string `connection must be closed`. +## set_proxy_options + +`syntax: httpc:set_proxy_options(opts)` + +Configure an http proxy to be used with this client instance. The `opts` is a table that accepts the following fields: + +* `http_proxy` - an URI to a proxy server to be used with http requests +* `https_proxy` - an URI to a proxy server to be used with https requests +* `no_proxy` - a comma separated list of hosts that should not be proxied. + +Note that proxy options are only applied when using the high-level `request_uri()` API. + ## get_reused_times `syntax: times, err = httpc:get_reused_times()` diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 49490255..b8bc0316 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -800,11 +800,65 @@ function _M.request_uri(self, uri, params) if not params.path then params.path = path end if not params.query then params.query = query end - local c, err = self:connect(host, port) + -- See if we should use a proxy to make this request + local proxy_host, proxy_port; + local proxy_uri = self:get_proxy_uri(scheme, host) + if proxy_uri then + local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) + if not parsed_proxy_uri then + return nil, err + end + + proxy_host, proxy_port = parsed_proxy_uri[2], parsed_proxy_uri[3] + end + + local c, err = self:connect(proxy_host or host, proxy_port or port) if not c then return nil, err end + if proxy_uri and scheme == "https" then + -- Make a CONNECT request to create a tunnel to the destination through + -- the proxy + local destination = host .. ":" .. port + local res, err = self:request({ + method = "CONNECT", + path = destination, + headers = { + ["Host"] = destination + } + }) + + if not res then + return nil, err + end + + if res.status < 200 or res.status > 299 then + return nil, "failed to establish a tunnel through a proxy: " .. res.status + end + + -- don't keep this connection alive as the next request could target + -- any host and re-using the tunnel for that is not possible + self.keepalive = false + end + + if proxy_uri and scheme == "http" then + -- http proxies expect to see the full URI in the request line + if port == 80 then + params.path = scheme .. "://" .. host .. path + else + params.path = scheme .. "://" .. host .. ":" .. port .. path + end + end + + if proxy_uri then + -- self:connect() set the host and port to point to the proxy server. As + -- the connection to the proxy has been established, set the host and port + -- to point to the actual remote endpoint at the other end of the tunnel + self.host = host + self.port = port + end + if scheme == "https" then local verify = true if params.ssl_verify == false then @@ -914,5 +968,52 @@ function _M.proxy_response(self, response, chunksize) until not chunk end +function _M.set_proxy_options(self, opts) + self.proxy_opts = opts +end + +function _M.get_proxy_uri(self, scheme, host) + if not self.proxy_opts then + return nil + end + + -- Check if the no_proxy option matches this host. Implementation adapted + -- from lua-http library (https://github.com/daurnimator/lua-http) + if self.proxy_opts.no_proxy then + if self.proxy_opts.no_proxy == "*" then + -- all hosts are excluded + return nil + end + + local no_proxy_set = {} + -- wget allows domains in no_proxy list to be prefixed by "." + -- e.g. no_proxy=.mit.edu + for host_suffix in self.proxy_opts.no_proxy:gmatch("%.?([^,]+)") do + no_proxy_set[host_suffix] = true + end + + -- From curl docs: + -- matched as either a domain which contains the hostname, or the + -- hostname itself. For example local.com would match local.com, + -- local.com:80, and www.local.com, but not www.notlocal.com. + for pos in host:gmatch("%f[^%z%.]()") do + local host_suffix = host:sub(pos, -1) + if no_proxy_set[host_suffix] then + return nil + end + end + end + + if scheme == "http" and self.proxy_opts.http_proxy then + return self.proxy_opts.http_proxy + end + + if scheme == "https" and self.proxy_opts.https_proxy then + return self.proxy_opts.https_proxy + end + + return nil +end + return _M diff --git a/t/14-host-header.t b/t/14-host-header.t index 62a6164c..6ea28ebf 100644 --- a/t/14-host-header.t +++ b/t/14-host-header.t @@ -165,3 +165,30 @@ GET /a [error] --- response_body Unable to generate a useful Host header for a unix domain socket. Please provide one. + +=== TEST 6: Host header is correct when http_proxy is used +--- http_config + lua_package_path "$TEST_NGINX_PWD/lib/?.lua;;"; + error_log logs/error.log debug; + resolver 8.8.8.8; + server { + listen *:8080; + } + +--- config + location /lua { + content_by_lua ' + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:8080" + }) + local res, err = httpc:request_uri("http://127.0.0.1:8081") + '; + } +--- request +GET /lua +--- no_error_log +[error] +--- error_log +Host: 127.0.0.1:8081 diff --git a/t/16-http-proxy.t b/t/16-http-proxy.t new file mode 100644 index 00000000..2f770482 --- /dev/null +++ b/t/16-http-proxy.t @@ -0,0 +1,247 @@ +use Test::Nginx::Socket; +use Cwd qw(cwd); + +plan tests => repeat_each() * (blocks() * 4); + +my $pwd = cwd(); + +$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; +$ENV{TEST_NGINX_PWD} ||= $pwd; +$ENV{TEST_COVERAGE} ||= 0; + +our $HttpConfig = qq{ + lua_package_path "$pwd/lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; + error_log logs/error.log debug; + resolver 8.8.8.8; + + init_by_lua_block { + if $ENV{TEST_COVERAGE} == 1 then + jit.off() + require("luacov.runner").init() + end + } +}; + +no_long_string(); +run_tests(); + +__DATA__ +=== TEST 1: get_proxy_uri returns nil if proxy is not configured +--- http_config eval: $::HttpConfig +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + ngx.say(httpc:get_proxy_uri("http", "example.com")) + } + } +--- request +GET /lua +--- response_body +nil +--- no_error_log +[error] +[warn] + +=== TEST 2: get_proxy_uri matches no_proxy hosts correctly +--- http_config eval: $::HttpConfig +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + + -- helper that verifies get_proxy_uri works correctly with the given + -- scheme, host and no_proxy list + function test_no_proxy(scheme, host, no_proxy) + httpc:set_proxy_options({ + http_proxy = "http://http_proxy.example.com", + https_proxy = "http://https_proxy.example.com", + no_proxy = no_proxy + }) + + local proxy_uri = httpc:get_proxy_uri(scheme, host) + ngx.say("scheme: ", scheme, ", host: ", host, ", no_proxy: ", no_proxy, ", proxy_uri: ", proxy_uri) + end + + -- All these match the no_proxy list + test_no_proxy("http", "example.com", nil) + test_no_proxy("http", "example.com", "*") + test_no_proxy("http", "example.com", "example.com") + test_no_proxy("http", "sub.example.com", "example.com") + test_no_proxy("http", "example.com", "example.com,example.org") + test_no_proxy("http", "example.com", "example.org,example.com") + + -- Same for https for good measure + test_no_proxy("https", "example.com", nil) + test_no_proxy("https", "example.com", "*") + test_no_proxy("https", "example.com", "example.com") + test_no_proxy("https", "sub.example.com", "example.com") + test_no_proxy("https", "example.com", "example.com,example.org") + test_no_proxy("https", "example.com", "example.org,example.com") + + -- Edge cases + + -- example.com should match .example.com in the no_proxy list (legacy behavior of wget) + test_no_proxy("http", "example.com", ".example.com") + + -- notexample.com should not match example.com in the no_proxy list (not a subdomain) + test_no_proxy("http", "notexample.com", "example.com") + } + } +--- request +GET /lua +--- response_body +scheme: http, host: example.com, no_proxy: nil, proxy_uri: http://http_proxy.example.com +scheme: http, host: example.com, no_proxy: *, proxy_uri: nil +scheme: http, host: example.com, no_proxy: example.com, proxy_uri: nil +scheme: http, host: sub.example.com, no_proxy: example.com, proxy_uri: nil +scheme: http, host: example.com, no_proxy: example.com,example.org, proxy_uri: nil +scheme: http, host: example.com, no_proxy: example.org,example.com, proxy_uri: nil +scheme: https, host: example.com, no_proxy: nil, proxy_uri: http://https_proxy.example.com +scheme: https, host: example.com, no_proxy: *, proxy_uri: nil +scheme: https, host: example.com, no_proxy: example.com, proxy_uri: nil +scheme: https, host: sub.example.com, no_proxy: example.com, proxy_uri: nil +scheme: https, host: example.com, no_proxy: example.com,example.org, proxy_uri: nil +scheme: https, host: example.com, no_proxy: example.org,example.com, proxy_uri: nil +scheme: http, host: example.com, no_proxy: .example.com, proxy_uri: nil +scheme: http, host: notexample.com, no_proxy: example.com, proxy_uri: http://http_proxy.example.com +--- no_error_log +[error] +[warn] + +=== TEST 3: get_proxy_uri returns correct proxy URIs for http and https URIs +--- http_config eval: $::HttpConfig +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + + -- helper that configures the proxy opts as proived and checks what + -- get_proxy_uri says for the given scheme / host pair + function test_get_proxy_uri(scheme, host, http_proxy, https_proxy) + httpc:set_proxy_options({ + http_proxy = http_proxy, + https_proxy = https_proxy + }) + + local proxy_uri = httpc:get_proxy_uri(scheme, host) + ngx.say( + "scheme: ", scheme, + ", host: ", host, + ", http_proxy: ", http_proxy, + ", https_proxy: ", https_proxy, + ", proxy_uri: ", proxy_uri + ) + end + + -- http + test_get_proxy_uri("http", "example.com", "http_proxy", "https_proxy") + test_get_proxy_uri("http", "example.com", nil, "https_proxy") + + -- https + test_get_proxy_uri("https", "example.com", "http_proxy", "https_proxy") + test_get_proxy_uri("https", "example.com", "http_proxy", nil) + } + } +--- request +GET /lua +--- response_body +scheme: http, host: example.com, http_proxy: http_proxy, https_proxy: https_proxy, proxy_uri: http_proxy +scheme: http, host: example.com, http_proxy: nil, https_proxy: https_proxy, proxy_uri: nil +scheme: https, host: example.com, http_proxy: http_proxy, https_proxy: https_proxy, proxy_uri: https_proxy +scheme: https, host: example.com, http_proxy: http_proxy, https_proxy: nil, proxy_uri: nil +--- no_error_log +[error] +[warn] + +=== TEST 4: request_uri uses http_proxy correctly for non-standard destination ports +--- http_config + lua_package_path "$TEST_NGINX_PWD/lib/?.lua;;"; + error_log logs/error.log debug; + resolver 8.8.8.8; + server { + listen *:8080; + + location / { + content_by_lua_block { + ngx.print(ngx.req.raw_header()) + } + } + } +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:8080", + https_proxy = "http://127.0.0.1:8080" + }) + + -- request should go to the proxy server + local res, err = httpc:request_uri("http://127.0.0.1:1234/target?a=1&b=2") + + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.status = res.status + ngx.say(res.body) + } + } +--- request +GET /lua +--- response_body_like +^GET http://127.0.0.1:1234/target\?a=1&b=2 HTTP/.+\r\nHost: 127.0.0.1:1234.+ +--- no_error_log +[error] +[warn] + +=== TEST 5: request_uri uses http_proxy correctly for standard destination port +--- http_config + lua_package_path "$TEST_NGINX_PWD/lib/?.lua;;"; + error_log logs/error.log debug; + resolver 8.8.8.8; + server { + listen *:8080; + + location / { + content_by_lua_block { + ngx.print(ngx.req.raw_header()) + } + } + } +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:8080", + https_proxy = "http://127.0.0.1:8080" + }) + + -- request should go to the proxy server + local res, err = httpc:request_uri("http://127.0.0.1/target?a=1&b=2") + + if not res then + ngx.log(ngx.ERR, err) + return + end + + -- the proxy echoed the raw request header and we shall pass it onwards + -- to the test harness + ngx.status = res.status + ngx.say(res.body) + } + } +--- request +GET /lua +--- response_body_like +^GET http://127.0.0.1/target\?a=1&b=2 HTTP/.+\r\nHost: 127.0.0.1.+ +--- no_error_log +[error] +[warn] From 5c96e1f381160d2ec853ba00b3b71a321c1ca388 Mon Sep 17 00:00:00 2001 From: Sami Jaktholm Date: Tue, 18 Jul 2017 21:33:53 +0300 Subject: [PATCH 002/130] followup: remove unneeded semicolon --- lib/resty/http.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index b8bc0316..ac4233f3 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -801,7 +801,7 @@ function _M.request_uri(self, uri, params) if not params.query then params.query = query end -- See if we should use a proxy to make this request - local proxy_host, proxy_port; + local proxy_host, proxy_port local proxy_uri = self:get_proxy_uri(scheme, host) if proxy_uri then local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) From b5f955961e42873725a345b752bc38ddad6c5203 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Mon, 24 Jul 2017 14:43:43 +0100 Subject: [PATCH 003/130] Version bump --- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...sty-http-0.10-0.rockspec => lua-resty-http-0.11-0.rockspec | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename lua-resty-http-0.10-0.rockspec => lua-resty-http-0.11-0.rockspec (93%) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 49490255..c59e6b6c 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -98,7 +98,7 @@ end local _M = { - _VERSION = '0.10', + _VERSION = '0.11', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 663ade6a..6ff26fb2 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.10', + _VERSION = '0.11', } diff --git a/lua-resty-http-0.10-0.rockspec b/lua-resty-http-0.11-0.rockspec similarity index 93% rename from lua-resty-http-0.10-0.rockspec rename to lua-resty-http-0.11-0.rockspec index 95cb42ab..e2088ac9 100644 --- a/lua-resty-http-0.10-0.rockspec +++ b/lua-resty-http-0.11-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.10-0" +version = "0.11-0" source = { url = "git://github.com/pintsized/lua-resty-http", - tag = "v0.10" + tag = "v0.11" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From b0e6fc42e47965b04a220b868d71e17859cbfb9e Mon Sep 17 00:00:00 2001 From: Sami Jaktholm Date: Tue, 22 Aug 2017 19:00:49 +0300 Subject: [PATCH 004/130] followup: separate proxy connection setup logic to a low-level connect_proxy() method This commit separates the logic that sets up the connection to the proxy server to a separate connect_proxy() method. This method is provided to users as a low-level API they can use similarly to the connect() method. The connect_proxy() will handle the connection establishment to the proxy server and performs the CONNECT request to setup a TCP tunnel to a https protected host. Similar to the connect() method, it is then up to the user to take care of the details that are relevant when using a proxy (i.e use absolute uris for http requests and perform a TLS handshake for https connections). There's also a new test case that verifies the CONNECT request is used properly to establish a tunnel to the remote server when TLS is used. Due to the limitations of the test framework, this case only considers the format of the outgoing CONNECT request and how the code handles errors sent by the proxy. Testing a full TLS tunnel is unfortunately not possible with the tools the test framework provides as it would require a real reverse proxy or a method of forwarding the TCP connection after the CONNECT request is received to a real web server that can talk TLS. --- README.md | 23 ++++++++- lib/resty/http.lua | 120 +++++++++++++++++++++++++++++---------------- t/16-http-proxy.t | 52 ++++++++++++++++++++ 3 files changed, 151 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 5f21ff2c..14dfdec1 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Production ready. * [new](#new) * [connect](#connect) +* [connect_proxy](#connect_proxy) * [set_timeout](#set_timeout) * [set_timeouts](#set_timeouts) * [ssl_handshake](#ssl_handshake) @@ -158,6 +159,24 @@ An optional Lua table can be specified as the last argument to this method to sp * `pool` : Specifies a custom name for the connection pool being used. If omitted, then the connection pool name will be generated from the string template `:` or ``. +## connect_proxy + +`syntax: ok, err = httpc:connect_proxy(proxy_uri, scheme, host, port)` + +Attempts to connect to the web server through the given proxy server. The method accepts the following arguments: + +* `proxy_uri` - Full URI of the proxy server to use (e.g. `http://proxy.example.com:3128/`). Note: Only `http` protocol is supported. +* `scheme` - The protocol to use between the proxy server and the remote host (`http` or `https`). If `https` is specified as the scheme, `connect_proxy()` makes a `CONNECT` request to establish a TCP tunnel to the remote host through the proxy server. +* `host` - The hostname of the remote host to connect to. +* `port` - The port of the remote host to connect to. + +If an error occurs during the connection attempt, this method returns `nil` with a string describing the error. If the connection was successfully established, the method returns `1`. + +There's a few key points to keep in mind when using this api: + +* If the scheme is `https`, you need to perform the TLS handshake with the remote server manually using the `ssl_handshake()` method before sending any requests through the proxy tunnel. +* If the scheme is `http`, you need to ensure that the requests you send through the connections conforms to [RFC 7230](https://tools.ietf.org/html/rfc7230) and especially [Section 5.3.2.](https://tools.ietf.org/html/rfc7230#section-5.3.2) which states that the request target must be in absolute form. In practice, this means that when you use `send_request()`, the `path` must be an absolute URI to the resource (e.g. `http://example.com/index.html` instead of just `/index.html`). + ## set_timeout `syntax: httpc:set_timeout(time)` @@ -244,7 +263,7 @@ When the request is successful, `res` will contain the following fields: * `status` The status code. * `reason` The status reason phrase. * `headers` A table of headers. Multiple headers with the same field name will be presented as a table of values. -* `has_body` A boolean flag indicating if there is a body to be read. +* `has_body` A boolean flag indicating if there is a body to be read. * `body_reader` An iterator function for reading the body in a streaming fashion. * `read_body` A method to read the entire body into a string. * `read_trailers` A method to merge any trailers underneath the headers, after reading the body. @@ -420,7 +439,7 @@ local res, err = httpc:request{ } ``` -If `sock` is specified, +If `sock` is specified, # Author diff --git a/lib/resty/http.lua b/lib/resty/http.lua index ac4233f3..afb00f98 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -787,7 +787,6 @@ function _M.request_pipeline(self, requests) return responses end - function _M.request_uri(self, uri, params) params = tbl_copy(params or {}) -- Take by value @@ -801,60 +800,50 @@ function _M.request_uri(self, uri, params) if not params.query then params.query = query end -- See if we should use a proxy to make this request - local proxy_host, proxy_port local proxy_uri = self:get_proxy_uri(scheme, host) - if proxy_uri then - local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) - if not parsed_proxy_uri then - return nil, err - end - proxy_host, proxy_port = parsed_proxy_uri[2], parsed_proxy_uri[3] + -- Make the connection either through the proxy or directly + -- to the remote host + local c, err + + if proxy_uri then + c, err = self:connect_proxy(proxy_uri, scheme, host, port) + else + c, err = self:connect(host, port) end - local c, err = self:connect(proxy_host or host, proxy_port or port) if not c then return nil, err end - if proxy_uri and scheme == "https" then - -- Make a CONNECT request to create a tunnel to the destination through - -- the proxy - local destination = host .. ":" .. port - local res, err = self:request({ - method = "CONNECT", - path = destination, - headers = { - ["Host"] = destination - } - }) - - if not res then - return nil, err - end - - if res.status < 200 or res.status > 299 then - return nil, "failed to establish a tunnel through a proxy: " .. res.status + if proxy_uri then + if scheme == "http" then + -- When a proxy is used, the target URI must be in absolute-form + -- (RFC 7230, Section 5.3.2.). That is, it must be an absolute URI + -- to the remote resource with the scheme, host and an optional port + -- in place. + -- + -- Since _format_request() constructs the request line by concatenating + -- params.path and params.query together, we need to modify the path + -- to also include the scheme, host and port so that the final form + -- in conformant to RFC 7230. + if port == 80 then + params.path = scheme .. "://" .. host .. path + else + params.path = scheme .. "://" .. host .. ":" .. port .. path + end end - -- don't keep this connection alive as the next request could target - -- any host and re-using the tunnel for that is not possible - self.keepalive = false - end - - if proxy_uri and scheme == "http" then - -- http proxies expect to see the full URI in the request line - if port == 80 then - params.path = scheme .. "://" .. host .. path - else - params.path = scheme .. "://" .. host .. ":" .. port .. path + if scheme == "https" then + -- don't keep this connection alive as the next request could target + -- any host and re-using the proxy tunnel for that is not possible + self.keepalive = false end - end - if proxy_uri then - -- self:connect() set the host and port to point to the proxy server. As + -- self:connect_uri() set the host and port to point to the proxy server. As -- the connection to the proxy has been established, set the host and port - -- to point to the actual remote endpoint at the other end of the tunnel + -- to point to the actual remote endpoint at the other end of the tunnel to + -- ensure the correct Host header added to the requests. self.host = host self.port = port end @@ -1016,4 +1005,51 @@ function _M.get_proxy_uri(self, scheme, host) end +function _M.connect_proxy(self, proxy_uri, scheme, host, port) + -- Parse the provided proxy URI + local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) + if not parsed_proxy_uri then + return nil, err + end + + -- Check that the scheme is http (https is not supported for + -- connections between the client and the proxy) + local proxy_scheme = parsed_proxy_uri[1] + if proxy_scheme ~= "http" then + return nil, "protocol " .. proxy_scheme .. " not supported for proxy connections" + end + + -- Make the connection to the given proxy + local proxy_host, proxy_port = parsed_proxy_uri[2], parsed_proxy_uri[3] + local c, err = self:connect(proxy_host, proxy_port) + if not c then + return nil, err + end + + if scheme == "https" then + -- Make a CONNECT request to create a tunnel to the destination through + -- the proxy. The request-target and the Host header must be in the + -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section + -- 4.3.6 for more details about the CONNECT request + local destination = host .. ":" .. port + local res, err = self:request({ + method = "CONNECT", + path = destination, + headers = { + ["Host"] = destination + } + }) + + if not res then + return nil, err + end + + if res.status < 200 or res.status > 299 then + return nil, "failed to establish a tunnel through a proxy: " .. res.status + end + end + + return c, nil +end + return _M diff --git a/t/16-http-proxy.t b/t/16-http-proxy.t index 2f770482..bdedded1 100644 --- a/t/16-http-proxy.t +++ b/t/16-http-proxy.t @@ -245,3 +245,55 @@ GET /lua --- no_error_log [error] [warn] + +=== TEST 6: request_uri makes a proper CONNECT request when proxying https resources +--- http_config eval: $::HttpConfig +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:12345", + https_proxy = "http://127.0.0.1:12345" + }) + + -- Slight Hack: temporarily change the module global user agent to make it + -- predictable for this test case + local ua = http._USER_AGENT + http._USER_AGENT = "test_ua" + local res, err = httpc:request_uri("https://127.0.0.1/target?a=1&b=2") + http._USER_AGENT = ua + + if not err then + -- The proxy request should fail as the TCP server listening returns + -- 403 response. We cannot really test the success case here as that + -- would require an actual reverse proxy to be implemented through + -- the limited functionality we have available in the raw TCP sockets + ngx.log(ngx.ERR, "unexpected success") + return + end + + ngx.status = 403 + ngx.say(err) + } + } +--- tcp_listen: 12345 +--- tcp_query eval_stdout +# Note: The incoming request contains CRLF line endings and print needs to +# be used here to get the same line breaks to the expected request +print "CONNECT 127.0.0.1:443 HTTP/1.1\r\nUser-Agent: test_ua\r\nHost: 127.0.0.1:443\r\n\r\n" + +# The reply cannot be successful or otherwise the client would start +# to do a TLS handshake with the proxied host and that we cannot +# do with these sockets +--- tcp_reply +HTTP/1.1 403 Forbidden +Connection: close + +--- request +GET /lua +--- error_code: 403 +--- no_error_log +[error] +[warn] From af8b08b7b96788c1be03e649b1fd01892d98c20d Mon Sep 17 00:00:00 2001 From: Sami Jaktholm Date: Sun, 22 Oct 2017 19:33:28 +0300 Subject: [PATCH 005/130] followup: minor fixes according to pr comments This includes: * Add set_proxy_options to README TOC. * Make a copy of proxy options gotten from the user. * Rewrite no_proxy matching to use ngx.re.gmatch and friends instead of Lua's built-in gmatch functions. Since the regular expression formats are completely different, the no_proxy matching process had to be modified a bit. * Fix 14-host-header.t on systems with ipv6 support by disabling ipv6 address resolution. * Make 16-http-proxy.t more reliable by ignoring the order of the headers in the CONNECT request (the order is not guaranteed). The new check uses a regular expression to check that the CONNECT line is correct and that the headers include correct Host in some position. --- README.md | 1 + lib/resty/http.lua | 29 +++++++++++++++++++---------- t/14-host-header.t | 2 +- t/16-http-proxy.t | 6 ++---- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 14dfdec1..c27ebf5c 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Production ready. * [new](#new) * [connect](#connect) * [connect_proxy](#connect_proxy) +* [set_proxy_options](#set_proxy_options) * [set_timeout](#set_timeout) * [set_timeouts](#set_timeouts) * [ssl_handshake](#ssl_handshake) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index afb00f98..19c01e22 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -15,6 +15,8 @@ local tbl_concat = table.concat local tbl_insert = table.insert local ngx_encode_args = ngx.encode_args local ngx_re_match = ngx.re.match +local ngx_re_gmatch = ngx.re.gmatch +local ngx_re_sub = ngx.re.sub local ngx_re_gsub = ngx.re.gsub local ngx_re_find = ngx.re.find local ngx_log = ngx.log @@ -958,7 +960,7 @@ function _M.proxy_response(self, response, chunksize) end function _M.set_proxy_options(self, opts) - self.proxy_opts = opts + self.proxy_opts = tbl_copy(opts) -- Take by value end function _M.get_proxy_uri(self, scheme, host) @@ -977,20 +979,27 @@ function _M.get_proxy_uri(self, scheme, host) local no_proxy_set = {} -- wget allows domains in no_proxy list to be prefixed by "." -- e.g. no_proxy=.mit.edu - for host_suffix in self.proxy_opts.no_proxy:gmatch("%.?([^,]+)") do - no_proxy_set[host_suffix] = true + for host_suffix in ngx_re_gmatch(self.proxy_opts.no_proxy, "\\.?([^,]+)") do + no_proxy_set[host_suffix[1]] = true end -- From curl docs: -- matched as either a domain which contains the hostname, or the -- hostname itself. For example local.com would match local.com, - -- local.com:80, and www.local.com, but not www.notlocal.com. - for pos in host:gmatch("%f[^%z%.]()") do - local host_suffix = host:sub(pos, -1) - if no_proxy_set[host_suffix] then - return nil - end - end + -- local.com:80, and www.local.com, but not www.notlocal.com. + -- + -- Therefore, we keep stripping subdomains from the host, compare + -- them to the ones in the no_proxy list and continue until we find + -- a match or until there's only the TLD left + repeat + if no_proxy_set[host] then + return nil + end + + -- Strip the next level from the domain and check if that one + -- is on the list + host = ngx_re_sub(host, "^[^.]+\\.", "") + until not ngx_re_find(host, "\\.") end if scheme == "http" and self.proxy_opts.http_proxy then diff --git a/t/14-host-header.t b/t/14-host-header.t index 6ea28ebf..110a6695 100644 --- a/t/14-host-header.t +++ b/t/14-host-header.t @@ -12,7 +12,7 @@ $ENV{TEST_COVERAGE} ||= 0; our $HttpConfig = qq{ lua_package_path "$pwd/lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; error_log logs/error.log debug; - resolver 8.8.8.8; + resolver 8.8.8.8 ipv6=off; init_by_lua_block { if $ENV{TEST_COVERAGE} == 1 then diff --git a/t/16-http-proxy.t b/t/16-http-proxy.t index bdedded1..a0bdf394 100644 --- a/t/16-http-proxy.t +++ b/t/16-http-proxy.t @@ -279,10 +279,8 @@ GET /lua } } --- tcp_listen: 12345 ---- tcp_query eval_stdout -# Note: The incoming request contains CRLF line endings and print needs to -# be used here to get the same line breaks to the expected request -print "CONNECT 127.0.0.1:443 HTTP/1.1\r\nUser-Agent: test_ua\r\nHost: 127.0.0.1:443\r\n\r\n" +--- tcp_query eval +qr/CONNECT 127.0.0.1:443 HTTP\/1.1\r\n.*Host: 127.0.0.1:443\r\n.*/s # The reply cannot be successful or otherwise the client would start # to do a TLS handshake with the proxied host and that we cannot From 01833131da3ba0b0c11991a2192846e88c857926 Mon Sep 17 00:00:00 2001 From: Matt Shen Date: Thu, 7 Dec 2017 09:10:53 -0800 Subject: [PATCH 006/130] fix: request_uri error leaves socket open For example, if a request times-out, the response may still be received because the socket is not closed and the next request using the socket will read the timed-out request's response and result in a "unread data in buffer" error from set_keepalive. --- lib/resty/http.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index c59e6b6c..39f85b87 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -818,11 +818,13 @@ function _M.request_uri(self, uri, params) local res, err = self:request(params) if not res then + self:close() return nil, err end local body, err = res:read_body() if not body then + self:close() return nil, err end From ed9a8fe4a620182c084425a78dc3bb8be83beebf Mon Sep 17 00:00:00 2001 From: "tianjia.ztj" Date: Tue, 19 Dec 2017 15:35:09 +0800 Subject: [PATCH 007/130] typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aee6028d..49343f15 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ Attempts to puts the current connection into the ngx_lua cosocket connection poo You can specify the max idle timeout (in ms) when the connection is in the pool and the maximal size of the pool every nginx worker process. -Only call this method in the place you would have called the `close` method instead. Calling this method will immediately turn the current http object into the `closed` state. Any subsequent operations other than `connect()` on the current objet will return the `closed` error. +Only call this method in the place you would have called the `close` method instead. Calling this method will immediately turn the current http object into the `closed` state. Any subsequent operations other than `connect()` on the current object will return the `closed` error. Note that calling this instead of `close` is "safe" in that it will conditionally close depending on the type of request. Specifically, a `1.0` request without `Connection: Keep-Alive` will be closed, as will a `1.1` request with `Connection: Close`. From c39cf8c48718668a52e230a2be3518a2912d203a Mon Sep 17 00:00:00 2001 From: James Hurst Date: Sat, 23 Dec 2017 14:21:55 +0000 Subject: [PATCH 008/130] Version bump --- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...sty-http-0.11-0.rockspec => lua-resty-http-0.12-0.rockspec | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename lua-resty-http-0.11-0.rockspec => lua-resty-http-0.12-0.rockspec (93%) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 13694bfb..b3cce434 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -100,7 +100,7 @@ end local _M = { - _VERSION = '0.11', + _VERSION = '0.12', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 6ff26fb2..56069eca 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.11', + _VERSION = '0.12', } diff --git a/lua-resty-http-0.11-0.rockspec b/lua-resty-http-0.12-0.rockspec similarity index 93% rename from lua-resty-http-0.11-0.rockspec rename to lua-resty-http-0.12-0.rockspec index e2088ac9..d727114b 100644 --- a/lua-resty-http-0.11-0.rockspec +++ b/lua-resty-http-0.12-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.11-0" +version = "0.12-0" source = { url = "git://github.com/pintsized/lua-resty-http", - tag = "v0.11" + tag = "v0.12" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From 931d3517ab41c092154e985e024b816b8f681e81 Mon Sep 17 00:00:00 2001 From: Robert Paprocki Date: Wed, 27 Dec 2017 11:21:11 -0800 Subject: [PATCH 009/130] Clarify request 'query' table element definition --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4dc1894e..ff066336 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,7 @@ The `params` table accepts the following fields: * `version` The HTTP version number, currently supporting 1.0 or 1.1. * `method` The HTTP method string. * `path` The path string. -* `query` The query string. +* `query` The query string, presented as either a literal string or Lua table.. * `headers` A table of request headers. * `body` The request body as a string, or an iterator function (see [get_client_body_reader](#get_client_body_reader)). * `ssl_verify` Verify SSL cert matches hostname From 573e192fbb2093b7483f664c3678acf4d503fe5d Mon Sep 17 00:00:00 2001 From: mengskysama Date: Wed, 11 Apr 2018 12:36:27 +0800 Subject: [PATCH 010/130] feat: request_uri() keepalive params --- README.md | 5 +++-- lib/resty/http.lua | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ff066336..9136e5ab 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,8 @@ server { body = "a=1&b=2", headers = { ["Content-Type"] = "application/x-www-form-urlencoded", - } + }, + keepalive_opts = {10000, 10} }) if not res then @@ -273,7 +274,7 @@ When the request is successful, `res` will contain the following fields: `syntax: res, err = httpc:request_uri(uri, params)` -The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself. +The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself, and you can set keepalive options with `keepalive_opts`. In this mode, there is no need to connect manually first. The connection is made on your behalf, suiting cases where you simply need to grab a URI without too much hassle. diff --git a/lib/resty/http.lua b/lib/resty/http.lua index b3cce434..004346a0 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -873,9 +873,21 @@ function _M.request_uri(self, uri, params) res.body = body - local ok, err = self:set_keepalive() - if not ok then - ngx_log(ngx_ERR, err) + if params.keepalive_opts == nil then + local ok, err = self:set_keepalive() + if not ok then + ngx_log(ngx_ERR, err) + end + elseif params.keepalive_opts == false then + local ok, err = self:close() + if not ok then + ngx_log(ngx_ERR, err) + end + else + local ok, err = self:set_keepalive(unpack(params.keepalive_opts)) + if not ok then + ngx_log(ngx_ERR, err) + end end return res, nil From 65ea5ca5029cd1730a44d2089b769147f2db2e8e Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Mon, 23 Jul 2018 15:34:33 +0200 Subject: [PATCH 011/130] Allow empty HTTP header values --- lib/resty/http.lua | 2 +- t/01-basic.t | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index b3cce434..a0f5fbcb 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -338,7 +338,7 @@ local function _receive_headers(sock) return nil, err end - local m, err = ngx_re_match(line, "([^:\\s]+):\\s*(.+)", "jo") + local m, err = ngx_re_match(line, "([^:\\s]+):\\s*(.*)", "jo") if not m then break end diff --git a/t/01-basic.t b/t/01-basic.t index 495f0526..64851147 100644 --- a/t/01-basic.t +++ b/t/01-basic.t @@ -371,3 +371,43 @@ scheme: http, host: example.com, port: 80, path: /foo/bar?a=1&b=2 --- no_error_log [error] [warn] + + +=== TEST 12: Allow empty HTTP header values (RFC7230) +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local httpc = require("resty.http").new() + + -- Create a TCP connection and return an raw HTTP-response because + -- there is no way to set an empty header value in nginx. + assert(httpc:connect("127.0.0.1", 12345), + "connect should return positively") + + local res = httpc:request({ path = "/b" }) + if res.headers["X-Header-Empty"] == "" then + ngx.say("Empty") + end + ngx.say(res.headers["X-Header-Test"]) + ngx.print(res:read_body()) + } + } +--- tcp_listen: 12345 +--- tcp_reply +HTTP/1.0 200 OK +Date: Mon, 23 Jul 2018 13:00:00 GMT +X-Header-Empty: +X-Header-Test: Test +Server: OpenResty + +OK +--- request +GET /a +--- response_body +Empty +Test +OK +--- no_error_log +[error] +[warn] \ No newline at end of file From ec90bf75ce6c192b9e08daaa17bf1cdb3d537146 Mon Sep 17 00:00:00 2001 From: mengskysama Date: Wed, 8 Aug 2018 17:15:12 +0800 Subject: [PATCH 012/130] instead of keepalive_opts --- README.md | 8 ++++++-- lib/resty/http.lua | 32 +++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9136e5ab..3ffd67eb 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,8 @@ server { headers = { ["Content-Type"] = "application/x-www-form-urlencoded", }, - keepalive_opts = {10000, 10} + keepalive_timeout = 60, + keepalive_pool = 10 }) if not res then @@ -274,7 +275,10 @@ When the request is successful, `res` will contain the following fields: `syntax: res, err = httpc:request_uri(uri, params)` -The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself, and you can set keepalive options with `keepalive_opts`. +The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself, Moreover you can set keepalive options with to fileds: + +* `keepalive_timeout` A value for tcpsock:setkeepalive (default is 0), Set `-1` to close connection immediately. +* `keepalive_pool` A value for tcpsock:setkeepalive (default is lua_socket_pool_size). In this mode, there is no need to connect manually first. The connection is made on your behalf, suiting cases where you simply need to grab a URI without too much hassle. diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 004346a0..942057c6 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -873,21 +873,31 @@ function _M.request_uri(self, uri, params) res.body = body - if params.keepalive_opts == nil then - local ok, err = self:set_keepalive() - if not ok then - ngx_log(ngx_ERR, err) - end - elseif params.keepalive_opts == false then - local ok, err = self:close() - if not ok then - ngx_log(ngx_ERR, err) - end + local ok, err + if params.keepalive_timeout == nil then + ok, err = self:set_keepalive() + if not ok then + ngx_log(ngx_ERR, err) + end else - local ok, err = self:set_keepalive(unpack(params.keepalive_opts)) + if params.keepalive_timeout == -1 then + ok, err = self:close() if not ok then ngx_log(ngx_ERR, err) end + else + if params.keepalive_pool then + ok, err = self:set_keepalive(params.keepalive_timeout, params.keepalive_pool) + if not ok then + ngx_log(ngx_ERR, err) + end + else + ok, err = self:set_keepalive(params.keepalive_timeout) + if not ok then + ngx_log(ngx_ERR, err) + end + end + end end return res, nil From 54b52db389e4f38ea122eac4106b587e15319f73 Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:12:56 +0100 Subject: [PATCH 013/130] Rework simple interface keepalive settings --- README.md | 9 ++++++--- lib/resty/http.lua | 29 ++++++++--------------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 3ffd67eb..944e20d5 100644 --- a/README.md +++ b/README.md @@ -275,10 +275,13 @@ When the request is successful, `res` will contain the following fields: `syntax: res, err = httpc:request_uri(uri, params)` -The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself, Moreover you can set keepalive options with to fileds: +The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself. -* `keepalive_timeout` A value for tcpsock:setkeepalive (default is 0), Set `-1` to close connection immediately. -* `keepalive_pool` A value for tcpsock:setkeepalive (default is lua_socket_pool_size). +There are 3 additional parmaters for controlling keepalives: + +* `keepalive` Set to `false` to disable keepalives and immediately close the connection. +* `keepalive_timeout` The maximal idle timeout (ms). Defaults to `lua_socket_keepalive_timeout`. +* `keepalive_pool` The maxmimal number of connections in the pool. Defaults to `lua_socket_pool_size`. In this mode, there is no need to connect manually first. The connection is made on your behalf, suiting cases where you simply need to grab a URI without too much hassle. diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 5b534b59..c69e1aab 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -873,31 +873,18 @@ function _M.request_uri(self, uri, params) res.body = body - local ok, err - if params.keepalive_timeout == nil then - ok, err = self:set_keepalive() - if not ok then - ngx_log(ngx_ERR, err) - end - else - if params.keepalive_timeout == -1 then - ok, err = self:close() + if params.keepalive == false then + local ok, err = self:close() if not ok then ngx_log(ngx_ERR, err) end - else - if params.keepalive_pool then - ok, err = self:set_keepalive(params.keepalive_timeout, params.keepalive_pool) - if not ok then - ngx_log(ngx_ERR, err) - end - else - ok, err = self:set_keepalive(params.keepalive_timeout) - if not ok then - ngx_log(ngx_ERR, err) - end + + else + local ok, err = self:set_keepalive(params.keepalive_timeout, params.keepalive_pool) + if not ok then + ngx_log(ngx_ERR, err) end - end + end return res, nil From a77292ea5cfb37eb00ae473206fbb659b44ecbe5 Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:17:34 +0100 Subject: [PATCH 014/130] Test: simple interface keepalive settings --- t/07-keepalive.t | 67 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/t/07-keepalive.t b/t/07-keepalive.t index 9a29d255..8298a4d2 100644 --- a/t/07-keepalive.t +++ b/t/07-keepalive.t @@ -244,3 +244,70 @@ connection must be closed --- no_error_log [error] [warn] + +=== TEST 6: Simple interface, override settings +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:"..ngx.var.server_port.."/b", + { + keepalive = false + } + ) + + ngx.say(res.headers["Connection"]) + + httpc:connect("127.0.0.1", ngx.var.server_port) + ngx.say(httpc:get_reused_times()) + httpc:close() + + local res, err = httpc:request_uri( + "http://127.0.0.1:"..ngx.var.server_port.."/b", + { + keepalive_timeout = 10 + } + ) + + ngx.say(res.headers["Connection"]) + + httpc:connect("127.0.0.1", ngx.var.server_port) + ngx.say(httpc:get_reused_times()) + httpc:close() + + local res, err = httpc:request_uri( + "http://127.0.0.1:"..ngx.var.server_port.."/b", + { + keepalive_timeout = 1 + } + ) + + ngx.say(res.headers["Connection"]) + + ngx.sleep(1.1) + + httpc:connect("127.0.0.1", ngx.var.server_port) + ngx.say(httpc:get_reused_times()) + httpc:close() + } + } + location = /b { + content_by_lua_block { + ngx.say("OK") + } + } +--- request +GET /a +--- response_body +keep-alive +0 +keep-alive +1 +keep-alive +0 +--- no_error_log +[error] +[warn] From 6e4fcff1f33027029fb854e3664a9bca8929c773 Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:23:28 +0100 Subject: [PATCH 015/130] Add luacheck config --- .luacheckrc | 2 ++ Makefile | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 .luacheckrc diff --git a/.luacheckrc b/.luacheckrc new file mode 100644 index 00000000..77ab5dfd --- /dev/null +++ b/.luacheckrc @@ -0,0 +1,2 @@ +std = "ngx_lua" +redefined = false diff --git a/Makefile b/Makefile index d9539f2a..f8399bc0 100644 --- a/Makefile +++ b/Makefile @@ -24,3 +24,6 @@ coverage: all PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH TEST_NGINX_NO_SHUFFLE=1 TEST_COVERAGE=1 prove -I../test-nginx/lib -r $(TEST_FILE) @luacov @tail -10 luacov.report.out + +check: + luacheck lib From 00e6cc53f2978d34719764f36c0c5ac76963a22f Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:23:46 +0100 Subject: [PATCH 016/130] Fix luacheck warnings, whitespace --- lib/resty/http.lua | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index c69e1aab..cadea4cc 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -6,7 +6,6 @@ local ngx_req = ngx.req local ngx_req_socket = ngx_req.socket local ngx_req_get_headers = ngx_req.get_headers local ngx_req_get_method = ngx_req.get_method -local str_gmatch = string.gmatch local str_lower = string.lower local str_upper = string.upper local str_find = string.find @@ -112,6 +111,7 @@ local HTTP = { [1.1] = " HTTP/1.1\r\n", } + local DEFAULT_PARAMS = { method = "GET", path = "/", @@ -119,7 +119,7 @@ local DEFAULT_PARAMS = { } -function _M.new(self) +function _M.new(_) local sock, err = ngx_socket_tcp() if not sock then return nil, err @@ -230,7 +230,7 @@ local function _should_receive_body(method, code) end -function _M.parse_uri(self, uri, query_in_path) +function _M.parse_uri(_, uri, query_in_path) if query_in_path == nil then query_in_path = true end local m, err = ngx_re_match(uri, [[^(?:(http[s]?):)?//([^:/\?]+)(?::(\d+))?([^\?]*)\??(.*)]], "jo") @@ -339,6 +339,8 @@ local function _receive_headers(sock) end local m, err = ngx_re_match(line, "([^:\\s]+):\\s*(.*)", "jo") + if err then ngx_log(ngx_ERR, err) end + if not m then break end @@ -563,7 +565,7 @@ end local function _handle_continue(sock, body) - local status, version, reason, err = _receive_status(sock) + local status, version, reason, err = _receive_status(sock) --luacheck: no unused if not status then return nil, nil, err end @@ -662,7 +664,7 @@ function _M.read_response(self, params) if not _status then return nil, _err elseif _status ~= 100 then - status, version, err = _status, _version, _err + status, version, err = _status, _version, _err -- luacheck: no unused end end @@ -789,6 +791,7 @@ function _M.request_pipeline(self, requests) return responses end + function _M.request_uri(self, uri, params) params = tbl_copy(params or {}) -- Take by value @@ -891,7 +894,7 @@ function _M.request_uri(self, uri, params) end -function _M.get_client_body_reader(self, chunksize, sock) +function _M.get_client_body_reader(_, chunksize, sock) chunksize = chunksize or 65536 if not sock then @@ -935,7 +938,7 @@ function _M.proxy_request(self, chunksize) end -function _M.proxy_response(self, response, chunksize) +function _M.proxy_response(_, response, chunksize) if not response then ngx_log(ngx_ERR, "no response provided") return @@ -968,10 +971,12 @@ function _M.proxy_response(self, response, chunksize) until not chunk end + function _M.set_proxy_options(self, opts) self.proxy_opts = tbl_copy(opts) -- Take by value end + function _M.get_proxy_uri(self, scheme, host) if not self.proxy_opts then return nil @@ -1070,4 +1075,5 @@ function _M.connect_proxy(self, proxy_uri, scheme, host, port) return c, nil end + return _M From db93fbfe7b61535ead8b19ed9c56753325be9a74 Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Thu, 9 Aug 2018 11:31:00 +0200 Subject: [PATCH 017/130] Improve keep-alive check --- lib/resty/http.lua | 4 ++-- t/07-keepalive.t | 58 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index cadea4cc..905d210a 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -685,8 +685,8 @@ function _M.read_response(self, params) -- keepalive is true by default. Determine if this is correct or not. local ok, connection = pcall(str_lower, res_headers["Connection"]) if ok then - if (version == 1.1 and connection == "close") or - (version == 1.0 and connection ~= "keep-alive") then + if (version == 1.1 and str_find(connection, "close", 1, true)) or + (version == 1.0 and not str_find(connection, "keep-alive", 1, true)) then self.keepalive = false end else diff --git a/t/07-keepalive.t b/t/07-keepalive.t index 8298a4d2..cae8e23c 100644 --- a/t/07-keepalive.t +++ b/t/07-keepalive.t @@ -311,3 +311,61 @@ keep-alive --- no_error_log [error] [warn] + +=== TEST 7: Generic interface, HTTP 1.1, Connection: Upgrade, close. Test we don't try to keepalive, but also that subsequent connections can keepalive. +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + + -- Create a TCP connection and return an raw HTTP-response because + -- there is no way to set an "Connection: Upgrade, close" header in nginx. + assert(httpc:connect("127.0.0.1", 12345), + "connect should return positively") + + local res = httpc:request({ + version = 1.1, + path = "/b", + }) + + local body = res:read_body() + ngx.print(body) + + ngx.say(res.headers["Connection"]) + + local r, e = httpc:set_keepalive() + ngx.say(r) + ngx.say(e) + + httpc:connect("127.0.0.1", ngx.var.server_port) + ngx.say(httpc:get_reused_times()) + + httpc:set_keepalive() + + httpc:connect("127.0.0.1", ngx.var.server_port) + ngx.say(httpc:get_reused_times()) + } + } +--- tcp_listen: 12345 +--- tcp_reply +HTTP/1.1 200 OK +Date: Wed, 08 Aug 2018 17:00:00 GMT +Server: Apache/2 +Upgrade: h2,h2c +Connection: Upgrade, close + +OK +--- request +GET /a +--- response_body +OK +Upgrade, close +2 +connection must be closed +0 +1 +--- no_error_log +[error] +[warn] From cf957cf1ed487045544de1b76d412ab1fae2cc04 Mon Sep 17 00:00:00 2001 From: Kleis Auke Wolthuizen Date: Thu, 9 Aug 2018 11:32:51 +0200 Subject: [PATCH 018/130] Fix whitespace issues --- lib/resty/http.lua | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 905d210a..eef32c84 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -594,7 +594,7 @@ function _M.send_request(self, params) if params_headers then -- We assign one by one so that the metatable can handle case insensitivity -- for us. You can blame the spec for this inefficiency. - for k,v in pairs(params_headers) do + for k, v in pairs(params_headers) do headers[k] = v end end @@ -738,7 +738,7 @@ end function _M.request(self, params) - params = tbl_copy(params) -- Take by value + params = tbl_copy(params) -- Take by value local res, err = self:send_request(params) if not res then return res, err @@ -749,7 +749,7 @@ end function _M.request_pipeline(self, requests) - requests = tbl_copy(requests) -- Take by value + requests = tbl_copy(requests) -- Take by value for _, params in ipairs(requests) do if params.headers and params.headers["Expect"] == "100-continue" then @@ -793,7 +793,7 @@ end function _M.request_uri(self, uri, params) - params = tbl_copy(params or {}) -- Take by value + params = tbl_copy(params or {}) -- Take by value local parsed_uri, err = self:parse_uri(uri, false) if not parsed_uri then @@ -923,18 +923,18 @@ function _M.get_client_body_reader(_, chunksize, sock) -- Not yet supported by ngx_lua but should just work... return _chunked_body_reader(sock, chunksize) else - return nil + return nil end end function _M.proxy_request(self, chunksize) - return self:request{ + return self:request({ method = ngx_req_get_method(), path = ngx_re_gsub(ngx_var.uri, "\\s", "%20", "jo") .. ngx_var.is_args .. (ngx_var.query_string or ""), body = self:get_client_body_reader(chunksize), headers = ngx_req_get_headers(), - } + }) end @@ -947,7 +947,7 @@ function _M.proxy_response(_, response, chunksize) ngx.status = response.status -- Filter out hop-by-hop headeres - for k,v in pairs(response.headers) do + for k, v in pairs(response.headers) do if not HOP_BY_HOP_HEADERS[str_lower(k)] then ngx_header[k] = v end @@ -973,7 +973,7 @@ end function _M.set_proxy_options(self, opts) - self.proxy_opts = tbl_copy(opts) -- Take by value + self.proxy_opts = tbl_copy(opts) -- Take by value end @@ -997,9 +997,9 @@ function _M.get_proxy_uri(self, scheme, host) no_proxy_set[host_suffix[1]] = true end - -- From curl docs: - -- matched as either a domain which contains the hostname, or the - -- hostname itself. For example local.com would match local.com, + -- From curl docs: + -- matched as either a domain which contains the hostname, or the + -- hostname itself. For example local.com would match local.com, -- local.com:80, and www.local.com, but not www.notlocal.com. -- -- Therefore, we keep stripping subdomains from the host, compare From fb03e457a92d4e97378a4266989a42b08bcb4ed1 Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:40:31 +0100 Subject: [PATCH 019/130] Close socket on handshake failure in simple mode --- lib/resty/http.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 6a17fd6f..fc703aed 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -855,13 +855,17 @@ function _M.request_uri(self, uri, params) if scheme == "https" then local verify = true + if params.ssl_verify == false then verify = false end + local ok, err = self:ssl_handshake(nil, host, verify) if not ok then + self:close() return nil, err end + end local res, err = self:request(params) From 5b867d2895a8605446b835203d86e5342e2337e4 Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:40:44 +0100 Subject: [PATCH 020/130] Test: simple interface closes socket on error --- t/06-simpleinterface.t | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/t/06-simpleinterface.t b/t/06-simpleinterface.t index bee21d32..451e5f2f 100644 --- a/t/06-simpleinterface.t +++ b/t/06-simpleinterface.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 6); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; @@ -232,3 +230,40 @@ OK --- no_error_log [error] [warn] + +=== TEST 6: Connection is closed on error +--- http_config eval: $::HttpConfig +--- config + lua_socket_read_timeout 100ms; + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:"..ngx.var.server_port.."/b?a=1&b=2") + + if not res then + ngx.log(ngx.ERR, err) + else + return ngx.say("BAD") + end + + local ok, err = httpc.sock:close() + ngx.say(ok, " ", err) + + } + } + location = /b { + content_by_lua_block { + ngx.say("1") + ngx.flush(true) + ngx.sleep(0.5) + ngx.say("2") + + } + } +--- request +GET /a +--- response_body +nil closed +--- error_log +lua tcp socket read timed out From 05b2b5340f4a1eb8e2f6032a311c134176ee9741 Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:41:58 +0100 Subject: [PATCH 021/130] Use content_by_lua_block in examples --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 944e20d5..1f809bf0 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ server { location /simpleinterface { resolver 8.8.8.8; # use Google's open DNS server for an example - content_by_lua ' + content_by_lua_block { -- For simple singleshot requests, use the URI interface. local http = require "resty.http" @@ -87,12 +87,12 @@ server { end ngx.say(res.body) - '; + } } location /genericinterface { - content_by_lua ' + content_by_lua_block { local http = require "resty.http" local httpc = http.new() @@ -134,7 +134,7 @@ server { ngx.say("failed to set keepalive: ", err) return end - '; + } } } ```` From 769e8b2a0209e92d32f75f93eb0a9fd5214f6224 Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:42:47 +0100 Subject: [PATCH 022/130] Typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1f809bf0..d140a297 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Production ready. * [Response](#response) * [body_reader](#resbody_reader) * [read_body](#resread_body) - * [read_trailes](#resread_trailers) + * [read_trailers](#resread_trailers) * [Proxy](#proxy) * [proxy_request](#proxy_request) * [proxy_response](#proxy_response) From fe5c10a47cf40440845c140a5d29cd0e0cd0208f Mon Sep 17 00:00:00 2001 From: Hamish Date: Thu, 9 Aug 2018 10:50:23 +0100 Subject: [PATCH 023/130] Remove lua-releng, luacheck catches those problems --- Makefile | 1 - util/lua-releng | 63 ------------------------------------------------- 2 files changed, 64 deletions(-) delete mode 100755 util/lua-releng diff --git a/Makefile b/Makefile index f8399bc0..a676a600 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,6 @@ install: all $(INSTALL) lib/resty/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/ test: all - util/lua-releng PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH TEST_NGINX_NO_SHUFFLE=1 prove -I../test-nginx/lib -r $(TEST_FILE) coverage: all diff --git a/util/lua-releng b/util/lua-releng deleted file mode 100755 index e3dea2d6..00000000 --- a/util/lua-releng +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -sub file_contains ($$); - -my $version; -for my $file (map glob, qw{ *.lua lib/*.lua lib/*/*.lua lib/*/*/*.lua lib/*/*/*/*.lua lib/*/*/*/*/*.lua }) { - # Check the sanity of each .lua file - open my $in, $file or - die "ERROR: Can't open $file for reading: $!\n"; - my $found_ver; - while (<$in>) { - my ($ver, $skipping); - if (/(?x) (?:_VERSION) \s* = .*? ([\d\.]*\d+) (.*? SKIP)?/) { - my $orig_ver = $ver = $1; - $found_ver = 1; - # $skipping = $2; - $ver =~ s{^(\d+)\.(\d{3})(\d{3})$}{join '.', int($1), int($2), int($3)}e; - warn "$file: $orig_ver ($ver)\n"; - - } elsif (/(?x) (?:_VERSION) \s* = \s* ([a-zA-Z_]\S*)/) { - warn "$file: $1\n"; - $found_ver = 1; - last; - } - - if ($ver and $version and !$skipping) { - if ($version ne $ver) { - # die "$file: $ver != $version\n"; - } - } elsif ($ver and !$version) { - $version = $ver; - } - } - if (!$found_ver) { - warn "WARNING: No \"_VERSION\" or \"version\" field found in `$file`.\n"; - } - close $in; - - #print "Checking use of Lua global variables in file $file ...\n"; - system("luac5.1 -p -l $file | grep ETGLOBAL | grep -vE '(require|type|tostring|error|ngx|ndk|jit|setmetatable|getmetatable|string|table|io|os|print|tonumber|math|pcall|xpcall|unpack|pairs|ipairs|assert|module|package|coroutine|[gs]etfenv|next|select|rawset|rawget|debug)\$'"); - #file_contains($file, "attempt to write to undeclared variable"); - system("grep -H -n -E --color '.{120}' $file"); -} - -sub file_contains ($$) { - my ($file, $regex) = @_; - open my $in, $file - or die "Cannot open $file fo reading: $!\n"; - my $content = do { local $/; <$in> }; - close $in; - #print "$content"; - return scalar ($content =~ /$regex/); -} - -if (-d 't') { - for my $file (map glob, qw{ t/*.t t/*/*.t t/*/*/*.t }) { - system(qq{grep -H -n --color -E '\\--- ?(ONLY|LAST)' $file}); - } -} - From a02a2ca2ca19bd1c7eed63bde3c70a2792efefbf Mon Sep 17 00:00:00 2001 From: Chris Kuehl Date: Wed, 29 Aug 2018 08:39:58 -0700 Subject: [PATCH 024/130] README.md: fix a couple minor typos --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d140a297..8fbedd75 100644 --- a/README.md +++ b/README.md @@ -190,13 +190,13 @@ Sets the timeout (in ms) protection for subsequent operations, including the `co `syntax: httpc:set_timeouts(connect_timeout, send_timeout, read_timeout)` -Sets the connect timeout thresold, send timeout threshold, and read timeout threshold, respetively, in milliseconds, for subsequent socket operations (connect, send, receive, and iterators returned from receiveuntil). +Sets the connect timeout threshold, send timeout threshold, and read timeout threshold, respectively, in milliseconds, for subsequent socket operations (connect, send, receive, and iterators returned from receiveuntil). ## ssl_handshake `syntax: session, err = httpc:ssl_handshake(session, host, verify)` -Performs an SSL handshake on the TCP connection, only availble in ngx_lua > v0.9.11 +Performs an SSL handshake on the TCP connection, only available in ngx_lua > v0.9.11 See docs for [ngx.socket.tcp](https://github.com/openresty/lua-nginx-module#ngxsockettcp) for details. @@ -212,7 +212,7 @@ Only call this method in the place you would have called the `close` method inst Note that calling this instead of `close` is "safe" in that it will conditionally close depending on the type of request. Specifically, a `1.0` request without `Connection: Keep-Alive` will be closed, as will a `1.1` request with `Connection: Close`. -In case of success, returns `1`. In case of errors, returns `nil, err`. In the case where the conneciton is conditionally closed as described above, returns `2` and the error string `connection must be closed`. +In case of success, returns `1`. In case of errors, returns `nil, err`. In the case where the connection is conditionally closed as described above, returns `2` and the error string `connection must be closed`. ## set_proxy_options @@ -277,11 +277,11 @@ When the request is successful, `res` will contain the following fields: The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself. -There are 3 additional parmaters for controlling keepalives: +There are 3 additional parameters for controlling keepalives: * `keepalive` Set to `false` to disable keepalives and immediately close the connection. * `keepalive_timeout` The maximal idle timeout (ms). Defaults to `lua_socket_keepalive_timeout`. -* `keepalive_pool` The maxmimal number of connections in the pool. Defaults to `lua_socket_pool_size`. +* `keepalive_pool` The maximum number of connections in the pool. Defaults to `lua_socket_pool_size`. In this mode, there is no need to connect manually first. The connection is made on your behalf, suiting cases where you simply need to grab a URI without too much hassle. @@ -409,7 +409,7 @@ Sets the current response based on the given `res`. Ensures that hop-by-hop head This is a convenience function allowing one to more easily use the generic interface, when the input data is a URI. -As of version `0.10`, the optional `query_in_path` parameter was added, which specifies whether the querystring is to be included in the `path` return value, or separately as its own return value. This defaults to `true` in order to maintain backwards compatability. When set to `false`, `path` will only include the path, and `query` will contain the URI args, not inluding the `?` delimeter. +As of version `0.10`, the optional `query_in_path` parameter was added, which specifies whether the querystring is to be included in the `path` return value, or separately as its own return value. This defaults to `true` in order to maintain backwards compatibility. When set to `false`, `path` will only include the path, and `query` will contain the URI args, not including the `?` delimiter. ## get_client_body_reader From 8b9acca4763a0dcff94d5e8f47a8cfcbf91e3e65 Mon Sep 17 00:00:00 2001 From: Hamish Date: Fri, 31 Aug 2018 10:51:31 +0100 Subject: [PATCH 025/130] test: multiple transfer-encoding headers --- t/02-chunked.t | 65 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/t/02-chunked.t b/t/02-chunked.t index 4c013aac..cb1f6f3a 100644 --- a/t/02-chunked.t +++ b/t/02-chunked.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; @@ -163,3 +161,64 @@ GET /a --- no_error_log [error] [warn] + + +=== TEST 4: Chunked. multiple-headers, mixed case +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:connect("127.0.0.1", ngx.var.server_port) + + local res, err = httpc:request{ + path = "/b" + } + + local chunks = {} + local c = 1 + repeat + local chunk, err = res.body_reader() + if chunk then + chunks[c] = chunk + c = c + 1 + end + until not chunk + + local body = table.concat(chunks) + + ngx.say(#body) + ngx.say(#chunks) + ngx.say(type(res.headers["Transfer-Encoding"])) + httpc:close() + } + } + location = /b { + header_filter_by_lua_block { + ngx.header["Transfer-Encoding"] = {"chUnked", "CHunked"} + } + content_by_lua_block { + local len = 32768 + local t = {} + for i=1,len do + t[i] = 0 + end + ngx.print(table.concat(t)) + local len = 32768 + local t = {} + for i=1,len do + t[i] = 0 + end + ngx.print(table.concat(t)) + } + } +--- request +GET /a +--- response_body +65536 +2 +table +--- no_error_log +[error] +[warn] From e5deba5bde1db31adc7aad574bae5e89f83fd973 Mon Sep 17 00:00:00 2001 From: Hamish Date: Fri, 31 Aug 2018 10:54:55 +0100 Subject: [PATCH 026/130] Handle multiple Transfer-Encoding headers, fixes #157 --- lib/resty/http.lua | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index beff8692..f2f80f4d 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -702,18 +702,35 @@ function _M.read_response(self, params) -- Receive the body_reader if _should_receive_body(params.method, status) then - local ok, encoding = pcall(str_lower, res_headers["Transfer-Encoding"]) - if ok and version == 1.1 and encoding == "chunked" then + has_body = true + + local te = res_headers["Transfer-Encoding"] + + -- Handle duplicate headers + -- This shouldn't happen but can in the real world + if type(te) == "table" then + te = tbl_concat(te, "") + end + + local ok, encoding = pcall(str_lower, te) + if not ok then + encoding = "" + end + + if version == 1.1 and str_find(encoding, "chunked", 1, true) ~= nil then body_reader, err = _chunked_body_reader(sock) - has_body = true - else + else local ok, length = pcall(tonumber, res_headers["Content-Length"]) - if ok then - body_reader, err = _body_reader(sock, length) - has_body = true + if not ok then + -- No content-length header, read until connection is closed by server + length = nil end + + body_reader, err = _body_reader(sock, length) + end + end if res_headers["Trailer"] then From 0723ea0db1fc7fef9fee86a92aa9d630e1e6b6b9 Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Sat, 1 Dec 2018 01:25:54 +0200 Subject: [PATCH 027/130] feat(http-proxy) add support for http/https proxy authorization --- README.md | 7 +- lib/resty/http.lua | 28 +++++- t/16-http-proxy.t | 225 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8fbedd75..1066a694 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ An optional Lua table can be specified as the last argument to this method to sp ## connect_proxy -`syntax: ok, err = httpc:connect_proxy(proxy_uri, scheme, host, port)` +`syntax: ok, err = httpc:connect_proxy(proxy_uri, scheme, host, port, proxy_authorization)` Attempts to connect to the web server through the given proxy server. The method accepts the following arguments: @@ -172,6 +172,7 @@ Attempts to connect to the web server through the given proxy server. The method * `scheme` - The protocol to use between the proxy server and the remote host (`http` or `https`). If `https` is specified as the scheme, `connect_proxy()` makes a `CONNECT` request to establish a TCP tunnel to the remote host through the proxy server. * `host` - The hostname of the remote host to connect to. * `port` - The port of the remote host to connect to. +* `proxy_authorization` - The `Proxy-Authorization` header value send to proxy server via `CONNECT` when the `scheme` is `https`. If an error occurs during the connection attempt, this method returns `nil` with a string describing the error. If the connection was successfully established, the method returns `1`. @@ -221,7 +222,11 @@ In case of success, returns `1`. In case of errors, returns `nil, err`. In the c Configure an http proxy to be used with this client instance. The `opts` is a table that accepts the following fields: * `http_proxy` - an URI to a proxy server to be used with http requests +* `http_proxy_authorization` - a value for `Proxy-Authorization` header to be used with `http_proxy`, e.g. `Basic ZGVtbzp0ZXN0`. + The `httpc:request_uri` can override the setting by specifying `Proxy-Authorization` header. * `https_proxy` - an URI to a proxy server to be used with https requests +* `https_proxy_authorization` - a value for `Proxy-Authorization` header to be used with `https_proxy`, e.g. `Basic ZGVtbzpwYXNz`. + The `httpc:request_uri` can override the setting by specifying `Proxy-Authorization` header. * `no_proxy` - a comma separated list of hosts that should not be proxied. Note that proxy options are only applied when using the high-level `request_uri()` API. diff --git a/lib/resty/http.lua b/lib/resty/http.lua index f2f80f4d..1cb97f0e 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -829,7 +829,16 @@ function _M.request_uri(self, uri, params) local c, err if proxy_uri then - c, err = self:connect_proxy(proxy_uri, scheme, host, port) + local proxy_authorization + if scheme == "https" then + if params.headers and params.headers["Proxy-Authorization"] then + proxy_authorization = params.headers["Proxy-Authorization"] + else + proxy_authorization = self.proxy_opts.https_proxy_authorization + end + end + + c, err = self:connect_proxy(proxy_uri, scheme, host, port, proxy_authorization) else c, err = self:connect(host, port) end @@ -854,9 +863,17 @@ function _M.request_uri(self, uri, params) else params.path = scheme .. "://" .. host .. ":" .. port .. path end - end - if scheme == "https" then + if self.proxy_opts.http_proxy_authorization then + if not params.headers then + params.headers = {} + end + + if not params.headers["Proxy-Authorization"] then + params.headers["Proxy-Authorization"] = self.proxy_opts.http_proxy_authorization + end + end + elseif scheme == "https" then -- don't keep this connection alive as the next request could target -- any host and re-using the proxy tunnel for that is not possible self.keepalive = false @@ -1051,7 +1068,7 @@ function _M.get_proxy_uri(self, scheme, host) end -function _M.connect_proxy(self, proxy_uri, scheme, host, port) +function _M.connect_proxy(self, proxy_uri, scheme, host, port, proxy_authorization) -- Parse the provided proxy URI local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) if not parsed_proxy_uri then @@ -1082,7 +1099,8 @@ function _M.connect_proxy(self, proxy_uri, scheme, host, port) method = "CONNECT", path = destination, headers = { - ["Host"] = destination + ["Host"] = destination, + ["Proxy-Authorization"] = proxy_authorization, } }) diff --git a/t/16-http-proxy.t b/t/16-http-proxy.t index a0bdf394..7393b644 100644 --- a/t/16-http-proxy.t +++ b/t/16-http-proxy.t @@ -26,6 +26,7 @@ no_long_string(); run_tests(); __DATA__ + === TEST 1: get_proxy_uri returns nil if proxy is not configured --- http_config eval: $::HttpConfig --- config @@ -44,6 +45,8 @@ nil [error] [warn] + + === TEST 2: get_proxy_uri matches no_proxy hosts correctly --- http_config eval: $::HttpConfig --- config @@ -111,6 +114,8 @@ scheme: http, host: notexample.com, no_proxy: example.com, proxy_uri: http://htt [error] [warn] + + === TEST 3: get_proxy_uri returns correct proxy URIs for http and https URIs --- http_config eval: $::HttpConfig --- config @@ -157,6 +162,8 @@ scheme: https, host: example.com, http_proxy: http_proxy, https_proxy: nil, prox [error] [warn] + + === TEST 4: request_uri uses http_proxy correctly for non-standard destination ports --- http_config lua_package_path "$TEST_NGINX_PWD/lib/?.lua;;"; @@ -200,6 +207,8 @@ GET /lua [error] [warn] + + === TEST 5: request_uri uses http_proxy correctly for standard destination port --- http_config lua_package_path "$TEST_NGINX_PWD/lib/?.lua;;"; @@ -246,6 +255,8 @@ GET /lua [error] [warn] + + === TEST 6: request_uri makes a proper CONNECT request when proxying https resources --- http_config eval: $::HttpConfig --- config @@ -295,3 +306,217 @@ GET /lua --- no_error_log [error] [warn] + + + +=== TEST 7: request_uri uses http_proxy_authorization option +--- http_config + lua_package_path "$TEST_NGINX_PWD/lib/?.lua;;"; + error_log logs/error.log debug; + resolver 8.8.8.8; + server { + listen *:8080; + + location / { + content_by_lua_block { + ngx.print(ngx.var.http_proxy_authorization or "no-header") + } + } + } +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:8080", + http_proxy_authorization = "Basic ZGVtbzp0ZXN0", + https_proxy = "http://127.0.0.1:8080", + https_proxy_authorization = "Basic ZGVtbzpwYXNz" + }) + + -- request should go to the proxy server + local res, err = httpc:request_uri("http://127.0.0.1/") + if not res then + ngx.log(ngx.ERR, err) + return + end + + -- the proxy echoed the proxy authorization header + -- to the test harness + ngx.status = res.status + ngx.say(res.body) + } + } +--- request +GET /lua +--- response_body +Basic ZGVtbzp0ZXN0 +--- no_error_log +[error] +[warn] + + + +=== TEST 8: request_uri uses https_proxy_authorization option +--- http_config eval: $::HttpConfig +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:12345", + http_proxy_authorization = "Basic ZGVtbzp0ZXN0", + https_proxy = "http://127.0.0.1:12345", + https_proxy_authorization = "Basic ZGVtbzpwYXNz" + }) + + -- Slight Hack: temporarily change the module global user agent to make it + -- predictable for this test case + local ua = http._USER_AGENT + http._USER_AGENT = "test_ua" + local res, err = httpc:request_uri("https://127.0.0.1/target?a=1&b=2") + http._USER_AGENT = ua + + if not err then + -- The proxy request should fail as the TCP server listening returns + -- 403 response. We cannot really test the success case here as that + -- would require an actual reverse proxy to be implemented through + -- the limited functionality we have available in the raw TCP sockets + ngx.log(ngx.ERR, "unexpected success") + return + end + + ngx.status = 403 + ngx.say(err) + } + } +--- tcp_listen: 12345 +--- tcp_query eval +qr/CONNECT 127.0.0.1:443 HTTP\/1.1\r\n.*Proxy-Authorization: Basic ZGVtbzpwYXNz\r\n.*Host: 127.0.0.1:443\r\n.*/s + +# The reply cannot be successful or otherwise the client would start +# to do a TLS handshake with the proxied host and that we cannot +# do with these sockets +--- tcp_reply +HTTP/1.1 403 Forbidden +Connection: close + +--- request +GET /lua +--- error_code: 403 +--- no_error_log +[error] +[warn] + + + +=== TEST 9: request_uri does not use http_proxy_authorization option when overridden +--- http_config + lua_package_path "$TEST_NGINX_PWD/lib/?.lua;;"; + error_log logs/error.log debug; + resolver 8.8.8.8; + server { + listen *:8080; + + location / { + content_by_lua_block { + ngx.print(ngx.var.http_proxy_authorization or "no-header") + } + } + } +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:8080", + http_proxy_authorization = "Basic ZGVtbzp0ZXN0", + https_proxy = "http://127.0.0.1:8080", + https_proxy_authorization = "Basic ZGVtbzpwYXNz" + }) + + -- request should go to the proxy server + local res, err = httpc:request_uri("http://127.0.0.1/", { + headers = { + ["Proxy-Authorization"] = "Basic ZGVtbzp3b3Jk" + } + }) + if not res then + ngx.log(ngx.ERR, err) + return + end + + -- the proxy echoed the proxy authorization header + -- to the test harness + ngx.status = res.status + ngx.say(res.body) + } + } +--- request +GET /lua +--- response_body +Basic ZGVtbzp3b3Jk +--- no_error_log +[error] +[warn] + + + +=== TEST 10: request_uri does not use https_proxy_authorization option when overridden +--- http_config eval: $::HttpConfig +--- config + location /lua { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:set_proxy_options({ + http_proxy = "http://127.0.0.1:12345", + http_proxy_authorization = "Basic ZGVtbzp0ZXN0", + https_proxy = "http://127.0.0.1:12345", + https_proxy_authorization = "Basic ZGVtbzpwYXNz" + }) + + -- Slight Hack: temporarily change the module global user agent to make it + -- predictable for this test case + local ua = http._USER_AGENT + http._USER_AGENT = "test_ua" + local res, err = httpc:request_uri("https://127.0.0.1/target?a=1&b=2", { + headers = { + ["Proxy-Authorization"] = "Basic ZGVtbzp3b3Jk" + } + }) + http._USER_AGENT = ua + + if not err then + -- The proxy request should fail as the TCP server listening returns + -- 403 response. We cannot really test the success case here as that + -- would require an actual reverse proxy to be implemented through + -- the limited functionality we have available in the raw TCP sockets + ngx.log(ngx.ERR, "unexpected success") + return + end + + ngx.status = 403 + ngx.say(err) + } + } +--- tcp_listen: 12345 +--- tcp_query eval +qr/CONNECT 127.0.0.1:443 HTTP\/1.1\r\n.*Proxy-Authorization: Basic ZGVtbzp3b3Jk\r\n.*Host: 127.0.0.1:443\r\n.*/s + +# The reply cannot be successful or otherwise the client would start +# to do a TLS handshake with the proxied host and that we cannot +# do with these sockets +--- tcp_reply +HTTP/1.1 403 Forbidden +Connection: close + +--- request +GET /lua +--- error_code: 403 +--- no_error_log +[error] +[warn] From 87c6bbebf433a1b3426566f9890d1b78fbd9cbd1 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Sun, 3 Feb 2019 12:28:11 +0000 Subject: [PATCH 028/130] Proxy-Authorization docs tweak --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1066a694..857484cf 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ Attempts to connect to the web server through the given proxy server. The method * `scheme` - The protocol to use between the proxy server and the remote host (`http` or `https`). If `https` is specified as the scheme, `connect_proxy()` makes a `CONNECT` request to establish a TCP tunnel to the remote host through the proxy server. * `host` - The hostname of the remote host to connect to. * `port` - The port of the remote host to connect to. -* `proxy_authorization` - The `Proxy-Authorization` header value send to proxy server via `CONNECT` when the `scheme` is `https`. +* `proxy_authorization` - The `Proxy-Authorization` header value sent to the proxy server via `CONNECT` when the `scheme` is `https`. If an error occurs during the connection attempt, this method returns `nil` with a string describing the error. If the connection was successfully established, the method returns `1`. @@ -222,11 +222,9 @@ In case of success, returns `1`. In case of errors, returns `nil, err`. In the c Configure an http proxy to be used with this client instance. The `opts` is a table that accepts the following fields: * `http_proxy` - an URI to a proxy server to be used with http requests -* `http_proxy_authorization` - a value for `Proxy-Authorization` header to be used with `http_proxy`, e.g. `Basic ZGVtbzp0ZXN0`. - The `httpc:request_uri` can override the setting by specifying `Proxy-Authorization` header. +* `http_proxy_authorization` - a default `Proxy-Authorization` header value to be used with `http_proxy`, e.g. `Basic ZGVtbzp0ZXN0`, which will be overriden if the `Proxy-Authorization` request header is present. * `https_proxy` - an URI to a proxy server to be used with https requests -* `https_proxy_authorization` - a value for `Proxy-Authorization` header to be used with `https_proxy`, e.g. `Basic ZGVtbzpwYXNz`. - The `httpc:request_uri` can override the setting by specifying `Proxy-Authorization` header. +* `https_proxy_authorization` - as `http_proxy_authorization` but for use with `https_proxy`. * `no_proxy` - a comma separated list of hosts that should not be proxied. Note that proxy options are only applied when using the high-level `request_uri()` API. From f71e9708a3fd0ff4179d4b0d770c91fcf98d0042 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Sun, 3 Feb 2019 12:32:31 +0000 Subject: [PATCH 029/130] Version bump to 0.13 --- dist.ini | 2 +- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...sty-http-0.12-0.rockspec => lua-resty-http-0.13-0.rockspec | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename lua-resty-http-0.12-0.rockspec => lua-resty-http-0.13-0.rockspec (93%) diff --git a/dist.ini b/dist.ini index 342193d0..c00bda88 100644 --- a/dist.ini +++ b/dist.ini @@ -5,5 +5,5 @@ is_original=yes license=2bsd lib_dir=lib doc_dir=lib -repo_link=https://github.com/pintsized/lua-resty-http +repo_link=https://github.com/ledgetech/lua-resty-http main_module=lib/resty/http.lua diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 1cb97f0e..9473c184 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -99,7 +99,7 @@ end local _M = { - _VERSION = '0.12', + _VERSION = '0.13', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 56069eca..247f26d5 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.12', + _VERSION = '0.13', } diff --git a/lua-resty-http-0.12-0.rockspec b/lua-resty-http-0.13-0.rockspec similarity index 93% rename from lua-resty-http-0.12-0.rockspec rename to lua-resty-http-0.13-0.rockspec index d727114b..ec5349d8 100644 --- a/lua-resty-http-0.12-0.rockspec +++ b/lua-resty-http-0.13-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.12-0" +version = "0.13-0" source = { url = "git://github.com/pintsized/lua-resty-http", - tag = "v0.12" + tag = "v0.13" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From c0c3b59537823496d854b3820a176dee08940b13 Mon Sep 17 00:00:00 2001 From: spacewander Date: Fri, 19 Apr 2019 17:34:23 +0800 Subject: [PATCH 030/130] optimize: save a table for headers which only have a single value. --- lib/resty/http.lua | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index b3cce434..3ac63637 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -301,13 +301,16 @@ local function _format_request(params) -- Append headers for key, values in pairs(headers) do - if type(values) ~= "table" then - values = {values} - end - key = tostring(key) - for _, value in pairs(values) do - req[c] = key .. ": " .. tostring(value) .. "\r\n" + + if type(values) == "table" then + for _, value in pairs(values) do + req[c] = key .. ": " .. tostring(value) .. "\r\n" + c = c + 1 + end + + else + req[c] = key .. ": " .. tostring(values) .. "\r\n" c = c + 1 end end From 70f7dd124867305a43d8b863c2b3c91c8879a238 Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Fri, 5 Jul 2019 18:46:31 +0300 Subject: [PATCH 031/130] Add Content-Length: 0 header on POST/PUT/PATCH when the body is absent Some servers are a bit strict about POST/PUT/PATCH request without body and without Content-Length header. According to: https://tools.ietf.org/html/rfc7230#section-3.3.2 > a Content-Length header field is normally sent in a POST request even > when the value is 0 (indicating an empty payload body) This PR will add Content-Length header on such case for POST/PUT/PATCH. --- lib/resty/http.lua | 16 ++++++++-- t/06-simpleinterface.t | 69 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index cb815604..9c8fbbe5 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -55,6 +55,13 @@ local HOP_BY_HOP_HEADERS = { } +local EXPECTING_BODY = { + POST = true, + PUT = true, + PATCH = true, +} + + -- Reimplemented coroutine.wrap, returning "nil, err" if the coroutine cannot -- be resumed. This protects user code from inifite loops when doing things like -- repeat @@ -603,8 +610,13 @@ function _M.send_request(self, params) end -- Ensure minimal headers are set - if type(body) == 'string' and not headers["Content-Length"] then - headers["Content-Length"] = #body + + if not headers["Content-Length"] then + if type(body) == 'string' then + headers["Content-Length"] = #body + elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then + headers["Content-Length"] = 0 + end end if not headers["Host"] then if (str_sub(self.host, 1, 5) == "unix:") then diff --git a/t/06-simpleinterface.t b/t/06-simpleinterface.t index 451e5f2f..c828c2d5 100644 --- a/t/06-simpleinterface.t +++ b/t/06-simpleinterface.t @@ -267,3 +267,72 @@ GET /a nil closed --- error_log lua tcp socket read timed out + + +=== TEST 7: Content-Length is set on POST/PUT/PATCH requests when body is absent +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + for i, method in ipairs({ "POST", "PUT", "PATCH" }) do + local http = require "resty.http" + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:"..ngx.var.server_port.."/b", { method = method }) + + if not res then + ngx.log(ngx.ERR, err) + end + + if i == 1 then + ngx.status = res.status + end + + ngx.print(res.body) + end + '; + } + location = /b { + content_by_lua ' + ngx.say(ngx.req.get_method(), " Content-Length: ", ngx.req.get_headers()["Content-Length"]) + '; + } +--- request +GET /a +--- response_body +POST Content-Length: 0 +PUT Content-Length: 0 +PATCH Content-Length: 0 +--- no_error_log +[error] +[warn] + + +=== TEST 8: Content-Length is not set on GET requests when body is absent +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local http = require "resty.http" + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:"..ngx.var.server_port.."/b") + + if not res then + ngx.log(ngx.ERR, err) + end + ngx.status = res.status + ngx.print(res.body) + '; + } + location = /b { + content_by_lua ' + ngx.say("Content-Length: ", type(ngx.req.get_headers()["Content-Length"])) + '; + } +--- request +GET /a +--- response_body +Content-Length: nil +--- no_error_log +[error] +[warn] + From b02f3dcacf18b2ef2b4a9cc0734980e5e09bb142 Mon Sep 17 00:00:00 2001 From: Hamish Date: Mon, 8 Jul 2019 10:18:02 +0100 Subject: [PATCH 032/130] Put debug logging behind a flag --- lib/resty/http.lua | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 9c8fbbe5..82d7e6df 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -126,6 +126,9 @@ local DEFAULT_PARAMS = { } +local DEBUG = false + + function _M.new(_) local sock, err = ngx_socket_tcp() if not sock then @@ -135,6 +138,11 @@ function _M.new(_) end +function _M.debug(d) + DEBUG = (d == true) +end + + function _M.set_timeout(self, timeout) local sock = self.sock if not sock then @@ -647,7 +655,7 @@ function _M.send_request(self, params) -- Format and send request local req = _format_request(params) - ngx_log(ngx_DEBUG, "\n", req) + if DEBUG then ngx_log(ngx_DEBUG, "\n", req) end local bytes, err = sock:send(req) if not bytes then From eef59b13e9bf00985397b351b0b1a53b73c4c1ef Mon Sep 17 00:00:00 2001 From: Hamish Date: Mon, 8 Jul 2019 10:18:20 +0100 Subject: [PATCH 033/130] Update tests, debug logging, global variable warnings --- t/01-basic.t | 6 +++--- t/14-host-header.t | 5 +++++ t/16-http-proxy.t | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/t/01-basic.t b/t/01-basic.t index 64851147..99ad1e69 100644 --- a/t/01-basic.t +++ b/t/01-basic.t @@ -304,7 +304,7 @@ bad uri: http:///example.com content_by_lua ' local http = require("resty.http").new() - function test_uri(uri) + local function test_uri(uri) local scheme, host, port, path, query = unpack(http:parse_uri(uri, false)) ngx.say("scheme: ", scheme, ", host: ", host, ", port: ", port, ", path: ", path, ", query: ", query) end @@ -342,7 +342,7 @@ scheme: http, host: example.com, port: 80, path: /foo/bar, query: a=1&b=2 content_by_lua ' local http = require("resty.http").new() - function test_uri(uri) + local function test_uri(uri) local scheme, host, port, path, query = unpack(http:parse_uri(uri)) ngx.say("scheme: ", scheme, ", host: ", host, ", port: ", port, ", path: ", path) end @@ -410,4 +410,4 @@ Test OK --- no_error_log [error] -[warn] \ No newline at end of file +[warn] diff --git a/t/14-host-header.t b/t/14-host-header.t index 110a6695..27d8747f 100644 --- a/t/14-host-header.t +++ b/t/14-host-header.t @@ -19,6 +19,8 @@ our $HttpConfig = qq{ jit.off() require("luacov.runner").init() end + + require("resty.http").debug(true) } }; @@ -89,6 +91,7 @@ Host: www.google.com --- config location /lua { content_by_lua ' + require("resty.http").debug(true) local http = require "resty.http" local httpc = http.new() @@ -117,6 +120,7 @@ Host: 127.0.0.1:8080 --- config location /lua { content_by_lua ' + require("resty.http").debug(true) local http = require "resty.http" local httpc = http.new() @@ -178,6 +182,7 @@ Unable to generate a useful Host header for a unix domain socket. Please provide --- config location /lua { content_by_lua ' + require("resty.http").debug(true) local http = require "resty.http" local httpc = http.new() httpc:set_proxy_options({ diff --git a/t/16-http-proxy.t b/t/16-http-proxy.t index 7393b644..0ce660c4 100644 --- a/t/16-http-proxy.t +++ b/t/16-http-proxy.t @@ -57,7 +57,7 @@ nil -- helper that verifies get_proxy_uri works correctly with the given -- scheme, host and no_proxy list - function test_no_proxy(scheme, host, no_proxy) + local function test_no_proxy(scheme, host, no_proxy) httpc:set_proxy_options({ http_proxy = "http://http_proxy.example.com", https_proxy = "http://https_proxy.example.com", @@ -126,7 +126,7 @@ scheme: http, host: notexample.com, no_proxy: example.com, proxy_uri: http://htt -- helper that configures the proxy opts as proived and checks what -- get_proxy_uri says for the given scheme / host pair - function test_get_proxy_uri(scheme, host, http_proxy, https_proxy) + local function test_get_proxy_uri(scheme, host, http_proxy, https_proxy) httpc:set_proxy_options({ http_proxy = http_proxy, https_proxy = https_proxy From 25181add467953338a0c349d6489dfdea1f40418 Mon Sep 17 00:00:00 2001 From: Hamish Date: Mon, 8 Jul 2019 11:43:33 +0100 Subject: [PATCH 034/130] v0.14 release --- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...sty-http-0.13-0.rockspec => lua-resty-http-0.14-0.rockspec | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename lua-resty-http-0.13-0.rockspec => lua-resty-http-0.14-0.rockspec (93%) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 82d7e6df..d9e05e6f 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -106,7 +106,7 @@ end local _M = { - _VERSION = '0.13', + _VERSION = '0.14', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 247f26d5..6771d1ed 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.13', + _VERSION = '0.14', } diff --git a/lua-resty-http-0.13-0.rockspec b/lua-resty-http-0.14-0.rockspec similarity index 93% rename from lua-resty-http-0.13-0.rockspec rename to lua-resty-http-0.14-0.rockspec index ec5349d8..40034997 100644 --- a/lua-resty-http-0.13-0.rockspec +++ b/lua-resty-http-0.14-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.13-0" +version = "0.14-0" source = { url = "git://github.com/pintsized/lua-resty-http", - tag = "v0.13" + tag = "v0.14" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From 984fdc26054376384e3df238fb0f7dfde01cacf1 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 13 Aug 2019 11:08:35 +0100 Subject: [PATCH 035/130] Updated luarocks URL with ledgetech namespace --- lua-resty-http-0.14-0.rockspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lua-resty-http-0.14-0.rockspec b/lua-resty-http-0.14-0.rockspec index 40034997..68b5dd62 100644 --- a/lua-resty-http-0.14-0.rockspec +++ b/lua-resty-http-0.14-0.rockspec @@ -1,12 +1,12 @@ package = "lua-resty-http" version = "0.14-0" source = { - url = "git://github.com/pintsized/lua-resty-http", + url = "git://github.com/ledgetech/lua-resty-http", tag = "v0.14" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", - homepage = "https://github.com/pintsized/lua-resty-http", + homepage = "https://github.com/ledgetech/lua-resty-http", license = "2-clause BSD", maintainer = "James Hurst " } From 41b2e822ce5c19f64e293b7dc2d5d244e511615d Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 13 Aug 2019 11:25:22 +0100 Subject: [PATCH 036/130] v0.15 for luarocks --- ...sty-http-0.14-0.rockspec => lua-resty-http-0.15-0.rockspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename lua-resty-http-0.14-0.rockspec => lua-resty-http-0.15-0.rockspec (93%) diff --git a/lua-resty-http-0.14-0.rockspec b/lua-resty-http-0.15-0.rockspec similarity index 93% rename from lua-resty-http-0.14-0.rockspec rename to lua-resty-http-0.15-0.rockspec index 68b5dd62..56ead74a 100644 --- a/lua-resty-http-0.14-0.rockspec +++ b/lua-resty-http-0.15-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.14-0" +version = "0.15-0" source = { url = "git://github.com/ledgetech/lua-resty-http", - tag = "v0.14" + tag = "v0.15" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From 97f3dad4d3d8e16deb2aa07fa11fc17f800ba0f1 Mon Sep 17 00:00:00 2001 From: beckjiang Date: Fri, 15 Nov 2019 14:17:19 +0800 Subject: [PATCH 037/130] fix pipeline test case typo --- t/08-pipeline.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/08-pipeline.t b/t/08-pipeline.t index a8b3b735..3fa4fecf 100644 --- a/t/08-pipeline.t +++ b/t/08-pipeline.t @@ -26,7 +26,7 @@ no_long_string(); run_tests(); __DATA__ -=== TEST 1 Test that pipelined reqests can be read correctly. +=== TEST 1 Test that pipelined requests can be read correctly. --- http_config eval: $::HttpConfig --- config location = /a { From 51b3630f3f060acf3eab6e002746be3637858996 Mon Sep 17 00:00:00 2001 From: zhangzheng Date: Thu, 2 Apr 2020 16:45:06 +0800 Subject: [PATCH 038/130] doc: improved the code examples in Synopsis 60 milliseconds is too short. When testing, it will be mistaken for keepalive not taking effect. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 857484cf..bf15f3ff 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ server { headers = { ["Content-Type"] = "application/x-www-form-urlencoded", }, - keepalive_timeout = 60, + keepalive_timeout = 60000, keepalive_pool = 10 }) From 859c572d6385866ea16b204b37c3bd0821f1cbf7 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 3 Apr 2020 12:18:46 +0100 Subject: [PATCH 039/130] Travis integration (#209) * Try to run tests in container * No need to select user * Add build status image --- .travis.yml | 66 ++--------------------------------------------------- README.md | 2 ++ 2 files changed, 4 insertions(+), 64 deletions(-) diff --git a/.travis.yml b/.travis.yml index e098e949..9ee0825c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,67 +1,5 @@ -sudo: required -dist: trusty - -os: linux - -language: c - -compiler: - - gcc - - clang - services: - - redis-server - -cache: - directories: - - download-cache - -env: - global: - - JOBS=3 - - NGX_BUILD_JOBS=$JOBS - - LUAJIT_PREFIX=/opt/luajit21 - - LUAJIT_LIB=$LUAJIT_PREFIX/lib - - LUAJIT_INC=$LUAJIT_PREFIX/include/luajit-2.1 - - LUA_INCLUDE_DIR=$LUAJIT_INC - - OPENSSL_PREFIX=/opt/ssl - - OPENSSL_LIB=$OPENSSL_PREFIX/lib - - OPENSSL_INC=$OPENSSL_PREFIX/include - - OPENSSL_VER=1.0.2h - - LD_LIBRARY_PATH=$LUAJIT_LIB:$LD_LIBRARY_PATH - - TEST_NGINX_SLEEP=0.006 - matrix: - - NGINX_VERSION=1.9.15 -# - NGINX_VERSION=1.10.0 - -install: - - if [ ! -d download-cache ]; then mkdir download-cache; fi - - if [ ! -f download-cache/openssl-$OPENSSL_VER.tar.gz ]; then wget -O download-cache/openssl-$OPENSSL_VER.tar.gz https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz; fi - - sudo apt-get install -qq -y cpanminus axel - - sudo cpanm --notest Test::Nginx > build.log 2>&1 || (cat build.log && exit 1) - - wget http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz - - git clone https://github.com/openresty/openresty.git ../openresty - - git clone https://github.com/openresty/nginx-devel-utils.git - - git clone https://github.com/openresty/lua-cjson.git - - git clone https://github.com/openresty/lua-nginx-module.git ../lua-nginx-module - - git clone https://github.com/openresty/echo-nginx-module.git ../echo-nginx-module - - git clone https://github.com/openresty/no-pool-nginx.git ../no-pool-nginx - - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git + - docker script: - - cd luajit2/ - - make -j$JOBS CCDEBUG=-g Q= PREFIX=$LUAJIT_PREFIX CC=$CC XCFLAGS='-DLUA_USE_APICHECK -DLUA_USE_ASSERT' > build.log 2>&1 || (cat build.log && exit 1) - - sudo make install PREFIX=$LUAJIT_PREFIX > build.log 2>&1 || (cat build.log && exit 1) - - cd ../lua-cjson && make && sudo PATH=$PATH make install && cd .. - - tar zxf download-cache/openssl-$OPENSSL_VER.tar.gz - - cd openssl-$OPENSSL_VER/ - - ./config shared --prefix=$OPENSSL_PREFIX -DPURIFY > build.log 2>&1 || (cat build.log && exit 1) - - make -j$JOBS > build.log 2>&1 || (cat build.log && exit 1) - - sudo make PATH=$PATH install_sw > build.log 2>&1 || (cat build.log && exit 1) - - cd .. - - export PATH=$PWD/work/nginx/sbin:$PWD/nginx-devel-utils:$PATH - - export NGX_BUILD_CC=$CC - - ngx-build $NGINX_VERSION --with-ipv6 --with-http_realip_module --with-http_ssl_module --add-module=../echo-nginx-module --add-module=../lua-nginx-module --with-debug - - nginx -V - - ldd `which nginx`|grep -E 'luajit|ssl|pcre' - - prove -r t + - docker run --rm -v "$PWD:/lua-resty-http" -w /lua-resty-http ledgetech/test-runner:latest /bin/bash -c "make coverage" diff --git a/README.md b/README.md index bf15f3ff..17a7cfea 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Lua HTTP client cosocket driver for [OpenResty](http://openresty.org/) / [ngx_lu Production ready. +[![Build Status](https://travis-ci.org/ledgetech/lua-resty-http.svg?branch=master)](https://travis-ci.org/ledgetech/lua-resty-http) + # Features * HTTP 1.0 and 1.1 From a6bd2e0eb1390e330e4fb10a48cced5a1f21fb66 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 3 Apr 2020 16:18:51 +0100 Subject: [PATCH 040/130] Fixed test coverage report and tidied up Makefile (#210) --- .luacov | 4 ++++ Makefile | 21 +++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 .luacov diff --git a/.luacov b/.luacov new file mode 100644 index 00000000..1580c7b6 --- /dev/null +++ b/.luacov @@ -0,0 +1,4 @@ +modules = { + ["http"] = "lib/resty/http.lua", + ["http_headers"] = "lib/resty/http_headers.lua", +} diff --git a/Makefile b/Makefile index a676a600..8fc7883f 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,16 @@ OPENRESTY_PREFIX=/usr/local/openresty -PREFIX ?= /usr/local +PREFIX ?= /usr/local LUA_INCLUDE_DIR ?= $(PREFIX)/include -LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION) -INSTALL ?= install -TEST_FILE ?= t +LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION) +INSTALL ?= install +TEST_FILE ?= t + +export PATH := $(OPENRESTY_PREFIX)/nginx/sbin:$(PATH) +PROVE=TEST_NGINX_NO_SHUFFLE=1 prove -I../test-nginx/lib -r $(TEST_FILE) + +# Keep in sync with .luacov, so that we show the right amount of output +LUACOV_NUM_MODULES ?= 2 .PHONY: all test install @@ -15,14 +21,13 @@ install: all $(INSTALL) lib/resty/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/ test: all - PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH TEST_NGINX_NO_SHUFFLE=1 prove -I../test-nginx/lib -r $(TEST_FILE) + $(PROVE) coverage: all - -@echo "Cleaning stats" @rm -f luacov.stats.out - PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH TEST_NGINX_NO_SHUFFLE=1 TEST_COVERAGE=1 prove -I../test-nginx/lib -r $(TEST_FILE) + TEST_COVERAGE=1 $(PROVE) @luacov - @tail -10 luacov.report.out + @tail -$$(( $(LUACOV_NUM_MODULES) + 5)) luacov.report.out check: luacheck lib From a5436ca9a937d1ca39e0d57a58da5ca3d60e7f21 Mon Sep 17 00:00:00 2001 From: Zhenzhen Zhao Date: Mon, 23 Nov 2020 15:16:06 +0800 Subject: [PATCH 041/130] fix(http): typo on comments --- lib/resty/http.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index d9e05e6f..8381d41c 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -63,7 +63,7 @@ local EXPECTING_BODY = { -- Reimplemented coroutine.wrap, returning "nil, err" if the coroutine cannot --- be resumed. This protects user code from inifite loops when doing things like +-- be resumed. This protects user code from infinite loops when doing things like -- repeat -- local chunk, err = res.body_reader() -- if chunk then -- <-- This could be a string msg in the core wrap function. @@ -631,7 +631,7 @@ function _M.send_request(self, params) return nil, "Unable to generate a useful Host header for a unix domain socket. Please provide one." end -- If we have a port (i.e. not connected to a unix domain socket), and this - -- port is non-standard, append it to the Host heaer. + -- port is non-standard, append it to the Host header. if self.port then if self.ssl and self.port ~= 443 then headers["Host"] = self.host .. ":" .. self.port From a707215dde71df8545f17f7680b35d03573175d8 Mon Sep 17 00:00:00 2001 From: vislee Date: Sun, 12 Apr 2020 13:28:08 +0800 Subject: [PATCH 042/130] Fix Doc: httpc:close() --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 17a7cfea..4344aa22 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ If the current connection does not come from the built-in connection pool, then ## close -`syntax: ok, err = http:close()` +`syntax: ok, err = httpc:close()` Closes the current connection and returns the status. From 902eecf9d44972e640377e1ec67d29ea15739389 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 11 Dec 2020 15:55:28 +0000 Subject: [PATCH 043/130] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..5f95a108 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: pintsized From 4bf2e9c52ed06fcb2e8c7ae6cfec824b4ce1677a Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 27 Mar 2020 11:47:31 +0100 Subject: [PATCH 044/130] feat(connect) new all-in-one connect method This covers connecting on tcp, ssl and proxy level --- .luacov | 1 + Makefile | 2 +- README.md | 44 +++++++ lib/resty/http.lua | 22 +++- lib/resty/http_connect.lua | 209 +++++++++++++++++++++++++++++++++ lua-resty-http-0.15-0.rockspec | 25 ++-- 6 files changed, 287 insertions(+), 16 deletions(-) create mode 100644 lib/resty/http_connect.lua diff --git a/.luacov b/.luacov index 1580c7b6..cd070eeb 100644 --- a/.luacov +++ b/.luacov @@ -1,4 +1,5 @@ modules = { ["http"] = "lib/resty/http.lua", ["http_headers"] = "lib/resty/http_headers.lua", + ["http_connect"] = "lib/resty/http_connect.lua", } diff --git a/Makefile b/Makefile index 8fc7883f..95894718 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ export PATH := $(OPENRESTY_PREFIX)/nginx/sbin:$(PATH) PROVE=TEST_NGINX_NO_SHUFFLE=1 prove -I../test-nginx/lib -r $(TEST_FILE) # Keep in sync with .luacov, so that we show the right amount of output -LUACOV_NUM_MODULES ?= 2 +LUACOV_NUM_MODULES ?= 3 .PHONY: all test install diff --git a/README.md b/README.md index 4344aa22..df24c2f3 100644 --- a/README.md +++ b/README.md @@ -151,14 +151,58 @@ Creates the http object. In case of failures, returns `nil` and a string describ ## connect +`syntax: ok, err = httpc:connect(aio_options_table)` + `syntax: ok, err = httpc:connect(host, port, options_table?)` `syntax: ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?)` Attempts to connect to the web server. +### all-in-one connect + +This version of connect uses the `aio_options` signature above. For the +other signatures see the [TCP only connect](#tcp-only-connect) below. + +This version of `connect` will connect to the remote end while incorporating the +following activities: + +- TCP connect +- SSL handshake +- HTTP-proxy + +Whilst doing this it will also create a distinct pool name that is safe to use +with SSL and/or Proxy based connections. + +The options table has the following fields: + +* `scheme`: scheme to use, or nil for unix domain socket +* `host`: target machine, or a unix domain socket +* `port`: port on target machine, will default to `80` or `443` based on scheme +* `pool`: connection pool name. Option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect), + except that the default will become a pool name constructed including all the + SSL/Proxy properties to make it safe to re-use. When in doubt, leave it blank! +* `pool_size`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) +* `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) +* `ssl`: sub-table. **NOTE**: ssl will be used when either `scheme == "https"`, or when `ssl` is truthy + * `server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) + * `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. + * `ctx`: NOT supported + +* `proxy`: sub-table. **NOTE**: a proxy will be used only if `proxy.uri` is provided + * `uri`: uri of the proxy to use, eg. `"http://myproxy.internal:123"` + * `authorization`: a "Proxy-Authorization" header value to be used + * `no_proxy`: comma separated string of domains bypassing proxy + + +### TCP only connect + Before actually resolving the host name and connecting to the remote backend, this method will always look up the connection pool for matched idle connections created by previous calls of this method. +NOTE: the default pool name will only incorporate IP and port information so is +unsafe to use in case of SSL and/or Proxy connections. Specify your own pool +name or use the [`all-in-one`](#all-in-one-connect) above. + An optional Lua table can be specified as the last argument to this method to specify various connect options: * `pool` diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 8381d41c..c58f0872 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -174,8 +174,24 @@ function _M.ssl_handshake(self, ...) return sock:sslhandshake(...) end +do + local aio_connect = require "resty.http_connect" + -- Function signatures to support: + -- ok, err = httpc:connect(options_table) + -- ok, err = httpc:connect(host, port, options_table?) + -- ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?) + function _M.connect(self, options, ...) + if type(options) == "table" then + -- all-in-one interface + return aio_connect(self, options) + else + -- backward compatible + return self:tcp_only_connect(options, ...) + end + end +end -function _M.connect(self, ...) +function _M.tcp_only_connect(self, ...) local sock = self.sock if not sock then return nil, "not initialized" @@ -863,7 +879,7 @@ function _M.request_uri(self, uri, params) c, err = self:connect_proxy(proxy_uri, scheme, host, port, proxy_authorization) else - c, err = self:connect(host, port) + c, err = self:tcp_only_connect(host, port) end if not c then @@ -1107,7 +1123,7 @@ function _M.connect_proxy(self, proxy_uri, scheme, host, port, proxy_authorizati -- Make the connection to the given proxy local proxy_host, proxy_port = parsed_proxy_uri[2], parsed_proxy_uri[3] - local c, err = self:connect(proxy_host, proxy_port) + local c, err = self:tcp_only_connect(proxy_host, proxy_port) if not c then return nil, err end diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua new file mode 100644 index 00000000..29018402 --- /dev/null +++ b/lib/resty/http_connect.lua @@ -0,0 +1,209 @@ +local ngx_re_gmatch = ngx.re.gmatch +local ngx_re_sub = ngx.re.sub +local ngx_re_find = ngx.re.find + +--[[ +A connection function that incorporates: + - tcp connect + - ssl handshake + - http proxy +Due to this it will be better at setting up a socket pool where connections can +be kept alive. + + +Call it with a single options table as follows: + +client:connect { + scheme = "https" -- scheme to use, or nil for unix domain socket + host = "myhost.com", -- target machine, or a unix domain socket + port = nil, -- port on target machine, will default to 80/443 based on scheme + pool = nil, -- connection pool name, leave blank! this function knows best! + pool_size = nil, -- options as per: https://github.com/openresty/lua-nginx-module#tcpsockconnect + backlog = nil, + + ssl = { -- ssl will be used when either scheme = https, or when ssl is truthy + server_name = nil, -- options as per: https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake + ssl_verify = true, -- defaults to true + ctx = nil, -- NOT supported + }, + + proxy = { -- proxy will be used only if "proxy.uri" is provided + uri = "http://myproxy.internal:123", -- uri of the proxy + authorization = nil, -- a "Proxy-Authorization" header value to be used + no_proxy = nil, -- comma separated string of domains bypassing proxy + }, +} +]] +local function connect(self, options) + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + local ok, err + local proxy_scheme = options.scheme -- scheme to use; http or https + local host = options.host -- remote host to connect to + local port = options.port -- remote port to connect to + local poolname = options.pool -- connection pool name to use + local pool_size = options.pool_size + local backlog = options.backlog + if proxy_scheme and not port then + port = (proxy_scheme == "https" and 443 or 80) + elseif port and not proxy_scheme then + return nil, "'scheme' is required when providing a port" + end + + -- ssl settings + local ssl, ssl_server_name, ssl_verify + if proxy_scheme ~= "http" then + -- either https or unix domain socket + ssl = options.ssl + if type(options.ssl) == "table" then + ssl_server_name = ssl.server_name + ssl_verify = (ssl.verify == nil) or (not not ssl.verify) -- default to true, and force to bool + ssl = true + else + if ssl then + ssl = true + ssl_verify = true -- default to true + else + ssl = false + end + end + else + -- plain http + ssl = false + end + + -- proxy related settings + local proxy, proxy_uri, proxy_uri_t, proxy_authorization, proxy_host, proxy_port + proxy = options.proxy + if proxy and proxy.no_proxy then + -- Check if the no_proxy option matches this host. Implementation adapted + -- from lua-http library (https://github.com/daurnimator/lua-http) + if proxy.no_proxy == "*" then + -- all hosts are excluded + proxy = nil + + else + local no_proxy_set = {} + -- wget allows domains in no_proxy list to be prefixed by "." + -- e.g. no_proxy=.mit.edu + for host_suffix in ngx_re_gmatch(proxy.no_proxy, "\\.?([^,]+)") do + no_proxy_set[host_suffix[1]] = true + end + + -- From curl docs: + -- matched as either a domain which contains the hostname, or the + -- hostname itself. For example local.com would match local.com, + -- local.com:80, and www.local.com, but not www.notlocal.com. + -- + -- Therefore, we keep stripping subdomains from the host, compare + -- them to the ones in the no_proxy list and continue until we find + -- a match or until there's only the TLD left + repeat + if no_proxy_set[host] then + proxy = nil + break + end + + -- Strip the next level from the domain and check if that one + -- is on the list + host = ngx_re_sub(host, "^[^.]+\\.", "") + until not ngx_re_find(host, "\\.") + end + end + + if proxy then + proxy_uri = proxy.uri -- full uri to proxy (only http supported) + proxy_authorization = proxy.authorization -- auth to send via CONNECT + proxy_uri_t, err = self:parse_uri(proxy_uri) + if not proxy_uri_t then + return nil, err + end + + local p_scheme = proxy_uri_t[1] + if p_scheme ~= "http" then + return nil, "protocol " .. p_scheme .. " not supported for proxy connections" + end + proxy_host = proxy_uri_t[2] + proxy_port = proxy_uri_t[3] + end + + -- construct a poolname unique within proxy and ssl info + if not poolname then + poolname = (proxy_scheme or "") + .. ":" .. host + .. ":" .. tostring(port) + .. ":" .. tostring(ssl) + .. ":" .. (ssl_server_name or "") + .. ":" .. tostring(ssl_verify) + .. ":" .. (proxy_uri or "") + .. ":" .. (proxy_authorization or "") + end + + -- do TCP level connection + local tcp_opts = { pool = poolname, pool_size = pool_size, backlog = backlog } + if proxy then + -- proxy based connection + ok, err = sock:connect(proxy_host, proxy_port, tcp_opts) + if not ok then + return nil, err + end + + if proxy and proxy_scheme == "https" and sock:getreusedtimes() == 0 then + -- Make a CONNECT request to create a tunnel to the destination through + -- the proxy. The request-target and the Host header must be in the + -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section + -- 4.3.6 for more details about the CONNECT request + local destination = host .. ":" .. port + local res, err = self:request({ + method = "CONNECT", + path = destination, + headers = { + ["Host"] = destination, + ["Proxy-Authorization"] = proxy_authorization, + } + }) + + if not res then + return nil, err + end + + if res.status < 200 or res.status > 299 then + return nil, "failed to establish a tunnel through a proxy: " .. res.status + end + end + + elseif not port then + -- non-proxy, without port -> unix domain socket + ok, err = sock:connect(host, tcp_opts) + if not ok then + return nil, err + end + + else + -- non-proxy, regular network tcp + ok, err = sock:connect(host, port, tcp_opts) + if not ok then + return nil, err + end + end + + -- Now do the ssl handshake + if ssl and sock:getreusedtimes() == 0 then + local ok, err = self:ssl_handshake(nil, ssl_server_name, ssl_verify) + if not ok then + self:close() + return nil, err + end + end + + self.host = host + self.port = port + self.keepalive = true + + return true +end + +return connect diff --git a/lua-resty-http-0.15-0.rockspec b/lua-resty-http-0.15-0.rockspec index 56ead74a..e1170d28 100644 --- a/lua-resty-http-0.15-0.rockspec +++ b/lua-resty-http-0.15-0.rockspec @@ -1,22 +1,23 @@ package = "lua-resty-http" version = "0.15-0" source = { - url = "git://github.com/ledgetech/lua-resty-http", - tag = "v0.15" + url = "git://github.com/ledgetech/lua-resty-http", + tag = "v0.15" } description = { - summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", - homepage = "https://github.com/ledgetech/lua-resty-http", - license = "2-clause BSD", - maintainer = "James Hurst " + summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", + homepage = "https://github.com/ledgetech/lua-resty-http", + license = "2-clause BSD", + maintainer = "James Hurst " } dependencies = { - "lua >= 5.1" + "lua >= 5.1" } build = { - type = "builtin", - modules = { - ["resty.http"] = "lib/resty/http.lua", - ["resty.http_headers"] = "lib/resty/http_headers.lua" - } + type = "builtin", + modules = { + ["resty.http"] = "lib/resty/http.lua", + ["resty.http_headers"] = "lib/resty/http_headers.lua" + ["resty.http_connect"] = "lib/resty/http_connect.lua" + } } From 04245d6d3b612d9578e7eaa8ca37980171967dc4 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 19 Feb 2021 14:56:21 +0100 Subject: [PATCH 045/130] fix(request) set ssl flag to enable generating proper host-header --- lib/resty/http.lua | 1 + lib/resty/http_connect.lua | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index c58f0872..b1e3ee20 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -206,6 +206,7 @@ function _M.tcp_only_connect(self, ...) end self.keepalive = true + self.ssl = false return sock:connect(...) end diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 29018402..0568c678 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -202,6 +202,7 @@ local function connect(self, options) self.host = host self.port = port self.keepalive = true + self.ssl = ssl return true end From b414a19d7ce9011b15dfb4f1db99fd31ecf501db Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Feb 2021 08:24:55 +0100 Subject: [PATCH 046/130] fix(proxy) use the global proxy options these options must be global since the https auth is used when connecting, and the http one only when requesting. So these settings span the process of connecting and requesting, and hence we should not pass them into 'connect', because we would have to ensure that the user would pass the same ones to 'request'. --- README.md | 11 +++-------- lib/resty/http_connect.lua | 28 ++++++++++++++++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index df24c2f3..9e64f331 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ following activities: - TCP connect - SSL handshake -- HTTP-proxy +- HTTP-proxy (options to be set using [set_proxy_options](#set_proxy_options)) Whilst doing this it will also create a distinct pool name that is safe to use with SSL and/or Proxy based connections. @@ -189,11 +189,6 @@ The options table has the following fields: * `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. * `ctx`: NOT supported -* `proxy`: sub-table. **NOTE**: a proxy will be used only if `proxy.uri` is provided - * `uri`: uri of the proxy to use, eg. `"http://myproxy.internal:123"` - * `authorization`: a "Proxy-Authorization" header value to be used - * `no_proxy`: comma separated string of domains bypassing proxy - ### TCP only connect @@ -270,10 +265,10 @@ Configure an http proxy to be used with this client instance. The `opts` is a ta * `http_proxy` - an URI to a proxy server to be used with http requests * `http_proxy_authorization` - a default `Proxy-Authorization` header value to be used with `http_proxy`, e.g. `Basic ZGVtbzp0ZXN0`, which will be overriden if the `Proxy-Authorization` request header is present. * `https_proxy` - an URI to a proxy server to be used with https requests -* `https_proxy_authorization` - as `http_proxy_authorization` but for use with `https_proxy`. +* `https_proxy_authorization` - as `http_proxy_authorization` but for use with `https_proxy` (since with https the authorisation is done when connecting, this one cannot be overridden by passing the `Proxy-Authorization` request header). * `no_proxy` - a comma separated list of hosts that should not be proxied. -Note that proxy options are only applied when using the high-level `request_uri()` API. +Note that proxy options are only applied when using the high-level `request_uri()` API, or when using the all-in-one connect. ## get_reused_times diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 0568c678..01f8d815 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -6,7 +6,7 @@ local ngx_re_find = ngx.re.find A connection function that incorporates: - tcp connect - ssl handshake - - http proxy + - http proxy (options to be set using "set_proxy_options") Due to this it will be better at setting up a socket pool where connections can be kept alive. @@ -26,12 +26,6 @@ client:connect { ssl_verify = true, -- defaults to true ctx = nil, -- NOT supported }, - - proxy = { -- proxy will be used only if "proxy.uri" is provided - uri = "http://myproxy.internal:123", -- uri of the proxy - authorization = nil, -- a "Proxy-Authorization" header value to be used - no_proxy = nil, -- comma separated string of domains bypassing proxy - }, } ]] local function connect(self, options) @@ -77,7 +71,21 @@ local function connect(self, options) -- proxy related settings local proxy, proxy_uri, proxy_uri_t, proxy_authorization, proxy_host, proxy_port - proxy = options.proxy + proxy = self.proxy_opts + + if proxy then + if proxy_scheme == "https" then + proxy_uri = proxy.https_proxy + proxy_authorization = proxy.https_proxy_authorization + else + proxy_uri = proxy.http_proxy + proxy_authorization = proxy.http_proxy_authorization + end + if not proxy_uri then + proxy = nil + end + end + if proxy and proxy.no_proxy then -- Check if the no_proxy option matches this host. Implementation adapted -- from lua-http library (https://github.com/daurnimator/lua-http) @@ -104,6 +112,8 @@ local function connect(self, options) repeat if no_proxy_set[host] then proxy = nil + proxy_uri = nil + proxy_authorization = nil break end @@ -115,8 +125,6 @@ local function connect(self, options) end if proxy then - proxy_uri = proxy.uri -- full uri to proxy (only http supported) - proxy_authorization = proxy.authorization -- auth to send via CONNECT proxy_uri_t, err = self:parse_uri(proxy_uri) if not proxy_uri_t then return nil, err From 6c86e43941a009dcc068124b42a73f486e5cb7d6 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Feb 2021 08:43:48 +0100 Subject: [PATCH 047/130] fix(proxy) localize a host variable that gets modified also renames other variables to be more descriptive --- lib/resty/http.lua | 2 ++ lib/resty/http_connect.lua | 49 +++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index b1e3ee20..c0ea6901 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -1053,6 +1053,8 @@ end function _M.set_proxy_options(self, opts) + -- TODO: parse and cache these options, instead of parsing them + -- on each request over and over again (lru-cache on module level) self.proxy_opts = tbl_copy(opts) -- Take by value end diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 01f8d815..10f0fded 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -35,21 +35,24 @@ local function connect(self, options) end local ok, err - local proxy_scheme = options.scheme -- scheme to use; http or https - local host = options.host -- remote host to connect to - local port = options.port -- remote port to connect to - local poolname = options.pool -- connection pool name to use + + local request_scheme = options.scheme + local request_host = options.host + local request_port = options.port + + local poolname = options.pool local pool_size = options.pool_size local backlog = options.backlog - if proxy_scheme and not port then - port = (proxy_scheme == "https" and 443 or 80) - elseif port and not proxy_scheme then + + if request_scheme and not request_port then + request_port = (request_scheme == "https" and 443 or 80) + elseif request_port and not request_scheme then return nil, "'scheme' is required when providing a port" end -- ssl settings local ssl, ssl_server_name, ssl_verify - if proxy_scheme ~= "http" then + if request_scheme ~= "http" then -- either https or unix domain socket ssl = options.ssl if type(options.ssl) == "table" then @@ -74,7 +77,7 @@ local function connect(self, options) proxy = self.proxy_opts if proxy then - if proxy_scheme == "https" then + if request_scheme == "https" then proxy_uri = proxy.https_proxy proxy_authorization = proxy.https_proxy_authorization else @@ -94,6 +97,7 @@ local function connect(self, options) proxy = nil else + local host = request_host local no_proxy_set = {} -- wget allows domains in no_proxy list to be prefixed by "." -- e.g. no_proxy=.mit.edu @@ -130,9 +134,10 @@ local function connect(self, options) return nil, err end - local p_scheme = proxy_uri_t[1] - if p_scheme ~= "http" then - return nil, "protocol " .. p_scheme .. " not supported for proxy connections" + local proxy_scheme = proxy_uri_t[1] + if proxy_scheme ~= "http" then + return nil, "protocol " .. tostring(proxy_scheme) .. + " not supported for proxy connections" end proxy_host = proxy_uri_t[2] proxy_port = proxy_uri_t[3] @@ -140,9 +145,9 @@ local function connect(self, options) -- construct a poolname unique within proxy and ssl info if not poolname then - poolname = (proxy_scheme or "") - .. ":" .. host - .. ":" .. tostring(port) + poolname = (request_scheme or "") + .. ":" .. request_host + .. ":" .. tostring(request_port) .. ":" .. tostring(ssl) .. ":" .. (ssl_server_name or "") .. ":" .. tostring(ssl_verify) @@ -159,12 +164,12 @@ local function connect(self, options) return nil, err end - if proxy and proxy_scheme == "https" and sock:getreusedtimes() == 0 then + if request_scheme == "https" and sock:getreusedtimes() == 0 then -- Make a CONNECT request to create a tunnel to the destination through -- the proxy. The request-target and the Host header must be in the -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section -- 4.3.6 for more details about the CONNECT request - local destination = host .. ":" .. port + local destination = request_host .. ":" .. request_port local res, err = self:request({ method = "CONNECT", path = destination, @@ -183,16 +188,16 @@ local function connect(self, options) end end - elseif not port then + elseif not request_port then -- non-proxy, without port -> unix domain socket - ok, err = sock:connect(host, tcp_opts) + ok, err = sock:connect(request_host, tcp_opts) if not ok then return nil, err end else -- non-proxy, regular network tcp - ok, err = sock:connect(host, port, tcp_opts) + ok, err = sock:connect(request_host, request_port, tcp_opts) if not ok then return nil, err end @@ -207,8 +212,8 @@ local function connect(self, options) end end - self.host = host - self.port = port + self.host = request_host + self.port = request_port self.keepalive = true self.ssl = ssl From 4d14be3e78d38826bcf9e94098e012b1f902cb4d Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Feb 2021 08:55:43 +0100 Subject: [PATCH 048/130] feat(ssl) support for 'send_status_req' option --- README.md | 1 + lib/resty/http_connect.lua | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9e64f331..2bf34b57 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,7 @@ The options table has the following fields: * `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) * `ssl`: sub-table. **NOTE**: ssl will be used when either `scheme == "https"`, or when `ssl` is truthy * `server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) + * `send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) * `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. * `ctx`: NOT supported diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 10f0fded..28b677f2 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -23,6 +23,7 @@ client:connect { ssl = { -- ssl will be used when either scheme = https, or when ssl is truthy server_name = nil, -- options as per: https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake + send_status_req = nil, ssl_verify = true, -- defaults to true ctx = nil, -- NOT supported }, @@ -51,12 +52,13 @@ local function connect(self, options) end -- ssl settings - local ssl, ssl_server_name, ssl_verify + local ssl, ssl_server_name, ssl_verify, send_status_req if request_scheme ~= "http" then -- either https or unix domain socket ssl = options.ssl if type(options.ssl) == "table" then ssl_server_name = ssl.server_name + send_status_req = ssl.send_status_req ssl_verify = (ssl.verify == nil) or (not not ssl.verify) -- default to true, and force to bool ssl = true else @@ -205,7 +207,7 @@ local function connect(self, options) -- Now do the ssl handshake if ssl and sock:getreusedtimes() == 0 then - local ok, err = self:ssl_handshake(nil, ssl_server_name, ssl_verify) + local ok, err = self:ssl_handshake(nil, ssl_server_name, ssl_verify, send_status_req) if not ok then self:close() return nil, err From d54bee06c86badc90300032067c8b502b1e69e83 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Feb 2021 11:13:35 +0100 Subject: [PATCH 049/130] chore(connect) reduce variable scope --- lib/resty/http_connect.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 28b677f2..8acfe2ce 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -75,7 +75,7 @@ local function connect(self, options) end -- proxy related settings - local proxy, proxy_uri, proxy_uri_t, proxy_authorization, proxy_host, proxy_port + local proxy, proxy_uri, proxy_authorization, proxy_host, proxy_port proxy = self.proxy_opts if proxy then @@ -131,7 +131,7 @@ local function connect(self, options) end if proxy then - proxy_uri_t, err = self:parse_uri(proxy_uri) + local proxy_uri_t, err = self:parse_uri(proxy_uri) if not proxy_uri_t then return nil, err end From 7678778ad0277c40d50a101f73406ee61372d853 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Feb 2021 11:20:39 +0100 Subject: [PATCH 050/130] fix(proxy) set proxy-auth when making request the connect method cannot deal with the auth header for a http request, only for https. Hence we store the value and inject is when making the request. We need it on connect, to construct the proper pool-name. We honor the existing request header if it is there, but that is a bad idea, since the poolname is based on the one provided to connect. So users should set the proxy options, and make requests without that header to be safe. --- lib/resty/http.lua | 3 +++ lib/resty/http_connect.lua | 2 ++ 2 files changed, 5 insertions(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index c0ea6901..bd81fef5 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -632,6 +632,9 @@ function _M.send_request(self, params) for k, v in pairs(params_headers) do headers[k] = v end + if not headers["Proxy-Authorization"] then + headers["Proxy-Authorization"] = self.http_proxy_auth + end end -- Ensure minimal headers are set diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 8acfe2ce..e8270704 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -218,6 +218,8 @@ local function connect(self, options) self.port = request_port self.keepalive = true self.ssl = ssl + -- set only for http, https has already been handled + self.http_proxy_auth = request_scheme ~= "https" and proxy_authorization or nil return true end From add64a5bf097b5cd8e320227943f6045c1d7bc5a Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Feb 2021 13:05:19 +0100 Subject: [PATCH 051/130] chore(todo) add some todo comments --- lib/resty/http.lua | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index bd81fef5..adaa75bc 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -277,6 +277,9 @@ function _M.parse_uri(_, uri, query_in_path) -- If the URI is schemaless (i.e. //example.com) try to use our current -- request scheme. if not m[1] then + -- TODO: remove this in next major version. This is undeterministic + -- behaviour. Scheme should be a required property. Best case is to + -- detect based on port 80/443. local scheme = ngx_var.scheme if scheme == "http" or scheme == "https" then m[1] = scheme @@ -633,6 +636,10 @@ function _M.send_request(self, params) headers[k] = v end if not headers["Proxy-Authorization"] then + -- TODO: next major, change this to always override the provided + -- header. Can't do that yet because it would be breaking. + -- The connect method uses self.http_proxy_auth in the poolname so + -- that should be leading. headers["Proxy-Authorization"] = self.http_proxy_auth end end @@ -876,6 +883,8 @@ function _M.request_uri(self, uri, params) if scheme == "https" then if params.headers and params.headers["Proxy-Authorization"] then proxy_authorization = params.headers["Proxy-Authorization"] + -- TODO: this is https, so we connect to the proxy first, so the actual + -- request would not need the Auth header. So we should clear that. else proxy_authorization = self.proxy_opts.https_proxy_authorization end From 7818846819ac0b20978c0441979eb337e229e7ac Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 20 Feb 2021 13:26:31 +0100 Subject: [PATCH 052/130] feat(proxy) allow overriding proxy_opts required to be able to use aio_connect from the request_uri method, since that method allows to use the proxy auth header for https proxy requests. --- README.md | 1 + lib/resty/http_connect.lua | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2bf34b57..20425cdc 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ The options table has the following fields: SSL/Proxy properties to make it safe to re-use. When in doubt, leave it blank! * `pool_size`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) * `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) +* `proxy_opts`: sub-table, defaults to the global proxy options set. * `ssl`: sub-table. **NOTE**: ssl will be used when either `scheme == "https"`, or when `ssl` is truthy * `server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) * `send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index e8270704..b8bbcd74 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -6,7 +6,7 @@ local ngx_re_find = ngx.re.find A connection function that incorporates: - tcp connect - ssl handshake - - http proxy (options to be set using "set_proxy_options") + - http proxy Due to this it will be better at setting up a socket pool where connections can be kept alive. @@ -27,6 +27,8 @@ client:connect { ssl_verify = true, -- defaults to true ctx = nil, -- NOT supported }, + + proxy_opts, -- proxy opts, defaults to global proxy options } ]] local function connect(self, options) @@ -76,7 +78,7 @@ local function connect(self, options) -- proxy related settings local proxy, proxy_uri, proxy_authorization, proxy_host, proxy_port - proxy = self.proxy_opts + proxy = options.proxy_opts or self.proxy_opts if proxy then if request_scheme == "https" then From 172fe618f5bc6b7e4dd3d7e41799f98f2fb15311 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sun, 21 Feb 2021 09:20:37 +0100 Subject: [PATCH 053/130] fix(connect) poolname update, add auth only when used --- lib/resty/http_connect.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index b8bbcd74..4dc01754 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -156,7 +156,11 @@ local function connect(self, options) .. ":" .. (ssl_server_name or "") .. ":" .. tostring(ssl_verify) .. ":" .. (proxy_uri or "") - .. ":" .. (proxy_authorization or "") + .. ":" .. (request_scheme == "https" and proxy_authorization or "") + -- in the above we only add the 'proxy_authorization' as part of the poolname + -- when the request is https. Because in that case the CONNECT request (which + -- carries the authorization header) is part of the connect procedure, whereas + -- with a plain http request the authorization is part of the actual request. end -- do TCP level connection From ea248b21b9d1f5cdf434bf0ea6d69c9fb450c1a9 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Mon, 22 Feb 2021 23:51:21 +0100 Subject: [PATCH 054/130] fix(path) move RFC7230 path construction --- lib/resty/http.lua | 11 ++++++----- lib/resty/http_connect.lua | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index adaa75bc..4da34777 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -309,7 +309,7 @@ function _M.parse_uri(_, uri, query_in_path) end -local function _format_request(params) +local function _format_request(self, params) local version = params.version local headers = params.headers or {} @@ -324,6 +324,7 @@ local function _format_request(params) local req = { str_upper(params.method), " ", + self.path_prefix or "", params.path, query, HTTP[version], @@ -332,7 +333,7 @@ local function _format_request(params) true, true, } - local c = 6 -- req table index it's faster to do this inline vs table.insert + local c = 7 -- req table index it's faster to do this inline vs table.insert -- Append headers for key, values in pairs(headers) do @@ -681,7 +682,7 @@ function _M.send_request(self, params) params.headers = headers -- Format and send request - local req = _format_request(params) + local req = _format_request(self, params) if DEBUG then ngx_log(ngx_DEBUG, "\n", req) end local bytes, err = sock:send(req) @@ -911,9 +912,9 @@ function _M.request_uri(self, uri, params) -- to also include the scheme, host and port so that the final form -- in conformant to RFC 7230. if port == 80 then - params.path = scheme .. "://" .. host .. path + self.path_prefix = scheme .. "://" .. host .. path else - params.path = scheme .. "://" .. host .. ":" .. port .. path + self.path_prefix = scheme .. "://" .. host .. ":" .. port .. path end if self.proxy_opts.http_proxy_authorization then diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 4dc01754..561d7e66 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -77,7 +77,7 @@ local function connect(self, options) end -- proxy related settings - local proxy, proxy_uri, proxy_authorization, proxy_host, proxy_port + local proxy, proxy_uri, proxy_authorization, proxy_host, proxy_port, path_prefix proxy = options.proxy_opts or self.proxy_opts if proxy then @@ -87,9 +87,21 @@ local function connect(self, options) else proxy_uri = proxy.http_proxy proxy_authorization = proxy.http_proxy_authorization + -- When a proxy is used, the target URI must be in absolute-form + -- (RFC 7230, Section 5.3.2.). That is, it must be an absolute URI + -- to the remote resource with the scheme, host and an optional port + -- in place. + -- + -- Since _format_request() constructs the request line by concatenating + -- params.path and params.query together, we need to modify the path + -- to also include the scheme, host and port so that the final form + -- in conformant to RFC 7230. + path_prefix = "http://" .. request_host .. (request_port == 80 and "" or (":" .. request_port)) end if not proxy_uri then proxy = nil + proxy_authorization = nil + path_prefix = nil end end @@ -226,6 +238,7 @@ local function connect(self, options) self.ssl = ssl -- set only for http, https has already been handled self.http_proxy_auth = request_scheme ~= "https" and proxy_authorization or nil + self.path_prefix = path_prefix return true end From b7c417fca784308cb44dcd54bef274930c21febb Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 24 Feb 2021 14:08:01 +0100 Subject: [PATCH 055/130] chore(*) move request_uri over to use the aio connect --- lib/resty/http.lua | 128 ++++++++++--------------------------- lib/resty/http_connect.lua | 23 ++----- t/07-keepalive.t | 28 ++++++-- 3 files changed, 62 insertions(+), 117 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 4da34777..0a1f6b23 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -629,20 +629,18 @@ function _M.send_request(self, params) local body = params.body local headers = http_headers.new() - local params_headers = params.headers - if params_headers then - -- We assign one by one so that the metatable can handle case insensitivity - -- for us. You can blame the spec for this inefficiency. - for k, v in pairs(params_headers) do - headers[k] = v - end - if not headers["Proxy-Authorization"] then - -- TODO: next major, change this to always override the provided - -- header. Can't do that yet because it would be breaking. - -- The connect method uses self.http_proxy_auth in the poolname so - -- that should be leading. - headers["Proxy-Authorization"] = self.http_proxy_auth - end + local params_headers = params.headers or {} + -- We assign one by one so that the metatable can handle case insensitivity + -- for us. You can blame the spec for this inefficiency. + for k, v in pairs(params_headers) do + headers[k] = v + end + if not headers["Proxy-Authorization"] then + -- TODO: next major, change this to always override the provided + -- header. Can't do that yet because it would be breaking. + -- The connect method uses self.http_proxy_auth in the poolname so + -- that should be leading. + headers["Proxy-Authorization"] = self.http_proxy_auth end -- Ensure minimal headers are set @@ -862,97 +860,39 @@ end function _M.request_uri(self, uri, params) params = tbl_copy(params or {}) -- Take by value - - local parsed_uri, err = self:parse_uri(uri, false) - if not parsed_uri then - return nil, err + if self.proxy_opts then + params.proxy_opts = tbl_copy(self.proxy_opts or {}) end - local scheme, host, port, path, query = unpack(parsed_uri) - if not params.path then params.path = path end - if not params.query then params.query = query end - - -- See if we should use a proxy to make this request - local proxy_uri = self:get_proxy_uri(scheme, host) - - -- Make the connection either through the proxy or directly - -- to the remote host - local c, err - - if proxy_uri then - local proxy_authorization - if scheme == "https" then - if params.headers and params.headers["Proxy-Authorization"] then - proxy_authorization = params.headers["Proxy-Authorization"] - -- TODO: this is https, so we connect to the proxy first, so the actual - -- request would not need the Auth header. So we should clear that. - else - proxy_authorization = self.proxy_opts.https_proxy_authorization - end + do + local parsed_uri, err = self:parse_uri(uri, false) + if not parsed_uri then + return nil, err end - c, err = self:connect_proxy(proxy_uri, scheme, host, port, proxy_authorization) - else - c, err = self:tcp_only_connect(host, port) + local path, query + params.scheme, params.host, params.port, path, query = unpack(parsed_uri) + params.path = params.path or path + params.query = params.query or query end - if not c then - return nil, err + if params.scheme == "https" and params.ssl_verify == false then + -- backward compat; params.ssl_verify --> params.ssl.ssl_verify + params.ssl = params.ssl or {} + params.ssl.ssl_verify = params.ssl_verify end - if proxy_uri then - if scheme == "http" then - -- When a proxy is used, the target URI must be in absolute-form - -- (RFC 7230, Section 5.3.2.). That is, it must be an absolute URI - -- to the remote resource with the scheme, host and an optional port - -- in place. - -- - -- Since _format_request() constructs the request line by concatenating - -- params.path and params.query together, we need to modify the path - -- to also include the scheme, host and port so that the final form - -- in conformant to RFC 7230. - if port == 80 then - self.path_prefix = scheme .. "://" .. host .. path - else - self.path_prefix = scheme .. "://" .. host .. ":" .. port .. path - end - - if self.proxy_opts.http_proxy_authorization then - if not params.headers then - params.headers = {} - end - - if not params.headers["Proxy-Authorization"] then - params.headers["Proxy-Authorization"] = self.proxy_opts.http_proxy_authorization - end - end - elseif scheme == "https" then - -- don't keep this connection alive as the next request could target - -- any host and re-using the proxy tunnel for that is not possible - self.keepalive = false + do + local proxy_auth = (params.headers or {})["Proxy-Authorization"] + if proxy_auth and params.proxy_opts then + params.proxy_opts.https_proxy_authorization = proxy_auth + params.proxy_opts.http_proxy_authorization = proxy_auth end - - -- self:connect_uri() set the host and port to point to the proxy server. As - -- the connection to the proxy has been established, set the host and port - -- to point to the actual remote endpoint at the other end of the tunnel to - -- ensure the correct Host header added to the requests. - self.host = host - self.port = port end - if scheme == "https" then - local verify = true - - if params.ssl_verify == false then - verify = false - end - - local ok, err = self:ssl_handshake(nil, host, verify) - if not ok then - self:close() - return nil, err - end - + local ok, err = self:connect(params) + if not ok then + return nil, err end local res, err = self:request(params) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 561d7e66..3898b7b0 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -55,25 +55,16 @@ local function connect(self, options) -- ssl settings local ssl, ssl_server_name, ssl_verify, send_status_req - if request_scheme ~= "http" then - -- either https or unix domain socket - ssl = options.ssl + ssl = (request_scheme == "https") or (not not options.ssl) + if ssl then + ssl_verify = true -- default if type(options.ssl) == "table" then - ssl_server_name = ssl.server_name - send_status_req = ssl.send_status_req - ssl_verify = (ssl.verify == nil) or (not not ssl.verify) -- default to true, and force to bool - ssl = true - else - if ssl then - ssl = true - ssl_verify = true -- default to true - else - ssl = false + ssl_server_name = options.ssl.server_name + send_status_req = options.ssl.send_status_req + if options.ssl.ssl_verify == false then + ssl_verify = false end end - else - -- plain http - ssl = false end -- proxy related settings diff --git a/t/07-keepalive.t b/t/07-keepalive.t index cae8e23c..d781f1f4 100644 --- a/t/07-keepalive.t +++ b/t/07-keepalive.t @@ -34,13 +34,15 @@ __DATA__ local http = require "resty.http" local httpc = http.new() local res, err = httpc:request_uri( - "http://127.0.0.1:"..ngx.var.server_port.."/b", { - } + "http://127.0.0.1:" .. ngx.var.server_port.."/b", {} ) - ngx.say(res.headers["Connection"]) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect { + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } ngx.say(httpc:get_reused_times()) '; } @@ -261,7 +263,11 @@ connection must be closed ngx.say(res.headers["Connection"]) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect { + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } ngx.say(httpc:get_reused_times()) httpc:close() @@ -274,7 +280,11 @@ connection must be closed ngx.say(res.headers["Connection"]) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect { + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } ngx.say(httpc:get_reused_times()) httpc:close() @@ -289,7 +299,11 @@ connection must be closed ngx.sleep(1.1) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect { + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } ngx.say(httpc:get_reused_times()) httpc:close() } From 7b09112f92f786ec731cac8a1fc90858299b1778 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Wed, 24 Feb 2021 14:17:43 +0100 Subject: [PATCH 056/130] fix(doc) ssl_verify parameter mentioned at wrong method --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 20425cdc..b8e5c32e 100644 --- a/README.md +++ b/README.md @@ -305,7 +305,6 @@ The `params` table accepts the following fields: * `query` The query string, presented as either a literal string or Lua table.. * `headers` A table of request headers. * `body` The request body as a string, or an iterator function (see [get_client_body_reader](#get_client_body_reader)). -* `ssl_verify` Verify SSL cert matches hostname When the request is successful, `res` will contain the following fields: @@ -323,8 +322,9 @@ When the request is successful, `res` will contain the following fields: The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself. -There are 3 additional parameters for controlling keepalives: +There are 4 additional parameters for controlling keepalives and ssl: +* `ssl_verify` Verify the SSL cert matches the hostname * `keepalive` Set to `false` to disable keepalives and immediately close the connection. * `keepalive_timeout` The maximal idle timeout (ms). Defaults to `lua_socket_keepalive_timeout`. * `keepalive_pool` The maximum number of connections in the pool. Defaults to `lua_socket_pool_size`. From 6b5ef7818f8385d8f2fc957bccedf5043cd91066 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 25 Feb 2021 09:16:15 +0100 Subject: [PATCH 057/130] chore(ssl) remove ssl subtable, use 'ssl_' prefix This provides better backward compatibility. --- README.md | 17 ++++++++--------- lib/resty/http.lua | 6 ------ lib/resty/http_connect.lua | 31 ++++++++++++++----------------- 3 files changed, 22 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index b8e5c32e..3cac3ba3 100644 --- a/README.md +++ b/README.md @@ -184,12 +184,10 @@ The options table has the following fields: SSL/Proxy properties to make it safe to re-use. When in doubt, leave it blank! * `pool_size`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) * `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) -* `proxy_opts`: sub-table, defaults to the global proxy options set. -* `ssl`: sub-table. **NOTE**: ssl will be used when either `scheme == "https"`, or when `ssl` is truthy - * `server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) - * `send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) - * `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. - * `ctx`: NOT supported +* `proxy_opts`: sub-table, defaults to the global proxy options set, see [set_proxy_options](#set-proxy-options). +* `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. +* `ssl_server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) +* `ssl_send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) ### TCP only connect @@ -320,11 +318,12 @@ When the request is successful, `res` will contain the following fields: `syntax: res, err = httpc:request_uri(uri, params)` -The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself. +The simple interface. Options supplied in the `params` table are the combined parameters +from the all-in-one connect interface, and the generic request interface. The parameters +will override components found in the uri itself. -There are 4 additional parameters for controlling keepalives and ssl: +There are 3 additional parameters for controlling keepalives: -* `ssl_verify` Verify the SSL cert matches the hostname * `keepalive` Set to `false` to disable keepalives and immediately close the connection. * `keepalive_timeout` The maximal idle timeout (ms). Defaults to `lua_socket_keepalive_timeout`. * `keepalive_pool` The maximum number of connections in the pool. Defaults to `lua_socket_pool_size`. diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 0a1f6b23..d614b9af 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -876,12 +876,6 @@ function _M.request_uri(self, uri, params) params.query = params.query or query end - if params.scheme == "https" and params.ssl_verify == false then - -- backward compat; params.ssl_verify --> params.ssl.ssl_verify - params.ssl = params.ssl or {} - params.ssl.ssl_verify = params.ssl_verify - end - do local proxy_auth = (params.headers or {})["Proxy-Authorization"] if proxy_auth and params.proxy_opts then diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 3898b7b0..ffd72af4 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -21,12 +21,11 @@ client:connect { pool_size = nil, -- options as per: https://github.com/openresty/lua-nginx-module#tcpsockconnect backlog = nil, - ssl = { -- ssl will be used when either scheme = https, or when ssl is truthy - server_name = nil, -- options as per: https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake - send_status_req = nil, - ssl_verify = true, -- defaults to true - ctx = nil, -- NOT supported - }, + -- ssl options as per: https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake + ssl_server_name = nil, + ssl_send_status_req = nil, + ssl_verify = true, -- NOTE: defaults to true + ctx = nil, -- NOTE: not supported proxy_opts, -- proxy opts, defaults to global proxy options } @@ -54,16 +53,14 @@ local function connect(self, options) end -- ssl settings - local ssl, ssl_server_name, ssl_verify, send_status_req - ssl = (request_scheme == "https") or (not not options.ssl) - if ssl then + local ssl, ssl_server_name, ssl_verify, ssl_send_status_req + if request_scheme == "https" then + ssl = true + ssl_server_name = options.ssl_server_name + ssl_send_status_req = options.ssl_send_status_req ssl_verify = true -- default - if type(options.ssl) == "table" then - ssl_server_name = options.ssl.server_name - send_status_req = options.ssl.send_status_req - if options.ssl.ssl_verify == false then - ssl_verify = false - end + if options.ssl_verify == false then + ssl_verify = false end end @@ -175,7 +172,7 @@ local function connect(self, options) return nil, err end - if request_scheme == "https" and sock:getreusedtimes() == 0 then + if ssl and sock:getreusedtimes() == 0 then -- Make a CONNECT request to create a tunnel to the destination through -- the proxy. The request-target and the Host header must be in the -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section @@ -216,7 +213,7 @@ local function connect(self, options) -- Now do the ssl handshake if ssl and sock:getreusedtimes() == 0 then - local ok, err = self:ssl_handshake(nil, ssl_server_name, ssl_verify, send_status_req) + local ok, err = self:ssl_handshake(nil, ssl_server_name, ssl_verify, ssl_send_status_req) if not ok then self:close() return nil, err From 80019d71eeeb7ecdd98594efe47481f37275d5b8 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 25 Feb 2021 09:26:57 +0100 Subject: [PATCH 058/130] fix(flaky) test 1 header only, header order is undetermined --- t/16-http-proxy.t | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/16-http-proxy.t b/t/16-http-proxy.t index 0ce660c4..385a0ac9 100644 --- a/t/16-http-proxy.t +++ b/t/16-http-proxy.t @@ -394,7 +394,7 @@ Basic ZGVtbzp0ZXN0 } --- tcp_listen: 12345 --- tcp_query eval -qr/CONNECT 127.0.0.1:443 HTTP\/1.1\r\n.*Proxy-Authorization: Basic ZGVtbzpwYXNz\r\n.*Host: 127.0.0.1:443\r\n.*/s +qr/CONNECT 127.0.0.1:443 HTTP\/1.1\r\n.*Proxy-Authorization: Basic ZGVtbzpwYXNz\r\n.*/s # The reply cannot be successful or otherwise the client would start # to do a TLS handshake with the proxied host and that we cannot @@ -505,7 +505,7 @@ Basic ZGVtbzp3b3Jk } --- tcp_listen: 12345 --- tcp_query eval -qr/CONNECT 127.0.0.1:443 HTTP\/1.1\r\n.*Proxy-Authorization: Basic ZGVtbzp3b3Jk\r\n.*Host: 127.0.0.1:443\r\n.*/s +qr/CONNECT 127.0.0.1:443 HTTP\/1.1\r\n.*Proxy-Authorization: Basic ZGVtbzp3b3Jk\r\n.*/s # The reply cannot be successful or otherwise the client would start # to do a TLS handshake with the proxied host and that we cannot From 34eca4f281a4daeb46ecff6a8eaca84939624a79 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 25 Feb 2021 19:24:51 +0100 Subject: [PATCH 059/130] fix(rockspec) fix typo in rockspec --- lua-resty-http-0.15-0.rockspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lua-resty-http-0.15-0.rockspec b/lua-resty-http-0.15-0.rockspec index e1170d28..71ba0abe 100644 --- a/lua-resty-http-0.15-0.rockspec +++ b/lua-resty-http-0.15-0.rockspec @@ -17,7 +17,7 @@ build = { type = "builtin", modules = { ["resty.http"] = "lib/resty/http.lua", - ["resty.http_headers"] = "lib/resty/http_headers.lua" + ["resty.http_headers"] = "lib/resty/http_headers.lua", ["resty.http_connect"] = "lib/resty/http_connect.lua" } } From e1090f9562b385db1b6b83f5e35b0f518bc63b75 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Sat, 27 Feb 2021 16:06:09 +0100 Subject: [PATCH 060/130] review comments --- README.md | 2 +- lib/resty/http.lua | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3cac3ba3..818f5d0a 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Attempts to connect to the web server. ### all-in-one connect -This version of connect uses the `aio_options` signature above. For the +This version of connect uses the `aio_options_table` signature above. For the other signatures see the [TCP only connect](#tcp-only-connect) below. This version of `connect` will connect to the remote end while incorporating the diff --git a/lib/resty/http.lua b/lib/resty/http.lua index d614b9af..44200990 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -277,9 +277,13 @@ function _M.parse_uri(_, uri, query_in_path) -- If the URI is schemaless (i.e. //example.com) try to use our current -- request scheme. if not m[1] then - -- TODO: remove this in next major version. This is undeterministic - -- behaviour. Scheme should be a required property. Best case is to - -- detect based on port 80/443. + -- Schema-less URIs can occur in client side code, implying "inherit + -- the schema from the current request". We support it for a fairly + -- specific case; if for example you are using the ESI parser in + -- ledge (https://github.com/ledgetech/ledge) to perform in-flight + -- sub requests on the edge based on instructions found in markup, + -- those URIs may also be schemaless with the intention that the + -- subrequest would inherit the schema just like JavaScript would. local scheme = ngx_var.scheme if scheme == "http" or scheme == "https" then m[1] = scheme @@ -1022,7 +1026,7 @@ function _M.get_proxy_uri(self, scheme, host) local no_proxy_set = {} -- wget allows domains in no_proxy list to be prefixed by "." -- e.g. no_proxy=.mit.edu - for host_suffix in ngx_re_gmatch(self.proxy_opts.no_proxy, "\\.?([^,]+)") do + for host_suffix in ngx_re_gmatch(self.proxy_opts.no_proxy, "\\.?([^,]+)", "jo") do no_proxy_set[host_suffix[1]] = true end @@ -1041,8 +1045,8 @@ function _M.get_proxy_uri(self, scheme, host) -- Strip the next level from the domain and check if that one -- is on the list - host = ngx_re_sub(host, "^[^.]+\\.", "") - until not ngx_re_find(host, "\\.") + host = ngx_re_sub(host, "^[^.]+\\.", "", "jo") + until not ngx_re_find(host, "\\.", "jo") end if scheme == "http" and self.proxy_opts.http_proxy then From beef9c2d3dd80c912fdf47a5df0da4c06f5c1887 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 2 Mar 2021 16:37:55 +0000 Subject: [PATCH 061/130] Reworked readme to emphasise new connection API Also deprecated certain features for clarity. --- README.md | 596 +++++++++++++++++++++++++----------------------------- 1 file changed, 273 insertions(+), 323 deletions(-) diff --git a/README.md b/README.md index 818f5d0a..6e1cbb99 100644 --- a/README.md +++ b/README.md @@ -1,291 +1,228 @@ # lua-resty-http -Lua HTTP client cosocket driver for [OpenResty](http://openresty.org/) / [ngx_lua](https://github.com/openresty/lua-nginx-module). +Lua HTTP client cosocket driver for [OpenResty](http://openresty.org/) / [ngx\_lua](https://github.com/openresty/lua-nginx-module). -# Status +## Status Production ready. [![Build Status](https://travis-ci.org/ledgetech/lua-resty-http.svg?branch=master)](https://travis-ci.org/ledgetech/lua-resty-http) -# Features +## Features * HTTP 1.0 and 1.1 * SSL * Streaming interface to the response body, for predictable memory usage -* Alternative simple interface for singleshot requests without manual connection step +* Alternative simple interface for single-shot requests without a manual connection step * Chunked and non-chunked transfer encodings -* Keepalive -* Pipelining +* Connection keepalives +* Request pipelining * Trailers +* HTTP proxy connections -# API +## API * [new](#new) * [connect](#connect) -* [connect_proxy](#connect_proxy) -* [set_proxy_options](#set_proxy_options) -* [set_timeout](#set_timeout) -* [set_timeouts](#set_timeouts) -* [ssl_handshake](#ssl_handshake) -* [set_keepalive](#set_keepalive) -* [get_reused_times](#get_reused_times) +* [set\_proxy\_options](#set_proxy_options) +* [set\_timeout](#set_timeout) +* [set\_timeouts](#set_timeouts) +* [set\_keepalive](#set_keepalive) +* [get\_reused\_times](#get_reused_times) * [close](#close) * [request](#request) -* [request_uri](#request_uri) -* [request_pipeline](#request_pipeline) +* [request\_uri](#request_uri) +* [request\_pipeline](#request_pipeline) +* [parse\_uri](#parse_uri) +* [get\_client\_body\_reader](#get_client_body_reader) * [Response](#response) - * [body_reader](#resbody_reader) - * [read_body](#resread_body) - * [read_trailers](#resread_trailers) -* [Proxy](#proxy) - * [proxy_request](#proxy_request) - * [proxy_response](#proxy_response) -* [Utility](#utility) - * [parse_uri](#parse_uri) - * [get_client_body_reader](#get_client_body_reader) - - -## Synopsis - -```` lua -lua_package_path "/path/to/lua-resty-http/lib/?.lua;;"; - -server { - - - location /simpleinterface { - resolver 8.8.8.8; # use Google's open DNS server for an example - - content_by_lua_block { - - -- For simple singleshot requests, use the URI interface. - local http = require "resty.http" - local httpc = http.new() - local res, err = httpc:request_uri("http://example.com/helloworld", { - method = "POST", - body = "a=1&b=2", - headers = { - ["Content-Type"] = "application/x-www-form-urlencoded", - }, - keepalive_timeout = 60000, - keepalive_pool = 10 - }) - - if not res then - ngx.say("failed to request: ", err) - return - end - - -- In this simple form, there is no manual connection step, so the body is read - -- all in one go, including any trailers, and the connection closed or keptalive - -- for you. - - ngx.status = res.status - - for k,v in pairs(res.headers) do - -- - end - - ngx.say(res.body) - } - } - - - location /genericinterface { - content_by_lua_block { - - local http = require "resty.http" - local httpc = http.new() - - -- The generic form gives us more control. We must connect manually. - httpc:set_timeout(500) - httpc:connect("127.0.0.1", 80) - - -- And request using a path, rather than a full URI. - local res, err = httpc:request({ - path = "/helloworld", - headers = { - ["Host"] = "example.com", - }, - }) - - if not res then - ngx.say("failed to request: ", err) - return - end - - -- Now we can use the body_reader iterator, to stream the body according to our desired chunk size. - local reader = res.body_reader - - repeat - local chunk, err = reader(8192) - if err then - ngx.log(ngx.ERR, err) - break - end - - if chunk then - -- process - end - until not chunk - - local ok, err = httpc:set_keepalive() - if not ok then - ngx.say("failed to set keepalive: ", err) - return - end - } - } -} -```` + * [body\_reader](#resbody_reader) + * [read\_body](#resread_body) + * [read\_trailers](#resread_trailers) -# Connection +### Deprecated -## new +These methods may be removed in future versions. -`syntax: httpc = http.new()` +* [connect\_proxy](#connect_proxy) +* [ssl\_handshake](#ssl_handshake) +* [proxy\_request](#proxy_request) +* [proxy\_response](#proxy_response) -Creates the http object. In case of failures, returns `nil` and a string describing the error. +## Usage -## connect +There are two basic modes of operation: -`syntax: ok, err = httpc:connect(aio_options_table)` +1. **Simple single-shot requests** which require no manual connection management but which buffer the entire response and leave the connection either closed or back in the connection pool. -`syntax: ok, err = httpc:connect(host, port, options_table?)` +2. **Streamed requests** where the connection is established separately, then the request is sent, the body stream read in chunks, and finally the connection is manually closed or kept alive. This technique requires a little more code but provides the ability to discard potentially large response bodies on the Lua side, as well as pipelining multiple requests over a single connection. -`syntax: ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?)` +### Single-shot request -Attempts to connect to the web server. +```lua +local httpc = require("resty.http").new() + +-- Single-shot requests use the `request_uri` interface. +local res, err = httpc:request_uri("http://example.com/helloworld", { + method = "POST", + body = "a=1&b=2", + headers = { + ["Content-Type"] = "application/x-www-form-urlencoded", + }, +}) +if not res then + ngx.log(ngx.ERR, "request failed: ", err) + return +end -### all-in-one connect +-- At this point, the entire request / response is complete and the connection +-- will be closed or back on the connection pool. -This version of connect uses the `aio_options_table` signature above. For the -other signatures see the [TCP only connect](#tcp-only-connect) below. +-- The `res` table contains the expeected `status`, `headers` and `body` fields. +local status = res.status +local length = res.headers["Content-Length"] +local body = res.body +``` -This version of `connect` will connect to the remote end while incorporating the -following activities: +### Streamed request -- TCP connect -- SSL handshake -- HTTP-proxy (options to be set using [set_proxy_options](#set_proxy_options)) +```lua +local httpc = require("resty.http").new() + +-- First establish a connection +local ok, err = httpc:connect({ + scheme = "https", + host = "127.0.0.1" + port = 8080, +}) +if not ok then + ngx.log(ngx.ERR, "connection failed: ", err) + return +end -Whilst doing this it will also create a distinct pool name that is safe to use -with SSL and/or Proxy based connections. +-- Then send using `request`, supplying a path and `Host` header instead of a +-- full URI. +local res, err = httpc:request({ + path = "/helloworld", + headers = { + ["Host"] = "example.com", + }, +}) +if not res then + ngx.log(ngx.ERR, "request failed: ", err) + return +end -The options table has the following fields: +-- At this point, the status and headers will be available to use in the `res` +-- table, but the body and any trailers will still be on the wire. -* `scheme`: scheme to use, or nil for unix domain socket -* `host`: target machine, or a unix domain socket -* `port`: port on target machine, will default to `80` or `443` based on scheme -* `pool`: connection pool name. Option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect), - except that the default will become a pool name constructed including all the - SSL/Proxy properties to make it safe to re-use. When in doubt, leave it blank! -* `pool_size`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) -* `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) -* `proxy_opts`: sub-table, defaults to the global proxy options set, see [set_proxy_options](#set-proxy-options). -* `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. -* `ssl_server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) -* `ssl_send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) +-- We can use the `body_reader` iterator, to stream the body according to our +-- desired buffer size. +local reader = res.body_reader +local buffer_size = 8192 +repeat + local buffer, err = reader(buffer_size) + if err then + ngx.log(ngx.ERR, err) + break + end + + if buffer then + -- process + end +until not buffer + +local ok, err = httpc:set_keepalive() +if not ok then + ngx.say("failed to set keepalive: ", err) + return +end -### TCP only connect +-- At this point, the connection will either be safely back in the pool, or closed. +```` -Before actually resolving the host name and connecting to the remote backend, this method will always look up the connection pool for matched idle connections created by previous calls of this method. +# Connection -NOTE: the default pool name will only incorporate IP and port information so is -unsafe to use in case of SSL and/or Proxy connections. Specify your own pool -name or use the [`all-in-one`](#all-in-one-connect) above. +## new -An optional Lua table can be specified as the last argument to this method to specify various connect options: +`syntax: httpc, err = http.new()` -* `pool` -: Specifies a custom name for the connection pool being used. If omitted, then the connection pool name will be generated from the string template `:` or ``. +Creates the HTTP connection object. In case of failures, returns `nil` and a string describing the error. -## connect_proxy +## connect -`syntax: ok, err = httpc:connect_proxy(proxy_uri, scheme, host, port, proxy_authorization)` +`syntax: ok, err = httpc:connect(options)` -Attempts to connect to the web server through the given proxy server. The method accepts the following arguments: +Attempts to connect to the web server while incorporating the following activities: -* `proxy_uri` - Full URI of the proxy server to use (e.g. `http://proxy.example.com:3128/`). Note: Only `http` protocol is supported. -* `scheme` - The protocol to use between the proxy server and the remote host (`http` or `https`). If `https` is specified as the scheme, `connect_proxy()` makes a `CONNECT` request to establish a TCP tunnel to the remote host through the proxy server. -* `host` - The hostname of the remote host to connect to. -* `port` - The port of the remote host to connect to. -* `proxy_authorization` - The `Proxy-Authorization` header value sent to the proxy server via `CONNECT` when the `scheme` is `https`. +- TCP connection +- SSL handshake +- HTTP proxy configuration -If an error occurs during the connection attempt, this method returns `nil` with a string describing the error. If the connection was successfully established, the method returns `1`. +In doing so it will create a distinct connection pool name that is safe to use with SSL and / or proxy based connections, and as such this syntax is strongly recommended over the original [deprecated connection syntax](#TCP_only_connect). -There's a few key points to keep in mind when using this api: +The options table has the following fields: -* If the scheme is `https`, you need to perform the TLS handshake with the remote server manually using the `ssl_handshake()` method before sending any requests through the proxy tunnel. -* If the scheme is `http`, you need to ensure that the requests you send through the connections conforms to [RFC 7230](https://tools.ietf.org/html/rfc7230) and especially [Section 5.3.2.](https://tools.ietf.org/html/rfc7230#section-5.3.2) which states that the request target must be in absolute form. In practice, this means that when you use `send_request()`, the `path` must be an absolute URI to the resource (e.g. `http://example.com/index.html` instead of just `/index.html`). +* `scheme`: scheme to use, or nil for unix domain socket +* `host`: target host, or path to a unix domain socket +* `port`: port on target host, will default to `80` or `443` based on the scheme +* `pool`: custom connection pool name. Option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect), except that the default will become a pool name constructed using the SSL / proxy properties, which is important for safe connection reuse. When in doubt, leave it blank! +* `pool_size`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) +* `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) +* `proxy_opts`: sub-table, defaults to the global proxy options set, see [set\_proxy\_options](#set-proxy-options). +* `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. +* `ssl_server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) +* `ssl_send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) -## set_timeout +## set\_timeout `syntax: httpc:set_timeout(time)` -Sets the timeout (in ms) protection for subsequent operations, including the `connect` method. +Sets the socket timeout (in ms) for subsequent operations. See [set\_timeouts](#set_timeouts) below for a more declarative approach. -## set_timeouts +## set\_timeouts `syntax: httpc:set_timeouts(connect_timeout, send_timeout, read_timeout)` Sets the connect timeout threshold, send timeout threshold, and read timeout threshold, respectively, in milliseconds, for subsequent socket operations (connect, send, receive, and iterators returned from receiveuntil). -## ssl_handshake - -`syntax: session, err = httpc:ssl_handshake(session, host, verify)` - -Performs an SSL handshake on the TCP connection, only available in ngx_lua > v0.9.11 - -See docs for [ngx.socket.tcp](https://github.com/openresty/lua-nginx-module#ngxsockettcp) for details. - -## set_keepalive +## set\_keepalive `syntax: ok, err = httpc:set_keepalive(max_idle_timeout, pool_size)` -Attempts to puts the current connection into the ngx_lua cosocket connection pool. - -You can specify the max idle timeout (in ms) when the connection is in the pool and the maximal size of the pool every nginx worker process. +Either places the current connection into the pool for future reuse, or closes the connection. Calling this instead of [close](#close) is "safe" in that it will conditionally close depending on the type of request. Specifically, a `1.0` request without `Connection: Keep-Alive` will be closed, as will a `1.1` request with `Connection: Close`. -Only call this method in the place you would have called the `close` method instead. Calling this method will immediately turn the current http object into the `closed` state. Any subsequent operations other than `connect()` on the current object will return the `closed` error. +In case of success, returns `1`. In case of errors, returns `nil, err`. In the case where the connection is conditionally closed as described above, returns `2` and the error string `connection must be closed`, so as to distinguish from unexpected errors. -Note that calling this instead of `close` is "safe" in that it will conditionally close depending on the type of request. Specifically, a `1.0` request without `Connection: Keep-Alive` will be closed, as will a `1.1` request with `Connection: Close`. +See [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksetkeepalive) for parameter documentation. -In case of success, returns `1`. In case of errors, returns `nil, err`. In the case where the connection is conditionally closed as described above, returns `2` and the error string `connection must be closed`. - -## set_proxy_options +## set\_proxy\_options `syntax: httpc:set_proxy_options(opts)` -Configure an http proxy to be used with this client instance. The `opts` is a table that accepts the following fields: +Configure an HTTP proxy to be used with this client instance. The `opts` table expects the following fields: -* `http_proxy` - an URI to a proxy server to be used with http requests -* `http_proxy_authorization` - a default `Proxy-Authorization` header value to be used with `http_proxy`, e.g. `Basic ZGVtbzp0ZXN0`, which will be overriden if the `Proxy-Authorization` request header is present. -* `https_proxy` - an URI to a proxy server to be used with https requests -* `https_proxy_authorization` - as `http_proxy_authorization` but for use with `https_proxy` (since with https the authorisation is done when connecting, this one cannot be overridden by passing the `Proxy-Authorization` request header). -* `no_proxy` - a comma separated list of hosts that should not be proxied. +* `http_proxy`: an URI to a proxy server to be used with HTTP requests +* `http_proxy_authorization`: a default `Proxy-Authorization` header value to be used with `http_proxy`, e.g. `Basic ZGVtbzp0ZXN0`, which will be overriden if the `Proxy-Authorization` request header is present. +* `https_proxy`: an URI to a proxy server to be used with HTTPS requests +* `https_proxy_authorization`: as `http_proxy_authorization` but for use with `https_proxy` (since with HTTPS the authorisation is done when connecting, this one cannot be overridden by passing the `Proxy-Authorization` request header). +* `no_proxy`: a comma separated list of hosts that should not be proxied. -Note that proxy options are only applied when using the high-level `request_uri()` API, or when using the all-in-one connect. +Note that this method has no effect when using the deprecated [TCP only connect](#TCP_only_connect) connection syntax. -## get_reused_times +## get\_reused\_times `syntax: times, err = httpc:get_reused_times()` -This method returns the (successfully) reused times for the current connection. In case of error, it returns `nil` and a string describing the error. - -If the current connection does not come from the built-in connection pool, then this method always returns `0`, that is, the connection has never been reused (yet). If the connection comes from the connection pool, then the return value is always non-zero. So this method can also be used to determine if the current connection comes from the pool. +See [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockgetreusedtimes). ## close `syntax: ok, err = httpc:close()` -Closes the current connection and returns the status. - -In case of success, returns `1`. In case of errors, returns `nil` with a string describing the error. - +See [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockclose). # Requesting @@ -293,74 +230,67 @@ In case of success, returns `1`. In case of errors, returns `nil` with a string `syntax: res, err = httpc:request(params)` -Returns a `res` table or `nil` and an error message. +Sends an HTTP request over an already established connection. Returns a `res` table or `nil` and an error message. -The `params` table accepts the following fields: +The `params` table expects the following fields: -* `version` The HTTP version number, currently supporting 1.0 or 1.1. -* `method` The HTTP method string. -* `path` The path string. -* `query` The query string, presented as either a literal string or Lua table.. -* `headers` A table of request headers. -* `body` The request body as a string, or an iterator function (see [get_client_body_reader](#get_client_body_reader)). +* `version`: The HTTP version number. Defaults to `1.1`. +* `method`: The HTTP method string. Defaults to `GET`. +* `path`: The path string. Defaults to `/`. +* `query`: The query string, presented as either a literal string or Lua table.. +* `headers`: A table of request headers. +* `body`: The request body as a string, or an iterator function (see [get\_client\_body\_reader](#get_client_body_reader)). When the request is successful, `res` will contain the following fields: -* `status` The status code. -* `reason` The status reason phrase. -* `headers` A table of headers. Multiple headers with the same field name will be presented as a table of values. -* `has_body` A boolean flag indicating if there is a body to be read. -* `body_reader` An iterator function for reading the body in a streaming fashion. -* `read_body` A method to read the entire body into a string. -* `read_trailers` A method to merge any trailers underneath the headers, after reading the body. +* `status`: The status code. +* `reason`: The status reason phrase. +* `headers`: A table of headers. Multiple headers with the same field name will be presented as a table of values. +* `has_body`: A boolean flag indicating if there is a body to be read. +* `body_reader`: An iterator function for reading the body in a streaming fashion. +* `read_body`: A method to read the entire body into a string. +* `read_trailers`: A method to merge any trailers underneath the headers, after reading the body. -## request_uri +## request\_uri `syntax: res, err = httpc:request_uri(uri, params)` -The simple interface. Options supplied in the `params` table are the combined parameters -from the all-in-one connect interface, and the generic request interface. The parameters -will override components found in the uri itself. +The single-shot interface (see [usage](#Usage)). Since this method performs an entire end-to-end request, options specified in the `params` can include anything found in both [connect](#connect) and [request](#request) documented above. Note also that fields in `params` will override relevant components of the `uri` if specified. There are 3 additional parameters for controlling keepalives: -* `keepalive` Set to `false` to disable keepalives and immediately close the connection. -* `keepalive_timeout` The maximal idle timeout (ms). Defaults to `lua_socket_keepalive_timeout`. -* `keepalive_pool` The maximum number of connections in the pool. Defaults to `lua_socket_pool_size`. +* `keepalive`: Set to `false` to disable keepalives and immediately close the connection. Defaults to `true`. +* `keepalive_timeout`: The maximal idle timeout (ms). Defaults to `lua_socket_keepalive_timeout`. +* `keepalive_pool`: The maximum number of connections in the pool. Defaults to `lua_socket_pool_size`. -In this mode, there is no need to connect manually first. The connection is made on your behalf, suiting cases where you simply need to grab a URI without too much hassle. +If the request is successful, `res` will contain the following fields: -Additionally there is no ability to stream the response body in this mode. If the request is successful, `res` will contain the following fields: +* `status`: The status code. +* `headers`: A table of headers. +* `body`: The entire response body as a string. -* `status` The status code. -* `headers` A table of headers. -* `body` The response body as a string. - -## request_pipeline +## request\_pipeline `syntax: responses, err = httpc:request_pipeline(params)` -This method works as per the [request](#request) method above, but `params` is instead a table of param tables. Each request is sent in order, and `responses` is returned as a table of response handles. For example: +This method works as per the [request](#request) method above, but `params` is instead a nested table of parameter tables. Each request is sent in order, and `responses` is returned as a table of response handles. For example: ```lua -local responses = httpc:request_pipeline{ - { - path = "/b", - }, - { - path = "/c", - }, - { - path = "/d", - } -} - -for i,r in ipairs(responses) do - if r.status then +local responses = httpc:request_pipeline({ + { path = "/b" }, + { path = "/c" }, + { path = "/d" }, +}) + +for _, r in ipairs(responses) do + if not r.status then + ngx.log(ngx.ERR, "socket read error") + break + end + ngx.say(r.status) ngx.say(r:read_body()) - end end ``` @@ -372,141 +302,161 @@ Be sure to test at least one field (such as status) before trying to use the oth # Response -## res.body_reader +## res.body\_reader The `body_reader` iterator can be used to stream the response body in chunk sizes of your choosing, as follows: ````lua local reader = res.body_reader +local buffer_size = 8192 repeat - local chunk, err = reader(8192) - if err then - ngx.log(ngx.ERR, err) - break - end - - if chunk then - -- process - end -until not chunk + local buffer, err = reader(buffer_size) + if err then + ngx.log(ngx.ERR, err) + break + end + + if buffer then + -- process + end +until not buffer ```` If the reader is called with no arguments, the behaviour depends on the type of connection. If the response is encoded as chunked, then the iterator will return the chunks as they arrive. If not, it will simply return the entire body. -Note that the size provided is actually a **maximum** size. So in the chunked transfer case, you may get chunks smaller than the size you ask, as a remainder of the actual HTTP chunks. +Note that the size provided is actually a **maximum** size. So in the chunked transfer case, you may get buffers smaller than the size you ask, as a remainder of the actual encoded chunks. -## res:read_body +## res:read\_body `syntax: body, err = res:read_body()` Reads the entire body into a local string. - -## res:read_trailers +## res:read\_trailers `syntax: res:read_trailers()` This merges any trailers underneath the `res.headers` table itself. Must be called after reading the body. +# Utility -# Proxy +## parse\_uri -There are two convenience methods for when one simply wishes to proxy the current request to the connected upstream, and safely send it downstream to the client, as a reverse proxy. A complete example: +`syntax: local scheme, host, port, path, query? = unpack(httpc:parse_uri(uri, query_in_path?))` + +This is a convenience function allowing one to more easily use the generic interface, when the input data is a URI. + +As of version `0.10`, the optional `query_in_path` parameter was added, which specifies whether the querystring is to be included in the `path` return value, or separately as its own return value. This defaults to `true` in order to maintain backwards compatibility. When set to `false`, `path` will only include the path, and `query` will contain the URI args, not including the `?` delimiter. + + +## get\_client\_body\_reader + +`syntax: reader, err = httpc:get_client_body_reader(chunksize?, sock?)` + +Returns an iterator function which can be used to read the downstream client request body in a streaming fashion. You may also specify an optional default chunksize (default is `65536`), or an already established socket in +place of the client request. + +Example: ```lua -local http = require "resty.http" -local httpc = http.new() +local req_reader = httpc:get_client_body_reader() +local buffer_size = 8192 -httpc:set_timeout(500) -local ok, err = httpc:connect(HOST, PORT) +repeat + local buffer, err = req_reader(buffer_size) + if err then + ngx.log(ngx.ERR, err) + break + end + + if buffer then + -- process + end +until not buffer +``` -if not ok then - ngx.log(ngx.ERR, err) - return -end +This iterator can also be used as the value for the body field in request params, allowing one to stream the request body into a proxied upstream request. + +```lua +local client_body_reader, err = httpc:get_client_body_reader() -httpc:set_timeout(2000) -httpc:proxy_response(httpc:proxy_request()) -httpc:set_keepalive() +local res, err = httpc:request({ + path = "/helloworld", + body = client_body_reader, +}) ``` +# Deprecated -## proxy_request +These features remain for backwards compatability, but may be removed in future releases. -`syntax: local res, err = httpc:proxy_request(request_body_chunk_size?)` +### TCP only connect -Performs a request using the current client request arguments, effectively proxying to the connected upstream. The request body will be read in a streaming fashion, according to `request_body_chunk_size` (see [documentation on the client body reader](#get_client_body_reader) below). +*The following versions of the `connect` method signature are deprecated in favour of the single `table` argument [documented above](#connect).* +`syntax: ok, err = httpc:connect(host, port, options_table?)` -## proxy_response +`syntax: ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?)` -`syntax: httpc:proxy_response(res, chunksize?)` +NOTE: the default pool name will only incorporate IP and port information so is unsafe to use in case of SSL and/or Proxy connections. Specify your own pool or, better still, do not use these signatures. -Sets the current response based on the given `res`. Ensures that hop-by-hop headers are not sent downstream, and will read the response according to `chunksize` (see [documentation on the body reader](#resbody_reader) above). +## connect\_proxy +`syntax: ok, err = httpc:connect_proxy(proxy_uri, scheme, host, port, proxy_authorization)` -# Utility +*Calling this method manually is no longer necessary since it is incorporated within [connect](#connect). It is retained for now for compatibility with users of the older [TCP only connect](#TCP_only_connect) syntax.* -## parse_uri +Attempts to connect to the web server through the given proxy server. The method accepts the following arguments: -`syntax: local scheme, host, port, path, query? = unpack(httpc:parse_uri(uri, query_in_path?))` +* `proxy_uri` - Full URI of the proxy server to use (e.g. `http://proxy.example.com:3128/`). Note: Only `http` protocol is supported. +* `scheme` - The protocol to use between the proxy server and the remote host (`http` or `https`). If `https` is specified as the scheme, `connect_proxy()` makes a `CONNECT` request to establish a TCP tunnel to the remote host through the proxy server. +* `host` - The hostname of the remote host to connect to. +* `port` - The port of the remote host to connect to. +* `proxy_authorization` - The `Proxy-Authorization` header value sent to the proxy server via `CONNECT` when the `scheme` is `https`. -This is a convenience function allowing one to more easily use the generic interface, when the input data is a URI. +If an error occurs during the connection attempt, this method returns `nil` with a string describing the error. If the connection was successfully established, the method returns `1`. -As of version `0.10`, the optional `query_in_path` parameter was added, which specifies whether the querystring is to be included in the `path` return value, or separately as its own return value. This defaults to `true` in order to maintain backwards compatibility. When set to `false`, `path` will only include the path, and `query` will contain the URI args, not including the `?` delimiter. +There's a few key points to keep in mind when using this api: +* If the scheme is `https`, you need to perform the TLS handshake with the remote server manually using the `ssl_handshake()` method before sending any requests through the proxy tunnel. +* If the scheme is `http`, you need to ensure that the requests you send through the connections conforms to [RFC 7230](https://tools.ietf.org/html/rfc7230) and especially [Section 5.3.2.](https://tools.ietf.org/html/rfc7230#section-5.3.2) which states that the request target must be in absolute form. In practice, this means that when you use `send_request()`, the `path` must be an absolute URI to the resource (e.g. `http://example.com/index.html` instead of just `/index.html`). -## get_client_body_reader +## ssl\_handshake -`syntax: reader, err = httpc:get_client_body_reader(chunksize?, sock?)` +`syntax: session, err = httpc:ssl_handshake(session, host, verify)` -Returns an iterator function which can be used to read the downstream client request body in a streaming fashion. You may also specify an optional default chunksize (default is `65536`), or an already established socket in -place of the client request. +*Calling this method manually is no longer necessary since it is incorporated within [connect](#connect). It is retained for now for compatibility with users of the older [TCP only connect](#TCP_only_connect) syntax.* -Example: +See [OpenResty docs](https://github.com/openresty/lua-nginx-module#ngxsockettcp). -```lua -local req_reader = httpc:get_client_body_reader() +## proxy\_request / proxy\_response -repeat - local chunk, err = req_reader(8192) - if err then - ngx.log(ngx.ERR, err) - break - end - - if chunk then - -- process - end -until not chunk -``` +*These two convenience methods were intended simply to demonstrate a common use case of implementing reverse proxying, and the author regrets their inclusion in the module. Users are encouraged to roll their own rather than depend on these functions, which may be removed in a subsequent release.* -This iterator can also be used as the value for the body field in request params, allowing one to stream the request body into a proxied upstream request. +### proxy\_request -```lua -local client_body_reader, err = httpc:get_client_body_reader() +`syntax: local res, err = httpc:proxy_request(request_body_chunk_size?)` -local res, err = httpc:request{ - path = "/helloworld", - body = client_body_reader, -} -``` +Performs a request using the current client request arguments, effectively proxying to the connected upstream. The request body will be read in a streaming fashion, according to `request_body_chunk_size` (see [documentation on the client body reader](#get_client_body_reader) below). -If `sock` is specified, -# Author +### proxy\_response + +`syntax: httpc:proxy_response(res, chunksize?)` + +Sets the current response based on the given `res`. Ensures that hop-by-hop headers are not sent downstream, and will read the response according to `chunksize` (see [documentation on the body reader](#resbody_reader) above). -James Hurst -Originally started life based on https://github.com/bakins/lua-resty-http-simple. Cosocket docs and implementation borrowed from the other lua-resty-* cosocket modules. +# Author +James Hurst , with contributions from @hamishforbes, @Tieske, @bungle et al. # Licence This module is licensed under the 2-clause BSD license. -Copyright (c) 2013-2016, James Hurst +Copyright (c) James Hurst All rights reserved. From b9548923e7ca021c446ce1bc4cda1c58aede8179 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 2 Mar 2021 16:48:17 +0000 Subject: [PATCH 062/130] Mark some functions as deprecated and WARN if used --- lib/resty/http.lua | 127 +++++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 44200990..d2322b49 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -21,7 +21,7 @@ local ngx_re_find = ngx.re.find local ngx_log = ngx.log local ngx_DEBUG = ngx.DEBUG local ngx_ERR = ngx.ERR -local ngx_var = ngx.var +local ngx_WARN = ngx.WARN local ngx_print = ngx.print local ngx_header = ngx.header local co_yield = coroutine.yield @@ -162,18 +162,6 @@ function _M.set_timeouts(self, connect_timeout, send_timeout, read_timeout) return sock:settimeouts(connect_timeout, send_timeout, read_timeout) end - -function _M.ssl_handshake(self, ...) - local sock = self.sock - if not sock then - return nil, "not initialized" - end - - self.ssl = true - - return sock:sslhandshake(...) -end - do local aio_connect = require "resty.http_connect" -- Function signatures to support: @@ -959,50 +947,6 @@ function _M.get_client_body_reader(_, chunksize, sock) end -function _M.proxy_request(self, chunksize) - return self:request({ - method = ngx_req_get_method(), - path = ngx_re_gsub(ngx_var.uri, "\\s", "%20", "jo") .. ngx_var.is_args .. (ngx_var.query_string or ""), - body = self:get_client_body_reader(chunksize), - headers = ngx_req_get_headers(), - }) -end - - -function _M.proxy_response(_, response, chunksize) - if not response then - ngx_log(ngx_ERR, "no response provided") - return - end - - ngx.status = response.status - - -- Filter out hop-by-hop headeres - for k, v in pairs(response.headers) do - if not HOP_BY_HOP_HEADERS[str_lower(k)] then - ngx_header[k] = v - end - end - - local reader = response.body_reader - repeat - local chunk, err = reader(chunksize) - if err then - ngx_log(ngx_ERR, err) - break - end - - if chunk then - local res, err = ngx_print(chunk) - if not res then - ngx_log(ngx_ERR, err) - break - end - end - until not chunk -end - - function _M.set_proxy_options(self, opts) -- TODO: parse and cache these options, instead of parsing them -- on each request over and over again (lru-cache on module level) @@ -1061,7 +1005,28 @@ function _M.get_proxy_uri(self, scheme, host) end +-- ---------------------------------------------------------------------------- +-- The following functions are considered DEPRECATED and may be REMOVED in +-- future releases. Please see the notes in `README.md`. +-- ---------------------------------------------------------------------------- + +function _M.ssl_handshake(self, ...) + ngx_log(ngx_WARN, "Use of deprecated function `ssl_handshake`") + + local sock = self.sock + if not sock then + return nil, "not initialized" + end + + self.ssl = true + + return sock:sslhandshake(...) +end + + function _M.connect_proxy(self, proxy_uri, scheme, host, port, proxy_authorization) + ngx_log(ngx_WARN, "Use of deprecated function `connect_proxy`") + -- Parse the provided proxy URI local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) if not parsed_proxy_uri then @@ -1110,4 +1075,52 @@ function _M.connect_proxy(self, proxy_uri, scheme, host, port, proxy_authorizati end +function _M.proxy_request(self, chunksize) + ngx_log(ngx_WARN, "Use of deprecated function `proxy_request`") + + return self:request({ + method = ngx_req_get_method(), + path = ngx_re_gsub(ngx_var.uri, "\\s", "%20", "jo") .. ngx_var.is_args .. (ngx_var.query_string or ""), + body = self:get_client_body_reader(chunksize), + headers = ngx_req_get_headers(), + }) +end + + +function _M.proxy_response(_, response, chunksize) + ngx_log(ngx_WARN, "Use of deprecated function `proxy_response`") + + if not response then + ngx_log(ngx_ERR, "no response provided") + return + end + + ngx.status = response.status + + -- Filter out hop-by-hop headeres + for k, v in pairs(response.headers) do + if not HOP_BY_HOP_HEADERS[str_lower(k)] then + ngx_header[k] = v + end + end + + local reader = response.body_reader + repeat + local chunk, err = reader(chunksize) + if err then + ngx_log(ngx_ERR, err) + break + end + + if chunk then + local res, err = ngx_print(chunk) + if not res then + ngx_log(ngx_ERR, err) + break + end + end + until not chunk +end + + return _M From c239936aaf38e5765ee1ac403bcedc5702225647 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 2 Mar 2021 16:52:04 +0000 Subject: [PATCH 063/130] Fix markdown link syntax --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6e1cbb99..fe3f14fe 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Attempts to connect to the web server while incorporating the following activiti - SSL handshake - HTTP proxy configuration -In doing so it will create a distinct connection pool name that is safe to use with SSL and / or proxy based connections, and as such this syntax is strongly recommended over the original [deprecated connection syntax](#TCP_only_connect). +In doing so it will create a distinct connection pool name that is safe to use with SSL and / or proxy based connections, and as such this syntax is strongly recommended over the original [deprecated connection syntax](#TCP-only-connect). The options table has the following fields: @@ -210,7 +210,7 @@ Configure an HTTP proxy to be used with this client instance. The `opts` table e * `https_proxy_authorization`: as `http_proxy_authorization` but for use with `https_proxy` (since with HTTPS the authorisation is done when connecting, this one cannot be overridden by passing the `Proxy-Authorization` request header). * `no_proxy`: a comma separated list of hosts that should not be proxied. -Note that this method has no effect when using the deprecated [TCP only connect](#TCP_only_connect) connection syntax. +Note that this method has no effect when using the deprecated [TCP only connect](#TCP-only-connect) connection syntax. ## get\_reused\_times @@ -405,7 +405,7 @@ NOTE: the default pool name will only incorporate IP and port information so is `syntax: ok, err = httpc:connect_proxy(proxy_uri, scheme, host, port, proxy_authorization)` -*Calling this method manually is no longer necessary since it is incorporated within [connect](#connect). It is retained for now for compatibility with users of the older [TCP only connect](#TCP_only_connect) syntax.* +*Calling this method manually is no longer necessary since it is incorporated within [connect](#connect). It is retained for now for compatibility with users of the older [TCP only connect](#TCP-only-connect) syntax.* Attempts to connect to the web server through the given proxy server. The method accepts the following arguments: @@ -426,7 +426,7 @@ There's a few key points to keep in mind when using this api: `syntax: session, err = httpc:ssl_handshake(session, host, verify)` -*Calling this method manually is no longer necessary since it is incorporated within [connect](#connect). It is retained for now for compatibility with users of the older [TCP only connect](#TCP_only_connect) syntax.* +*Calling this method manually is no longer necessary since it is incorporated within [connect](#connect). It is retained for now for compatibility with users of the older [TCP only connect](#TCP-only-connect) syntax.* See [OpenResty docs](https://github.com/openresty/lua-nginx-module#ngxsockettcp). From dd9c9a03d847037a92237f722eff642cfa94f643 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 2 Mar 2021 16:53:59 +0000 Subject: [PATCH 064/130] Make link description more consistent --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fe3f14fe..05b9ffef 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Attempts to connect to the web server while incorporating the following activiti - SSL handshake - HTTP proxy configuration -In doing so it will create a distinct connection pool name that is safe to use with SSL and / or proxy based connections, and as such this syntax is strongly recommended over the original [deprecated connection syntax](#TCP-only-connect). +In doing so it will create a distinct connection pool name that is safe to use with SSL and / or proxy based connections, and as such this syntax is strongly recommended over the original (now deprecated) [TCP only connection syntax](#TCP-only-connect). The options table has the following fields: From acf675cf352e4839d13dedb9a527e76739b4234d Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 2 Mar 2021 17:00:35 +0000 Subject: [PATCH 065/130] Add warning if using old `connect` syntax --- lib/resty/http.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index d2322b49..e8c34d7e 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -180,6 +180,8 @@ do end function _M.tcp_only_connect(self, ...) + ngx_log(ngx_WARN, "Use of deprecated `connect` method signature") + local sock = self.sock if not sock then return nil, "not initialized" From 52a87d4aeff6ca4a651da1829561e02e2e3a9a59 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 2 Mar 2021 17:32:31 +0000 Subject: [PATCH 066/130] Revert accidental local removal of ngx_var --- lib/resty/http.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index e8c34d7e..d3c80c69 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -22,6 +22,7 @@ local ngx_log = ngx.log local ngx_DEBUG = ngx.DEBUG local ngx_ERR = ngx.ERR local ngx_WARN = ngx.WARN +local ngx_var = ngx.var local ngx_print = ngx.print local ngx_header = ngx.header local co_yield = coroutine.yield From e80cca772ac34294c5ce511ca4db50669c18d8e1 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 2 Mar 2021 17:59:30 +0000 Subject: [PATCH 067/130] Update tests to use new connect syntax --- t/01-basic.t | 53 ++++++++++++++++++++------- t/02-chunked.t | 24 ++++++++++--- t/03-requestbody.t | 24 ++++++++++--- t/04-trailers.t | 12 +++++-- t/05-stream.t | 54 +++++++++++++++++++++++----- t/07-keepalive.t | 79 +++++++++++++++++++++++++++++++++-------- t/08-pipeline.t | 12 +++++-- t/10-clientbodyreader.t | 6 +++- t/11-proxy.t | 28 ++++++++++++--- 9 files changed, 239 insertions(+), 53 deletions(-) diff --git a/t/01-basic.t b/t/01-basic.t index 99ad1e69..5f5402e9 100644 --- a/t/01-basic.t +++ b/t/01-basic.t @@ -33,7 +33,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } local res, err = httpc:request{ path = "/b" @@ -64,7 +68,11 @@ OK content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } local res, err = httpc:request{ version = 1.0, @@ -96,7 +104,11 @@ OK content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + local ok, err = httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } local res, err = httpc:request{ path = "/b" @@ -133,7 +145,11 @@ OK content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } local res, err = httpc:request{ path = "/b" @@ -167,7 +183,11 @@ x-value content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } local res, err = httpc:request{ query = { @@ -212,7 +232,11 @@ X-Header-B: 2 content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } local res, err = httpc:request{ method = "HEAD", @@ -245,15 +269,16 @@ GET /a content_by_lua ' local http = require "resty.http" - local res, err = http:connect("127.0.0.1", 1984) + local res, err = http:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } if not res then ngx.say(err) end local res, err = http:set_timeout(500) if not res then ngx.say(err) end - local res, err = http:ssl_handshake() - if not res then ngx.say(err) end - local res, err = http:set_keepalive() if not res then ngx.say(err) end @@ -272,7 +297,6 @@ not initialized not initialized not initialized not initialized -not initialized --- no_error_log [error] [warn] @@ -382,8 +406,11 @@ scheme: http, host: example.com, port: 80, path: /foo/bar?a=1&b=2 -- Create a TCP connection and return an raw HTTP-response because -- there is no way to set an empty header value in nginx. - assert(httpc:connect("127.0.0.1", 12345), - "connect should return positively") + assert(httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = 12345, + }, "connect should return positively") local res = httpc:request({ path = "/b" }) if res.headers["X-Header-Empty"] == "" then diff --git a/t/02-chunked.t b/t/02-chunked.t index cb1f6f3a..dda47c4b 100644 --- a/t/02-chunked.t +++ b/t/02-chunked.t @@ -31,7 +31,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b" @@ -70,7 +74,11 @@ GET /a content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b" @@ -126,7 +134,11 @@ GET /a content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b" @@ -170,7 +182,11 @@ GET /a content_by_lua_block { local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b" diff --git a/t/03-requestbody.t b/t/03-requestbody.t index f0168327..6ead034f 100644 --- a/t/03-requestbody.t +++ b/t/03-requestbody.t @@ -33,7 +33,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ body = "a=1&b=2&c=3", @@ -74,7 +78,11 @@ c: 3 content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ method = "POST", @@ -119,7 +127,11 @@ c: 3 content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ body = "a=1&b=2&c=3", @@ -161,7 +173,11 @@ c: 3 content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", diff --git a/t/04-trailers.t b/t/04-trailers.t index 7b6a6e98..295d0575 100644 --- a/t/04-trailers.t +++ b/t/04-trailers.t @@ -33,7 +33,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -101,7 +105,11 @@ OK content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", diff --git a/t/05-stream.t b/t/05-stream.t index 9228bbf8..544e1e79 100644 --- a/t/05-stream.t +++ b/t/05-stream.t @@ -33,7 +33,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -81,7 +85,11 @@ chunked content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -132,7 +140,11 @@ nil content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -181,7 +193,11 @@ Buffer size not specified, bailing content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -233,7 +249,11 @@ nil content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -287,7 +307,11 @@ nil content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -350,7 +374,11 @@ GET /a content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", @@ -466,7 +494,11 @@ foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarba content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local reader, err = httpc:get_client_body_reader(64) @@ -509,7 +541,11 @@ foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarba content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b", diff --git a/t/07-keepalive.t b/t/07-keepalive.t index d781f1f4..64659c92 100644 --- a/t/07-keepalive.t +++ b/t/07-keepalive.t @@ -77,12 +77,20 @@ keep-alive } ) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) httpc:set_keepalive() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) '; } @@ -108,7 +116,11 @@ GET /a content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ path = "/b" @@ -119,7 +131,11 @@ GET /a ngx.say(res.headers["Connection"]) ngx.say(httpc:set_keepalive()) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) '; } @@ -146,7 +162,11 @@ keep-alive content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local res, err = httpc:request{ version = 1.0, @@ -163,12 +183,20 @@ keep-alive ngx.say(r) ngx.say(e) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) httpc:set_keepalive() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) '; } @@ -197,7 +225,11 @@ connection must be closed content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", 12345) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = 12345, + }) local res, err = httpc:request{ version = 1.0, @@ -213,12 +245,20 @@ connection must be closed ngx.say(r) ngx.say(e) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) httpc:set_keepalive() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) '; } @@ -336,8 +376,11 @@ keep-alive -- Create a TCP connection and return an raw HTTP-response because -- there is no way to set an "Connection: Upgrade, close" header in nginx. - assert(httpc:connect("127.0.0.1", 12345), - "connect should return positively") + assert(httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = 12345, + }), "connect should return positively") local res = httpc:request({ version = 1.1, @@ -353,12 +396,20 @@ keep-alive ngx.say(r) ngx.say(e) - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) httpc:set_keepalive() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) ngx.say(httpc:get_reused_times()) } } diff --git a/t/08-pipeline.t b/t/08-pipeline.t index 3fa4fecf..bd92bd43 100644 --- a/t/08-pipeline.t +++ b/t/08-pipeline.t @@ -33,7 +33,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) local responses = httpc:request_pipeline{ { @@ -101,7 +105,11 @@ D content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) httpc:set_timeout(1) local responses = httpc:request_pipeline{ diff --git a/t/10-clientbodyreader.t b/t/10-clientbodyreader.t index 15542346..1b364518 100644 --- a/t/10-clientbodyreader.t +++ b/t/10-clientbodyreader.t @@ -36,7 +36,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port, + }) local res, err = httpc:request{ path = "/c", diff --git a/t/11-proxy.t b/t/11-proxy.t index fb40252d..31a9427f 100644 --- a/t/11-proxy.t +++ b/t/11-proxy.t @@ -35,7 +35,11 @@ __DATA__ content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port, + }) httpc:proxy_response(httpc:proxy_request()) httpc:set_keepalive() '; @@ -56,6 +60,7 @@ X-Test: foo --- error_code: 200 --- no_error_log [error] +--- error_log [warn] @@ -67,7 +72,11 @@ X-Test: foo content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port, + }) httpc:proxy_response(httpc:proxy_request()) httpc:set_keepalive() '; @@ -93,6 +102,7 @@ X-Test: foo --- error_code: 404 --- no_error_log [error] +--- error_log [warn] @@ -104,7 +114,11 @@ X-Test: foo content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port, + }) httpc:proxy_response(httpc:proxy_request()) httpc:set_keepalive() '; @@ -124,6 +138,7 @@ OK --- error_code: 200 --- no_error_log [error] +--- error_log [warn] @@ -135,7 +150,11 @@ OK content_by_lua ' local http = require "resty.http" local httpc = http.new() - httpc:connect("127.0.0.1", ngx.var.server_port) + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port, + }) httpc:proxy_response(httpc:proxy_request()) httpc:set_keepalive() '; @@ -156,4 +175,5 @@ X-Test: foo --- error_code: 200 --- no_error_log [error] +--- error_log [warn] From 87dced906eb1e868d76c02612a246e48b4bbf086 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Wed, 3 Mar 2021 14:15:54 +0000 Subject: [PATCH 068/130] Use DEBUG level logging for deprecation warnings In time, this can be raised but for now DEBUG should be enough to prompt developers. --- lib/resty/http.lua | 10 +++++----- t/11-proxy.t | 11 ++++++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index d3c80c69..7c7c4d01 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -181,7 +181,7 @@ do end function _M.tcp_only_connect(self, ...) - ngx_log(ngx_WARN, "Use of deprecated `connect` method signature") + ngx_log(ngx_DEBUG, "Use of deprecated `connect` method signature") local sock = self.sock if not sock then @@ -1014,7 +1014,7 @@ end -- ---------------------------------------------------------------------------- function _M.ssl_handshake(self, ...) - ngx_log(ngx_WARN, "Use of deprecated function `ssl_handshake`") + ngx_log(ngx_DEBUG, "Use of deprecated function `ssl_handshake`") local sock = self.sock if not sock then @@ -1028,7 +1028,7 @@ end function _M.connect_proxy(self, proxy_uri, scheme, host, port, proxy_authorization) - ngx_log(ngx_WARN, "Use of deprecated function `connect_proxy`") + ngx_log(ngx_DEBUG, "Use of deprecated function `connect_proxy`") -- Parse the provided proxy URI local parsed_proxy_uri, err = self:parse_uri(proxy_uri, false) @@ -1079,7 +1079,7 @@ end function _M.proxy_request(self, chunksize) - ngx_log(ngx_WARN, "Use of deprecated function `proxy_request`") + ngx_log(ngx_DEBUG, "Use of deprecated function `proxy_request`") return self:request({ method = ngx_req_get_method(), @@ -1091,7 +1091,7 @@ end function _M.proxy_response(_, response, chunksize) - ngx_log(ngx_WARN, "Use of deprecated function `proxy_response`") + ngx_log(ngx_DEBUG, "Use of deprecated function `proxy_response`") if not response then ngx_log(ngx_ERR, "no response provided") diff --git a/t/11-proxy.t b/t/11-proxy.t index 31a9427f..9da6d271 100644 --- a/t/11-proxy.t +++ b/t/11-proxy.t @@ -60,8 +60,9 @@ X-Test: foo --- error_code: 200 --- no_error_log [error] ---- error_log [warn] +--- error_log +[debug] === TEST 2: Proxy POST request and response @@ -104,6 +105,8 @@ X-Test: foo [error] --- error_log [warn] +--- error_log +[debug] === TEST 3: Proxy multiple headers @@ -138,8 +141,9 @@ OK --- error_code: 200 --- no_error_log [error] ---- error_log [warn] +--- error_log +[debug] === TEST 4: Proxy still works with spaces in URI @@ -175,5 +179,6 @@ X-Test: foo --- error_code: 200 --- no_error_log [error] ---- error_log [warn] +--- error_log +[debug] From 945b6765524f748c7e6b937127fedad2ef69b321 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Wed, 3 Mar 2021 14:21:11 +0000 Subject: [PATCH 069/130] Remove fixed sized test plans --- t/01-basic.t | 4 +--- t/03-requestbody.t | 4 +--- t/04-trailers.t | 4 +--- t/05-stream.t | 4 +--- t/07-keepalive.t | 4 +--- t/08-pipeline.t | 4 +--- t/09-ssl.t | 4 +--- t/10-clientbodyreader.t | 4 +--- t/11-proxy.t | 4 +--- t/13-default-path.t | 4 +--- t/14-host-header.t | 4 +--- t/16-http-proxy.t | 4 +--- 12 files changed, 12 insertions(+), 36 deletions(-) diff --git a/t/01-basic.t b/t/01-basic.t index 5f5402e9..15e36c9d 100644 --- a/t/01-basic.t +++ b/t/01-basic.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4) + 1; - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/03-requestbody.t b/t/03-requestbody.t index 6ead034f..1e28c83a 100644 --- a/t/03-requestbody.t +++ b/t/03-requestbody.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/04-trailers.t b/t/04-trailers.t index 295d0575..bb9d8009 100644 --- a/t/04-trailers.t +++ b/t/04-trailers.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/05-stream.t b/t/05-stream.t index 544e1e79..7b90d2b2 100644 --- a/t/05-stream.t +++ b/t/05-stream.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4) - 1; - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/07-keepalive.t b/t/07-keepalive.t index 64659c92..573bbf0d 100644 --- a/t/07-keepalive.t +++ b/t/07-keepalive.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/08-pipeline.t b/t/08-pipeline.t index bd92bd43..4384bc8c 100644 --- a/t/08-pipeline.t +++ b/t/08-pipeline.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/09-ssl.t b/t/09-ssl.t index bac29d06..7c1e5da0 100644 --- a/t/09-ssl.t +++ b/t/09-ssl.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/10-clientbodyreader.t b/t/10-clientbodyreader.t index 1b364518..21d0b1e0 100644 --- a/t/10-clientbodyreader.t +++ b/t/10-clientbodyreader.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/11-proxy.t b/t/11-proxy.t index 9da6d271..a56203c9 100644 --- a/t/11-proxy.t +++ b/t/11-proxy.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 5); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/13-default-path.t b/t/13-default-path.t index fe68ac95..b5bb52e9 100644 --- a/t/13-default-path.t +++ b/t/13-default-path.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 3); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/14-host-header.t b/t/14-host-header.t index 27d8747f..685813c0 100644 --- a/t/14-host-header.t +++ b/t/14-host-header.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 3); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; diff --git a/t/16-http-proxy.t b/t/16-http-proxy.t index 385a0ac9..d5912181 100644 --- a/t/16-http-proxy.t +++ b/t/16-http-proxy.t @@ -1,8 +1,6 @@ -use Test::Nginx::Socket; +use Test::Nginx::Socket 'no_plan'; use Cwd qw(cwd); -plan tests => repeat_each() * (blocks() * 4); - my $pwd = cwd(); $ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; From 02b8815e4fc019464438f9391c7e97046aab91dd Mon Sep 17 00:00:00 2001 From: James Hurst Date: Wed, 3 Mar 2021 14:27:24 +0000 Subject: [PATCH 070/130] Add a test using the old connect syntax --- t/17-deprecated.t | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 t/17-deprecated.t diff --git a/t/17-deprecated.t b/t/17-deprecated.t new file mode 100644 index 00000000..4a0156c4 --- /dev/null +++ b/t/17-deprecated.t @@ -0,0 +1,58 @@ +use Test::Nginx::Socket 'no_plan'; +use Cwd qw(cwd); + +my $pwd = cwd(); + +$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; +$ENV{TEST_NGINX_PWD} ||= $pwd; +$ENV{TEST_COVERAGE} ||= 0; + +our $HttpConfig = qq{ + lua_package_path "$pwd/lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; + error_log logs/error.log debug; + resolver 8.8.8.8; + + init_by_lua_block { + if $ENV{TEST_COVERAGE} == 1 then + jit.off() + require("luacov.runner").init() + end + } +}; + +no_long_string(); +run_tests(); + +__DATA__ +=== TEST 1: Old connect syntax still works +--- http_config eval: $::HttpConfig +--- config + location /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + local ok, err = httpc:connect("127.0.0.1", ngx.var.server_port) + assert(ok, err) + + local res, err = httpc:request{ + path = "/b" + } + + ngx.status = res.status + ngx.print(res:read_body()) + + httpc:close() + } + } + location = /b { + echo "OK"; + } +--- request +GET /a +--- response_body +OK +--- no_error_log +[error] +[warn] +--- error_log +[debug] From c2faeb4ae91fc4167280f7cc1cd78cf169da4ef9 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 4 Mar 2021 09:18:45 +0000 Subject: [PATCH 071/130] Ensure partial chunks are printed on error --- lib/resty/http.lua | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 7c7c4d01..5aeb379c 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -1109,19 +1109,21 @@ function _M.proxy_response(_, response, chunksize) local reader = response.body_reader repeat - local chunk, err = reader(chunksize) + local chunk, read_err = reader(chunksize) if err then - ngx_log(ngx_ERR, err) - break + ngx_log(ngx_ERR, read_err) end if chunk then - local res, err = ngx_print(chunk) + local res, print_err = ngx_print(chunk) if not res then - ngx_log(ngx_ERR, err) - break + ngx_log(ngx_ERR, print_err) end end + + if read_err or print_err then + break + end until not chunk end From 654ea1589a9cb62ebcbb626f27b81da0883b58e4 Mon Sep 17 00:00:00 2001 From: Mikhail Kirichenko Date: Tue, 25 Apr 2017 15:33:19 +0300 Subject: [PATCH 072/130] Support literal ipv6 addresses --- lib/resty/http.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 5aeb379c..ce140baf 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -256,7 +256,7 @@ end function _M.parse_uri(_, uri, query_in_path) if query_in_path == nil then query_in_path = true end - local m, err = ngx_re_match(uri, [[^(?:(http[s]?):)?//([^:/\?]+)(?::(\d+))?([^\?]*)\??(.*)]], "jo") + local m, err = ngx_re_match(uri, [[^(?:(http[s]?):)?//((?:[^\[\]:/\?]+)|(?:\[.+\]))(?::(\d+))?([^\?]*)\??(.*)]], "jo") if not m then if err then From f986d8eb8efb3bd71ce767ac2ac3d2c3d2d51fa8 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 4 Mar 2021 10:10:59 +0000 Subject: [PATCH 073/130] Move parse_uri tests to separate file and add ipv6 tests --- t/01-basic.t | 95 ---------------------------- t/18-parse_uri.t | 157 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 95 deletions(-) create mode 100644 t/18-parse_uri.t diff --git a/t/01-basic.t b/t/01-basic.t index 15e36c9d..0439a8a0 100644 --- a/t/01-basic.t +++ b/t/01-basic.t @@ -300,101 +300,6 @@ not initialized [warn] -=== TEST 9: Parse URI errors if malformed ---- http_config eval: $::HttpConfig ---- config - location = /a { - content_by_lua ' - local http = require("resty.http").new() - local parts, err = http:parse_uri("http:///example.com") - if not parts then ngx.say(err) end - '; - } ---- request -GET /a ---- response_body -bad uri: http:///example.com ---- no_error_log -[error] -[warn] - - -=== TEST 10: Parse URI fills in defaults correctly ---- http_config eval: $::HttpConfig ---- config - location = /a { - content_by_lua ' - local http = require("resty.http").new() - - local function test_uri(uri) - local scheme, host, port, path, query = unpack(http:parse_uri(uri, false)) - ngx.say("scheme: ", scheme, ", host: ", host, ", port: ", port, ", path: ", path, ", query: ", query) - end - - test_uri("http://example.com") - test_uri("http://example.com/") - test_uri("https://example.com/foo/bar") - test_uri("https://example.com/foo/bar?a=1&b=2") - test_uri("http://example.com?a=1&b=2") - test_uri("//example.com") - test_uri("//example.com?a=1&b=2") - test_uri("//example.com/foo/bar?a=1&b=2") - '; - } ---- request -GET /a ---- response_body -scheme: http, host: example.com, port: 80, path: /, query: -scheme: http, host: example.com, port: 80, path: /, query: -scheme: https, host: example.com, port: 443, path: /foo/bar, query: -scheme: https, host: example.com, port: 443, path: /foo/bar, query: a=1&b=2 -scheme: http, host: example.com, port: 80, path: /, query: a=1&b=2 -scheme: http, host: example.com, port: 80, path: /, query: -scheme: http, host: example.com, port: 80, path: /, query: a=1&b=2 -scheme: http, host: example.com, port: 80, path: /foo/bar, query: a=1&b=2 ---- no_error_log -[error] -[warn] - - -=== TEST 11: Parse URI fills in defaults correctly, using backwards compatible mode ---- http_config eval: $::HttpConfig ---- config - location = /a { - content_by_lua ' - local http = require("resty.http").new() - - local function test_uri(uri) - local scheme, host, port, path, query = unpack(http:parse_uri(uri)) - ngx.say("scheme: ", scheme, ", host: ", host, ", port: ", port, ", path: ", path) - end - - test_uri("http://example.com") - test_uri("http://example.com/") - test_uri("https://example.com/foo/bar") - test_uri("https://example.com/foo/bar?a=1&b=2") - test_uri("http://example.com?a=1&b=2") - test_uri("//example.com") - test_uri("//example.com?a=1&b=2") - test_uri("//example.com/foo/bar?a=1&b=2") - '; - } ---- request -GET /a ---- response_body -scheme: http, host: example.com, port: 80, path: / -scheme: http, host: example.com, port: 80, path: / -scheme: https, host: example.com, port: 443, path: /foo/bar -scheme: https, host: example.com, port: 443, path: /foo/bar?a=1&b=2 -scheme: http, host: example.com, port: 80, path: /?a=1&b=2 -scheme: http, host: example.com, port: 80, path: / -scheme: http, host: example.com, port: 80, path: /?a=1&b=2 -scheme: http, host: example.com, port: 80, path: /foo/bar?a=1&b=2 ---- no_error_log -[error] -[warn] - - === TEST 12: Allow empty HTTP header values (RFC7230) --- http_config eval: $::HttpConfig --- config diff --git a/t/18-parse_uri.t b/t/18-parse_uri.t new file mode 100644 index 00000000..2343d9b8 --- /dev/null +++ b/t/18-parse_uri.t @@ -0,0 +1,157 @@ +use Test::Nginx::Socket 'no_plan'; +use Cwd qw(cwd); + +my $pwd = cwd(); + +$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8'; +$ENV{TEST_COVERAGE} ||= 0; + +our $HttpConfig = qq{ + lua_package_path "$pwd/lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; + error_log logs/error.log debug; + + init_by_lua_block { + if $ENV{TEST_COVERAGE} == 1 then + jit.off() + require("luacov.runner").init() + end + } +}; + +no_long_string(); +#no_diff(); + +run_tests(); + +__DATA__ +=== TEST 1: Parse URI errors if malformed +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local http = require("resty.http").new() + local parts, err = http:parse_uri("http:///example.com") + if not parts then ngx.say(err) end + '; + } +--- request +GET /a +--- response_body +bad uri: http:///example.com +--- no_error_log +[error] +[warn] + + +=== TEST 2: Parse URI fills in defaults correctly +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local http = require("resty.http").new() + + local function test_uri(uri) + local scheme, host, port, path, query = unpack(http:parse_uri(uri, false)) + ngx.say("scheme: ", scheme, ", host: ", host, ", port: ", port, ", path: ", path, ", query: ", query) + end + + test_uri("http://example.com") + test_uri("http://example.com/") + test_uri("https://example.com/foo/bar") + test_uri("https://example.com/foo/bar?a=1&b=2") + test_uri("http://example.com?a=1&b=2") + test_uri("//example.com") + test_uri("//example.com?a=1&b=2") + test_uri("//example.com/foo/bar?a=1&b=2") + '; + } +--- request +GET /a +--- response_body +scheme: http, host: example.com, port: 80, path: /, query: +scheme: http, host: example.com, port: 80, path: /, query: +scheme: https, host: example.com, port: 443, path: /foo/bar, query: +scheme: https, host: example.com, port: 443, path: /foo/bar, query: a=1&b=2 +scheme: http, host: example.com, port: 80, path: /, query: a=1&b=2 +scheme: http, host: example.com, port: 80, path: /, query: +scheme: http, host: example.com, port: 80, path: /, query: a=1&b=2 +scheme: http, host: example.com, port: 80, path: /foo/bar, query: a=1&b=2 +--- no_error_log +[error] +[warn] + + +=== TEST 3: Parse URI fills in defaults correctly, using backwards compatible mode +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local http = require("resty.http").new() + + local function test_uri(uri) + local scheme, host, port, path, query = unpack(http:parse_uri(uri)) + ngx.say("scheme: ", scheme, ", host: ", host, ", port: ", port, ", path: ", path) + end + + test_uri("http://example.com") + test_uri("http://example.com/") + test_uri("https://example.com/foo/bar") + test_uri("https://example.com/foo/bar?a=1&b=2") + test_uri("http://example.com?a=1&b=2") + test_uri("//example.com") + test_uri("//example.com?a=1&b=2") + test_uri("//example.com/foo/bar?a=1&b=2") + '; + } +--- request +GET /a +--- response_body +scheme: http, host: example.com, port: 80, path: / +scheme: http, host: example.com, port: 80, path: / +scheme: https, host: example.com, port: 443, path: /foo/bar +scheme: https, host: example.com, port: 443, path: /foo/bar?a=1&b=2 +scheme: http, host: example.com, port: 80, path: /?a=1&b=2 +scheme: http, host: example.com, port: 80, path: / +scheme: http, host: example.com, port: 80, path: /?a=1&b=2 +scheme: http, host: example.com, port: 80, path: /foo/bar?a=1&b=2 +--- no_error_log +[error] +[warn] + + +=== TEST 4: IPv6 notation +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local http = require("resty.http").new() + + local function test_uri(uri) + local scheme, host, port, path, query = unpack(http:parse_uri(uri, false)) + ngx.say("scheme: ", scheme, ", host: ", host, ", port: ", port, ", path: ", path, ", query: ", query) + end + + test_uri("http://[::1]") + test_uri("http://[::1]/") + test_uri("https://[::1]/foo/bar") + test_uri("https://[::1]/foo/bar?a=1&b=2") + test_uri("http://[::1]?a=1&b=2") + test_uri("//[0:0:0:0:0:0:0:0]") + test_uri("//[0:0:0:0:0:0:0:0]?a=1&b=2") + test_uri("//[0:0:0:0:0:0:0:0]/foo/bar?a=1&b=2") + '; + } +--- request +GET /a +--- response_body +scheme: http, host: [::1], port: 80, path: /, query: +scheme: http, host: [::1], port: 80, path: /, query: +scheme: https, host: [::1], port: 443, path: /foo/bar, query: +scheme: https, host: [::1], port: 443, path: /foo/bar, query: a=1&b=2 +scheme: http, host: [::1], port: 80, path: /, query: a=1&b=2 +scheme: http, host: [0:0:0:0:0:0:0:0], port: 80, path: /, query: +scheme: http, host: [0:0:0:0:0:0:0:0], port: 80, path: /, query: a=1&b=2 +scheme: http, host: [0:0:0:0:0:0:0:0], port: 80, path: /foo/bar, query: a=1&b=2 +--- no_error_log +[error] +[warn] From 6fa99394c3a8c0d6edaca91244e68171887a6200 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 4 Mar 2021 20:23:28 +0000 Subject: [PATCH 074/130] Ensure luacheck runs on CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9ee0825c..7d701d70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,4 +2,4 @@ services: - docker script: - - docker run --rm -v "$PWD:/lua-resty-http" -w /lua-resty-http ledgetech/test-runner:latest /bin/bash -c "make coverage" + - docker run --rm -v "$PWD:/lua-resty-http" -w /lua-resty-http ledgetech/test-runner:latest /bin/bash -c "make check; make coverage" From e102a38d1641da306021563c4d66468fdf7c0d2a Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 4 Mar 2021 20:25:16 +0000 Subject: [PATCH 075/130] Remove unused module local --- lib/resty/http.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index ce140baf..d9869dad 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -21,7 +21,6 @@ local ngx_re_find = ngx.re.find local ngx_log = ngx.log local ngx_DEBUG = ngx.DEBUG local ngx_ERR = ngx.ERR -local ngx_WARN = ngx.WARN local ngx_var = ngx.var local ngx_print = ngx.print local ngx_header = ngx.header From 0fbe7ec419e920a6537b5d11c18d4b18c227e0f3 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 4 Mar 2021 20:26:53 +0000 Subject: [PATCH 076/130] Fix long line length --- lib/resty/http.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index d9869dad..6b7b9bd5 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -255,7 +255,11 @@ end function _M.parse_uri(_, uri, query_in_path) if query_in_path == nil then query_in_path = true end - local m, err = ngx_re_match(uri, [[^(?:(http[s]?):)?//((?:[^\[\]:/\?]+)|(?:\[.+\]))(?::(\d+))?([^\?]*)\??(.*)]], "jo") + local m, err = ngx_re_match( + uri, + [[^(?:(http[s]?):)?//((?:[^\[\]:/\?]+)|(?:\[.+\]))(?::(\d+))?([^\?]*)\??(.*)]], + "jo" + ) if not m then if err then From fa6b730c28478401c34cdfd6cf0812b1e9f1a972 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 4 Mar 2021 20:42:19 +0000 Subject: [PATCH 077/130] Fix embarrassing variable scope errors --- lib/resty/http.lua | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 6b7b9bd5..5e3e4a88 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -1111,15 +1111,18 @@ function _M.proxy_response(_, response, chunksize) end local reader = response.body_reader + repeat - local chunk, read_err = reader(chunksize) - if err then + local chunk, ok, read_err, print_err + + chunk, read_err = reader(chunksize) + if read_err then ngx_log(ngx_ERR, read_err) end if chunk then - local res, print_err = ngx_print(chunk) - if not res then + ok, print_err = ngx_print(chunk) + if not ok then ngx_log(ngx_ERR, print_err) end end From b4cfb2b1a52eb09145244e60d6bb9b0927b2f0d1 Mon Sep 17 00:00:00 2001 From: Hamish Date: Fri, 5 Mar 2021 10:19:27 +1300 Subject: [PATCH 078/130] Add GH Actions --- .github/workflows/test.yml | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..e4d5dbce --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,66 @@ +name: Test + +on: + push: + paths-ignore: # Skip if only docs are updated + - '*.md' + +jobs: + luacheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: leafo/gh-actions-lua@v8 + with: + luaVersion: "luajit-openresty" + - uses: leafo/gh-actions-luarocks@v4 + - run: luarocks install luacheck + - run: luacheck lib + + run_tests: + strategy: + matrix: + openresty_version: + - 1.17.8.1 + - 1.19.3.1 + + runs-on: ubuntu-latest + container: + image: openresty/openresty:${{ matrix.openresty_version }}-alpine-fat + # --init runs tinit as PID 1 and prevents the 'WARNING: killing the child process' spam from the test suite + options: --init + + steps: + - name: Install deps + run: | + apk add --no-cache curl perl bash wget git perl-dev libarchive-tools + ln -s /usr/bin/bsdtar /usr/bin/tar + + - name: Install CPAN + run: curl -s -L http://xrl.us/cpanm > /bin/cpanm && chmod +x /bin/cpanm + + - name: Cache + uses: actions/cache@v2 + with: + path: | + ~/.cpan + ~/.cache + key: ${{ runner.os }}-${{ matrix.openresty_version }}-cache + + - name: Install Test::Nginx + run: cpanm -q -n Test::Nginx + + - name: Install Luacov + run: luarocks install luacov + + - uses: actions/checkout@v2 + + - name: Run tests + env: + TEST_COVERAGE: '1' + run: /usr/bin/prove -I../test-nginx/lib -r t/ + + - name: Coverage + run: | + luacov + tail -n 8 luacov.report.out From 794b6397d9af867f7e2586035deeb75752d15672 Mon Sep 17 00:00:00 2001 From: Hamish Date: Fri, 5 Mar 2021 10:19:54 +1300 Subject: [PATCH 079/130] Remove travis config --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7d701d70..00000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -services: - - docker - -script: - - docker run --rm -v "$PWD:/lua-resty-http" -w /lua-resty-http ledgetech/test-runner:latest /bin/bash -c "make check; make coverage" From a74d9942181c8f57594e1df080e969465d56f199 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Thu, 4 Mar 2021 22:42:23 +0100 Subject: [PATCH 080/130] fix(readme) fix a broken link (#228) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05b9ffef..4f1b59cf 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ The options table has the following fields: * `pool`: custom connection pool name. Option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect), except that the default will become a pool name constructed using the SSL / proxy properties, which is important for safe connection reuse. When in doubt, leave it blank! * `pool_size`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) * `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) -* `proxy_opts`: sub-table, defaults to the global proxy options set, see [set\_proxy\_options](#set-proxy-options). +* `proxy_opts`: sub-table, defaults to the global proxy options set, see [set\_proxy\_options](#set_proxy_options). * `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. * `ssl_server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) * `ssl_send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) From 0d94f154d756d74888f272267eff81da8fd0463d Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 5 Mar 2021 10:33:01 +0000 Subject: [PATCH 081/130] Update ci status badge (#229) * Use Github actions status badge * Fix badge URI * Linkify status badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f1b59cf..78dddbc4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Lua HTTP client cosocket driver for [OpenResty](http://openresty.org/) / [ngx\_l Production ready. -[![Build Status](https://travis-ci.org/ledgetech/lua-resty-http.svg?branch=master)](https://travis-ci.org/ledgetech/lua-resty-http) +[![Test](https://github.com/ledgetech/lua-resty-http/actions/workflows/test.yml/badge.svg)]((https://github.com/ledgetech/lua-resty-http/actions) ## Features From fa054c348ee3d4608e20443d9d428825064ed538 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 5 Mar 2021 10:34:18 +0000 Subject: [PATCH 082/130] Fix CI badge syntax --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78dddbc4..c9fe91b7 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Lua HTTP client cosocket driver for [OpenResty](http://openresty.org/) / [ngx\_l Production ready. -[![Test](https://github.com/ledgetech/lua-resty-http/actions/workflows/test.yml/badge.svg)]((https://github.com/ledgetech/lua-resty-http/actions) +[![Test](https://github.com/ledgetech/lua-resty-http/actions/workflows/test.yml/badge.svg)](https://github.com/ledgetech/lua-resty-http/actions) ## Features From 4cbf7b66252e48c9d986a673ad27ce5fd7841169 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 16 Mar 2021 12:06:24 +0000 Subject: [PATCH 083/130] Ensure request body Content-Length is set correctly (#230) * Avoid sending non-string body values Fixes #217 * Add newline at EOF * Document support for table of strings in request body * Revert change preventing non string request bodies * Add tests for different request body types * Ensure request body Content-Length is set correctly * Don't blindly set request content length * Add more explicit support for chunked request body iterators * Use new transfer encoding utility * Add tests for client body reader Fails on chunked input until supported by ngx_lua * Improve transfer encoding check Co-authored-by: Thijs Schreijer Co-authored-by: Thijs Schreijer --- README.md | 2 +- lib/resty/http.lua | 65 +++++++++++------ t/02-chunked.t | 32 ++++++++ t/03-requestbody.t | 158 ++++++++++++++++++++++++++++++++++++++++ t/10-clientbodyreader.t | 56 ++++++++++++++ 5 files changed, 288 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c9fe91b7..a1d233df 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ The `params` table expects the following fields: * `path`: The path string. Defaults to `/`. * `query`: The query string, presented as either a literal string or Lua table.. * `headers`: A table of request headers. -* `body`: The request body as a string, or an iterator function (see [get\_client\_body\_reader](#get_client_body_reader)). +* `body`: The request body as a string, a table of strings, or an iterator function yielding strings until nil when exhausted. Note that you must specify a `Content-Length` for the request body, or specify `Transfer-Encoding: chunked` and have your function implement the encoding. See also: [get\_client\_body\_reader](#get_client_body_reader)). When the request is successful, `res` will contain the following fields: diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 5e3e4a88..98acb8ef 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -398,6 +398,23 @@ local function _receive_headers(sock) end +local function transfer_encoding_is_chunked(headers) + local te = headers["Transfer-Encoding"] + if not te then + return false + end + + -- Handle duplicate headers + -- This shouldn't happen but can in the real world + if type(te) ~= "string" then + te = tbl_concat(te, ",") + end + + return str_find(str_lower(te), "chunked", 1, true) ~= nil +end +_M.transfer_encoding_is_chunked = transfer_encoding_is_chunked + + local function _chunked_body_reader(sock, default_chunk_size) return co_wrap(function(max_chunk_size) local remaining = 0 @@ -575,7 +592,7 @@ end local function _send_body(sock, body) - if type(body) == 'function' then + if type(body) == "function" then repeat local chunk, err, partial = body() @@ -627,12 +644,13 @@ function _M.send_request(self, params) local body = params.body local headers = http_headers.new() - local params_headers = params.headers or {} - -- We assign one by one so that the metatable can handle case insensitivity + -- We assign one-by-one so that the metatable can handle case insensitivity -- for us. You can blame the spec for this inefficiency. + local params_headers = params.headers or {} for k, v in pairs(params_headers) do headers[k] = v end + if not headers["Proxy-Authorization"] then -- TODO: next major, change this to always override the provided -- header. Can't do that yet because it would be breaking. @@ -644,12 +662,28 @@ function _M.send_request(self, params) -- Ensure minimal headers are set if not headers["Content-Length"] then - if type(body) == 'string' then - headers["Content-Length"] = #body + local body_type = type(body) + + if body_type == "function" then + if not transfer_encoding_is_chunked(headers) then + return nil, "Request body is a function but a length or chunked encoding is not specified" + end + + elseif body_type == "table" then + local length = 0 + for _, v in ipairs(body) do + length = length + #tostring(v) + end + headers["Content-Length"] = length + elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then headers["Content-Length"] = 0 + + elseif body ~= nil then + headers["Content-Length"] = #tostring(body) end end + if not headers["Host"] then if (str_sub(self.host, 1, 5) == "unix:") then return nil, "Unable to generate a useful Host header for a unix domain socket. Please provide one." @@ -751,22 +785,8 @@ function _M.read_response(self, params) if _should_receive_body(params.method, status) then has_body = true - local te = res_headers["Transfer-Encoding"] - - -- Handle duplicate headers - -- This shouldn't happen but can in the real world - if type(te) == "table" then - te = tbl_concat(te, "") - end - - local ok, encoding = pcall(str_lower, te) - if not ok then - encoding = "" - end - - if version == 1.1 and str_find(encoding, "chunked", 1, true) ~= nil then + if version == 1.1 and transfer_encoding_is_chunked(res_headers) then body_reader, err = _chunked_body_reader(sock) - else local ok, length = pcall(tonumber, res_headers["Content-Length"]) if not ok then @@ -775,9 +795,7 @@ function _M.read_response(self, params) end body_reader, err = _body_reader(sock, length) - end - end if res_headers["Trailer"] then @@ -941,10 +959,9 @@ function _M.get_client_body_reader(_, chunksize, sock) local headers = ngx_req_get_headers() local length = headers.content_length - local encoding = headers.transfer_encoding if length then return _body_reader(sock, tonumber(length), chunksize) - elseif encoding and str_lower(encoding) == 'chunked' then + elseif transfer_encoding_is_chunked(headers) then -- Not yet supported by ngx_lua but should just work... return _chunked_body_reader(sock, chunksize) else diff --git a/t/02-chunked.t b/t/02-chunked.t index dda47c4b..92d0ecc0 100644 --- a/t/02-chunked.t +++ b/t/02-chunked.t @@ -238,3 +238,35 @@ table --- no_error_log [error] [warn] + + +=== TEST 5: transfer_encoding_is_chunked utility. +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http_headers = require("resty.http_headers") + local http = require("resty.http") + + local headers = http_headers:new() + assert(http.transfer_encoding_is_chunked(headers) == false, + "empty headers should return false") + + headers["Transfer-Encoding"] = "chunked" + assert(http.transfer_encoding_is_chunked(headers) == true, + "te set to `chunked` should return true`") + + headers["Transfer-Encoding"] = " ChuNkEd " + assert(http.transfer_encoding_is_chunked(headers) == true, + "te set to ` ChuNkEd ` should return true`") + + headers["Transfer-Encoding"] = { "chunked", " ChuNkEd " } + assert(http.transfer_encoding_is_chunked(headers) == true, + "te set to table values containing `chunked` should return true`") + } + } +--- request +GET /a +--- no_error_log +[error] +[warn] diff --git a/t/03-requestbody.t b/t/03-requestbody.t index 1e28c83a..c1fdd241 100644 --- a/t/03-requestbody.t +++ b/t/03-requestbody.t @@ -203,3 +203,161 @@ Expectation Failed --- no_error_log [error] [warn] + + +=== TEST 5: Non string request bodies are converted with correct length +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local httpc = require("resty.http").new() + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/b" + + for _, body in ipairs({ 12345, + true, + "string", + { "tab", "le" }, + { "mix", 123, "ed", "tab", "le" } }) do + + local res, err = assert(httpc:request_uri(uri, { + body = body, + })) + + ngx.say(res.body) + end + '; + } + location = /b { + content_by_lua ' + ngx.req.read_body() + ngx.print(ngx.req.get_body_data()) + '; + } +--- request +GET /a +--- response_body +12345 +true +string +table +mix123edtable +--- no_error_log +[error] +[warn] + + +=== TEST 6: Request body as iterator +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local httpc = require("resty.http").new() + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/b" + + local res, err = assert(httpc:request_uri(uri, { + body = coroutine.wrap(function() + coroutine.yield("foo") + coroutine.yield("bar") + end), + headers = { + ["Content-Length"] = 6 + } + })) + + ngx.say(res.body) + '; + } + location = /b { + content_by_lua ' + ngx.req.read_body() + ngx.print(ngx.req.get_body_data()) + '; + } +--- request +GET /a +--- response_body +foobar +--- no_error_log +[error] +[warn] + + +=== TEST 7: Request body as iterator, errors with missing length +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local httpc = require("resty.http").new() + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/b" + + local res, err = httpc:request_uri(uri, { + body = coroutine.wrap(function() + coroutine.yield("foo") + coroutine.yield("bar") + end), + }) + + assert(not res) + ngx.say(err) + '; + } + location = /b { + content_by_lua ' + ngx.req.read_body() + ngx.print(ngx.req.get_body_data()) + '; + } +--- request +GET /a +--- response_body +Request body is a function but a length or chunked encoding is not specified +--- no_error_log +[error] +[warn] + + +=== TEST 8: Request body as iterator with chunked encoding +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local httpc = require("resty.http").new() + local yield = coroutine.yield + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/b" + + local res, err = assert(httpc:request_uri(uri, { + body = coroutine.wrap(function() + yield("3\r\n") + yield("foo\r\n") + + yield("3\r\n") + yield("bar\r\n") + + yield("0\r\n") + yield("\r\n") + end), + headers = { + ["Transfer-Encoding"] = "chunked" + } + })) + + ngx.say(res.body) + } + } + location = /b { + content_by_lua ' + ngx.req.read_body() + ngx.print(ngx.req.get_body_data()) + '; + } +--- request +GET /a +--- response_body +foobar +--- no_error_log +[error] +[warn] diff --git a/t/10-clientbodyreader.t b/t/10-clientbodyreader.t index 21d0b1e0..6380c4a0 100644 --- a/t/10-clientbodyreader.t +++ b/t/10-clientbodyreader.t @@ -64,3 +64,59 @@ OK [error] [warn] + +=== TEST 2: Read request body +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local httpc = require("resty.http").new() + + local reader, err = assert(httpc:get_client_body_reader()) + + repeat + local buffer, err = reader() + if err then + ngx.log(ngx.ERR, err) + end + + if buffer then + ngx.print(buffer) + end + until not buffer + } + } +--- request +POST /a +foobar +--- response_body: foobar +--- no_error_log +[error] +[warn] + + +=== TEST 2: Read chunked request body, errors as not yet supported +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local httpc = require("resty.http").new() + local _, err = httpc:get_client_body_reader() + ngx.log(ngx.ERR, err) + } + } +--- more_headers +Transfer-Encoding: chunked +--- request eval +"POST /a +3\r +foo\r +3\r +bar\r +0\r +\r +" +--- error_log +chunked request bodies not supported yet +--- no_error_log +[warn] From d83df5df6e658dac5dddb0bda7be176688a468b6 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 23 Mar 2021 10:17:59 +0000 Subject: [PATCH 084/130] Allow actions to run on pr as well as push events (#233) Also removed avoiding builds on doc only, since checks are required for merging. --- .github/workflows/test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4d5dbce..7708e04a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,9 +1,6 @@ name: Test -on: - push: - paths-ignore: # Skip if only docs are updated - - '*.md' +on: [push, pull_request] jobs: luacheck: From 1120af703db606045c36c6832033522f846820f6 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Tue, 23 Mar 2021 01:57:02 +0100 Subject: [PATCH 085/130] fix(deprecated) do not use deprecated method --- lib/resty/http_connect.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index ffd72af4..cbb102ac 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -213,7 +213,7 @@ local function connect(self, options) -- Now do the ssl handshake if ssl and sock:getreusedtimes() == 0 then - local ok, err = self:ssl_handshake(nil, ssl_server_name, ssl_verify, ssl_send_status_req) + local ok, err = sock:sslhandshake(nil, ssl_server_name, ssl_verify, ssl_send_status_req) if not ok then self:close() return nil, err From d7f636c28c12cd5ec9a5c033f0c99f9ee943c450 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Mon, 29 Mar 2021 11:38:38 +0100 Subject: [PATCH 086/130] fix/drop content length when chunked (#234) * Make flakey test a little less flakey * Ensure Content-Length is dropped if Transfer-Encoding is specified https://tools.ietf.org/html/rfc7230#section-3.3.3 --- lib/resty/http.lua | 41 ++++++++++++++++++++++-------------- t/02-chunked.t | 52 ++++++++++++++++++++++++++++++++++++++++++++++ t/14-host-header.t | 1 + 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 98acb8ef..058f5a74 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -659,28 +659,37 @@ function _M.send_request(self, params) headers["Proxy-Authorization"] = self.http_proxy_auth end - -- Ensure minimal headers are set + -- Ensure we have appropriate message length or encoding. + do + local is_chunked = transfer_encoding_is_chunked(headers) + + if is_chunked then + -- If we have both Transfer-Encoding and Content-Length we MUST + -- drop the Content-Length, to help prevent request smuggling. + -- https://tools.ietf.org/html/rfc7230#section-3.3.3 + headers["Content-Length"] = nil - if not headers["Content-Length"] then - local body_type = type(body) + elseif not headers["Content-Length"] then + -- A length was not given, try to calculate one. - if body_type == "function" then - if not transfer_encoding_is_chunked(headers) then + local body_type = type(body) + + if body_type == "function" then return nil, "Request body is a function but a length or chunked encoding is not specified" - end - elseif body_type == "table" then - local length = 0 - for _, v in ipairs(body) do - length = length + #tostring(v) - end - headers["Content-Length"] = length + elseif body_type == "table" then + local length = 0 + for _, v in ipairs(body) do + length = length + #tostring(v) + end + headers["Content-Length"] = length - elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then - headers["Content-Length"] = 0 + elseif body == nil and EXPECTING_BODY[str_upper(params.method)] then + headers["Content-Length"] = 0 - elseif body ~= nil then - headers["Content-Length"] = #tostring(body) + elseif body ~= nil then + headers["Content-Length"] = #tostring(body) + end end end diff --git a/t/02-chunked.t b/t/02-chunked.t index 92d0ecc0..26b75f23 100644 --- a/t/02-chunked.t +++ b/t/02-chunked.t @@ -263,10 +263,62 @@ table headers["Transfer-Encoding"] = { "chunked", " ChuNkEd " } assert(http.transfer_encoding_is_chunked(headers) == true, "te set to table values containing `chunked` should return true`") + + headers["Transfer-Encoding"] = "chunked" + headers["Content-Length"] = 10 + assert(http.transfer_encoding_is_chunked(headers) == true, + "transfer encoding should override content-length`") + } + } +--- request +GET /a +--- no_error_log +[error] +[warn] + + +=== TEST 6: Don't send Content-Length if Transfer-Encoding is specified +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local httpc = require("resty.http").new() + local yield = coroutine.yield + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/b" + + local res, err = assert(httpc:request_uri(uri, { + body = coroutine.wrap(function() + yield("3\r\n") + yield("foo\r\n") + + yield("3\r\n") + yield("bar\r\n") + + yield("0\r\n") + yield("\r\n") + end), + headers = { + ["Transfer-Encoding"] = "chunked", + ["Content-Length"] = 42, + }, + })) + + ngx.say(res.body) + } + } + location = /b { + content_by_lua_block { + ngx.req.read_body() + ngx.say(ngx.req.get_headers()["Content-Length"]) + ngx.print(ngx.req.get_body_data()) } } --- request GET /a +--- response_body +nil +foobar --- no_error_log [error] [warn] diff --git a/t/14-host-header.t b/t/14-host-header.t index 685813c0..39e4d3ad 100644 --- a/t/14-host-header.t +++ b/t/14-host-header.t @@ -67,6 +67,7 @@ Host: www.google.com local http = require "resty.http" local httpc = http.new() + httpc:set_timeouts(300, 1000, 1000) local res, err = httpc:request_uri("https://www.google.com:443", { ssl_verify = false }) '; } From faf7b45f27588af98e409e6870cb59691cea13f3 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Tue, 6 Apr 2021 13:20:23 +0100 Subject: [PATCH 087/130] release/0.16 (#235) * Bump version * Don't ignore luarock spec files --- .gitignore | 2 +- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...sty-http-0.15-0.rockspec => lua-resty-http-0.16-0.rockspec | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename lua-resty-http-0.15-0.rockspec => lua-resty-http-0.16-0.rockspec (94%) diff --git a/.gitignore b/.gitignore index 51b26b94..422ea976 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ t/servroot/ t/error.log -lua-resty-http* luacov* +*.src.rock diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 058f5a74..7f4dbc3a 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -106,7 +106,7 @@ end local _M = { - _VERSION = '0.14', + _VERSION = '0.16', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 6771d1ed..7b13ba3a 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.14', + _VERSION = '0.16', } diff --git a/lua-resty-http-0.15-0.rockspec b/lua-resty-http-0.16-0.rockspec similarity index 94% rename from lua-resty-http-0.15-0.rockspec rename to lua-resty-http-0.16-0.rockspec index 71ba0abe..5e79ae93 100644 --- a/lua-resty-http-0.15-0.rockspec +++ b/lua-resty-http-0.16-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.15-0" +version = "0.16-0" source = { url = "git://github.com/ledgetech/lua-resty-http", - tag = "v0.15" + tag = "v0.16" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From 865b4c6e96dc85beb64ee1482dd6f1953c3d3356 Mon Sep 17 00:00:00 2001 From: Thijs Schreijer Date: Fri, 9 Apr 2021 18:02:16 +0200 Subject: [PATCH 088/130] fix(request_uri) use uri host for ssl-server-name (#237) also fixes an incorrect note in the docs fixes #236 --- README.md | 2 +- lib/resty/http.lua | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a1d233df..d6ef64bc 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ When the request is successful, `res` will contain the following fields: `syntax: res, err = httpc:request_uri(uri, params)` -The single-shot interface (see [usage](#Usage)). Since this method performs an entire end-to-end request, options specified in the `params` can include anything found in both [connect](#connect) and [request](#request) documented above. Note also that fields in `params` will override relevant components of the `uri` if specified. +The single-shot interface (see [usage](#Usage)). Since this method performs an entire end-to-end request, options specified in the `params` can include anything found in both [connect](#connect) and [request](#request) documented above. Note also that fields `path`, and `query`, in `params` will override relevant components of the `uri` if specified (`scheme`, `host`, and `port` will always be taken from the `uri`). There are 3 additional parameters for controlling keepalives: diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 7f4dbc3a..46386fb7 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -899,6 +899,7 @@ function _M.request_uri(self, uri, params) params.scheme, params.host, params.port, path, query = unpack(parsed_uri) params.path = params.path or path params.query = params.query or query + params.ssl_server_name = params.ssl_server_name or params.host end do From 9bf951dfe162dd9710a0e1f4525738d4902e9d20 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 9 Apr 2021 17:11:35 +0100 Subject: [PATCH 089/130] release/0.16.1 (#238) * Ignore luarocks build artefacts * Bump version to 1.16.1 --- .gitignore | 4 +++- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...y-http-0.16-0.rockspec => lua-resty-http-0.16.1-0.rockspec | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) rename lua-resty-http-0.16-0.rockspec => lua-resty-http-0.16.1-0.rockspec (93%) diff --git a/.gitignore b/.gitignore index 422ea976..44834f8e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ t/servroot/ t/error.log luacov* -*.src.rock +lua-resty-http-*/ +lua-resty-http-*.src.rock +lua-resty-http-*.tar.gz diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 46386fb7..85d83c12 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -106,7 +106,7 @@ end local _M = { - _VERSION = '0.16', + _VERSION = '0.16.1', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 7b13ba3a..d9272da5 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.16', + _VERSION = '0.16.1', } diff --git a/lua-resty-http-0.16-0.rockspec b/lua-resty-http-0.16.1-0.rockspec similarity index 93% rename from lua-resty-http-0.16-0.rockspec rename to lua-resty-http-0.16.1-0.rockspec index 5e79ae93..22726ffa 100644 --- a/lua-resty-http-0.16-0.rockspec +++ b/lua-resty-http-0.16.1-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.16-0" +version = "0.16.1-0" source = { url = "git://github.com/ledgetech/lua-resty-http", - tag = "v0.16" + tag = "v0.16.1" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From f754eb7ecc8c969657b357ae82f93a05a29dd395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=B6=E3=83=BC=E3=81=A8=20/=20Yoshiaki=20Ueda?= Date: Sat, 24 Apr 2021 00:26:09 +0900 Subject: [PATCH 090/130] fix readme (#239) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6ef64bc..12baa6bc 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ local httpc = require("resty.http").new() -- First establish a connection local ok, err = httpc:connect({ scheme = "https", - host = "127.0.0.1" + host = "127.0.0.1", port = 8080, }) if not ok then From 0ce55d6d15da140ecc5966fa848204c6fd9074e8 Mon Sep 17 00:00:00 2001 From: Arthur Khashaev Date: Fri, 6 Aug 2021 12:14:38 +0300 Subject: [PATCH 091/130] Add error message for broken response status line (#242) * Add test to reproduce #241 * Add error message for broken response status line Fixes #241 * Add test for an explicit error message string See #241 --- lib/resty/http.lua | 16 +++++++++++++++- t/01-basic.t | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 85d83c12..5f7ed1af 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -362,7 +362,21 @@ local function _receive_status(sock) return nil, nil, nil, err end - return tonumber(str_sub(line, 10, 12)), tonumber(str_sub(line, 6, 8)), str_sub(line, 14) + local version = tonumber(str_sub(line, 6, 8)) + if not version then + return nil, nil, nil, + "couldn't parse HTTP version from response status line: " .. line + end + + local status = tonumber(str_sub(line, 10, 12)) + if not status then + return nil, nil, nil, + "couldn't parse status code from response status line: " .. line + end + + local reason = str_sub(line, 14) + + return status, version, reason end diff --git a/t/01-basic.t b/t/01-basic.t index 0439a8a0..f3c413d7 100644 --- a/t/01-basic.t +++ b/t/01-basic.t @@ -341,3 +341,51 @@ OK --- no_error_log [error] [warn] + +=== TEST 13: Should return error on invalid HTTP version in response status line +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:12345") + + assert(err == "couldn't parse HTTP version from response status line: TEAPOT/1.1 OMG") + } + } +--- tcp_listen: 12345 +--- tcp_reply +TEAPOT/1.1 OMG +Server: Teapot + +OK +--- request +GET /a +--- no_error_log +[error] +[warn] + +=== TEST 14: Should return error on invalid status code in response status line +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:12345") + + assert(err == "couldn't parse status code from response status line: HTTP/1.1 OMG") + } + } +--- tcp_listen: 12345 +--- tcp_reply +HTTP/1.1 OMG +Server: Teapot + +OK +--- request +GET /a +--- no_error_log +[error] +[warn] From 26dfd76b5fe2a20ad4979267de86e80ced2e34d2 Mon Sep 17 00:00:00 2001 From: Guy Lewin Date: Mon, 22 Nov 2021 11:20:04 -0500 Subject: [PATCH 092/130] Feature/connect improvements (#254) * feat: pass ssl_reused_session into http_connect * style: don't redeclare variables * feat: return ssl_session upon successful SSL connection * doc: update README with changed APIs * chore(tests) update expired test cert * test(ssl_reused_session) add tests Co-authored-by: James Hurst --- README.md | 5 +- lib/resty/http.lua | 2 +- lib/resty/http_connect.lua | 17 +++-- t/19-ssl_reused_session.t | 136 +++++++++++++++++++++++++++++++++++++ t/cert/test.crt | 47 +++++++------ t/cert/test.key | 79 +++++++++++++-------- 6 files changed, 228 insertions(+), 58 deletions(-) create mode 100644 t/19-ssl_reused_session.t diff --git a/README.md b/README.md index 12baa6bc..9c97b4b5 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ local body = res.body local httpc = require("resty.http").new() -- First establish a connection -local ok, err = httpc:connect({ +local ok, err, ssl_session = httpc:connect({ scheme = "https", host = "127.0.0.1", port = 8080, @@ -153,7 +153,7 @@ Creates the HTTP connection object. In case of failures, returns `nil` and a str ## connect -`syntax: ok, err = httpc:connect(options)` +`syntax: ok, err, ssl_session = httpc:connect(options)` Attempts to connect to the web server while incorporating the following activities: @@ -172,6 +172,7 @@ The options table has the following fields: * `pool_size`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) * `backlog`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsockconnect) * `proxy_opts`: sub-table, defaults to the global proxy options set, see [set\_proxy\_options](#set_proxy_options). +* `ssl_reused_session`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) * `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. * `ssl_server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) * `ssl_send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 5f7ed1af..d5753443 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -165,7 +165,7 @@ end do local aio_connect = require "resty.http_connect" -- Function signatures to support: - -- ok, err = httpc:connect(options_table) + -- ok, err, ssl_session = httpc:connect(options_table) -- ok, err = httpc:connect(host, port, options_table?) -- ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?) function _M.connect(self, options, ...) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index cbb102ac..25c47041 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -22,6 +22,7 @@ client:connect { backlog = nil, -- ssl options as per: https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake + ssl_reused_session = nil ssl_server_name = nil, ssl_send_status_req = nil, ssl_verify = true, -- NOTE: defaults to true @@ -53,9 +54,10 @@ local function connect(self, options) end -- ssl settings - local ssl, ssl_server_name, ssl_verify, ssl_send_status_req + local ssl, ssl_reused_session, ssl_server_name, ssl_verify, ssl_send_status_req if request_scheme == "https" then ssl = true + ssl_reused_session = options.ssl_reused_session ssl_server_name = options.ssl_server_name ssl_send_status_req = options.ssl_send_status_req ssl_verify = true -- default @@ -133,7 +135,8 @@ local function connect(self, options) end if proxy then - local proxy_uri_t, err = self:parse_uri(proxy_uri) + local proxy_uri_t + proxy_uri_t, err = self:parse_uri(proxy_uri) if not proxy_uri_t then return nil, err end @@ -178,7 +181,8 @@ local function connect(self, options) -- authority-form of RFC 7230 Section 5.3.3. See also RFC 7231 Section -- 4.3.6 for more details about the CONNECT request local destination = request_host .. ":" .. request_port - local res, err = self:request({ + local res + res, err = self:request({ method = "CONNECT", path = destination, headers = { @@ -211,10 +215,11 @@ local function connect(self, options) end end + local ssl_session -- Now do the ssl handshake if ssl and sock:getreusedtimes() == 0 then - local ok, err = sock:sslhandshake(nil, ssl_server_name, ssl_verify, ssl_send_status_req) - if not ok then + ssl_session, err = sock:sslhandshake(ssl_reused_session, ssl_server_name, ssl_verify, ssl_send_status_req) + if not ssl_session then self:close() return nil, err end @@ -228,7 +233,7 @@ local function connect(self, options) self.http_proxy_auth = request_scheme ~= "https" and proxy_authorization or nil self.path_prefix = path_prefix - return true + return true, nil, ssl_session end return connect diff --git a/t/19-ssl_reused_session.t b/t/19-ssl_reused_session.t new file mode 100644 index 00000000..45f9e8fb --- /dev/null +++ b/t/19-ssl_reused_session.t @@ -0,0 +1,136 @@ +use Test::Nginx::Socket::Lua 'no_plan'; +use Cwd qw(abs_path realpath); +use File::Basename; + +$ENV{TEST_NGINX_HTML_DIR} ||= html_dir(); + +$ENV{TEST_NGINX_RESOLVER} ||= '8.8.8.8'; +$ENV{TEST_NGINX_CERT_DIR} ||= dirname(realpath(abs_path(__FILE__))); +$ENV{TEST_COVERAGE} ||= 0; + +my $realpath = realpath(); + +our $HttpConfig = qq{ + lua_package_path "$realpath/lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; + + init_by_lua_block { + if $ENV{TEST_COVERAGE} == 1 then + jit.off() + require("luacov.runner").init() + end + + TEST_SERVER_SOCK = "unix:/$ENV{TEST_NGINX_HTML_DIR}/nginx.sock" + + num_handshakes = 0 + } + + server { + listen unix:$ENV{TEST_NGINX_HTML_DIR}/nginx.sock ssl; + server_name example.com; + ssl_certificate $ENV{TEST_NGINX_CERT_DIR}/cert/test.crt; + ssl_certificate_key $ENV{TEST_NGINX_CERT_DIR}/cert/test.key; + ssl_session_tickets off; + + server_tokens off; + } +}; + +no_long_string(); +#no_diff(); + +run_tests(); + +__DATA__ + +=== TEST 1: connect returns session userdata +--- http_config eval: $::HttpConfig +--- config + server_tokens off; + resolver $TEST_NGINX_RESOLVER ipv6=off; + lua_ssl_trusted_certificate $TEST_NGINX_CERT_DIR/cert/test.crt; + + location /t { + content_by_lua_block { + local httpc = assert(require("resty.http").new()) + local ok, err, session = assert(httpc:connect { + scheme = "https", + host = TEST_SERVER_SOCK, + }) + + assert(type(session) == "userdata", "expected session to be userdata") + assert(httpc:close()) + } + } + +--- request +GET /t +--- no_error_log +[error] +[alert] + + +=== TEST 2: ssl_reused_session false does not return session userdata +--- http_config eval: $::HttpConfig +--- config + server_tokens off; + resolver $TEST_NGINX_RESOLVER ipv6=off; + lua_ssl_trusted_certificate $TEST_NGINX_CERT_DIR/cert/test.crt; + + location /t { + content_by_lua_block { + local httpc = assert(require("resty.http").new()) + local ok, err, session = assert(httpc:connect { + scheme = "https", + host = TEST_SERVER_SOCK, + ssl_reused_session = false, + }) + + assert(type(session) == "boolean", "expected session to be a boolean") + assert(session == true, "expected session to be true") + assert(httpc:close()) + } + } + +--- request +GET /t +--- no_error_log +[error] +[alert] + + +=== TEST 3: ssl_reused_session accepts userdata +--- http_config eval: $::HttpConfig +--- config + server_tokens off; + resolver $TEST_NGINX_RESOLVER ipv6=off; + lua_ssl_trusted_certificate $TEST_NGINX_CERT_DIR/cert/test.crt; + + location /t { + content_by_lua_block { + local httpc = assert(require("resty.http").new()) + local ok, err, session = assert(httpc:connect { + scheme = "https", + host = TEST_SERVER_SOCK, + }) + + assert(type(session) == "userdata", "expected session to be userdata") + + local httpc2 = assert(require("resty.http").new()) + local ok, err, session2 = assert(httpc2:connect { + scheme = "https", + host = TEST_SERVER_SOCK, + ssl_reused_session = session, + }) + + assert(type(session2) == "userdata", "expected session2 to be userdata") + + assert(httpc:close()) + assert(httpc2:close()) + } + } + +--- request +GET /t +--- no_error_log +[error] +[alert] diff --git a/t/cert/test.crt b/t/cert/test.crt index ae113446..477ddc2a 100644 --- a/t/cert/test.crt +++ b/t/cert/test.crt @@ -1,24 +1,27 @@ -----BEGIN CERTIFICATE----- -MIID8DCCAtigAwIBAgIJALL9eJPZ6neGMA0GCSqGSIb3DQEBBQUAMFgxCzAJBgNV -BAYTAkdCMQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MQ0wCwYDVQQKEwRU -ZXN0MQ0wCwYDVQQLEwRUZXN0MQ0wCwYDVQQDEwR0ZXN0MB4XDTE1MTAyMTE2MjQ1 -NloXDTE1MTEyMDE2MjQ1NlowWDELMAkGA1UEBhMCR0IxDTALBgNVBAgTBFRlc3Qx -DTALBgNVBAcTBFRlc3QxDTALBgNVBAoTBFRlc3QxDTALBgNVBAsTBFRlc3QxDTAL -BgNVBAMTBHRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDz/AoE -c+TPdm+Aqcchq8fLNWksFQZqbsCBGnq8rUG1b6MsVlAOkDUQGRlNPs9v0/+pzgX7 -IYXPCFcV7YONNsTUfvBYTq43mfOycmAdb3SX6kBygxdhYsDRZR+vCAIkjoRmRB20 -meh1motqM58spq3IcT8VADTRJl1OI48VTnxmXdCtmkOymU948DcauMoxm03eL/hU -6eniNEujbnbB305noNG0W5c3h6iz9CvqUAD1kwyjick+f1atB2YYn1bymA+db6YN -3iTo0v2raWmIc7D+qqpkNaCRxgMb2HN6X3/SfkijtNJidjqHMbs2ftlKJ5/lODPZ -rCPQOcYK6TT8MIZ1AgMBAAGjgbwwgbkwHQYDVR0OBBYEFFUC1GrAhUp7IvJH5iyf -+fJQliEIMIGJBgNVHSMEgYEwf4AUVQLUasCFSnsi8kfmLJ/58lCWIQihXKRaMFgx -CzAJBgNVBAYTAkdCMQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MQ0wCwYD -VQQKEwRUZXN0MQ0wCwYDVQQLEwRUZXN0MQ0wCwYDVQQDEwR0ZXN0ggkAsv14k9nq -d4YwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAtaUQOr3Qn87KXmmP -GbSvCLSl+bScE09VYZsYaB6iq0pGN9y+Vh4/HjBUUsFexopw1dY25MEEJXEVi1xV -2krLYAsfKCM6c1QBVmdqfVuxUvxpXwr+CNRNAlzz6PhjkeY/Ds/j4sg7EqN8hMmT -gu8GuogX7+ZCgrzRSMMclWej+W8D1xSIuCC+rqv4w9SZdtVb3XGpCyizpTNsQAuV -ACXvq9KXkEEj+XNvKrNdWd4zG715RdMnVm+WM53d9PLp63P+4/kwhwHULYhXygQ3 -DzzVPaojBBdw3VaHbbPHnv73FtAzOb7ky6zJ01DlmEPxEahCFpklMkY9T2uCdpj9 -oOzaNA== +MIIEpDCCAowCCQDgS40Q0UNIKTANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls +b2NhbGhvc3QwHhcNMjExMTIyMTQ0MDM2WhcNMjIxMTIyMTQ0MDM2WjAUMRIwEAYD +VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9 +OVsD7qyres7qkLZPY4MyC00/ByESpUVFgWiTU2nBeNC/kBvOOOVTvrVHxlTjZ9JG +rCdYeIGkpkaJQ2QylHKRJolxh8x309/VkpKJYL357NuNeAk2l3tW4BMImXUGdVjD +0qT8WTyApEE7lyfOXKYKbbEwfdks0eBcEZBBn4QTaZ9lVj7yScurto8jDYMkBfO/ +4QeuVe4ijgWU35Yd89Ya0HZr+1QBU1dMzbtf9mYPedRwRZgzKogZ6mZXs7GJ0ADM +L+fg/djROd01eA3ynAfxJbRv1R7S73ljQJhD+oYf3rJaHU2Ko5heF9K9bd771Ze/ +sHGHZdLy3JZrPNdDPHG61H9m4zOUkNZvVkvHF9wg+uNs4VJxrkG6undOEyCyBefE +XRknEBvMLlWF9c9KT5tZz/vI6/e5xrpNCdUjthn5YFEf43MVqWWcU8t9ViK+8+rG +PQNKmxRiCOOmd6NhjWJInuMO3QiPJFqVs4UNpmE/4hi3LtwV7qH4jHWX5hRG27gz +8VbfkBkIoy0Esm2Dv5lErZztPhycpFUodpBXGtQjXQLvYVm+yKwWlD6gTRH0LXj3 +3lp69m5/V9so3NsYWMxoNGvI4MqZuwGaiwxgkCWjUqQEX1y4i7Us01UDFpK2WuwG +MUQNzGV2jX9urnjndFPHjszqo5BsmJuHwihGOFvbpwIDAQABMA0GCSqGSIb3DQEB +CwUAA4ICAQAhXhf95xbkSw1fknoGhdgTAb3A+pKzcejZjqmQi3BJku8EILe06LGF +Pli4WlN7KMk20A/TO/L46tsWztFl0vhSs+ed/Z3/0ugB+DO0InpwDxutPTrVjkmy +W5HAKnW7MQAA9tcTcAPbiHIalwswQ1pab+kxl9fbsaC1CrSypa1eK6Hh/qJKxe2O +06ovTTHuZ/yWJBaIwmvP8lCMfCrLGOCGToM4FWtDPc47VVxpk0Ks0kRL8Bqo4CsG +rM+PpfsFBdPO/FftfVeMdzZCcK1CLWuE/uBdjDtU10qMVhQTxGG4eFv24xVQuCmH +AB5hTt62DLxHAs6kgvafZz9ST6aVDu0kuPABC4z4JQVBCRbw0mL75R4ziEYH3CIe +tmgYEXHfj3D3BDgX3ki6qqX7VtmkT0+VyrpQs+dhkCXlnqGmNM+gSsKve3ZH6a9J +fFeKjiJH4YjlPmLlCxpnDoBgjGjcUmVEBcpZ7H8lNC3ldPVfDq3PDrHgFMH5lmth +PZHbWROudoHVX5EBY2n08uzMWtl0xFwAfLG3Hk7xoBWBpebHxhw/yzm/iWHeBdi+ +8uSMLM3/TjmKhh+mF7KRKSYKetKKYiwB0/yIQJLMLxpUc3Aae7i0djH/kFz80HU/ +alsGmliI6PGyCt11DYS7Spl44pxl6N7pj2CJpyPOMwDI3VG097cYDQ== -----END CERTIFICATE----- diff --git a/t/cert/test.key b/t/cert/test.key index 4fcd0dae..5b64ce1d 100644 --- a/t/cert/test.key +++ b/t/cert/test.key @@ -1,27 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA8/wKBHPkz3ZvgKnHIavHyzVpLBUGam7AgRp6vK1BtW+jLFZQ -DpA1EBkZTT7Pb9P/qc4F+yGFzwhXFe2DjTbE1H7wWE6uN5nzsnJgHW90l+pAcoMX -YWLA0WUfrwgCJI6EZkQdtJnodZqLajOfLKatyHE/FQA00SZdTiOPFU58Zl3QrZpD -splPePA3GrjKMZtN3i/4VOnp4jRLo252wd9OZ6DRtFuXN4eos/Qr6lAA9ZMMo4nJ -Pn9WrQdmGJ9W8pgPnW+mDd4k6NL9q2lpiHOw/qqqZDWgkcYDG9hzel9/0n5Io7TS -YnY6hzG7Nn7ZSief5Tgz2awj0DnGCuk0/DCGdQIDAQABAoIBAGjKc7L94+SHRdTJ -FtILacCJrCZW0W6dKulIajbnYzV+QWMlnzTiEyha31ciBw5My54u8rqt5z7Ioj60 -yK+6OkfaTXhgMsuGv/iAz29VE4q7/fow+7XEKHTHLhiLJAB3hb42u1t6TzFTs1Vl -3pPa8wEIQsPOVuENzT1mYGoST7PW+LBIMr9ScMnRHfC0MNdV/ntQiXideOAd5PkA -4O7fNgYZ8CTAZ8rOLYTMFF76/c/jLiqfeghqbIhqMykk36kd7Lud//FRykVsn1aJ -REUva/SjVEth5kITot1hpMC4SIElWpha2YxiiZFoSXSaUbtHpymiUGV01cYtMWk0 -MZ5HN3ECgYEA/74U8DpwPxd4up9syKyNqOqrCrYnhEEC/tdU/W5wECi4y5kppjdd -88lZzICVPzk2fezYXlCO9HiSHU1UfcEsY3u16qNCvylK7Qz1OqXV/Ncj59891Q5Z -K0UBcbnrv+YD6muZuhlHEbyDPqYO091G9Gf/BbL5JIBDzg1qFO9Dh9cCgYEA9Drt -O9PJ5Sjz3mXQVtVHpwyhOVnd7CUv8a1zkUQCK5uQeaiF5kal1FIo7pLOr3KAvG0C -pXbm/TobwlfAfcERQN88aPN8Z/l1CB0oKV6ipBMD2/XLzDRtx8lpTeh/BB8jIhrz -+FDJY54HCzLfW0P5kT+Cyw51ofjziPnFdO/Z6pMCgYEAon17gEchGnUnWCwDSl2Y -hELV+jBSW02TQag/b+bDfQDiqTnfpKR5JXRBghYQveL0JH5f200EB4C0FboUfPJH -6c2ogDTLK/poiMU66tCDbeqj/adx+fTr4votOL0QdRUIV+GWAxAcf8BvA1cvBJ4L -fy60ckKM2gxFCJ6tUC/VkHECgYBoMDNAUItSnXPbrmeAg5/7naGxy6qmsP6RBUPF -9tNOMyEhJUlqAT2BJEOd8zcFFb3hpEd6uwyzfnSVJcZSX2iy2gj1ZNnvqTXJ7lZR -v7N2dz4wOd1lEgC7OCsaN1LoOThNtl3Z0uz2+FVc66jpUEhJNGThpxt7q66JArS/ -vAqkzQKBgFkzqA6QpnH5KhOCoZcuLQ4MtvnNHOx1xSm2B0gKDVJzGkHexTmOJvwM -ZhHXRl9txS4icejS+AGUXNBzCWEusfhDaZpZqS6zt6UxEjMsLj/Te7z++2KQn4t/ -aI77jClydW1pJvICtqm5v+sukVZvQTTJza9ujta6fj7u2s671np9 ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC9OVsD7qyres7q +kLZPY4MyC00/ByESpUVFgWiTU2nBeNC/kBvOOOVTvrVHxlTjZ9JGrCdYeIGkpkaJ +Q2QylHKRJolxh8x309/VkpKJYL357NuNeAk2l3tW4BMImXUGdVjD0qT8WTyApEE7 +lyfOXKYKbbEwfdks0eBcEZBBn4QTaZ9lVj7yScurto8jDYMkBfO/4QeuVe4ijgWU +35Yd89Ya0HZr+1QBU1dMzbtf9mYPedRwRZgzKogZ6mZXs7GJ0ADML+fg/djROd01 +eA3ynAfxJbRv1R7S73ljQJhD+oYf3rJaHU2Ko5heF9K9bd771Ze/sHGHZdLy3JZr +PNdDPHG61H9m4zOUkNZvVkvHF9wg+uNs4VJxrkG6undOEyCyBefEXRknEBvMLlWF +9c9KT5tZz/vI6/e5xrpNCdUjthn5YFEf43MVqWWcU8t9ViK+8+rGPQNKmxRiCOOm +d6NhjWJInuMO3QiPJFqVs4UNpmE/4hi3LtwV7qH4jHWX5hRG27gz8VbfkBkIoy0E +sm2Dv5lErZztPhycpFUodpBXGtQjXQLvYVm+yKwWlD6gTRH0LXj33lp69m5/V9so +3NsYWMxoNGvI4MqZuwGaiwxgkCWjUqQEX1y4i7Us01UDFpK2WuwGMUQNzGV2jX9u +rnjndFPHjszqo5BsmJuHwihGOFvbpwIDAQABAoICABB0Y2DFKYjD5ihpqyDeM3Nv +nikD8rFPY+W2aiSdlU24ttZhrrlRI0gUBmmWap1X0uZIZCeCWyu5NdsL3DO1yvyq +UDtqJrKo3wcQduOxYPPR0AnOTWbM53HXjHAsAAwuuihVMfmrvOIm5nFLJLACSIIR +pd3ko7UNDyiScmvydibGAZFrXY/uMOLevjW7IBNK3TZrWCKl4E5q8TtP8hrqsEym +ohWjxltTJv1LyqZ+o8Nmwb19n82bPjiatImd26tzRg57f/uFt4wCLyVOKlBVly82 +Kymqa8LGBlVGMbd/mpg84l+Just6jG5qPe4xFrnpigemTUkL5rstyXrL0KBNKkqB +jtuow1nGphZJSxis4tAQqqG9t0xtu7zCS1gWYEpUd1Y8wfTIM37hFojVVBOE8n86 +VEEx9xSt8diuY3X8He9W4IeXcDGe111Hbic4zuPQGsu+SeGoR/AJeZORCpf/PA9l +uX6gk48rEm8kzLyhqikYPJAfvRbRYxGKti2M7nDfUzhXs1CwnBWHMUzALr29//8D +I2hNTnAOmv42uQe+eiQw19HXLNMyKE1lgpbW9eAIZWCLJPUrYr2Fpdv1yoqyW5J5 +FVJE6iZSG31oUCIkymJ/xz0RqjdGD4zUnAhcqT7srMRQ7tRrifHvOahj7OLO/Yc3 +iBfeJZqOU7difsWYQqLxAoIBAQDlFkMDLb3ouTvtD1HWPWiRLpUEs3tGXsmZY1Xs +Z0FVHmkA7JLAj+NLr5J7lNzyP7oJkfrGuMHwO4AdXh+cQwZwhGq+TKOb67u8B0dx +CaZEPIKY13tl9WpZI/LANuzgnmo4rbWB8Z/crft/LjdpjZPUVFX8WBdi9c0wcPLJ +y7FYAfSGMLjgDI3UiI79Q6UjX8HvP5KPGm2D4rwPeuDCf24efj+b0YhsH+0b+9G0 +5T1PM2VTB3SPwtRnCR+A5W5XxxSylBAjjcPTJEsc4coWpuhH7GaLAuxmHV+UFUOC +CCA+Xq+lfE/2A0mwmrPbp+y9JuLclMTkoPbi+CXy6s5dlbjZAoIBAQDTdDnEqn3Z +fMs5YQZCaBmCPxbTAZnHbJmfkYyI/iLmWKpe0NQAHX4J+s8NqQcJFkJwUxjWfRBR +do1OYm0CK5pc06YwKk5W69WYvDr4/h3OsXXbtZT70i9Hdip1KMJk+umYExqk4pG6 +VA4WhIBVDFPEkAgP6uGj+rzpH/1UKiMRhUy37tY9MrNZaJVxzl7ect+K6YfAXAIR +4/1fb7sYUyaUFzr6u4oX7pojUUzuHqeVTZIpVLfGfUT0Wh4JxzHY2aBhjeboEpmc +fzltpd+EqAtOjvh7k2NKb3fkPzXp6EN2RYLZzM06fn+koHfpQ5UoFusG+EDG6PWw +r0yCA+vhxWh/AoIBAHD+Qv1dYW9ZdhJeXQoj7eC2LjBMasBx3lP26BmbcGEQh9A5 +38R09DKPnduwuC4Qeq1fwGamGJpqbq2NkF/du15iIdhNKuGsK0P5/yXlEYpUokHK +/wVyQAtJrOFb9ghweooMPBuk1ync8tBvNkus1j6Dfonh1V8tSBDdlRMtkgKLrFVH +0NhJuOccmBKZe+lvwNNF8v78lQnZGtMz04p7mhAWSz+K4RQbxzLnSS9FZrEa8545 +bie+fE9583z7LSrEehP+7drrgKwzGY1cXPZBSw62rnlgbsOLN2pt9oc7hNPelho0 +r6fzRTSDibUNJbrCZGFSS69NKZkXtngTksgcIjkCggEBAJZ3jx/2HiWkGszS0Oxy +PSykji5KZk41Zj/ZB4L4LIdzMpDT0vfeBLE/SxUyYTJDJ7XcO8FtLy54yOatPVqd +AnPpm8mMYUCTAuRzlRdXZIiPWQml9cUX9NLKMD48adAHiC/R8FTCcOaBJ6E4WkWq +xvUJdOM2xae1JbZSMCcy8M9n5obAKq1TpbIKrMVWD8YFQjMMKLBmCIcIkGTYEe+m +Q75/pVWEHCkZfY+CK9cJ80BX+Zj1kuxoeafTdKqKl1ryyaCBpH0htzNYiNoQfbRd +C+az7/enkrEGDu9ZV5kB0PeZURqBFMz59QSFt65CjCIB5O6JuzaP0T0QEMY32iNH +ISsCggEAP/2+C8hpkl530l002QnWzxhVdm5sCkk05jZJ2Rm1X2CXlGTOMxUhUamn +hYr0u+GUsORBjsJ6CnGkoXSiJOGn1+iOEZ8aAxRWxQdveWCEi2NITEIoK4jXFcsj +ewUzXLX+4OrruLsPVCQLjVYj84mH+X0rSctylJUayEOPbnnKtfMegQ2Ruw8XKEmL +aDssBizn2R3fUHcml1/pIM7bCv5AyBBlL2fAAHkB5d7XBW9lIUmY8YATaElQuF2/ +9xZrxMYyNWpVQWfAu4NvwunFmqzfXK8OFs3gwiSwRgmvFgAwDE5WbRdGH3F8J/Pc +xmyWhwxzeZdweoZDR6um1BRcJVJzdA== +-----END PRIVATE KEY----- From 7552ac4a20d91240c03a6df111d94fc237c06d37 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Wed, 24 Nov 2021 10:42:09 +0000 Subject: [PATCH 093/130] fix(ci) allow nektos/act to run workflow locally This requires nodejs to be added to the image. Also cleaned a few things up and bumped openresty patch versions. --- .github/workflows/test.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7708e04a..f78eb83a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,8 @@ jobs: strategy: matrix: openresty_version: - - 1.17.8.1 - - 1.19.3.1 + - 1.17.8.2 + - 1.19.9.1 runs-on: ubuntu-latest container: @@ -28,9 +28,10 @@ jobs: options: --init steps: + - uses: actions/checkout@v2 - name: Install deps run: | - apk add --no-cache curl perl bash wget git perl-dev libarchive-tools + apk add --no-cache curl perl bash wget git perl-dev libarchive-tools nodejs ln -s /usr/bin/bsdtar /usr/bin/tar - name: Install CPAN @@ -48,14 +49,12 @@ jobs: run: cpanm -q -n Test::Nginx - name: Install Luacov - run: luarocks install luacov + run: /usr/local/openresty/luajit/bin/luarocks install luacov - uses: actions/checkout@v2 - name: Run tests - env: - TEST_COVERAGE: '1' - run: /usr/bin/prove -I../test-nginx/lib -r t/ + run: make coverage - name: Coverage run: | From 28191b0a13ead386037c576a36300f45c1e3b950 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 11 Feb 2022 16:11:55 +0000 Subject: [PATCH 094/130] Add experimental mTLS support (#263) * Add experimental support for mTLS Note this requires patches to OpenResty not yet available in mainline. * Loosen test cases to expect either cdata or userdata * Skip mTLS test for CI * Improve connection error messages --- lib/resty/http_connect.lua | 43 +++++++- t/19-ssl_reused_session.t | 6 +- t/20-mtls.t | 211 +++++++++++++++++++++++++++++++++++++ t/cert/mtls_ca.crt | 33 ++++++ t/cert/mtls_client.crt | 111 +++++++++++++++++++ t/cert/mtls_client.key | 27 +++++ t/cert/mtls_server.crt | 112 ++++++++++++++++++++ t/cert/mtls_server.key | 27 +++++ 8 files changed, 563 insertions(+), 7 deletions(-) create mode 100644 t/20-mtls.t create mode 100644 t/cert/mtls_ca.crt create mode 100644 t/cert/mtls_client.crt create mode 100644 t/cert/mtls_client.key create mode 100644 t/cert/mtls_server.crt create mode 100644 t/cert/mtls_server.key diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 25c47041..18a74b1a 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -1,6 +1,8 @@ local ngx_re_gmatch = ngx.re.gmatch local ngx_re_sub = ngx.re.sub local ngx_re_find = ngx.re.find +local ngx_log = ngx.log +local ngx_WARN = ngx.WARN --[[ A connection function that incorporates: @@ -28,6 +30,19 @@ client:connect { ssl_verify = true, -- NOTE: defaults to true ctx = nil, -- NOTE: not supported + -- mTLS options (experimental!) + -- + -- !!! IMPORTANT !!! These options require support for mTLS in cosockets, + -- which is currently only available in the following unmerged PRs. + -- + -- * https://github.com/openresty/lua-nginx-module/pull/1602 + -- * https://github.com/openresty/lua-resty-core/pull/278 + -- + -- The details of this feature may change. You have been warned! + -- + ssl_client_cert = nil, + ssl_client_priv_key = nil, + proxy_opts, -- proxy opts, defaults to global proxy options } ]] @@ -54,7 +69,8 @@ local function connect(self, options) end -- ssl settings - local ssl, ssl_reused_session, ssl_server_name, ssl_verify, ssl_send_status_req + local ssl, ssl_reused_session, ssl_server_name + local ssl_verify, ssl_send_status_req, ssl_client_cert, ssl_client_priv_key if request_scheme == "https" then ssl = true ssl_reused_session = options.ssl_reused_session @@ -64,6 +80,8 @@ local function connect(self, options) if options.ssl_verify == false then ssl_verify = false end + ssl_client_cert = options.ssl_client_cert + ssl_client_priv_key = options.ssl_client_priv_key end -- proxy related settings @@ -138,7 +156,7 @@ local function connect(self, options) local proxy_uri_t proxy_uri_t, err = self:parse_uri(proxy_uri) if not proxy_uri_t then - return nil, err + return nil, "uri parse error: ", err end local proxy_scheme = proxy_uri_t[1] @@ -172,7 +190,9 @@ local function connect(self, options) -- proxy based connection ok, err = sock:connect(proxy_host, proxy_port, tcp_opts) if not ok then - return nil, err + return nil, "failed to connect to: " .. (proxy_host or "") .. + ":" .. (proxy_port or "") .. + ": ", err end if ssl and sock:getreusedtimes() == 0 then @@ -192,7 +212,7 @@ local function connect(self, options) }) if not res then - return nil, err + return nil, "failed to issue CONNECT to proxy:", err end if res.status < 200 or res.status > 299 then @@ -218,6 +238,21 @@ local function connect(self, options) local ssl_session -- Now do the ssl handshake if ssl and sock:getreusedtimes() == 0 then + + -- Experimental mTLS support + if ssl_client_cert and ssl_client_priv_key then + if type(sock.setclientcert) ~= "function" then + ngx_log(ngx_WARN, "cannot use SSL client cert and key without mTLS support") + + else + -- currently no return value + ok, err = sock:setclientcert(ssl_client_cert, ssl_client_priv_key) + if not ok then + ngx_log(ngx_WARN, "could not set client certificate: ", err) + end + end + end + ssl_session, err = sock:sslhandshake(ssl_reused_session, ssl_server_name, ssl_verify, ssl_send_status_req) if not ssl_session then self:close() diff --git a/t/19-ssl_reused_session.t b/t/19-ssl_reused_session.t index 45f9e8fb..d9b69236 100644 --- a/t/19-ssl_reused_session.t +++ b/t/19-ssl_reused_session.t @@ -57,7 +57,7 @@ __DATA__ host = TEST_SERVER_SOCK, }) - assert(type(session) == "userdata", "expected session to be userdata") + assert(type(session) == "userdata" or type(session) == "cdata", "expected session to be userdata or cdata") assert(httpc:close()) } } @@ -113,7 +113,7 @@ GET /t host = TEST_SERVER_SOCK, }) - assert(type(session) == "userdata", "expected session to be userdata") + assert(type(session) == "userdata" or type(session) == "cdata", "expected session to be userdata or cdata") local httpc2 = assert(require("resty.http").new()) local ok, err, session2 = assert(httpc2:connect { @@ -122,7 +122,7 @@ GET /t ssl_reused_session = session, }) - assert(type(session2) == "userdata", "expected session2 to be userdata") + assert(type(session2) == "userdata" or type(session2) == "cdata", "expected session2 to be userdata or cdata") assert(httpc:close()) assert(httpc2:close()) diff --git a/t/20-mtls.t b/t/20-mtls.t new file mode 100644 index 00000000..e6395bdd --- /dev/null +++ b/t/20-mtls.t @@ -0,0 +1,211 @@ +use Test::Nginx::Socket::Lua 'no_plan'; + + +#$ENV{TEST_NGINX_RESOLVER} ||= '8.8.8.8'; +#$ENV{TEST_COVERAGE} ||= 0; + +log_level 'debug'; + +no_long_string(); +#no_diff(); + +sub read_file { + my $infile = shift; + open my $in, $infile + or die "cannot open $infile for reading: $!"; + my $cert = do { local $/; <$in> }; + close $in; + $cert; +} + +our $MTLSCA = read_file("t/cert/mtls_ca.crt"); +our $MTLSClient = read_file("t/cert/mtls_client.crt"); +our $MTLSClientKey = read_file("t/cert/mtls_client.key"); +our $TestCert = read_file("t/cert/test.crt"); +our $TestKey = read_file("t/cert/test.key"); + +our $HtmlDir = html_dir; + +use Cwd qw(cwd); +my $pwd = cwd(); + +our $mtls_http_config = <<"_EOC_"; + lua_package_path "$pwd/lib/?.lua;/usr/local/share/lua/5.1/?.lua;;"; +server { + listen unix:$::HtmlDir/mtls.sock ssl; + + ssl_certificate $::HtmlDir/test.crt; + ssl_certificate_key $::HtmlDir/test.key; + ssl_client_certificate $::HtmlDir/mtls_ca.crt; + ssl_verify_client on; + server_tokens off; + server_name example.com; + + location / { + echo -n "hello, \$ssl_client_s_dn"; + } +} +_EOC_ + +our $mtls_user_files = <<"_EOC_"; +>>> mtls_ca.crt +$::MTLSCA +>>> mtls_client.key +$::MTLSClientKey +>>> mtls_client.crt +$::MTLSClient +>>> test.crt +$::TestCert +>>> test.key +$::TestKey +_EOC_ + +run_tests(); + +__DATA__ + +=== TEST 1: Connection fails during handshake without client cert and key +--- http_config eval: $::mtls_http_config +--- config eval +" +lua_ssl_trusted_certificate $::HtmlDir/test.crt; +location /t { + content_by_lua_block { + local httpc = assert(require('resty.http').new()) + + local ok, err = httpc:connect { + scheme = 'https', + host = 'unix:$::HtmlDir/mtls.sock', + } + + if ok and not err then + local res, err = assert(httpc:request { + method = 'GET', + path = '/', + headers = { + ['Host'] = 'example.com', + }, + }) + + ngx.status = res.status -- expect 400 + end + + httpc:close() + } +} +" +--- user_files eval: $::mtls_user_files +--- request +GET /t +--- error_code: 400 +--- no_error_log +[error] +[warn] + + +=== TEST 2: Connection fails during handshake with not priv_key +--- http_config eval: $::mtls_http_config +--- SKIP +--- config eval +" +lua_ssl_trusted_certificate $::HtmlDir/test.crt; +location /t { + content_by_lua_block { + local f = assert(io.open('$::HtmlDir/mtls_client.crt')) + local cert_data = f:read('*a') + f:close() + + local ssl = require('ngx.ssl') + + local cert = assert(ssl.parse_pem_cert(cert_data)) + + local httpc = assert(require('resty.http').new()) + + local ok, err = httpc:connect { + scheme = 'https', + host = 'unix:$::HtmlDir/mtls.sock', + ssl_client_cert = cert, + ssl_client_priv_key = 'foo', + } + + if ok and not err then + local res, err = assert(httpc:request { + method = 'GET', + path = '/', + headers = { + ['Host'] = 'example.com', + }, + }) + + ngx.say(res:read_body()) + end + + httpc:close() + } +} +" +--- user_files eval: $::mtls_user_files +--- request +GET /t +--- error_code: 200 +--- error_log +could not set client certificate: bad client pkey type +--- response_body_unlike: hello, CN=foo@example.com,O=OpenResty,ST=California,C=US + + +=== TEST 3: Connection succeeds with client cert and key. SKIP'd for CI until feature is merged. +--- SKIP +--- http_config eval: $::mtls_http_config +--- config eval +" +lua_ssl_trusted_certificate $::HtmlDir/test.crt; + +location /t { + content_by_lua_block { + local f = assert(io.open('$::HtmlDir/mtls_client.crt')) + local cert_data = f:read('*a') + f:close() + + f = assert(io.open('$::HtmlDir/mtls_client.key')) + local key_data = f:read('*a') + f:close() + + local ssl = require('ngx.ssl') + + local cert = assert(ssl.parse_pem_cert(cert_data)) + local key = assert(ssl.parse_pem_priv_key(key_data)) + + local httpc = assert(require('resty.http').new()) + + local ok, err = httpc:connect { + scheme = 'https', + host = 'unix:$::HtmlDir/mtls.sock', + ssl_client_cert = cert, + ssl_client_priv_key = key, + } + + if ok and not err then + local res, err = assert(httpc:request { + method = 'GET', + path = '/', + headers = { + ['Host'] = 'example.com', + }, + }) + + ngx.say(res:read_body()) + end + + httpc:close() + } +} +" +--- user_files eval: $::mtls_user_files +--- request +GET /t +--- no_error_log +[error] +[warn] +--- response_body +hello, CN=foo@example.com,O=OpenResty,ST=California,C=US + diff --git a/t/cert/mtls_ca.crt b/t/cert/mtls_ca.crt new file mode 100644 index 00000000..99c8ec83 --- /dev/null +++ b/t/cert/mtls_ca.crt @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFpTCCA42gAwIBAgIUfTh89NyxxmbVwNZ/YFddssWc+WkwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAoM +CU9wZW5SZXN0eTEiMCAGA1UEAwwZT3BlblJlc3R5IFRlc3RpbmcgUm9vdCBDQTAe +Fw0xOTA5MTMyMjI4MTJaFw0zOTA5MDgyMjI4MTJaMFoxCzAJBgNVBAYTAlVTMRMw +EQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQKDAlPcGVuUmVzdHkxIjAgBgNVBAMM +GU9wZW5SZXN0eSBUZXN0aW5nIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQDcMg2DeV8z+0E2ZiXUax111lKzhAbMCK0RJlV9tAi+YdcsDR/t +zvvAZNGONUoewUuz/7E88oweh+Xi1GJtvd0DjB70y7tgpf5PUXovstWVwy7s5jZo +kgn62yi9ZOOZpjwnYTBviirtRTnZRwkzL6wF0xMyJjAbKBJuPMrMiyFdh82lt7wI +NS4mhyEdM0UiVVxfC2uzsddTOcOJURfGbW7UZm4Xohzq4QZ8geQj2OT5YTqw7dZ7 +Xxre5H7IcNcAh+vIk5SEBV1WE+S5MnFly7gaLYNc49OSfz5Hcpv59Vr+4bZ+olbW +nQ/uU8BQovtkW6pjuT8nC4OKs2e8osoAZuk0rFS1uC501C+yES48mzaU8ttAidu6 +nb/JgsdkrnJQeTc5rAoER4M2ne5kqtEXN8wzf3/sazo2PLywbfrUXUTV6kJilrGr +RkBN+fr6HTBkf+ooQMBOQPTojUdwbR86CLCyiJov2bzmBfGcOgSakv59S+uvUZFp +FLTiahuzLfcgYsG3UKQA47pYlNdUqP8vCCaf1nwmqjx2KS3Z/YFnO/gQgtY+f0Bh +UpnUDv+zBxpVFfVCyxByEsDPdwDkqLSwB6+YZINl36S48iXpoPhNXIYmO6GnhNWV +k2/RyCDTxEO+MbXHVg6iyIVHJWth7m18vl4uuSK/LbJHV9Q9Z7G99DQ0NwIDAQAB +o2MwYTAdBgNVHQ4EFgQUuoo+ehdlDFcQU+j5qONMKh0NtFQwHwYDVR0jBBgwFoAU +uoo+ehdlDFcQU+j5qONMKh0NtFQwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E +BAMCAYYwDQYJKoZIhvcNAQELBQADggIBAM7c9q41K8306lfAVLqtbtaeETy+xxYG +XE2HfFW1IuukrXQ8d/JH4stL/HcHJzzhHPf5p3ja3Snu9zPmTk3pgUPDYPZf57tR +NCqwxjn6blwXWlzQqSavto9KAx3IWHuj0OTrZz/a1KPb9NGvatBhgthyRCRTbvhL +OA5tveuYSHb724cp3NZ1xaTQmDZsSgHCoCJ/7RnlbcJ7RsKCOzCWNFRomH410vdv +TajkUBlEC4OC1RIvxuVePHHb1ogbbe93SA/9mzw/E5SfoeF3mvByN4Ay8awXbNlH +26RfuIdGc4fZRc/87s4yPwhYScZBG+pHO0gn42E0FyiG6Jp3rhHMH5Sa2hNlPMpn +hYMaA6zQI4n/3AeFNM0VGxA+Yg/Al2WpXEJARrZqMW/qcrdMcPj5WeY6Tb6er04S +kfImwhMIajl3nNc9tHoad8r2VuMWMltH/dnWuEdo+pPdIY3fdJdyQeoLQDDLEQwL +AYrFy4uzKfQogfQBIHRdIMZTJh5v3mAFDpK59I5yzSt1GtUnFMC5MVOg+LbOo5UW +FCtwaW5EZiTszmakvvWMMZe9HwZMYNCeSGGtiPA/GA2zNci/n2TEcB11HgiY52y2 +E/40nS61oL81zMwhV7l5psgJxQ2ORsKRJPHjADwvwh3xyCEJgVyBRCDX7J3PpAUO +79DprjVU8t7p +-----END CERTIFICATE----- diff --git a/t/cert/mtls_client.crt b/t/cert/mtls_client.crt new file mode 100644 index 00000000..c6ba07cc --- /dev/null +++ b/t/cert/mtls_client.crt @@ -0,0 +1,111 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4096 (0x1000) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=OpenResty, CN=OpenResty Testing Root CA + Validity + Not Before: Sep 13 22:30:49 2019 GMT + Not After : Sep 10 22:30:49 2029 GMT + Subject: C=US, ST=California, O=OpenResty, CN=foo@example.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:d0:8c:1e:2c:25:7f:00:9d:8a:3d:8a:f3:b5:1d: + 6b:24:f5:ac:35:7d:cd:b7:d0:af:db:88:7c:ee:82: + 46:16:47:f3:43:08:a1:04:6e:0d:3e:ce:69:fa:d5: + 89:14:82:20:f1:47:f2:38:c8:ab:ea:2f:1e:f0:15: + 04:c0:f4:8b:3c:c3:d4:78:56:87:4c:f1:70:ac:11: + 86:2e:c4:6a:6d:10:84:27:81:ca:2a:8b:85:3e:62: + 13:5e:40:6c:19:e4:49:3d:f3:de:aa:e8:5e:11:a1: + f2:66:83:6a:40:d1:34:c5:bf:b8:cb:97:7c:6a:ea: + 46:bf:17:be:32:8d:a8:31:56:e5:8b:6d:08:03:d0: + 44:69:b9:af:1e:15:1d:a5:64:9e:12:84:83:db:d9: + c6:71:90:3b:c2:7b:41:21:57:af:70:15:0b:56:59: + 21:a6:4e:46:71:66:90:f1:ef:bc:b2:48:f9:8b:ea: + e5:72:4a:ba:4a:ae:2d:74:0b:33:03:f6:2e:47:0f: + 56:a4:00:e8:1e:62:cb:b8:af:9c:98:1a:89:7c:d0: + a3:7a:5a:e1:84:50:64:e4:5d:a5:70:a4:69:54:c4: + f4:76:44:a2:be:1b:ef:dc:a3:d8:1d:0d:30:a2:d4: + 79:fb:39:76:ab:b7:18:f2:f7:92:f8:81:83:94:b8: + 11:b1 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Cert Type: + SSL Client, S/MIME + Netscape Comment: + OpenSSL Generated Client Certificate + X509v3 Subject Key Identifier: + D2:E4:F5:21:1C:17:A4:FF:13:F4:1A:28:A8:A7:DC:C6:DE:89:A0:31 + X509v3 Authority Key Identifier: + keyid:BA:8A:3E:7A:17:65:0C:57:10:53:E8:F9:A8:E3:4C:2A:1D:0D:B4:54 + + X509v3 Key Usage: critical + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Client Authentication, E-mail Protection + X509v3 Subject Alternative Name: + email:foo@example.com, email:bar@example.com + Signature Algorithm: sha256WithRSAEncryption + 47:77:51:37:00:f7:28:da:c4:d9:4e:38:c4:ec:ea:24:c9:83: + 36:4c:90:93:a7:b2:2b:10:bf:75:df:0b:72:d8:e7:4f:4f:68: + e5:32:2f:35:89:17:95:5c:bb:43:fc:70:89:46:08:43:61:ac: + 41:62:84:01:94:88:d1:dc:8a:bd:30:2c:18:eb:51:79:0b:b7: + 1b:b6:49:df:c9:85:55:f6:73:9f:b7:83:99:52:23:fe:e6:bd: + 09:da:90:b9:e2:9b:68:c4:df:bd:fe:23:94:55:34:be:0d:7d: + 84:0c:53:69:2a:0f:3c:47:68:34:3f:2a:3f:89:3f:3e:d3:26: + ce:b7:58:bc:d0:6f:ee:f8:bd:5d:c6:48:ae:a0:6c:1f:6d:e0: + 66:93:7d:db:3c:07:e6:15:ae:aa:e3:d0:3d:ef:04:b6:dd:53: + 16:93:61:70:e9:af:c0:e9:1d:ff:2b:e5:0a:03:56:48:3f:1c: + dc:fe:1b:a6:6d:f6:54:ab:41:e5:3b:5b:ab:f5:81:10:46:26: + bb:ea:d7:0e:33:b1:5e:30:4d:81:86:63:9a:4a:4f:1e:44:b9: + c2:c6:08:4e:da:fa:3a:55:da:96:7c:01:f6:d5:e8:3b:ba:e9: + 31:3b:1c:51:39:1a:59:f0:e0:c7:17:2e:f6:18:9d:ec:a7:48: + 30:b8:4c:6d:e5:4a:4f:43:41:cb:0e:6b:ac:ad:87:44:90:76: + 85:23:2b:eb:8f:97:4b:22:13:60:20:3a:37:a4:dc:74:7d:85: + 3d:a1:f5:1a:03:f6:d5:78:c7:bc:9b:09:f2:c8:05:27:43:2a: + ac:50:21:3a:ee:83:2d:db:02:6f:c7:91:de:63:d6:36:7d:7a: + 9f:1f:fb:48:62:f4:fb:8e:3a:ea:61:9b:3c:03:f9:f8:a5:df: + 1b:02:14:2c:de:e6:e3:47:d2:44:65:94:1a:c6:e1:fd:ba:8d: + b6:f8:93:a9:46:46:26:79:b0:bf:57:a8:a2:20:66:56:7e:c9: + f5:a4:0b:5e:76:70:0a:47:a4:db:45:2e:15:99:69:f9:6b:14: + 93:2a:0a:b6:ee:53:a6:b9:02:9b:a2:25:37:1e:37:70:a2:7c: + 7f:c3:ce:98:17:2f:9b:5b:fa:6f:ae:d8:0e:d4:6a:b2:03:5a: + fe:ba:4b:7f:f6:98:20:ea:cb:be:17:34:e0:43:74:d1:0c:e5: + d4:cc:5d:13:41:d3:5e:a4:f6:94:f7:15:b8:15:a9:65:f8:28: + 3f:da:ef:b2:30:34:6d:96:3a:7a:f4:20:ec:9e:62:13:36:f1: + a7:04:e1:7a:d2:33:20:f6:61:4a:68:44:cb:92:d7:62:f0:e4: + 70:f0:a5:e3:dd:2f:e2:a3 +-----BEGIN CERTIFICATE----- +MIIFGTCCAwGgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwWjELMAkGA1UEBhMCVVMx +EzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAoMCU9wZW5SZXN0eTEiMCAGA1UE +AwwZT3BlblJlc3R5IFRlc3RpbmcgUm9vdCBDQTAeFw0xOTA5MTMyMjMwNDlaFw0y +OTA5MTAyMjMwNDlaMFAxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRIwEAYDVQQKDAlPcGVuUmVzdHkxGDAWBgNVBAMMD2Zvb0BleGFtcGxlLmNvbTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANCMHiwlfwCdij2K87UdayT1 +rDV9zbfQr9uIfO6CRhZH80MIoQRuDT7OafrViRSCIPFH8jjIq+ovHvAVBMD0izzD +1HhWh0zxcKwRhi7Eam0QhCeByiqLhT5iE15AbBnkST3z3qroXhGh8maDakDRNMW/ +uMuXfGrqRr8XvjKNqDFW5YttCAPQRGm5rx4VHaVknhKEg9vZxnGQO8J7QSFXr3AV +C1ZZIaZORnFmkPHvvLJI+Yvq5XJKukquLXQLMwP2LkcPVqQA6B5iy7ivnJgaiXzQ +o3pa4YRQZORdpXCkaVTE9HZEor4b79yj2B0NMKLUefs5dqu3GPL3kviBg5S4EbEC +AwEAAaOB8jCB7zAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIFoDAzBglghkgB +hvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRpZmljYXRlMB0G +A1UdDgQWBBTS5PUhHBek/xP0Giiop9zG3omgMTAfBgNVHSMEGDAWgBS6ij56F2UM +VxBT6Pmo40wqHQ20VDAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYwFAYIKwYBBQUH +AwIGCCsGAQUFBwMEMCsGA1UdEQQkMCKBD2Zvb0BleGFtcGxlLmNvbYEPYmFyQGV4 +YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4ICAQBHd1E3APco2sTZTjjE7OokyYM2 +TJCTp7IrEL913wty2OdPT2jlMi81iReVXLtD/HCJRghDYaxBYoQBlIjR3Iq9MCwY +61F5C7cbtknfyYVV9nOft4OZUiP+5r0J2pC54ptoxN+9/iOUVTS+DX2EDFNpKg88 +R2g0Pyo/iT8+0ybOt1i80G/u+L1dxkiuoGwfbeBmk33bPAfmFa6q49A97wS23VMW +k2Fw6a/A6R3/K+UKA1ZIPxzc/humbfZUq0HlO1ur9YEQRia76tcOM7FeME2BhmOa +Sk8eRLnCxghO2vo6VdqWfAH21eg7uukxOxxRORpZ8ODHFy72GJ3sp0gwuExt5UpP +Q0HLDmusrYdEkHaFIyvrj5dLIhNgIDo3pNx0fYU9ofUaA/bVeMe8mwnyyAUnQyqs +UCE67oMt2wJvx5HeY9Y2fXqfH/tIYvT7jjrqYZs8A/n4pd8bAhQs3ubjR9JEZZQa +xuH9uo22+JOpRkYmebC/V6iiIGZWfsn1pAtednAKR6TbRS4VmWn5axSTKgq27lOm +uQKboiU3Hjdwonx/w86YFy+bW/pvrtgO1GqyA1r+ukt/9pgg6su+FzTgQ3TRDOXU +zF0TQdNepPaU9xW4Fall+Cg/2u+yMDRtljp69CDsnmITNvGnBOF60jMg9mFKaETL +ktdi8ORw8KXj3S/iow== +-----END CERTIFICATE----- diff --git a/t/cert/mtls_client.key b/t/cert/mtls_client.key new file mode 100644 index 00000000..34b301f0 --- /dev/null +++ b/t/cert/mtls_client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0IweLCV/AJ2KPYrztR1rJPWsNX3Nt9Cv24h87oJGFkfzQwih +BG4NPs5p+tWJFIIg8UfyOMir6i8e8BUEwPSLPMPUeFaHTPFwrBGGLsRqbRCEJ4HK +KouFPmITXkBsGeRJPfPequheEaHyZoNqQNE0xb+4y5d8aupGvxe+Mo2oMVbli20I +A9BEabmvHhUdpWSeEoSD29nGcZA7wntBIVevcBULVlkhpk5GcWaQ8e+8skj5i+rl +ckq6Sq4tdAszA/YuRw9WpADoHmLLuK+cmBqJfNCjelrhhFBk5F2lcKRpVMT0dkSi +vhvv3KPYHQ0wotR5+zl2q7cY8veS+IGDlLgRsQIDAQABAoIBAEpoY++OZVT74LH6 +nN+XIn5qZUokm7yk6cnjVefndUhH3aSiNIkXFwS8sxV7ENDPaR+NcwANoUEKFPjG +Fw8dcXx5xpo1DUtHrdLG4eBX1j0Zsn1CErbBVwYeChkL1UYbrII9O8ow5DdYV9t5 +sfR0cGbJ9A43+31OH3XY69SvtD39xItgxK0Wg4Ciz475kCvG51Q1iBWAkm4koXhz +VCha4wghs81wJ28HRMFZAFf2C+72rk6EypMUX2dYirvW/+7zONirk298NDMAOSBh +mRWyPV8qipYx42hBQ9vSVm0UVb0ZbqVomKKUZfj11LL6Ad/OzCyVAiNLXyZREV6r +d324Bu0CgYEA/YPsE6p6H3MTPIoVTnsyyQw0pNmXtIAuYTgSveeuPoTYn3ZBWoGN +iLpbnW4EH3xNKfrdMjbqqLls1iwm7/ZAP5klAuL4s10onrcjMt65fyfa3Lw1gavG +SUFFdsueH2k3FohqNsbQUSXZILVQnXsRoldi38b7NKrAqABcEMAIqXMCgYEA0pde +nt4aMmrGBRPLnjCs1UlC5PbXzCE8XxxQ7HZKx4Sy5ErQ0EW1wzF6c0fEYI7i+j1/ +ESKqekzc5ue0T8acoioB+VUybO1oxQZsZUPY7roqXOYwZH9LQOdPYUOh9k33CZHw +6KFfx8bKCpdXn7FkwR2UUtCSp/6CZcyYr89Qn0sCgYAQ0L5I86bUDTL6cgJFyWAt ++7RGNvScEWCCLFD57bMeDHu93/8nvK4hopLPF2wIlpsbrLsdSI06EcqJTjZq9j9+ +uG6/CUULyKMYG/emuSU+rOsUdxtpdXZah4zO+2SKmtT/lp7M8VUB/OuxArXNLEuY +JAm35B/nd2f9/MAekE5CxwKBgQCV660w7G0590mB09XhiEWCkeVNm22FpSOVklMK +BCy4XX/9hkWh//6mN1M1PqJPG2n7PEx5pnQ3HQEmYU28fWiFCeLd3glIArvTh/8j +GGoXifEescFByl2IlyOr2roy3s4/weX/tuK5Fow/ff6jcWaJFMXDLzk437d1QXJx +tuVugQKBgByfr2eakXFQvAVGJUfVXA3M2BoBODZEPYTgryVMoEEduFy0HZiw4xKi +Dngwewy6/UJMAGA+8ak9Ca367FxnegZU9knm6ujYVyhU5WzbKpR8v7OaUP8d5icq +rCZZtglG0c8XfVpJjR4FsKA/qrFvKZpu5NdEw3o5/LSrV4HjqZQ6 +-----END RSA PRIVATE KEY----- diff --git a/t/cert/mtls_server.crt b/t/cert/mtls_server.crt new file mode 100644 index 00000000..c4d320f4 --- /dev/null +++ b/t/cert/mtls_server.crt @@ -0,0 +1,112 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 4097 (0x1001) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=California, O=OpenResty, CN=OpenResty Testing Root CA + Validity + Not Before: Sep 13 22:32:03 2019 GMT + Not After : Sep 10 22:32:03 2029 GMT + Subject: C=US, ST=California, O=OpenResty, CN=example.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:bb:69:2b:30:43:b0:b4:cc:84:3f:22:39:65:65: + 6a:bd:75:a4:2b:7d:f7:ec:e4:12:e8:d1:c4:ce:7e: + 4e:54:5c:22:cc:d2:18:7f:3b:9e:cb:70:d9:7d:79: + 8f:05:93:b6:9f:2f:d5:33:d7:98:a2:ed:c5:00:93: + e4:ca:bc:cb:f0:e1:63:3e:07:6b:38:6f:4d:09:45: + f1:a1:3b:a3:ca:c0:47:c1:a1:0a:f8:c9:bb:c7:da: + 26:9d:d3:0b:35:24:01:3e:16:14:2e:44:38:8c:c9: + 09:02:41:9e:b6:fb:0c:aa:fc:d6:44:5e:27:ab:aa: + d5:c3:68:e1:dd:57:06:6c:4f:f6:24:33:a8:2b:49: + 60:82:0e:15:aa:55:9f:61:cc:74:39:7e:9f:a6:4f: + 71:4a:8b:eb:43:dd:c2:f7:90:38:df:a6:a6:a8:f6: + 77:bc:9e:54:69:30:83:4c:2a:eb:b8:62:7c:c7:14: + 84:9e:f3:e1:4a:15:33:51:65:a3:af:9d:09:c6:b8: + 89:30:a3:d2:18:e9:dc:5d:6b:ea:68:ca:8b:5c:e4: + 3b:fe:32:7f:48:c3:4c:f0:b5:06:f6:23:97:3e:f2: + 50:90:68:26:39:6d:b2:e2:53:89:71:6a:48:f0:f1: + fc:89:3c:6d:db:87:6c:79:23:ed:87:5d:c5:fa:8a: + 0d:b9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Cert Type: + SSL Server + Netscape Comment: + OpenSSL Generated Server Certificate + X509v3 Subject Key Identifier: + 8C:8E:18:2C:13:84:C9:2A:61:6B:73:3F:18:76:A4:85:55:5F:5C:5F + X509v3 Authority Key Identifier: + keyid:BA:8A:3E:7A:17:65:0C:57:10:53:E8:F9:A8:E3:4C:2A:1D:0D:B4:54 + DirName:/C=US/ST=California/O=OpenResty/CN=OpenResty Testing Root CA + serial:7D:38:7C:F4:DC:B1:C6:66:D5:C0:D6:7F:60:57:5D:B2:C5:9C:F9:69 + + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication + Signature Algorithm: sha256WithRSAEncryption + 80:9f:3e:f5:8b:50:ee:cb:e4:c2:f0:16:01:07:a1:af:76:bc: + da:c8:cd:24:e9:63:df:d3:47:28:8c:7f:58:14:5d:d4:fd:44: + 16:c3:06:15:be:90:ec:1c:8f:78:34:11:e7:cc:86:d8:2a:a2: + e5:99:70:83:76:4a:65:a4:e1:9a:68:20:29:c0:7a:c6:4a:08: + b3:74:c3:53:b5:7b:79:92:f1:99:b5:a1:a3:90:ce:9a:cb:26: + a5:a6:33:de:74:98:99:ec:18:d1:1e:41:be:f8:c3:d2:8d:aa: + 07:de:9a:97:28:0d:bf:70:ac:2b:cf:b7:ff:bc:ac:e4:16:0c: + 1c:03:a7:5a:2d:64:0d:90:16:bd:97:c3:1f:f5:bf:a9:fa:15: + d1:e0:d4:0d:f7:b3:51:23:ce:ad:16:4f:41:72:17:aa:01:d5: + 44:e2:9e:d5:ce:ea:54:98:04:43:14:2e:51:4b:c7:d9:21:4f: + e1:a4:fa:dd:e0:f0:82:ec:6f:9f:be:a2:3c:3b:85:f7:6d:96: + ee:0d:e6:08:2b:1b:be:06:a4:b7:5f:a3:f2:f2:b9:d0:5a:8f: + 90:86:1a:f4:7a:9f:c8:ae:09:1d:60:a2:8b:e0:0b:f6:00:21: + d9:df:33:4b:39:75:b6:64:9b:c7:df:e4:85:7a:ae:df:72:8c: + 8b:7e:98:8e:47:0a:27:1f:8e:2c:11:7f:7b:fc:a0:db:1b:6e: + f6:de:4e:85:ac:30:e6:e8:6a:7a:e6:f9:f4:18:0a:c6:ad:1c: + e1:0c:dd:e0:e0:8d:5a:d7:08:34:e7:22:b4:44:bd:99:39:b1: + 71:74:3f:7c:aa:65:f5:37:46:85:d3:79:f7:a8:35:8d:2b:30: + 99:d2:47:ce:a6:74:eb:f3:9f:d3:9a:4e:99:96:50:7b:ba:22: + c8:72:47:d4:da:6e:9a:73:01:3c:89:e9:3f:56:17:b7:ba:22: + 71:db:66:a2:d2:fb:33:51:36:f6:b6:f2:5b:32:70:9d:e7:e3: + 36:d6:ae:cb:9b:62:ef:69:c7:f7:ba:95:49:16:f5:7c:d9:29: + bb:0a:02:b1:6b:72:15:ab:2c:27:7b:c8:bc:f6:15:1f:fa:ae: + 08:fd:e0:11:36:b1:ab:9c:c8:11:d1:d3:0d:7d:49:4e:ca:e6: + 73:ee:0d:c3:8d:6f:f5:a4:fe:a1:af:6b:91:f7:53:fd:10:df: + 77:dd:ef:ec:7b:cf:32:75:df:04:8a:d1:a1:f7:36:68:ee:65: + e3:43:90:37:43:e8:d1:a8:e2:90:5c:1c:75:0a:29:94:4a:6a: + 9b:89:28:43:bd:85:56:0d:f1:2b:44:bd:e6:7a:4c:b7:85:10: + 77:b7:a8:0f:33:29:a7:26 +-----BEGIN CERTIFICATE----- +MIIFWTCCA0GgAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwWjELMAkGA1UEBhMCVVMx +EzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAoMCU9wZW5SZXN0eTEiMCAGA1UE +AwwZT3BlblJlc3R5IFRlc3RpbmcgUm9vdCBDQTAeFw0xOTA5MTMyMjMyMDNaFw0y +OTA5MTAyMjMyMDNaMEwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh +MRIwEAYDVQQKDAlPcGVuUmVzdHkxFDASBgNVBAMMC2V4YW1wbGUuY29tMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu2krMEOwtMyEPyI5ZWVqvXWkK333 +7OQS6NHEzn5OVFwizNIYfzuey3DZfXmPBZO2ny/VM9eYou3FAJPkyrzL8OFjPgdr +OG9NCUXxoTujysBHwaEK+Mm7x9omndMLNSQBPhYULkQ4jMkJAkGetvsMqvzWRF4n +q6rVw2jh3VcGbE/2JDOoK0lggg4VqlWfYcx0OX6fpk9xSovrQ93C95A436amqPZ3 +vJ5UaTCDTCrruGJ8xxSEnvPhShUzUWWjr50JxriJMKPSGOncXWvqaMqLXOQ7/jJ/ +SMNM8LUG9iOXPvJQkGgmOW2y4lOJcWpI8PH8iTxt24dseSPth13F+ooNuQIDAQAB +o4IBNTCCATEwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4 +QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNV +HQ4EFgQUjI4YLBOEySpha3M/GHakhVVfXF8wgZcGA1UdIwSBjzCBjIAUuoo+ehdl +DFcQU+j5qONMKh0NtFShXqRcMFoxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxp +Zm9ybmlhMRIwEAYDVQQKDAlPcGVuUmVzdHkxIjAgBgNVBAMMGU9wZW5SZXN0eSBU +ZXN0aW5nIFJvb3QgQ0GCFH04fPTcscZm1cDWf2BXXbLFnPlpMA4GA1UdDwEB/wQE +AwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEAgJ8+ +9YtQ7svkwvAWAQehr3a82sjNJOlj39NHKIx/WBRd1P1EFsMGFb6Q7ByPeDQR58yG +2Cqi5Zlwg3ZKZaThmmggKcB6xkoIs3TDU7V7eZLxmbWho5DOmssmpaYz3nSYmewY +0R5BvvjD0o2qB96alygNv3CsK8+3/7ys5BYMHAOnWi1kDZAWvZfDH/W/qfoV0eDU +DfezUSPOrRZPQXIXqgHVROKe1c7qVJgEQxQuUUvH2SFP4aT63eDwguxvn76iPDuF +922W7g3mCCsbvgakt1+j8vK50FqPkIYa9HqfyK4JHWCii+AL9gAh2d8zSzl1tmSb +x9/khXqu33KMi36YjkcKJx+OLBF/e/yg2xtu9t5Ohaww5uhqeub59BgKxq0c4Qzd +4OCNWtcINOcitES9mTmxcXQ/fKpl9TdGhdN596g1jSswmdJHzqZ06/Of05pOmZZQ +e7oiyHJH1NpumnMBPInpP1YXt7oicdtmotL7M1E29rbyWzJwnefjNtauy5ti72nH +97qVSRb1fNkpuwoCsWtyFassJ3vIvPYVH/quCP3gETaxq5zIEdHTDX1JTsrmc+4N +w41v9aT+oa9rkfdT/RDfd93v7HvPMnXfBIrRofc2aO5l40OQN0Po0ajikFwcdQop +lEpqm4koQ72FVg3xK0S95npMt4UQd7eoDzMppyY= +-----END CERTIFICATE----- diff --git a/t/cert/mtls_server.key b/t/cert/mtls_server.key new file mode 100644 index 00000000..3509beb6 --- /dev/null +++ b/t/cert/mtls_server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAu2krMEOwtMyEPyI5ZWVqvXWkK3337OQS6NHEzn5OVFwizNIY +fzuey3DZfXmPBZO2ny/VM9eYou3FAJPkyrzL8OFjPgdrOG9NCUXxoTujysBHwaEK ++Mm7x9omndMLNSQBPhYULkQ4jMkJAkGetvsMqvzWRF4nq6rVw2jh3VcGbE/2JDOo +K0lggg4VqlWfYcx0OX6fpk9xSovrQ93C95A436amqPZ3vJ5UaTCDTCrruGJ8xxSE +nvPhShUzUWWjr50JxriJMKPSGOncXWvqaMqLXOQ7/jJ/SMNM8LUG9iOXPvJQkGgm +OW2y4lOJcWpI8PH8iTxt24dseSPth13F+ooNuQIDAQABAoIBABnT/KfCLHA+X1t0 +FATtXTCPLfjwe2KibBi6EC2FKrZlnEYuDkI6rT/MZaztO9DA8sItjWx/ogGSUzwp +JbbrHhAsf8jkrNoyPKOyiAJ4fbJLnZgJ4cE3zDFW10uY8kp4k9NCp7VYoZKFgkBV +WtJM9wn5nm39q+n0uVEc+0PN4oy6m54Aqb1HqVCyXFp+/pVhL6PtgaClbqJd3oKV +0/HLWfWaI3nvV6ltAphUfPoCmYIUtSl90sRgSeEaJ61UZXh0OkhhtD7Iw/JUlHDk +a0J7owrh0Wf1kDsaSn+j1ba8MELsspFYYVm0gAMKAvRXbVrYgUMdb+HVdZ0odULl +ezFWeAECgYEA9nnoZs+PzKWNxTzYPtgvvrSLmpXLzNrs/p41JUhy6GjQIixTSBFy +WHkjwu0k2fvRgODfcaetAyK6sV4uTpRgqUhtFSeyNMelZ2yiIEqvhUrtHoVov1C+ +BqwwlUnmkQZNQODXOpKCvnqnOaPwMILKLtxDGmPtW0tCTR2dVVaht6ECgYEAwqb/ +h0Fh3YtykOnhV8wOZRrpVr8jS1RIgg/hklt2xh6+OYtL16sKFaLBF/BhzZRBapqd +fB2Cx3B6rxZ5PLTse8yjEvjt6Ly7TusYWpaKbYKFnnEbmdsm5sBepuLUv4AoMYbk +99ZejFcQI2gNbzX7eIrFitCQGxT+Wu7Gncv+vxkCgYBvAYCVrS2KcZVkG38Y7qyy +KwYk3QoofQD3u7Eb1YFLAsmaWnQ3pQPmrMhaZguO0UcN0DlSKr5VBzMl5tDcOx89 +noziVjqAYtovtlFeUcSzN4eLk3IVl/u9bZeD5QCemEP60Eie7JVNzFe8MgVfE8iT +Skg+fnrL/x0hNhFB+f5jgQKBgDgOEX4o5P8A3nA++gbnm6mgE1xI1OgnkG3sFuCn ++E9boRo/NAsalV/fq82yCuhB7oi9l+abNQMsMBhl12oVDBkmuDuJdjHUz/gNGclU +mu6obMRQ/ErVYqGG+nsCzZOMW4bPuvZoRHgTxnD70QqauB1hkTvFjgpOhGU5Z/cf +PPBZAoGBAJQK7NF6VoF9nm5CT8vufrQ2vvp2aiLdOLLx5JXt/6seEnPZWtEvjp8/ ++ExIsfOIaU5elhv8ze8iKmRP9f04XdWpbRm6k6AR5cOkkQQ1oO7N9abU7KbD/gqX +pJIWOlaUrbKO4Dprx7HyMYYPs9mu/UoF0Dvd/+bYXM5ZKiFrQ3Ly +-----END RSA PRIVATE KEY----- From 5e70eaa420abf44f3ea4e8efb2feccdf8422ed9f Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 11 Feb 2022 16:18:03 +0000 Subject: [PATCH 095/130] Bump version to 0.17.0-beta.1 --- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...16.1-0.rockspec => lua-resty-http-0.17.0-beta.1-0.rockspec | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename lua-resty-http-0.16.1-0.rockspec => lua-resty-http-0.17.0-beta.1-0.rockspec (91%) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index d5753443..70c3bee9 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -106,7 +106,7 @@ end local _M = { - _VERSION = '0.16.1', + _VERSION = '0.17.0-beta.1', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index d9272da5..97e81575 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.16.1', + _VERSION = '0.17.0-beta.1', } diff --git a/lua-resty-http-0.16.1-0.rockspec b/lua-resty-http-0.17.0-beta.1-0.rockspec similarity index 91% rename from lua-resty-http-0.16.1-0.rockspec rename to lua-resty-http-0.17.0-beta.1-0.rockspec index 22726ffa..a14c518e 100644 --- a/lua-resty-http-0.16.1-0.rockspec +++ b/lua-resty-http-0.17.0-beta.1-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.16.1-0" +version = "0.17.0-beta.1-0" source = { url = "git://github.com/ledgetech/lua-resty-http", - tag = "v0.16.1" + tag = "v0.17.0-beta.1" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From 8cb73c4cc2118f0c62d9132e3b3b14aa36192e34 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Fri, 11 Feb 2022 16:39:25 +0000 Subject: [PATCH 096/130] Fix rockspec version format --- ...beta.1-0.rockspec => lua-resty-http-0.17.0.beta.1-0.rockspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename lua-resty-http-0.17.0-beta.1-0.rockspec => lua-resty-http-0.17.0.beta.1-0.rockspec (95%) diff --git a/lua-resty-http-0.17.0-beta.1-0.rockspec b/lua-resty-http-0.17.0.beta.1-0.rockspec similarity index 95% rename from lua-resty-http-0.17.0-beta.1-0.rockspec rename to lua-resty-http-0.17.0.beta.1-0.rockspec index a14c518e..d72353e9 100644 --- a/lua-resty-http-0.17.0-beta.1-0.rockspec +++ b/lua-resty-http-0.17.0.beta.1-0.rockspec @@ -1,5 +1,5 @@ package = "lua-resty-http" -version = "0.17.0-beta.1-0" +version = "0.17.0.beta.1-0" source = { url = "git://github.com/ledgetech/lua-resty-http", tag = "v0.17.0-beta.1" From 2af9925acf5cbb5b387539cd4463552ebbd5efee Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 14 Feb 2022 11:44:11 +0800 Subject: [PATCH 097/130] remove old comment in setclientcert --- lib/resty/http_connect.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 18a74b1a..4da98bcb 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -245,7 +245,6 @@ local function connect(self, options) ngx_log(ngx_WARN, "cannot use SSL client cert and key without mTLS support") else - -- currently no return value ok, err = sock:setclientcert(ssl_client_cert, ssl_client_priv_key) if not ok then ngx_log(ngx_WARN, "could not set client certificate: ", err) From b2806bda92e576aacddf887ab3161daad3a95ca7 Mon Sep 17 00:00:00 2001 From: Alex Dowad Date: Tue, 6 Dec 2022 16:15:11 +0200 Subject: [PATCH 098/130] Documentation is clearer about using same connection for multiple requests via 'request' function If a previous HTTP response had a response body, and the body was never read, then a subsequent call to 'request' using the same connection will fail. This is because the unread response body will still be sitting in the TCP socket buffers. As a result, when the subsequent call to 'request' tries to read the first line of its response to parse out the HTTP version and status code, it will get the first line of the unread response body instead, which will almost certainly not be a valid HTTP status code line. The documentation was not at all clear about this; make it explicitly state what the user needs to do. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9c97b4b5..98b125a9 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,8 @@ When the request is successful, `res` will contain the following fields: * `read_body`: A method to read the entire body into a string. * `read_trailers`: A method to merge any trailers underneath the headers, after reading the body. +If the response has a body, then before the same connection can be used for another request, you must read the body using `read_body` or `body_reader`. + ## request\_uri `syntax: res, err = httpc:request_uri(uri, params)` From 1bfdafa3f5f119bf20a5982944c7d5c43b0382d8 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 15 Dec 2022 11:51:04 +0000 Subject: [PATCH 099/130] chore(tests) renew test certificate --- t/cert/test.crt | 44 +++++++++++++--------------- t/cert/test.key | 76 +++++++++++++++++-------------------------------- 2 files changed, 45 insertions(+), 75 deletions(-) diff --git a/t/cert/test.crt b/t/cert/test.crt index 477ddc2a..f20fa956 100644 --- a/t/cert/test.crt +++ b/t/cert/test.crt @@ -1,27 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIEpDCCAowCCQDgS40Q0UNIKTANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls -b2NhbGhvc3QwHhcNMjExMTIyMTQ0MDM2WhcNMjIxMTIyMTQ0MDM2WjAUMRIwEAYD -VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9 -OVsD7qyres7qkLZPY4MyC00/ByESpUVFgWiTU2nBeNC/kBvOOOVTvrVHxlTjZ9JG -rCdYeIGkpkaJQ2QylHKRJolxh8x309/VkpKJYL357NuNeAk2l3tW4BMImXUGdVjD -0qT8WTyApEE7lyfOXKYKbbEwfdks0eBcEZBBn4QTaZ9lVj7yScurto8jDYMkBfO/ -4QeuVe4ijgWU35Yd89Ya0HZr+1QBU1dMzbtf9mYPedRwRZgzKogZ6mZXs7GJ0ADM -L+fg/djROd01eA3ynAfxJbRv1R7S73ljQJhD+oYf3rJaHU2Ko5heF9K9bd771Ze/ -sHGHZdLy3JZrPNdDPHG61H9m4zOUkNZvVkvHF9wg+uNs4VJxrkG6undOEyCyBefE -XRknEBvMLlWF9c9KT5tZz/vI6/e5xrpNCdUjthn5YFEf43MVqWWcU8t9ViK+8+rG -PQNKmxRiCOOmd6NhjWJInuMO3QiPJFqVs4UNpmE/4hi3LtwV7qH4jHWX5hRG27gz -8VbfkBkIoy0Esm2Dv5lErZztPhycpFUodpBXGtQjXQLvYVm+yKwWlD6gTRH0LXj3 -3lp69m5/V9so3NsYWMxoNGvI4MqZuwGaiwxgkCWjUqQEX1y4i7Us01UDFpK2WuwG -MUQNzGV2jX9urnjndFPHjszqo5BsmJuHwihGOFvbpwIDAQABMA0GCSqGSIb3DQEB -CwUAA4ICAQAhXhf95xbkSw1fknoGhdgTAb3A+pKzcejZjqmQi3BJku8EILe06LGF -Pli4WlN7KMk20A/TO/L46tsWztFl0vhSs+ed/Z3/0ugB+DO0InpwDxutPTrVjkmy -W5HAKnW7MQAA9tcTcAPbiHIalwswQ1pab+kxl9fbsaC1CrSypa1eK6Hh/qJKxe2O -06ovTTHuZ/yWJBaIwmvP8lCMfCrLGOCGToM4FWtDPc47VVxpk0Ks0kRL8Bqo4CsG -rM+PpfsFBdPO/FftfVeMdzZCcK1CLWuE/uBdjDtU10qMVhQTxGG4eFv24xVQuCmH -AB5hTt62DLxHAs6kgvafZz9ST6aVDu0kuPABC4z4JQVBCRbw0mL75R4ziEYH3CIe -tmgYEXHfj3D3BDgX3ki6qqX7VtmkT0+VyrpQs+dhkCXlnqGmNM+gSsKve3ZH6a9J -fFeKjiJH4YjlPmLlCxpnDoBgjGjcUmVEBcpZ7H8lNC3ldPVfDq3PDrHgFMH5lmth -PZHbWROudoHVX5EBY2n08uzMWtl0xFwAfLG3Hk7xoBWBpebHxhw/yzm/iWHeBdi+ -8uSMLM3/TjmKhh+mF7KRKSYKetKKYiwB0/yIQJLMLxpUc3Aae7i0djH/kFz80HU/ -alsGmliI6PGyCt11DYS7Spl44pxl6N7pj2CJpyPOMwDI3VG097cYDQ== +MIIDYzCCAkugAwIBAgIUeNQ159lFQKe26wCGGDxXKGtpVXowDQYJKoZIhvcNAQEL +BQAwQTELMAkGA1UEBhMCUFQxDjAMBgNVBAgMBVBvcnRvMQ4wDAYDVQQHDAVQb3J0 +bzESMBAGA1UECgwJbGVkZ2V0ZWNoMB4XDTIyMTIxNTExNDgzOFoXDTIzMDExNDEx +NDgzOFowQTELMAkGA1UEBhMCUFQxDjAMBgNVBAgMBVBvcnRvMQ4wDAYDVQQHDAVQ +b3J0bzESMBAGA1UECgwJbGVkZ2V0ZWNoMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAtglmxqYDZG/u7bw929y4TfBgfJqGcEm8vyioEbGIJA3g3QTRyBP4 +VwHN1t7pDp3/kpHR3yu9/LCNbGcrkGUsALkPG3FewDapJHvIH+NHeQcrM8aBJsNh +Wh/KJoHsVOgF083aLpM0LRtqSrRJWAufBHFYlEXANPR3eFSThhLotja8uARDq1RH +jcj2WD15iDxoGKMjXJdfksUrD07R2k8Fgm48lall07uXtUoImW6eR8fwOSde4fAD +tmEDJVZaPE1Qh8Pp8OkvVSxig/l1aWnjQoNhsKia3HdJqKJUHD7zF8GSunoU0Mn7 +96pQa2cUtqgY6L1SW/hf8FXynVhTHHj6XwIDAQABo1MwUTAdBgNVHQ4EFgQUodoq +6WIebDtvDFvYRi/Y1B86HnYwHwYDVR0jBBgwFoAUodoq6WIebDtvDFvYRi/Y1B86 +HnYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAeLTpIyBs7W3m +6ZJfZBKV8eT3R/HlkQEIDTDqmA8ZPLsdPSDg2XE6JO0zkHke14imycJIit3ViNGo +A7+F425jJxrGrur6X1reYCXlEERRadwmod2nIlW9DTAP9yOeRNoWUDdnCMkJ67qT +YO1DCnnWn3jgIyh+slo+IrKeNoAAcfJhsIM4riKXtiHT5gvqY/FxxwawUa8E4GK4 +rpEJcOxgJ3/+r24n93vZjVB682v1dBag27fTZ+jyJMLowpDodEs5JniVh44EKJDp +DNlqqRS9Usv2dlS7FcH352JKH88aF57Md0vVE6+BGx0k1UergQeLE/hqJ0tmQ6Ao +kxdZy7udxA== -----END CERTIFICATE----- diff --git a/t/cert/test.key b/t/cert/test.key index 5b64ce1d..893e56a8 100644 --- a/t/cert/test.key +++ b/t/cert/test.key @@ -1,52 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC9OVsD7qyres7q -kLZPY4MyC00/ByESpUVFgWiTU2nBeNC/kBvOOOVTvrVHxlTjZ9JGrCdYeIGkpkaJ -Q2QylHKRJolxh8x309/VkpKJYL357NuNeAk2l3tW4BMImXUGdVjD0qT8WTyApEE7 -lyfOXKYKbbEwfdks0eBcEZBBn4QTaZ9lVj7yScurto8jDYMkBfO/4QeuVe4ijgWU -35Yd89Ya0HZr+1QBU1dMzbtf9mYPedRwRZgzKogZ6mZXs7GJ0ADML+fg/djROd01 -eA3ynAfxJbRv1R7S73ljQJhD+oYf3rJaHU2Ko5heF9K9bd771Ze/sHGHZdLy3JZr -PNdDPHG61H9m4zOUkNZvVkvHF9wg+uNs4VJxrkG6undOEyCyBefEXRknEBvMLlWF -9c9KT5tZz/vI6/e5xrpNCdUjthn5YFEf43MVqWWcU8t9ViK+8+rGPQNKmxRiCOOm -d6NhjWJInuMO3QiPJFqVs4UNpmE/4hi3LtwV7qH4jHWX5hRG27gz8VbfkBkIoy0E -sm2Dv5lErZztPhycpFUodpBXGtQjXQLvYVm+yKwWlD6gTRH0LXj33lp69m5/V9so -3NsYWMxoNGvI4MqZuwGaiwxgkCWjUqQEX1y4i7Us01UDFpK2WuwGMUQNzGV2jX9u -rnjndFPHjszqo5BsmJuHwihGOFvbpwIDAQABAoICABB0Y2DFKYjD5ihpqyDeM3Nv -nikD8rFPY+W2aiSdlU24ttZhrrlRI0gUBmmWap1X0uZIZCeCWyu5NdsL3DO1yvyq -UDtqJrKo3wcQduOxYPPR0AnOTWbM53HXjHAsAAwuuihVMfmrvOIm5nFLJLACSIIR -pd3ko7UNDyiScmvydibGAZFrXY/uMOLevjW7IBNK3TZrWCKl4E5q8TtP8hrqsEym -ohWjxltTJv1LyqZ+o8Nmwb19n82bPjiatImd26tzRg57f/uFt4wCLyVOKlBVly82 -Kymqa8LGBlVGMbd/mpg84l+Just6jG5qPe4xFrnpigemTUkL5rstyXrL0KBNKkqB -jtuow1nGphZJSxis4tAQqqG9t0xtu7zCS1gWYEpUd1Y8wfTIM37hFojVVBOE8n86 -VEEx9xSt8diuY3X8He9W4IeXcDGe111Hbic4zuPQGsu+SeGoR/AJeZORCpf/PA9l -uX6gk48rEm8kzLyhqikYPJAfvRbRYxGKti2M7nDfUzhXs1CwnBWHMUzALr29//8D -I2hNTnAOmv42uQe+eiQw19HXLNMyKE1lgpbW9eAIZWCLJPUrYr2Fpdv1yoqyW5J5 -FVJE6iZSG31oUCIkymJ/xz0RqjdGD4zUnAhcqT7srMRQ7tRrifHvOahj7OLO/Yc3 -iBfeJZqOU7difsWYQqLxAoIBAQDlFkMDLb3ouTvtD1HWPWiRLpUEs3tGXsmZY1Xs -Z0FVHmkA7JLAj+NLr5J7lNzyP7oJkfrGuMHwO4AdXh+cQwZwhGq+TKOb67u8B0dx -CaZEPIKY13tl9WpZI/LANuzgnmo4rbWB8Z/crft/LjdpjZPUVFX8WBdi9c0wcPLJ -y7FYAfSGMLjgDI3UiI79Q6UjX8HvP5KPGm2D4rwPeuDCf24efj+b0YhsH+0b+9G0 -5T1PM2VTB3SPwtRnCR+A5W5XxxSylBAjjcPTJEsc4coWpuhH7GaLAuxmHV+UFUOC -CCA+Xq+lfE/2A0mwmrPbp+y9JuLclMTkoPbi+CXy6s5dlbjZAoIBAQDTdDnEqn3Z -fMs5YQZCaBmCPxbTAZnHbJmfkYyI/iLmWKpe0NQAHX4J+s8NqQcJFkJwUxjWfRBR -do1OYm0CK5pc06YwKk5W69WYvDr4/h3OsXXbtZT70i9Hdip1KMJk+umYExqk4pG6 -VA4WhIBVDFPEkAgP6uGj+rzpH/1UKiMRhUy37tY9MrNZaJVxzl7ect+K6YfAXAIR -4/1fb7sYUyaUFzr6u4oX7pojUUzuHqeVTZIpVLfGfUT0Wh4JxzHY2aBhjeboEpmc -fzltpd+EqAtOjvh7k2NKb3fkPzXp6EN2RYLZzM06fn+koHfpQ5UoFusG+EDG6PWw -r0yCA+vhxWh/AoIBAHD+Qv1dYW9ZdhJeXQoj7eC2LjBMasBx3lP26BmbcGEQh9A5 -38R09DKPnduwuC4Qeq1fwGamGJpqbq2NkF/du15iIdhNKuGsK0P5/yXlEYpUokHK -/wVyQAtJrOFb9ghweooMPBuk1ync8tBvNkus1j6Dfonh1V8tSBDdlRMtkgKLrFVH -0NhJuOccmBKZe+lvwNNF8v78lQnZGtMz04p7mhAWSz+K4RQbxzLnSS9FZrEa8545 -bie+fE9583z7LSrEehP+7drrgKwzGY1cXPZBSw62rnlgbsOLN2pt9oc7hNPelho0 -r6fzRTSDibUNJbrCZGFSS69NKZkXtngTksgcIjkCggEBAJZ3jx/2HiWkGszS0Oxy -PSykji5KZk41Zj/ZB4L4LIdzMpDT0vfeBLE/SxUyYTJDJ7XcO8FtLy54yOatPVqd -AnPpm8mMYUCTAuRzlRdXZIiPWQml9cUX9NLKMD48adAHiC/R8FTCcOaBJ6E4WkWq -xvUJdOM2xae1JbZSMCcy8M9n5obAKq1TpbIKrMVWD8YFQjMMKLBmCIcIkGTYEe+m -Q75/pVWEHCkZfY+CK9cJ80BX+Zj1kuxoeafTdKqKl1ryyaCBpH0htzNYiNoQfbRd -C+az7/enkrEGDu9ZV5kB0PeZURqBFMz59QSFt65CjCIB5O6JuzaP0T0QEMY32iNH -ISsCggEAP/2+C8hpkl530l002QnWzxhVdm5sCkk05jZJ2Rm1X2CXlGTOMxUhUamn -hYr0u+GUsORBjsJ6CnGkoXSiJOGn1+iOEZ8aAxRWxQdveWCEi2NITEIoK4jXFcsj -ewUzXLX+4OrruLsPVCQLjVYj84mH+X0rSctylJUayEOPbnnKtfMegQ2Ruw8XKEmL -aDssBizn2R3fUHcml1/pIM7bCv5AyBBlL2fAAHkB5d7XBW9lIUmY8YATaElQuF2/ -9xZrxMYyNWpVQWfAu4NvwunFmqzfXK8OFs3gwiSwRgmvFgAwDE5WbRdGH3F8J/Pc -xmyWhwxzeZdweoZDR6um1BRcJVJzdA== +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC2CWbGpgNkb+7t +vD3b3LhN8GB8moZwSby/KKgRsYgkDeDdBNHIE/hXAc3W3ukOnf+SkdHfK738sI1s +ZyuQZSwAuQ8bcV7ANqkke8gf40d5ByszxoEmw2FaH8omgexU6AXTzdoukzQtG2pK +tElYC58EcViURcA09Hd4VJOGEui2Nry4BEOrVEeNyPZYPXmIPGgYoyNcl1+SxSsP +TtHaTwWCbjyVqWXTu5e1SgiZbp5Hx/A5J17h8AO2YQMlVlo8TVCHw+nw6S9VLGKD ++XVpaeNCg2GwqJrcd0moolQcPvMXwZK6ehTQyfv3qlBrZxS2qBjovVJb+F/wVfKd +WFMcePpfAgMBAAECggEAWUeSIyJ/7m4hinK1ChSeRgd7YxFslzzq9/xKNedS5I/z +9aJr+wukVJP7bwOD3VnrY2jmPGTZOMokioKtMJQWOUKKIMUBGheU7GJTPCR9xOOZ +sjqM1ESmBgwqWqmFDf2AAinndMlYoTjdcEcs46RLeOPYdFwjUTWHNLqSYgL7JIYs +0TJRpI+ipRSfiEIgDG9Ca1iBdJY9DcRfLSplxafhtZoa6PorEPR/SlYBr3Lu0txj +oKsdSzmILlSm6MMiCinocp3wk3K80VPCbDZwnU9ZcJJiRka5OQEpXZ/HyKywXFUv +bWSJaSDn79AD4+PV/CuGz+Eiolmpt4ayMQOqMBOoPQKBgQD73c+fTEr6kZj8glzS +XK0WnLInsoZn6rLrXKNOmUGTVcIiU6hzllIHvYftJAppUO8pQIVtzESbkvyNABHl +4pKrr1HeEesyts/L+YBUwGG9PIvNi0+8rWwEEcgceUujpB0+L6YxOGVCe1JPw/W5 +cX3qaZ1aDy5S8U5HwgPGIDtxCwKBgQC5BjVmZWvyojvcsbWYI/68PWqXU47Wd+Dk +n4ngACwWLSK8+Prx0yMZI2fOIp+BMMFVbq9YqihgAvRHdk9nQB6aFtUjucmM2AUx ++hnlpB86Tj0aMhclA7fH2dU9f7ALrWeBy9CIl1n/8jopbN6sujYh7utiTM+PytCu +udERDiVYfQKBgQDddqqLbbA4JzIYRiCMMEVaLqpQUZ/xW+O7JJfMEuIgRTYQrPLf +v/udJsCQejJVbH7/DObGJujkyxgae0dBljpVkDEyTRs8WpbYyihXXVoT5VsgfzC7 +vNnPsjr9HoKt/y5fcdJ2URponLtBvBMi3nJTXk6n09VtxRSJ8EUVP15FBQKBgQCd +9r8AfPZ8YCXbdZ0kFxcUfxrRt3Trhc1rYwnDRGJfTImr7KXScAA3JhMp2kVzdryY +w7bQa/WsgtkracbAz+1QgM3Xgnag2I0W7f/OHyDKFqH0C2/XGiV9amyz5J/YWGNW +XTvcYdtkuNkPJPO3ZANzVReBBTLwAKfJhLsJZJpdwQKBgCjuE1/4yDx+KQRqktha +t4oOdWjUk3Yiwa2tvmsWt66RE9ID/wI5+8UIyFWYEwzUZ7+o0WA8wQ4V7rKvAEAf +C1EiTZwi15pgemqpWJ5N+dBvBOiY5yG5YsCtkWTnMjWykPeRu7qNb60UkV7Fg80S +5IJNZnDErLb8UnBuinxkZe1t -----END PRIVATE KEY----- From 3fc7ced4be20cf012ec97818caf77f90a84e1fa1 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 15 Dec 2022 12:04:56 +0000 Subject: [PATCH 100/130] chore(tests) extend expiration date of test certs to 10 years --- t/cert/test.crt | 34 ++++++++++++++++---------------- t/cert/test.key | 52 ++++++++++++++++++++++++------------------------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/t/cert/test.crt b/t/cert/test.crt index f20fa956..6efde069 100644 --- a/t/cert/test.crt +++ b/t/cert/test.crt @@ -1,21 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIDYzCCAkugAwIBAgIUeNQ159lFQKe26wCGGDxXKGtpVXowDQYJKoZIhvcNAQEL +MIIDYzCCAkugAwIBAgIUXCmnoPKJ60jFkycVZ04mVj3B8aswDQYJKoZIhvcNAQEL BQAwQTELMAkGA1UEBhMCUFQxDjAMBgNVBAgMBVBvcnRvMQ4wDAYDVQQHDAVQb3J0 -bzESMBAGA1UECgwJbGVkZ2V0ZWNoMB4XDTIyMTIxNTExNDgzOFoXDTIzMDExNDEx -NDgzOFowQTELMAkGA1UEBhMCUFQxDjAMBgNVBAgMBVBvcnRvMQ4wDAYDVQQHDAVQ +bzESMBAGA1UECgwJbGVkZ2V0ZWNoMB4XDTIyMTIxNTEyMDQzMVoXDTMyMDkxMzEy +MDQzMVowQTELMAkGA1UEBhMCUFQxDjAMBgNVBAgMBVBvcnRvMQ4wDAYDVQQHDAVQ b3J0bzESMBAGA1UECgwJbGVkZ2V0ZWNoMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAtglmxqYDZG/u7bw929y4TfBgfJqGcEm8vyioEbGIJA3g3QTRyBP4 -VwHN1t7pDp3/kpHR3yu9/LCNbGcrkGUsALkPG3FewDapJHvIH+NHeQcrM8aBJsNh -Wh/KJoHsVOgF083aLpM0LRtqSrRJWAufBHFYlEXANPR3eFSThhLotja8uARDq1RH -jcj2WD15iDxoGKMjXJdfksUrD07R2k8Fgm48lall07uXtUoImW6eR8fwOSde4fAD -tmEDJVZaPE1Qh8Pp8OkvVSxig/l1aWnjQoNhsKia3HdJqKJUHD7zF8GSunoU0Mn7 -96pQa2cUtqgY6L1SW/hf8FXynVhTHHj6XwIDAQABo1MwUTAdBgNVHQ4EFgQUodoq -6WIebDtvDFvYRi/Y1B86HnYwHwYDVR0jBBgwFoAUodoq6WIebDtvDFvYRi/Y1B86 -HnYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAeLTpIyBs7W3m -6ZJfZBKV8eT3R/HlkQEIDTDqmA8ZPLsdPSDg2XE6JO0zkHke14imycJIit3ViNGo -A7+F425jJxrGrur6X1reYCXlEERRadwmod2nIlW9DTAP9yOeRNoWUDdnCMkJ67qT -YO1DCnnWn3jgIyh+slo+IrKeNoAAcfJhsIM4riKXtiHT5gvqY/FxxwawUa8E4GK4 -rpEJcOxgJ3/+r24n93vZjVB682v1dBag27fTZ+jyJMLowpDodEs5JniVh44EKJDp -DNlqqRS9Usv2dlS7FcH352JKH88aF57Md0vVE6+BGx0k1UergQeLE/hqJ0tmQ6Ao -kxdZy7udxA== +MIIBCgKCAQEAqzJINgQtchR799ahPpu2gweHCmx7cytQ59ZFW4J3QBRIB+DFnLSM +fJa797s0Sl6f+t9pT3QLYjbdNK+R60HGd5tM31cEIa518AcDokfkzGo/clY+dbs4 +OcFe10HxAlXpu6S5/yiQ0u4CIf1uQSsS53kNpaRWcEz6Z/sw80ksCEnYSfD3YXEh +sJm41i5Hd4F2/Y1WrMg82YGZG7Y81cM9LgUxKcikTm4JnEn9G1yg56hSbKs1a0E0 +J9Gk84lufWpiUqpX9ASsUPttnYzgljo24x7zeNEaXsVOKgu+88Cc+bBn7AORGXM4 +YtYbxVGhAgPf0vOalNl/kDwfE1jwBz0r6QIDAQABo1MwUTAdBgNVHQ4EFgQUxuo7 +FdlD/iQ7cMnORpKFQ0JW9I4wHwYDVR0jBBgwFoAUxuo7FdlD/iQ7cMnORpKFQ0JW +9I4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAfINyfrlE/f3W +cTNFiVMzqCa+oeNtvlkaCKgtjJdow8xwAc/Mxae76dkN3Mh8yegKV9+QRY6rw6q0 +dfma/Rg0tJgrI9El64XdGRcMpwGdtujZbE/bCGTJwLpcIM051cr/NCtYYduM2RU8 +i5uc8z5sQbMIJHdwDDm4PevA4WrqteUo5bXFQp9jYessDIkjIg7n5hGNvSNtfpVV +0fGDYPD9yNydgRMe6EwqQ0Z9p6yfq4o60JceYt4MfbbcGxzwrQc41ou2wKM/iKnQ +FBwCOaa+imgn01Qno/PdisV05KM7uSv/dK1v33nrk4xPSVG8u0aGe440U1CcGXbB +jy3ACMw/ZQ== -----END CERTIFICATE----- diff --git a/t/cert/test.key b/t/cert/test.key index 893e56a8..7d61246a 100644 --- a/t/cert/test.key +++ b/t/cert/test.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC2CWbGpgNkb+7t -vD3b3LhN8GB8moZwSby/KKgRsYgkDeDdBNHIE/hXAc3W3ukOnf+SkdHfK738sI1s -ZyuQZSwAuQ8bcV7ANqkke8gf40d5ByszxoEmw2FaH8omgexU6AXTzdoukzQtG2pK -tElYC58EcViURcA09Hd4VJOGEui2Nry4BEOrVEeNyPZYPXmIPGgYoyNcl1+SxSsP -TtHaTwWCbjyVqWXTu5e1SgiZbp5Hx/A5J17h8AO2YQMlVlo8TVCHw+nw6S9VLGKD -+XVpaeNCg2GwqJrcd0moolQcPvMXwZK6ehTQyfv3qlBrZxS2qBjovVJb+F/wVfKd -WFMcePpfAgMBAAECggEAWUeSIyJ/7m4hinK1ChSeRgd7YxFslzzq9/xKNedS5I/z -9aJr+wukVJP7bwOD3VnrY2jmPGTZOMokioKtMJQWOUKKIMUBGheU7GJTPCR9xOOZ -sjqM1ESmBgwqWqmFDf2AAinndMlYoTjdcEcs46RLeOPYdFwjUTWHNLqSYgL7JIYs -0TJRpI+ipRSfiEIgDG9Ca1iBdJY9DcRfLSplxafhtZoa6PorEPR/SlYBr3Lu0txj -oKsdSzmILlSm6MMiCinocp3wk3K80VPCbDZwnU9ZcJJiRka5OQEpXZ/HyKywXFUv -bWSJaSDn79AD4+PV/CuGz+Eiolmpt4ayMQOqMBOoPQKBgQD73c+fTEr6kZj8glzS -XK0WnLInsoZn6rLrXKNOmUGTVcIiU6hzllIHvYftJAppUO8pQIVtzESbkvyNABHl -4pKrr1HeEesyts/L+YBUwGG9PIvNi0+8rWwEEcgceUujpB0+L6YxOGVCe1JPw/W5 -cX3qaZ1aDy5S8U5HwgPGIDtxCwKBgQC5BjVmZWvyojvcsbWYI/68PWqXU47Wd+Dk -n4ngACwWLSK8+Prx0yMZI2fOIp+BMMFVbq9YqihgAvRHdk9nQB6aFtUjucmM2AUx -+hnlpB86Tj0aMhclA7fH2dU9f7ALrWeBy9CIl1n/8jopbN6sujYh7utiTM+PytCu -udERDiVYfQKBgQDddqqLbbA4JzIYRiCMMEVaLqpQUZ/xW+O7JJfMEuIgRTYQrPLf -v/udJsCQejJVbH7/DObGJujkyxgae0dBljpVkDEyTRs8WpbYyihXXVoT5VsgfzC7 -vNnPsjr9HoKt/y5fcdJ2URponLtBvBMi3nJTXk6n09VtxRSJ8EUVP15FBQKBgQCd -9r8AfPZ8YCXbdZ0kFxcUfxrRt3Trhc1rYwnDRGJfTImr7KXScAA3JhMp2kVzdryY -w7bQa/WsgtkracbAz+1QgM3Xgnag2I0W7f/OHyDKFqH0C2/XGiV9amyz5J/YWGNW -XTvcYdtkuNkPJPO3ZANzVReBBTLwAKfJhLsJZJpdwQKBgCjuE1/4yDx+KQRqktha -t4oOdWjUk3Yiwa2tvmsWt66RE9ID/wI5+8UIyFWYEwzUZ7+o0WA8wQ4V7rKvAEAf -C1EiTZwi15pgemqpWJ5N+dBvBOiY5yG5YsCtkWTnMjWykPeRu7qNb60UkV7Fg80S -5IJNZnDErLb8UnBuinxkZe1t +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrMkg2BC1yFHv3 +1qE+m7aDB4cKbHtzK1Dn1kVbgndAFEgH4MWctIx8lrv3uzRKXp/632lPdAtiNt00 +r5HrQcZ3m0zfVwQhrnXwBwOiR+TMaj9yVj51uzg5wV7XQfECVem7pLn/KJDS7gIh +/W5BKxLneQ2lpFZwTPpn+zDzSSwISdhJ8PdhcSGwmbjWLkd3gXb9jVasyDzZgZkb +tjzVwz0uBTEpyKRObgmcSf0bXKDnqFJsqzVrQTQn0aTziW59amJSqlf0BKxQ+22d +jOCWOjbjHvN40RpexU4qC77zwJz5sGfsA5EZczhi1hvFUaECA9/S85qU2X+QPB8T +WPAHPSvpAgMBAAECggEADrAtoUwCQpRYWIv9LkJ5sJjmPX8DDn9Fnaz099d9qy1L +5GuLRbLF9PCRdKO55k5jenvnOXs5RZym2QNEfHaA9GuKKmvF8N/awmFpSE46M4Gp +xOhHMdG1qFroOLhkv1cNfhB+XTjzrm1GS8VGZU5jr+5Jo4/lUxUZTO+XCq1mAXK0 +io58x7K3Z7P26/46yeuuqNggbFCxeHfHSK8SXYg6QxgjlZyA/A8tHDuvhDCH81q/ +AISHxIFmly8ZQ2KxuVCSyHXQkPwODPh2F/4UpOq5EdN781CuV/W4e73WKBwcDUQM +aEX5KF8D2C0nXjk592SS9SFmXqw9ZD77sAL2JmbE4wKBgQDGoKT8i3xn3x4nAju0 +yTxK6jaV72Lrdn5HmOZyJb8irAUr9B0t33zaubzXlM1LpuS/JAQwtppShGo1jPlt +T8iE2+JR9UKi6siHV/3EKC15i5UT3p30txdR2KS+hAasb7qXvM6sgM0nwzoS9PUy +LbKnRN8UYTvJmEKU1vuyeFQAQwKBgQDcpUQpNhtf4GrT7EoBh8E24yOO2k9XludW +yhaEddTYe2FXmqf9MGuPPX+dBj3G1SY6RNXvjmVjr3uvFN5qz+mtOuAkm7g99KkI +bKjkVuCgx6mpiH89dt7d/3uTcKXEkai0F7JgSPd1mcSxjRybhyna4qC4CuCHjICU +Ug00LAqGYwKBgGjX6N6JPgySABd1HVDrG9ErWc7AwkUpkbR3J8S+2eoSRNSTkUdi +fUPy4JQmrkqteHbQKwoPiNvfmzRTCmHByEUgz5CVViwqo9iVAJUm5AIRRIptapD+ +h+ei5CrQA7nHbAWmGq2Be0juytuwwzBOYMvcFahrPqTFovdvlwH4c9aDAoGAYAk6 +4qkfPxrhxH3rNEFPUsGIX4wbzqbq6DarmFnlG5iQJN420hf6KO1+luz5hIqPyfre +Fxemf74Imor9yAXY0sJ2fticV7Mew4Dv/frmaHSfHyA/KZSMqpmhwunb7PPtNv29 +cPUxaClWmGUwF228RP4xMAnj8nuwF16jSpsEtbsCgYEAxEaEzpuknqV4AOnlJFBQ +7JqSkvW7w4Zy8hnUNmqhOOX1KwB0YudfirHceGtOpjxWQmHqjCHZqau0nEU02+ev +qUQsdiGKeJvWHYyqEldOdNd49qaux19IPQBeKRJQGWheGHDWGlg1aSDw3jY0sESS +3VqdbL5Z3XRXozQdwmNyfbw= -----END PRIVATE KEY----- From ffa23c652943d968c74f3f1331a8e21895abe439 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Thu, 9 Mar 2023 15:21:58 +0800 Subject: [PATCH 101/130] fix 100 response with headers --- lib/resty/http.lua | 19 ++++-- t/03-requestbody.t | 156 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 164 insertions(+), 11 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 70c3bee9..ac6309ff 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -635,18 +635,23 @@ end local function _handle_continue(sock, body) local status, version, reason, err = _receive_status(sock) --luacheck: no unused if not status then - return nil, nil, err + return nil, nil, nil, err end -- Only send body if we receive a 100 Continue if status == 100 then - local ok, err = sock:receive("*l") -- Read carriage return + -- Read headers + local headers, err = _receive_headers(sock) + if not headers then + return nil, nil, nil, err + end + + local ok, err, partial = _send_body(sock, body) if not ok then - return nil, nil, err + return nil, nil, nil, err end - _send_body(sock, body) end - return status, version, err + return status, version, reason, err end @@ -764,11 +769,11 @@ function _M.read_response(self, params) -- If we expect: continue, we need to handle this, sending the body if allowed. -- If we don't get 100 back, then status is the actual status. if params.headers["Expect"] == "100-continue" then - local _status, _version, _err = _handle_continue(sock, params.body) + local _status, _version, _reason, _err = _handle_continue(sock, params.body) if not _status then return nil, _err elseif _status ~= 100 then - status, version, err = _status, _version, _err -- luacheck: no unused + status, version, reason, err = _status, _version, _reason, _err -- luacheck: no unused end end diff --git a/t/03-requestbody.t b/t/03-requestbody.t index c1fdd241..86fbf358 100644 --- a/t/03-requestbody.t +++ b/t/03-requestbody.t @@ -164,6 +164,7 @@ c: 3 [error] [warn] + === TEST 4: Return non-100 status to user --- http_config eval: $::HttpConfig --- config @@ -205,7 +206,154 @@ Expectation Failed [warn] -=== TEST 5: Non string request bodies are converted with correct length +=== TEST 5: Return 100 Continue with headers +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) + + local res, err = httpc:request{ + body = "a=1&b=2&c=3", + path = "/b", + headers = { + ["Expect"] = "100-continue", + ["Content-Type"] = "application/x-www-form-urlencoded", + } + } + + if not res then + ngx.log(ngx.ERR, "httpc:request failed: ", err) + end + + ngx.say(res.status) + ngx.say(res:read_body()) + httpc:close() + } + } + location = /b { + content_by_lua_block { + local len = ngx.req.get_headers()["Content-Length"] + + local sock, err = ngx.req.socket(true) + if not sock then + ngx.log(ngx.ERR, "server: failed to get raw req socket: ", err) + return + end + + -- with additional header + ngx.log(ngx.NOTICE, "before send 100") + local ok, err = sock:send("HTTP/1.1 100 Continue\r\nConnection: keep-alive\r\n\r\n") + if not ok then + ngx.log(ngx.ERR, "failed to send 100 response: ", err) + end + + local data, err = sock:receive(len) + if not data then + ngx.log(ngx.ERR, "failed to receive: ", err) + return + end + + local ok, err = sock:send("HTTP/1.1 200 OK\r\n" .. + "Content-Length: " .. len .. "\r\n" .. + "Content-Type: application/x-www-form-urlencoded\r\n\r\n" .. + data) + if not ok then + ngx.log(ngx.ERR, "failed to send 200 response: ", err) + return + end + } + } +--- request +GET /a +--- response_body +200 +a=1&b=2&c=3 +--- no_error_log +[error] +[warn] + + +=== TEST 6: Return 100 Continue without headers +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + }) + + local res, err = httpc:request{ + body = "a=1&b=2&c=3", + path = "/b", + headers = { + ["Expect"] = "100-continue", + ["Content-Type"] = "application/x-www-form-urlencoded", + } + } + + if not res then + ngx.log(ngx.ERR, "httpc:request failed: ", err) + end + + ngx.say(res.status) + ngx.say(res:read_body()) + httpc:close() + } + } + location = /b { + content_by_lua_block { + local len = ngx.req.get_headers()["Content-Length"] + + local sock, err = ngx.req.socket(true) + if not sock then + ngx.log(ngx.ERR, "server: failed to get raw req socket: ", err) + return + end + + -- without additional headers + local ok, err = sock:send("HTTP/1.1 100 Continue\r\n\r\n") + if not ok then + ngx.log(ngx.ERR, "failed to send 100 response: ", err) + end + + local data, err = sock:receive(len) + if not data then + ngx.log(ngx.ERR, "failed to receive: ", err) + return + end + + local ok, err = sock:send("HTTP/1.1 200 OK\r\n" .. + "Content-Length: " .. len .. "\r\n" .. + "Content-Type: application/x-www-form-urlencoded\r\n\r\n" .. + data) + if not ok then + ngx.log(ngx.ERR, "failed to send 200 response: ", err) + return + end + } + } +--- request +GET /a +--- response_body +200 +a=1&b=2&c=3 +--- no_error_log +[error] +[warn] + + +=== TEST 7: Non string request bodies are converted with correct length --- http_config eval: $::HttpConfig --- config location = /a { @@ -247,7 +395,7 @@ mix123edtable [warn] -=== TEST 6: Request body as iterator +=== TEST 8: Request body as iterator --- http_config eval: $::HttpConfig --- config location = /a { @@ -284,7 +432,7 @@ foobar [warn] -=== TEST 7: Request body as iterator, errors with missing length +=== TEST 9: Request body as iterator, errors with missing length --- http_config eval: $::HttpConfig --- config location = /a { @@ -319,7 +467,7 @@ Request body is a function but a length or chunked encoding is not specified [warn] -=== TEST 8: Request body as iterator with chunked encoding +=== TEST 10: Request body as iterator with chunked encoding --- http_config eval: $::HttpConfig --- config location = /a { From 18c437dfce1b826c2a69f180bccf466c80003c5f Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Thu, 9 Mar 2023 15:43:06 +0800 Subject: [PATCH 102/130] update --- t/03-requestbody.t | 2 -- 1 file changed, 2 deletions(-) diff --git a/t/03-requestbody.t b/t/03-requestbody.t index 86fbf358..50b876ba 100644 --- a/t/03-requestbody.t +++ b/t/03-requestbody.t @@ -164,7 +164,6 @@ c: 3 [error] [warn] - === TEST 4: Return non-100 status to user --- http_config eval: $::HttpConfig --- config @@ -248,7 +247,6 @@ Expectation Failed end -- with additional header - ngx.log(ngx.NOTICE, "before send 100") local ok, err = sock:send("HTTP/1.1 100 Continue\r\nConnection: keep-alive\r\n\r\n") if not ok then ngx.log(ngx.ERR, "failed to send 100 response: ", err) From d4eaf884e5fc5b26a1a3914ad10b02cc7d0d7c38 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Thu, 9 Mar 2023 19:12:01 +0800 Subject: [PATCH 103/130] remove unused variable --- lib/resty/http.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index ac6309ff..e97c456b 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -646,7 +646,7 @@ local function _handle_continue(sock, body) return nil, nil, nil, err end - local ok, err, partial = _send_body(sock, body) + local ok, err = _send_body(sock, body) if not ok then return nil, nil, nil, err end From f94c18cf50b7530e3eee9713d151cd724a593f2c Mon Sep 17 00:00:00 2001 From: suika-kong Date: Tue, 14 Mar 2023 15:04:39 +0800 Subject: [PATCH 104/130] fix: no trailing "?" for empty query table --- lib/resty/http.lua | 6 ++++-- t/01-basic.t | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index e97c456b..9eeeeb5c 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -313,8 +313,10 @@ local function _format_request(self, params) local query = params.query or "" if type(query) == "table" then - query = "?" .. ngx_encode_args(query) - elseif query ~= "" and str_sub(query, 1, 1) ~= "?" then + query = ngx_encode_args(query) + end + + if query ~= "" and str_sub(query, 1, 1) ~= "?" then query = "?" .. query end diff --git a/t/01-basic.t b/t/01-basic.t index f3c413d7..472bed15 100644 --- a/t/01-basic.t +++ b/t/01-basic.t @@ -389,3 +389,45 @@ GET /a --- no_error_log [error] [warn] + + + +=== TEST 14: Empty query +--- http_config eval: $::HttpConfig +--- config + location = /a { + content_by_lua ' + local http = require "resty.http" + local httpc = http.new() + httpc:connect{ + scheme = "http", + host = "127.0.0.1", + port = ngx.var.server_port + } + + local res, err = httpc:request{ + query = {}, + path = "/b" + } + + ngx.status = res.status + + ngx.print(ngx.header.test) + + httpc:close() + '; + } + location = /b { + content_by_lua ' + ngx.header.test = ngx.var.request_uri + '; + } +--- request +GET /a +--- response_headers +/b +--- no_error_log +[error] +[warn] + + From e22651cdcb49c6e2b96f7fc102244effd7218b91 Mon Sep 17 00:00:00 2001 From: Xumin <100666470+StarlightIbuki@users.noreply.github.com> Date: Tue, 14 Mar 2023 18:06:47 +0800 Subject: [PATCH 105/130] Update lib/resty/http.lua --- lib/resty/http.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 9eeeeb5c..0cd56036 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -315,7 +315,6 @@ local function _format_request(self, params) if type(query) == "table" then query = ngx_encode_args(query) end - if query ~= "" and str_sub(query, 1, 1) ~= "?" then query = "?" .. query end From 3ad9ff688b0f1cf943234962c140ec27e779fdc5 Mon Sep 17 00:00:00 2001 From: Xumin <100666470+StarlightIbuki@users.noreply.github.com> Date: Tue, 14 Mar 2023 18:21:27 +0800 Subject: [PATCH 106/130] Update lib/resty/http.lua --- lib/resty/http.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index 0cd56036..ceccbcf2 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -315,6 +315,7 @@ local function _format_request(self, params) if type(query) == "table" then query = ngx_encode_args(query) end + if query ~= "" and str_sub(query, 1, 1) ~= "?" then query = "?" .. query end From f9c302a55d29e3c556e3e282f335059b0f0a2167 Mon Sep 17 00:00:00 2001 From: James Hurst Date: Wed, 22 Mar 2023 09:36:22 +0000 Subject: [PATCH 107/130] Update mTLS documentation --- README.md | 3 +++ lib/resty/http_connect.lua | 12 ++---------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 98b125a9..59c02f5a 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Production ready. * Request pipelining * Trailers * HTTP proxy connections +* mTLS (requires `ngx_lua_http_module` >= v0.10.23) ## API @@ -176,6 +177,8 @@ The options table has the following fields: * `ssl_verify`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake), except that it defaults to `true`. * `ssl_server_name`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) * `ssl_send_status_req`: option as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake) +* `ssl_client_cert`: will be passed to `tcpsock:setclientcert`. Requires `ngx_lua_http_module` >= v0.10.23. +* `ssl_client_priv_key`: as above. ## set\_timeout diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 4da98bcb..d89452a6 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -30,16 +30,8 @@ client:connect { ssl_verify = true, -- NOTE: defaults to true ctx = nil, -- NOTE: not supported - -- mTLS options (experimental!) - -- - -- !!! IMPORTANT !!! These options require support for mTLS in cosockets, - -- which is currently only available in the following unmerged PRs. - -- - -- * https://github.com/openresty/lua-nginx-module/pull/1602 - -- * https://github.com/openresty/lua-resty-core/pull/278 - -- - -- The details of this feature may change. You have been warned! - -- + -- mTLS options: These require support for mTLS in cosockets, which first + -- appeared in `ngx_http_lua_module` v0.10.23. ssl_client_cert = nil, ssl_client_priv_key = nil, From 4ab4269cf442ba52507aa2c718f606054452fcad Mon Sep 17 00:00:00 2001 From: James Hurst Date: Wed, 22 Mar 2023 09:45:52 +0000 Subject: [PATCH 108/130] Bump version to 0.17.1 --- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...17.0.beta.1-0.rockspec => lua-resty-http-0.17.1-0.rockspec | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename lua-resty-http-0.17.0.beta.1-0.rockspec => lua-resty-http-0.17.1-0.rockspec (91%) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index ceccbcf2..bf75683b 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -106,7 +106,7 @@ end local _M = { - _VERSION = '0.17.0-beta.1', + _VERSION = '0.17.1', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 97e81575..125b304c 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.17.0-beta.1', + _VERSION = '0.17.1', } diff --git a/lua-resty-http-0.17.0.beta.1-0.rockspec b/lua-resty-http-0.17.1-0.rockspec similarity index 91% rename from lua-resty-http-0.17.0.beta.1-0.rockspec rename to lua-resty-http-0.17.1-0.rockspec index d72353e9..1d286d52 100644 --- a/lua-resty-http-0.17.0.beta.1-0.rockspec +++ b/lua-resty-http-0.17.1-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.17.0.beta.1-0" +version = "0.17.1-0" source = { url = "git://github.com/ledgetech/lua-resty-http", - tag = "v0.17.0-beta.1" + tag = "v0.17.1" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.", From 8194773cb664fbf5557a739381a8a0c3b265a800 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Mon, 8 Jan 2024 14:54:47 +0800 Subject: [PATCH 109/130] fix poolname to include the digest of the cert for mTLS --- lib/resty/http_connect.lua | 55 ++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index d89452a6..052d20ec 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -1,8 +1,12 @@ +local ffi = require "ffi" local ngx_re_gmatch = ngx.re.gmatch local ngx_re_sub = ngx.re.sub local ngx_re_find = ngx.re.find local ngx_log = ngx.log local ngx_WARN = ngx.WARN +local to_hex = require("resty.string").to_hex +local ffi_gc = ffi.gc +local string_format = string.format --[[ A connection function that incorporates: @@ -160,16 +164,51 @@ local function connect(self, options) proxy_port = proxy_uri_t[3] end + local cert_hash + if ssl and ssl_client_cert and ssl_client_priv_key then + local status, res = xpcall(function() + return require("resty.openssl.x509") + end, debug.traceback) + + if status then + local x509 = res + local cert, err = x509.new(ssl_client_cert) + if not cert then + return nil, err + end + -- should not free the cdata passed in + ffi_gc(cert.ctx, nil) + + cert_hash, err = cert:digest("sha256") + if cert_hash then + cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + + else + return nil, err + end + + else + if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\.x509)' not found") then + ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS.") + + else + return nil, "failed to load module 'resty.openssl.x509':\n" .. res + end + end + end + -- construct a poolname unique within proxy and ssl info if not poolname then - poolname = (request_scheme or "") - .. ":" .. request_host - .. ":" .. tostring(request_port) - .. ":" .. tostring(ssl) - .. ":" .. (ssl_server_name or "") - .. ":" .. tostring(ssl_verify) - .. ":" .. (proxy_uri or "") - .. ":" .. (request_scheme == "https" and proxy_authorization or "") + poolname = string_format("%s:%s:%s:%s:%s:%s:%s:%s:%s", + request_scheme or "", + request_host, + tostring(request_port), + tostring(ssl), + ssl_server_name or "", + tostring(ssl_verify), + proxy_uri or "", + request_scheme == "https" and proxy_authorization or "", + cert_hash or "") -- in the above we only add the 'proxy_authorization' as part of the poolname -- when the request is https. Because in that case the CONNECT request (which -- carries the authorization header) is part of the connect procedure, whereas From 7598cbe99c140518c7906188f78514badc9e6dc0 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Mon, 8 Jan 2024 14:56:57 +0800 Subject: [PATCH 110/130] check the private key in order to make sure the caller is indeed the holder of the cert --- lib/resty/http_connect.lua | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 052d20ec..91d16ba0 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -167,11 +167,14 @@ local function connect(self, options) local cert_hash if ssl and ssl_client_cert and ssl_client_priv_key then local status, res = xpcall(function() - return require("resty.openssl.x509") + local x509 = require("resty.openssl.x509") + local pkey = require("resty.openssl.pkey") + return { x509, pkey } end, debug.traceback) if status then - local x509 = res + local x509 = res[1] + local pkey = res[2] local cert, err = x509.new(ssl_client_cert) if not cert then return nil, err @@ -179,6 +182,19 @@ local function connect(self, options) -- should not free the cdata passed in ffi_gc(cert.ctx, nil) + local key, err = pkey.new(ssl_client_priv_key) + if not key then + return nil, err + end + -- should not free the cdata passed in + ffi_gc(key.ctx, nil) + + -- check the private key in order to make sure the caller is indeed the holder of the cert + ok, err = cert:check_private_key(key) + if not ok then + return nil, "failed to match the private key with the certificate: " .. err + end + cert_hash, err = cert:digest("sha256") if cert_hash then cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable @@ -188,11 +204,11 @@ local function connect(self, options) end else - if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\.x509)' not found") then + if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\.(x509|pkey)' not found") then ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS.") else - return nil, "failed to load module 'resty.openssl.x509':\n" .. res + return nil, "failed to load module 'resty.openssl.*':\n" .. res end end end From c83c9e27bb9f63af7f797ceeb018ecce27ad3fab Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Mon, 8 Jan 2024 16:17:40 +0800 Subject: [PATCH 111/130] add test --- t/20-mtls.t | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/t/20-mtls.t b/t/20-mtls.t index e6395bdd..f48c70ed 100644 --- a/t/20-mtls.t +++ b/t/20-mtls.t @@ -209,3 +209,101 @@ GET /t --- response_body hello, CN=foo@example.com,O=OpenResty,ST=California,C=US + +=== TEST 4: users with different client certs should not share the same pool. +--- SKIP +--- http_config eval: $::mtls_http_config +--- config eval +" +lua_ssl_trusted_certificate $::HtmlDir/test.crt; + +location /t { + content_by_lua_block { + local f = assert(io.open('$::HtmlDir/mtls_client.crt')) + local cert_data = f:read('*a') + f:close() + + f = assert(io.open('$::HtmlDir/mtls_client.key')) + local key_data = f:read('*a') + f:close() + + local ssl = require('ngx.ssl') + + local cert = assert(ssl.parse_pem_cert(cert_data)) + local key = assert(ssl.parse_pem_priv_key(key_data)) + + f = assert(io.open('$::HtmlDir/test.crt')) + local invalid_cert_data = f:read('*a') + f:close() + + f = assert(io.open('$::HtmlDir/test.key')) + local invalid_key_data = f:read('*a') + f:close() + + local invalid_cert = assert(ssl.parse_pem_cert(invalid_cert_data)) + local invalid_key = assert(ssl.parse_pem_priv_key(invalid_key_data)) + + local httpc = assert(require('resty.http').new()) + + local ok, err = httpc:connect { + scheme = 'https', + host = 'unix:$::HtmlDir/mtls.sock', + ssl_client_cert = cert, + ssl_client_priv_key = key, + } + + if ok and not err then + local res, err = assert(httpc:request { + method = 'GET', + path = '/', + headers = { + ['Host'] = 'example.com', + }, + }) + + ngx.say(res:read_body()) + end + + httpc:set_keepalive() + + local httpc = assert(require('resty.http').new()) + + local ok, err = httpc:connect { + scheme = 'https', + host = 'unix:$::HtmlDir/mtls.sock', + ssl_client_cert = invalid_cert, + ssl_client_priv_key = invalid_key, + } + + ngx.say(httpc:get_reused_times()) + ngx.say(ok) + ngx.say(err) + + if ok and not err then + local res, err = assert(httpc:request { + method = 'GET', + path = '/', + headers = { + ['Host'] = 'example.com', + }, + }) + + ngx.say(res.status) -- expect 400 + end + + httpc:close() + } +} +" +--- user_files eval: $::mtls_user_files +--- request +GET /t +--- no_error_log +[error] +[warn] +--- response_body +hello, CN=foo@example.com,O=OpenResty,ST=California,C=US +0 +true +nil +400 From a4873ce7d05b24ca641f2aa24ad60e963a22cfce Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Mon, 8 Jan 2024 17:13:29 +0800 Subject: [PATCH 112/130] fix: ssl_client_cert is a chain of x509 instead of a x509 --- lib/resty/http_connect.lua | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 91d16ba0..91eb18b0 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -167,20 +167,30 @@ local function connect(self, options) local cert_hash if ssl and ssl_client_cert and ssl_client_priv_key then local status, res = xpcall(function() + local chain = require("resty.openssl.chain") local x509 = require("resty.openssl.x509") local pkey = require("resty.openssl.pkey") - return { x509, pkey } + return { chain, x509, pkey } end, debug.traceback) if status then - local x509 = res[1] - local pkey = res[2] - local cert, err = x509.new(ssl_client_cert) + local chain = res[1] + local x509 = res[2] + local pkey = res[3] + + local cert_chain, err = chain.dup(ssl_client_cert) + if not cert_chain then + return nil, err + end + + if #cert_chain < 1 then + return nil, "no cert in the chain" + end + + local cert, err = x509.dup(cert_chain[1].ctx) if not cert then return nil, err end - -- should not free the cdata passed in - ffi_gc(cert.ctx, nil) local key, err = pkey.new(ssl_client_priv_key) if not key then @@ -204,8 +214,8 @@ local function connect(self, options) end else - if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\.(x509|pkey)' not found") then - ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS.") + if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\.(chain|x509|pkey)' not found") then + ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS." .. res) else return nil, "failed to load module 'resty.openssl.*':\n" .. res From af9d005c65605545871e2808b01e64cf60bd1e86 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Wed, 10 Jan 2024 11:09:40 +0800 Subject: [PATCH 113/130] fix: use the correct chain path and add `ffi.cast` --- lib/resty/http_connect.lua | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 91eb18b0..560534dc 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -6,6 +6,7 @@ local ngx_log = ngx.log local ngx_WARN = ngx.WARN local to_hex = require("resty.string").to_hex local ffi_gc = ffi.gc +local ffi_cast = ffi.cast local string_format = string.format --[[ @@ -167,7 +168,7 @@ local function connect(self, options) local cert_hash if ssl and ssl_client_cert and ssl_client_priv_key then local status, res = xpcall(function() - local chain = require("resty.openssl.chain") + local chain = require("resty.openssl.x509.chain") local x509 = require("resty.openssl.x509") local pkey = require("resty.openssl.pkey") return { chain, x509, pkey } @@ -178,7 +179,9 @@ local function connect(self, options) local x509 = res[2] local pkey = res[3] - local cert_chain, err = chain.dup(ssl_client_cert) + + -- convert from `void*` to `OPENSSL_STACK*` + local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) if not cert_chain then return nil, err end @@ -192,7 +195,8 @@ local function connect(self, options) return nil, err end - local key, err = pkey.new(ssl_client_priv_key) + -- convert from `void*` to `EVP_PKEY*` + local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) if not key then return nil, err end @@ -214,7 +218,7 @@ local function connect(self, options) end else - if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\.(chain|x509|pkey)' not found") then + if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS." .. res) else From 08ad5111d113f3bddc1a4fe416dd97ba6d23f4e6 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Wed, 10 Jan 2024 11:42:14 +0800 Subject: [PATCH 114/130] add the latest openresty versions and skip mtls tests only when nginx version < 1.21.4 --- .github/workflows/test.yml | 6 +++++- t/20-mtls.t | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f78eb83a..1fe8a0b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,8 @@ jobs: openresty_version: - 1.17.8.2 - 1.19.9.1 + - 1.21.4.3 + - 1.25.3.1 runs-on: ubuntu-latest container: @@ -49,7 +51,9 @@ jobs: run: cpanm -q -n Test::Nginx - name: Install Luacov - run: /usr/local/openresty/luajit/bin/luarocks install luacov + run: | + /usr/local/openresty/luajit/bin/luarocks install luacov + /usr/local/openresty/luajit/bin/luarocks install lua-resty-openssl - uses: actions/checkout@v2 diff --git a/t/20-mtls.t b/t/20-mtls.t index f48c70ed..9384dc6a 100644 --- a/t/20-mtls.t +++ b/t/20-mtls.t @@ -105,7 +105,6 @@ GET /t === TEST 2: Connection fails during handshake with not priv_key --- http_config eval: $::mtls_http_config ---- SKIP --- config eval " lua_ssl_trusted_certificate $::HtmlDir/test.crt; @@ -151,10 +150,10 @@ GET /t --- error_log could not set client certificate: bad client pkey type --- response_body_unlike: hello, CN=foo@example.com,O=OpenResty,ST=California,C=US - +--- skip_nginx +4: < 1.21.4 === TEST 3: Connection succeeds with client cert and key. SKIP'd for CI until feature is merged. ---- SKIP --- http_config eval: $::mtls_http_config --- config eval " @@ -208,10 +207,10 @@ GET /t [warn] --- response_body hello, CN=foo@example.com,O=OpenResty,ST=California,C=US - +--- skip_nginx +4: < 1.21.4 === TEST 4: users with different client certs should not share the same pool. ---- SKIP --- http_config eval: $::mtls_http_config --- config eval " @@ -307,3 +306,5 @@ hello, CN=foo@example.com,O=OpenResty,ST=California,C=US true nil 400 +--- skip_nginx +4: < 1.21.4 From be15d2ae6ea08ded202ca1eb3ac1a60972cd6991 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Wed, 10 Jan 2024 14:06:09 +0800 Subject: [PATCH 115/130] add type check for client cert and key --- lib/resty/http_connect.lua | 7 +++++++ t/20-mtls.t | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 560534dc..04fa5bcd 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -179,6 +179,13 @@ local function connect(self, options) local x509 = res[2] local pkey = res[3] + if type(ssl_client_cert) ~= "cdata" then + return nil, "bad ssl_client_cert: cdata expected, got " .. type(ssl_client_cert) + end + + if type(ssl_client_priv_key) ~= "cdata" then + return nil, "bad ssl_client_priv_key: cdata expected, got " .. type(ssl_client_priv_key) + end -- convert from `void*` to `OPENSSL_STACK*` local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) diff --git a/t/20-mtls.t b/t/20-mtls.t index 9384dc6a..761185a7 100644 --- a/t/20-mtls.t +++ b/t/20-mtls.t @@ -148,7 +148,7 @@ location /t { GET /t --- error_code: 200 --- error_log -could not set client certificate: bad client pkey type +bad ssl_client_priv_key: cdata expected, got string --- response_body_unlike: hello, CN=foo@example.com,O=OpenResty,ST=California,C=US --- skip_nginx 4: < 1.21.4 From a939c7efe2aebdf799aa6bf8470841af5248f6be Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Wed, 10 Jan 2024 14:42:56 +0800 Subject: [PATCH 116/130] fallback to non-mTLS when any cert/key error and log a warning --- lib/resty/http_connect.lua | 124 +++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 52 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 04fa5bcd..afe00add 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -167,71 +167,91 @@ local function connect(self, options) local cert_hash if ssl and ssl_client_cert and ssl_client_priv_key then - local status, res = xpcall(function() - local chain = require("resty.openssl.x509.chain") - local x509 = require("resty.openssl.x509") - local pkey = require("resty.openssl.pkey") - return { chain, x509, pkey } - end, debug.traceback) - - if status then - local chain = res[1] - local x509 = res[2] - local pkey = res[3] - - if type(ssl_client_cert) ~= "cdata" then - return nil, "bad ssl_client_cert: cdata expected, got " .. type(ssl_client_cert) + -- fallback to non-mTLS when any error + repeat + local cert_type = type(ssl_client_cert) + local key_type = type(ssl_client_priv_key) + + if cert_type ~= "cdata" then + ngx_log(ngx_WARN, "bad ssl_client_cert: cdata expected, got " .. cert_type) + break end - if type(ssl_client_priv_key) ~= "cdata" then - return nil, "bad ssl_client_priv_key: cdata expected, got " .. type(ssl_client_priv_key) + if key_type ~= "cdata" then + ngx_log(ngx_WARN, "bad ssl_client_priv_key: cdata expected, got " .. key_type) + break end - -- convert from `void*` to `OPENSSL_STACK*` - local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) - if not cert_chain then - return nil, err - end - - if #cert_chain < 1 then - return nil, "no cert in the chain" - end + local status, res = xpcall(function() + local chain = require("resty.openssl.x509.chain") + local x509 = require("resty.openssl.x509") + local pkey = require("resty.openssl.pkey") + return { chain, x509, pkey } + end, debug.traceback) + + if status then + local chain = res[1] + local x509 = res[2] + local pkey = res[3] + + -- convert from `void*` to `OPENSSL_STACK*` + local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) + if not cert_chain then + ngx_log(ngx_WARN, "failed to dup the ssl_client_cert, falling back to non-mTLS: " .. err) + break + end - local cert, err = x509.dup(cert_chain[1].ctx) - if not cert then - return nil, err - end + if #cert_chain < 1 then + ngx_log(ngx_WARN, "no cert in ssl_client_cert, falling back to non-mTLS: " .. err) + break + end - -- convert from `void*` to `EVP_PKEY*` - local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) - if not key then - return nil, err - end - -- should not free the cdata passed in - ffi_gc(key.ctx, nil) + local cert, err = x509.dup(cert_chain[1].ctx) + if not cert then + ngx_log(ngx_WARN, "failed to dup the x509, falling back to non-mTLS: " .. err) + break + end - -- check the private key in order to make sure the caller is indeed the holder of the cert - ok, err = cert:check_private_key(key) - if not ok then - return nil, "failed to match the private key with the certificate: " .. err - end + -- convert from `void*` to `EVP_PKEY*` + local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) + if not key then + ngx_log(ngx_WARN, "failed to new the pkey, falling back to non-mTLS: " .. err) + break + end + -- should not free the cdata passed in + ffi_gc(key.ctx, nil) - cert_hash, err = cert:digest("sha256") - if cert_hash then - cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + -- check the private key in order to make sure the caller is indeed the holder of the cert + ok, err = cert:check_private_key(key) + if not ok then + ngx_log(ngx_WARN, "the private key doesn't match the cert, falling back to non-mTLS: " .. err) + break + end - else - return nil, err - end + cert_hash, err = cert:digest("sha256") + if cert_hash then + cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable - else - if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then - ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS." .. res) + else + ngx_log(ngx_WARN, "failed to calculate the digest of the cert, falling back to non-mTLS: " .. err) + break + end else - return nil, "failed to load module 'resty.openssl.*':\n" .. res + if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then + ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS:\n " + .. res) + + else + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, falling back to non-mTLS:\n" .. res) + end end - end + until true + end + + if not cert_hash then + ssl_client_cert = nil + ssl_client_priv_key = nil end -- construct a poolname unique within proxy and ssl info From bdaf80adc5c573c93b7ecc0c361473e763207802 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Mon, 15 Jan 2024 16:53:28 +0800 Subject: [PATCH 117/130] add debug log to print poolname --- lib/resty/http_connect.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index afe00add..10add14c 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -4,6 +4,7 @@ local ngx_re_sub = ngx.re.sub local ngx_re_find = ngx.re.find local ngx_log = ngx.log local ngx_WARN = ngx.WARN +local ngx_DEBUG = ngx.DEBUG local to_hex = require("resty.string").to_hex local ffi_gc = ffi.gc local ffi_cast = ffi.cast @@ -272,6 +273,8 @@ local function connect(self, options) -- with a plain http request the authorization is part of the actual request. end + ngx_log(ngx_DEBUG, "poolname: " .. poolname) + -- do TCP level connection local tcp_opts = { pool = poolname, pool_size = pool_size, backlog = backlog } if proxy then From 5455eee45058d9365f092af8807137ff96fe4ce8 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 14:10:22 +0800 Subject: [PATCH 118/130] apply the comments --- lib/resty/http_connect.lua | 119 +++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 10add14c..f1df20cc 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -9,6 +9,14 @@ local to_hex = require("resty.string").to_hex local ffi_gc = ffi.gc local ffi_cast = ffi.cast local string_format = string.format +local type = type + +local function require_openssl_libs() + local chain = require("resty.openssl.x509.chain") + local x509 = require("resty.openssl.x509") + local pkey = require("resty.openssl.pkey") + return { chain, x509, pkey } +end --[[ A connection function that incorporates: @@ -174,79 +182,76 @@ local function connect(self, options) local key_type = type(ssl_client_priv_key) if cert_type ~= "cdata" then - ngx_log(ngx_WARN, "bad ssl_client_cert: cdata expected, got " .. cert_type) + ngx_log(ngx_WARN, "bad ssl_client_cert: cdata expected, got ", cert_type) break end if key_type ~= "cdata" then - ngx_log(ngx_WARN, "bad ssl_client_priv_key: cdata expected, got " .. key_type) + ngx_log(ngx_WARN, "bad ssl_client_priv_key: cdata expected, got ", key_type) break end - local status, res = xpcall(function() - local chain = require("resty.openssl.x509.chain") - local x509 = require("resty.openssl.x509") - local pkey = require("resty.openssl.pkey") - return { chain, x509, pkey } - end, debug.traceback) - - if status then - local chain = res[1] - local x509 = res[2] - local pkey = res[3] - - -- convert from `void*` to `OPENSSL_STACK*` - local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) - if not cert_chain then - ngx_log(ngx_WARN, "failed to dup the ssl_client_cert, falling back to non-mTLS: " .. err) - break - end + local status, res = xpcall(require_openssl_libs, debug.traceback) - if #cert_chain < 1 then - ngx_log(ngx_WARN, "no cert in ssl_client_cert, falling back to non-mTLS: " .. err) - break - end + if not status then + if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then + ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS:\n " + , res) - local cert, err = x509.dup(cert_chain[1].ctx) - if not cert then - ngx_log(ngx_WARN, "failed to dup the x509, falling back to non-mTLS: " .. err) - break + else + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, falling back to non-mTLS:\n", res) end - -- convert from `void*` to `EVP_PKEY*` - local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) - if not key then - ngx_log(ngx_WARN, "failed to new the pkey, falling back to non-mTLS: " .. err) - break - end - -- should not free the cdata passed in - ffi_gc(key.ctx, nil) + break + end - -- check the private key in order to make sure the caller is indeed the holder of the cert - ok, err = cert:check_private_key(key) - if not ok then - ngx_log(ngx_WARN, "the private key doesn't match the cert, falling back to non-mTLS: " .. err) - break - end + local chain = res[1] + local x509 = res[2] + local pkey = res[3] - cert_hash, err = cert:digest("sha256") - if cert_hash then - cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + -- convert from `void*` to `OPENSSL_STACK*` + local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) + if not cert_chain then + ngx_log(ngx_WARN, "failed to dup the ssl_client_cert, falling back to non-mTLS: ", err) + break + end - else - ngx_log(ngx_WARN, "failed to calculate the digest of the cert, falling back to non-mTLS: " .. err) - break - end + if #cert_chain < 1 then + ngx_log(ngx_WARN, "no cert in ssl_client_cert, falling back to non-mTLS: ", err) + break + end - else - if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then - ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS:\n " - .. res) + local cert, err = x509.dup(cert_chain[1].ctx) + if not cert then + ngx_log(ngx_WARN, "failed to dup the x509, falling back to non-mTLS: ", err) + break + end - else - ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, falling back to non-mTLS:\n" .. res) - end + -- convert from `void*` to `EVP_PKEY*` + local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) + if not key then + ngx_log(ngx_WARN, "failed to new the pkey, falling back to non-mTLS: ", err) + break end + -- should not free the cdata passed in + ffi_gc(key.ctx, nil) + + -- check the private key in order to make sure the caller is indeed the holder of the cert + ok, err = cert:check_private_key(key) + if not ok then + ngx_log(ngx_WARN, "the private key doesn't match the cert, falling back to non-mTLS: ", err) + break + end + + cert_hash, err = cert:digest("sha256") + if cert_hash then + cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + + else + ngx_log(ngx_WARN, "failed to calculate the digest of the cert, falling back to non-mTLS: ", err) + break + end + until true end @@ -273,7 +278,7 @@ local function connect(self, options) -- with a plain http request the authorization is part of the actual request. end - ngx_log(ngx_DEBUG, "poolname: " .. poolname) + ngx_log(ngx_DEBUG, "poolname: ", poolname) -- do TCP level connection local tcp_opts = { pool = poolname, pool_size = pool_size, backlog = backlog } From 57dae629a8b89caadcc1fbeb2ca85fdbda5d9c70 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 19:32:26 +0800 Subject: [PATCH 119/130] apply comments --- lib/resty/http_connect.lua | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index f1df20cc..e7594eec 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -244,14 +244,13 @@ local function connect(self, options) end cert_hash, err = cert:digest("sha256") - if cert_hash then - cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable - - else + if not cert_hash then ngx_log(ngx_WARN, "failed to calculate the digest of the cert, falling back to non-mTLS: ", err) break end + cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + until true end @@ -265,10 +264,10 @@ local function connect(self, options) poolname = string_format("%s:%s:%s:%s:%s:%s:%s:%s:%s", request_scheme or "", request_host, - tostring(request_port), - tostring(ssl), + request_port, + ssl, ssl_server_name or "", - tostring(ssl_verify), + ssl_verify, proxy_uri or "", request_scheme == "https" and proxy_authorization or "", cert_hash or "") From 836569d0e00175f34c1e60ed263724e1899e4430 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 20:16:07 +0800 Subject: [PATCH 120/130] apply the comment --- lib/resty/http_connect.lua | 128 +++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index e7594eec..abae7122 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -175,84 +175,86 @@ local function connect(self, options) end local cert_hash - if ssl and ssl_client_cert and ssl_client_priv_key then - -- fallback to non-mTLS when any error - repeat - local cert_type = type(ssl_client_cert) - local key_type = type(ssl_client_priv_key) - - if cert_type ~= "cdata" then - ngx_log(ngx_WARN, "bad ssl_client_cert: cdata expected, got ", cert_type) - break - end + -- fallback to non-mTLS when any error + repeat + if not ssl or not ssl_client_cert or not ssl_client_priv_key then + break + end - if key_type ~= "cdata" then - ngx_log(ngx_WARN, "bad ssl_client_priv_key: cdata expected, got ", key_type) - break - end + local cert_type = type(ssl_client_cert) + local key_type = type(ssl_client_priv_key) - local status, res = xpcall(require_openssl_libs, debug.traceback) + if cert_type ~= "cdata" then + ngx_log(ngx_WARN, "bad ssl_client_cert: cdata expected, got ", cert_type) + break + end - if not status then - if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then - ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS:\n " - , res) + if key_type ~= "cdata" then + ngx_log(ngx_WARN, "bad ssl_client_priv_key: cdata expected, got ", key_type) + break + end - else - ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, falling back to non-mTLS:\n", res) - end + local status, res = xpcall(require_openssl_libs, debug.traceback) - break + if not status then + if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then + ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS:\n " + , res) + + else + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, falling back to non-mTLS:\n", res) end - local chain = res[1] - local x509 = res[2] - local pkey = res[3] + break + end - -- convert from `void*` to `OPENSSL_STACK*` - local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) - if not cert_chain then - ngx_log(ngx_WARN, "failed to dup the ssl_client_cert, falling back to non-mTLS: ", err) - break - end + local chain = res[1] + local x509 = res[2] + local pkey = res[3] - if #cert_chain < 1 then - ngx_log(ngx_WARN, "no cert in ssl_client_cert, falling back to non-mTLS: ", err) - break - end + -- convert from `void*` to `OPENSSL_STACK*` + local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) + if not cert_chain then + ngx_log(ngx_WARN, "failed to dup the ssl_client_cert, falling back to non-mTLS: ", err) + break + end - local cert, err = x509.dup(cert_chain[1].ctx) - if not cert then - ngx_log(ngx_WARN, "failed to dup the x509, falling back to non-mTLS: ", err) - break - end + if #cert_chain < 1 then + ngx_log(ngx_WARN, "no cert in ssl_client_cert, falling back to non-mTLS: ", err) + break + end - -- convert from `void*` to `EVP_PKEY*` - local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) - if not key then - ngx_log(ngx_WARN, "failed to new the pkey, falling back to non-mTLS: ", err) - break - end - -- should not free the cdata passed in - ffi_gc(key.ctx, nil) + local cert, err = x509.dup(cert_chain[1].ctx) + if not cert then + ngx_log(ngx_WARN, "failed to dup the x509, falling back to non-mTLS: ", err) + break + end - -- check the private key in order to make sure the caller is indeed the holder of the cert - ok, err = cert:check_private_key(key) - if not ok then - ngx_log(ngx_WARN, "the private key doesn't match the cert, falling back to non-mTLS: ", err) - break - end + -- convert from `void*` to `EVP_PKEY*` + local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) + if not key then + ngx_log(ngx_WARN, "failed to new the pkey, falling back to non-mTLS: ", err) + break + end + -- should not free the cdata passed in + ffi_gc(key.ctx, nil) - cert_hash, err = cert:digest("sha256") - if not cert_hash then - ngx_log(ngx_WARN, "failed to calculate the digest of the cert, falling back to non-mTLS: ", err) - break - end + -- check the private key in order to make sure the caller is indeed the holder of the cert + ok, err = cert:check_private_key(key) + if not ok then + ngx_log(ngx_WARN, "the private key doesn't match the cert, falling back to non-mTLS: ", err) + break + end - cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + cert_hash, err = cert:digest("sha256") + if not cert_hash then + ngx_log(ngx_WARN, "failed to calculate the digest of the cert, falling back to non-mTLS: ", err) + break + end - until true - end + cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable + + until true if not cert_hash then ssl_client_cert = nil From 310e78dd829c65fc2fd3e62146ca2d02b42ab812 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 22:07:02 +0800 Subject: [PATCH 121/130] load resty.openssl.* at the module level and error out if the client cert set isn't valid --- lib/resty/http_connect.lua | 51 ++++++++++++++------------------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index abae7122..dfed31e5 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -11,11 +11,15 @@ local ffi_cast = ffi.cast local string_format = string.format local type = type -local function require_openssl_libs() - local chain = require("resty.openssl.x509.chain") - local x509 = require("resty.openssl.x509") - local pkey = require("resty.openssl.pkey") - return { chain, x509, pkey } +local lib_chain, lib_x509, lib_pkey +local openssl_available, res = xpcall(function() + lib_chain = require("resty.openssl.x509.chain") + lib_x509 = require("resty.openssl.x509") + lib_pkey = require("resty.openssl.pkey") +end, debug.traceback) + +if not openssl_available then + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, mTLS isn't supported without lua-resty-openssl :\n", res) end --[[ @@ -175,7 +179,7 @@ local function connect(self, options) end local cert_hash - -- fallback to non-mTLS when any error + -- fallback to non-mTLS if it's not an error due to the caller repeat if not ssl or not ssl_client_cert or not ssl_client_priv_key then break @@ -185,53 +189,37 @@ local function connect(self, options) local key_type = type(ssl_client_priv_key) if cert_type ~= "cdata" then - ngx_log(ngx_WARN, "bad ssl_client_cert: cdata expected, got ", cert_type) - break + return nil, string_format("bad ssl_client_cert: cdata expected, got %s", cert_type) end if key_type ~= "cdata" then - ngx_log(ngx_WARN, "bad ssl_client_priv_key: cdata expected, got ", key_type) - break + return nil, string_format("bad ssl_client_priv_key: cdata expected, got %s", key_type) end - local status, res = xpcall(require_openssl_libs, debug.traceback) - - if not status then - if type(res) == "string" and ngx_re_find(res, "module 'resty\\.openssl\\..+' not found") then - ngx_log(ngx_WARN, "can't use mTLS without module `lua-resty-openssl`, falling back to non-mTLS:\n " - , res) - - else - ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, falling back to non-mTLS:\n", res) - end - + if not openssl_available then + ngx_log(ngx_WARN, "module `resty.openssl.*` not available, falling back to non-mTLS:\n") break end - local chain = res[1] - local x509 = res[2] - local pkey = res[3] - -- convert from `void*` to `OPENSSL_STACK*` - local cert_chain, err = chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) + local cert_chain, err = lib_chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) if not cert_chain then ngx_log(ngx_WARN, "failed to dup the ssl_client_cert, falling back to non-mTLS: ", err) break end if #cert_chain < 1 then - ngx_log(ngx_WARN, "no cert in ssl_client_cert, falling back to non-mTLS: ", err) - break + return nil, "no cert in ssl_client_cert" end - local cert, err = x509.dup(cert_chain[1].ctx) + local cert, err = lib_x509.dup(cert_chain[1].ctx) if not cert then ngx_log(ngx_WARN, "failed to dup the x509, falling back to non-mTLS: ", err) break end -- convert from `void*` to `EVP_PKEY*` - local key, err = pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) + local key, err = lib_pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) if not key then ngx_log(ngx_WARN, "failed to new the pkey, falling back to non-mTLS: ", err) break @@ -242,8 +230,7 @@ local function connect(self, options) -- check the private key in order to make sure the caller is indeed the holder of the cert ok, err = cert:check_private_key(key) if not ok then - ngx_log(ngx_WARN, "the private key doesn't match the cert, falling back to non-mTLS: ", err) - break + return nil, string_format("the private key doesn't match the cert: %s", err) end cert_hash, err = cert:digest("sha256") From a62778ca2433fe058188e2ce36d9b35db657c5da Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 22:29:38 +0800 Subject: [PATCH 122/130] error out directly for other cases as well and fix test --- lib/resty/http_connect.lua | 38 +++++++++++--------------------------- t/20-mtls.t | 13 +++++++++---- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index dfed31e5..43996efe 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -179,12 +179,7 @@ local function connect(self, options) end local cert_hash - -- fallback to non-mTLS if it's not an error due to the caller - repeat - if not ssl or not ssl_client_cert or not ssl_client_priv_key then - break - end - + if ssl and ssl_client_cert and ssl_client_priv_key then local cert_type = type(ssl_client_cert) local key_type = type(ssl_client_priv_key) @@ -197,15 +192,13 @@ local function connect(self, options) end if not openssl_available then - ngx_log(ngx_WARN, "module `resty.openssl.*` not available, falling back to non-mTLS:\n") - break + return nil, "module `resty.openssl.*` not available, mTLS isn't supported with lua-resty-openssl" end -- convert from `void*` to `OPENSSL_STACK*` local cert_chain, err = lib_chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) if not cert_chain then - ngx_log(ngx_WARN, "failed to dup the ssl_client_cert, falling back to non-mTLS: ", err) - break + return nil, string_format("failed to dup the ssl_client_cert: %s", err) end if #cert_chain < 1 then @@ -214,15 +207,13 @@ local function connect(self, options) local cert, err = lib_x509.dup(cert_chain[1].ctx) if not cert then - ngx_log(ngx_WARN, "failed to dup the x509, falling back to non-mTLS: ", err) - break + return nil, string_format("failed to dup the x509: %s", err) end -- convert from `void*` to `EVP_PKEY*` local key, err = lib_pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) if not key then - ngx_log(ngx_WARN, "failed to new the pkey, falling back to non-mTLS: ", err) - break + return nil, string_format("failed to new the pkey: %s": err) end -- should not free the cdata passed in ffi_gc(key.ctx, nil) @@ -235,17 +226,10 @@ local function connect(self, options) cert_hash, err = cert:digest("sha256") if not cert_hash then - ngx_log(ngx_WARN, "failed to calculate the digest of the cert, falling back to non-mTLS: ", err) - break + return nil, string_format("failed to calculate the digest of the cert: %s", err) end cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable - - until true - - if not cert_hash then - ssl_client_cert = nil - ssl_client_priv_key = nil end -- construct a poolname unique within proxy and ssl info @@ -326,13 +310,13 @@ local function connect(self, options) -- Experimental mTLS support if ssl_client_cert and ssl_client_priv_key then if type(sock.setclientcert) ~= "function" then - ngx_log(ngx_WARN, "cannot use SSL client cert and key without mTLS support") + return nil, "cannot use SSL client cert and key without mTLS support" else - ok, err = sock:setclientcert(ssl_client_cert, ssl_client_priv_key) - if not ok then - ngx_log(ngx_WARN, "could not set client certificate: ", err) - end + ok, err = sock:setclientcert(ssl_client_cert, ssl_client_priv_key) + if not ok then + return nil, string_format("could not set client certificate: %s", err) + end end end diff --git a/t/20-mtls.t b/t/20-mtls.t index 761185a7..7508dbb9 100644 --- a/t/20-mtls.t +++ b/t/20-mtls.t @@ -137,6 +137,9 @@ location /t { }) ngx.say(res:read_body()) + + else + ngx.say("failed to connect: " .. err or "") end httpc:close() @@ -147,13 +150,15 @@ location /t { --- request GET /t --- error_code: 200 ---- error_log -bad ssl_client_priv_key: cdata expected, got string ---- response_body_unlike: hello, CN=foo@example.com,O=OpenResty,ST=California,C=US +--- no_error_log +[error] +[warn] +--- response_body +failed to connect: bad ssl_client_priv_key: cdata expected, got string --- skip_nginx 4: < 1.21.4 -=== TEST 3: Connection succeeds with client cert and key. SKIP'd for CI until feature is merged. +=== TEST 3: Connection succeeds with client cert and key. --- http_config eval: $::mtls_http_config --- config eval " From f7a7af18d72ff0388693690b66e3f35ae0de5f54 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 22:31:36 +0800 Subject: [PATCH 123/130] fix typo --- lib/resty/http_connect.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 43996efe..049d4ec2 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -213,7 +213,7 @@ local function connect(self, options) -- convert from `void*` to `EVP_PKEY*` local key, err = lib_pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) if not key then - return nil, string_format("failed to new the pkey: %s": err) + return nil, string_format("failed to new the pkey: %s", err) end -- should not free the cdata passed in ffi_gc(key.ctx, nil) From 814f2aeb5a41ee3d3c1a50f904390b6400a5e8da Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 22:44:57 +0800 Subject: [PATCH 124/130] fix test --- t/20-mtls.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/20-mtls.t b/t/20-mtls.t index 7508dbb9..e5b6df6a 100644 --- a/t/20-mtls.t +++ b/t/20-mtls.t @@ -139,7 +139,7 @@ location /t { ngx.say(res:read_body()) else - ngx.say("failed to connect: " .. err or "") + ngx.say('failed to connect: ' .. (err or '')) end httpc:close() @@ -158,6 +158,7 @@ failed to connect: bad ssl_client_priv_key: cdata expected, got string --- skip_nginx 4: < 1.21.4 + === TEST 3: Connection succeeds with client cert and key. --- http_config eval: $::mtls_http_config --- config eval From 7624d2094e47de6422962111c7ef67728bae7c63 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 22:49:37 +0800 Subject: [PATCH 125/130] fix error message --- lib/resty/http_connect.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 049d4ec2..4ea7999b 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -192,7 +192,7 @@ local function connect(self, options) end if not openssl_available then - return nil, "module `resty.openssl.*` not available, mTLS isn't supported with lua-resty-openssl" + return nil, "module `resty.openssl.*` not available, mTLS isn't supported without lua-resty-openssl" end -- convert from `void*` to `OPENSSL_STACK*` From 52f83bbdefb9043816f4be278a686bc5af2dfdf8 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Tue, 27 Feb 2024 22:52:47 +0800 Subject: [PATCH 126/130] fix error message --- lib/resty/http_connect.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 4ea7999b..e05cda0b 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -19,7 +19,7 @@ local openssl_available, res = xpcall(function() end, debug.traceback) if not openssl_available then - ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, mTLS isn't supported without lua-resty-openssl :\n", res) + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, mTLS isn't supported without lua-resty-openssl:\n", res) end --[[ From dbc6ba21bbe07e04d8168abcd2242494958bed6c Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Wed, 28 Feb 2024 11:03:18 +0800 Subject: [PATCH 127/130] use `..` concat operation to keep the original style --- lib/resty/http_connect.lua | 39 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index e05cda0b..5da3524b 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -8,7 +8,6 @@ local ngx_DEBUG = ngx.DEBUG local to_hex = require("resty.string").to_hex local ffi_gc = ffi.gc local ffi_cast = ffi.cast -local string_format = string.format local type = type local lib_chain, lib_x509, lib_pkey @@ -19,7 +18,8 @@ local openssl_available, res = xpcall(function() end, debug.traceback) if not openssl_available then - ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, mTLS isn't supported without lua-resty-openssl:\n", res) + ngx_log(ngx_WARN, "failed to load module `resty.openssl.*`, \z + mTLS isn't supported without lua-resty-openssl:\n", res) end --[[ @@ -184,11 +184,11 @@ local function connect(self, options) local key_type = type(ssl_client_priv_key) if cert_type ~= "cdata" then - return nil, string_format("bad ssl_client_cert: cdata expected, got %s", cert_type) + return nil, "bad ssl_client_cert: cdata expected, got " .. cert_type end if key_type ~= "cdata" then - return nil, string_format("bad ssl_client_priv_key: cdata expected, got %s", key_type) + return nil, "bad ssl_client_priv_key: cdata expected, got " .. key_type end if not openssl_available then @@ -198,7 +198,7 @@ local function connect(self, options) -- convert from `void*` to `OPENSSL_STACK*` local cert_chain, err = lib_chain.dup(ffi_cast("OPENSSL_STACK*", ssl_client_cert)) if not cert_chain then - return nil, string_format("failed to dup the ssl_client_cert: %s", err) + return nil, "failed to dup the ssl_client_cert: " .. err end if #cert_chain < 1 then @@ -207,26 +207,27 @@ local function connect(self, options) local cert, err = lib_x509.dup(cert_chain[1].ctx) if not cert then - return nil, string_format("failed to dup the x509: %s", err) + return nil, "failed to dup the x509: " .. err end -- convert from `void*` to `EVP_PKEY*` local key, err = lib_pkey.new(ffi_cast("EVP_PKEY*", ssl_client_priv_key)) if not key then - return nil, string_format("failed to new the pkey: %s", err) + return nil, "failed to new the pkey: " .. err end + -- should not free the cdata passed in ffi_gc(key.ctx, nil) -- check the private key in order to make sure the caller is indeed the holder of the cert ok, err = cert:check_private_key(key) if not ok then - return nil, string_format("the private key doesn't match the cert: %s", err) + return nil, "the private key doesn't match the cert: " .. err end cert_hash, err = cert:digest("sha256") if not cert_hash then - return nil, string_format("failed to calculate the digest of the cert: %s", err) + return nil, "failed to calculate the digest of the cert: " .. err end cert_hash = to_hex(cert_hash) -- convert to hex so that it's printable @@ -234,16 +235,14 @@ local function connect(self, options) -- construct a poolname unique within proxy and ssl info if not poolname then - poolname = string_format("%s:%s:%s:%s:%s:%s:%s:%s:%s", - request_scheme or "", - request_host, - request_port, - ssl, - ssl_server_name or "", - ssl_verify, - proxy_uri or "", - request_scheme == "https" and proxy_authorization or "", - cert_hash or "") + poolname = (request_scheme or "") + .. ":" .. request_host + .. ":" .. tostring(request_port) + .. ":" .. tostring(ssl) + .. ":" .. (ssl_server_name or "") + .. ":" .. tostring(ssl_verify) + .. ":" .. (proxy_uri or "") + .. ":" .. (request_scheme == "https" and proxy_authorization or "") -- in the above we only add the 'proxy_authorization' as part of the poolname -- when the request is https. Because in that case the CONNECT request (which -- carries the authorization header) is part of the connect procedure, whereas @@ -315,7 +314,7 @@ local function connect(self, options) else ok, err = sock:setclientcert(ssl_client_cert, ssl_client_priv_key) if not ok then - return nil, string_format("could not set client certificate: %s", err) + return nil, "could not set client certificate: " .. err end end end From f2f7460dbbd97c7240a8d54234a0d4506f126702 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Wed, 28 Feb 2024 11:07:51 +0800 Subject: [PATCH 128/130] fixup --- lib/resty/http_connect.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 5da3524b..65039850 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -243,6 +243,7 @@ local function connect(self, options) .. ":" .. tostring(ssl_verify) .. ":" .. (proxy_uri or "") .. ":" .. (request_scheme == "https" and proxy_authorization or "") + .. ":" .. (cert_hash or "") -- in the above we only add the 'proxy_authorization' as part of the poolname -- when the request is https. Because in that case the CONNECT request (which -- carries the authorization header) is part of the connect procedure, whereas From bf3e411a9f52cae5ceddf53ee8fded3c2027a8b9 Mon Sep 17 00:00:00 2001 From: Zhefeng Chen Date: Wed, 28 Feb 2024 11:50:15 +0800 Subject: [PATCH 129/130] fix some return value errors --- lib/resty/http_connect.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/resty/http_connect.lua b/lib/resty/http_connect.lua index 65039850..83b6d2f4 100644 --- a/lib/resty/http_connect.lua +++ b/lib/resty/http_connect.lua @@ -166,7 +166,7 @@ local function connect(self, options) local proxy_uri_t proxy_uri_t, err = self:parse_uri(proxy_uri) if not proxy_uri_t then - return nil, "uri parse error: ", err + return nil, "uri parse error: " .. err end local proxy_scheme = proxy_uri_t[1] @@ -260,7 +260,7 @@ local function connect(self, options) if not ok then return nil, "failed to connect to: " .. (proxy_host or "") .. ":" .. (proxy_port or "") .. - ": ", err + ": " .. err end if ssl and sock:getreusedtimes() == 0 then @@ -280,7 +280,7 @@ local function connect(self, options) }) if not res then - return nil, "failed to issue CONNECT to proxy:", err + return nil, "failed to issue CONNECT to proxy: " .. err end if res.status < 200 or res.status > 299 then From 183310324026120ab7eaf5dd82b9be90ae63aadf Mon Sep 17 00:00:00 2001 From: James Hurst Date: Thu, 29 Feb 2024 19:41:02 +0000 Subject: [PATCH 130/130] Bump version --- lib/resty/http.lua | 2 +- lib/resty/http_headers.lua | 2 +- ...http-0.17.1-0.rockspec => lua-resty-http-0.17.2-0.rockspec | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename lua-resty-http-0.17.1-0.rockspec => lua-resty-http-0.17.2-0.rockspec (93%) diff --git a/lib/resty/http.lua b/lib/resty/http.lua index bf75683b..a85f85af 100644 --- a/lib/resty/http.lua +++ b/lib/resty/http.lua @@ -106,7 +106,7 @@ end local _M = { - _VERSION = '0.17.1', + _VERSION = '0.17.2', } _M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version diff --git a/lib/resty/http_headers.lua b/lib/resty/http_headers.lua index 125b304c..6394e619 100644 --- a/lib/resty/http_headers.lua +++ b/lib/resty/http_headers.lua @@ -4,7 +4,7 @@ local rawget, rawset, setmetatable = local str_lower = string.lower local _M = { - _VERSION = '0.17.1', + _VERSION = '0.17.2', } diff --git a/lua-resty-http-0.17.1-0.rockspec b/lua-resty-http-0.17.2-0.rockspec similarity index 93% rename from lua-resty-http-0.17.1-0.rockspec rename to lua-resty-http-0.17.2-0.rockspec index 1d286d52..c1d81136 100644 --- a/lua-resty-http-0.17.1-0.rockspec +++ b/lua-resty-http-0.17.2-0.rockspec @@ -1,8 +1,8 @@ package = "lua-resty-http" -version = "0.17.1-0" +version = "0.17.2-0" source = { url = "git://github.com/ledgetech/lua-resty-http", - tag = "v0.17.1" + tag = "v0.17.2" } description = { summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.",