From 0e9c40a9aaaaffcbef59f604cf4c719c0eb04d91 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 17 Jun 2021 18:56:08 +1200 Subject: [PATCH 1/6] Add support for proxy. Fixes #18. --- .github/workflows/development.yml | 8 +++- lib/async/http/faraday/adapter.rb | 23 ++++++++- spec/async/http/faraday/adapter/proxy_spec.rb | 48 +++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 spec/async/http/faraday/adapter/proxy_spec.rb diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 9762ab3..6e9fc66 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -11,7 +11,7 @@ jobs: matrix: os: - ubuntu - - macos + # - macos ruby: - "2.6" @@ -32,6 +32,10 @@ jobs: ruby: head experimental: true + services: + squid: + image: sameersbn/squid + steps: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 @@ -41,4 +45,6 @@ jobs: - name: Run tests timeout-minutes: 5 + env: + PROXY_URL: "http://localhost:3128" run: ${{matrix.env}} bundle exec rspec diff --git a/lib/async/http/faraday/adapter.rb b/lib/async/http/faraday/adapter.rb index dc4cc37..07d7e69 100644 --- a/lib/async/http/faraday/adapter.rb +++ b/lib/async/http/faraday/adapter.rb @@ -60,8 +60,29 @@ def call(env) super Sync do + endpoint = Endpoint.parse(env[:url].to_s) + + if proxy = env[:proxy] + proxy_endpoint = Endpoint.parse(proxy) + client = @internet.client_for(proxy_endpoint).proxied_endpoint(endpoint) + else + client = @internet.client_for(endpoint) + end + + if body = env[:body] + body = Body::Buffered.wrap(body) + end + + if headers = env[:request_headers] + headers = ::Protocol::HTTP::Headers[headers] + end + + method = env[:method].to_s.upcase + + request = ::Protocol::HTTP::Request.new(endpoint.scheme, endpoint.authority, method, endpoint.path, nil, headers, body) + with_timeout do - response = @internet.call(env[:method].to_s.upcase, env[:url].to_s, env[:request_headers], env[:body] || []) + response = client.call(request) save_response(env, response.status, response.read, response.headers) end diff --git a/spec/async/http/faraday/adapter/proxy_spec.rb b/spec/async/http/faraday/adapter/proxy_spec.rb new file mode 100644 index 0000000..7bd427f --- /dev/null +++ b/spec/async/http/faraday/adapter/proxy_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# Copyright, 2017, by Samuel G. D. Williams. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +require 'async/http/faraday' +require 'async/http/server' +require 'async/http/endpoint' + +require 'async' + +RSpec.describe Async::HTTP::Faraday::Adapter, if: ENV.key?('PROXY_URL') do + include_context Async::RSpec::Reactor + + def get_response(url = endpoint.url, path = '/index', adapter_options: {}) + connection = Faraday.new(url: url) do |faraday| + faraday.response :logger + faraday.adapter :async_http, **adapter_options + end + + connection.get(path, proxy: ENV['PROXY_URL']) + end + + it "can get remote resource via proxy" do + Async do + response = get_response('http://www.google.com', '/search?q=cats') + + expect(response).to be_success + end + end +end From 9e58479a5cd374283399ef58f62e34e579a45d17 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 17 Jun 2021 19:57:10 +1200 Subject: [PATCH 2/6] Use dedicated clients cache. --- lib/async/http/faraday/adapter.rb | 74 +++++++++++++++---- spec/async/http/faraday/adapter/proxy_spec.rb | 6 +- spec/async/http/faraday/adapter_spec.rb | 16 +--- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/lib/async/http/faraday/adapter.rb b/lib/async/http/faraday/adapter.rb index 07d7e69..8f20c2a 100644 --- a/lib/async/http/faraday/adapter.rb +++ b/lib/async/http/faraday/adapter.rb @@ -23,7 +23,9 @@ require 'faraday' require 'faraday/adapter' require 'kernel/sync' -require 'async/http/internet' + +require 'async/http/client' +require 'async/http/proxy' require_relative 'agent' @@ -44,40 +46,82 @@ class Adapter < ::Faraday::Adapter SocketError ].freeze - def initialize(*arguments, **options, &block) - super + def initialize(*arguments, timeout: nil, **options, &block) + super(*arguments, **options) + + @timeout = timeout + + @clients = {} + @proxy_clients = {} + + @options = options + end + + def make_client(endpoint) + Client.new(endpoint, **@connection_options) + end + + def host_key(endpoint) + url = endpoint.url.dup + + url.path = "" + url.fragment = nil + url.query = nil - @internet = Async::HTTP::Internet.new - @persistent = options.fetch(:persistent, true) - @timeout = options[:timeout] + return url + end + + def client_for(endpoint) + key = host_key(endpoint) + + @clients.fetch(key) do + @clients[key] = make_client(endpoint) + end + end + + def proxy_client_for(proxy_endpoint, endpoint) + key = host_key(endpoint) + + @proxy_clients.fetch(key) do + client = client_for(proxy_endpoint) + @proxy_clients[key] = client.proxied_client(endpoint) + end end def close - @internet.close + # The order of operations here is to avoid a race condition between iterating over clients (#close may yield) and creating new clients. + proxy_clients = @proxy_clients.values + clients = @clients.values + + @proxy_clients.clear + @clients.clear + + proxy_clients.each(&:close) + clients.each(&:close) end def call(env) super Sync do - endpoint = Endpoint.parse(env[:url].to_s) + endpoint = Endpoint.new(env.url) - if proxy = env[:proxy] - proxy_endpoint = Endpoint.parse(proxy) - client = @internet.client_for(proxy_endpoint).proxied_endpoint(endpoint) + if proxy = env.request.proxy + proxy_endpoint = Endpoint.new(proxy.uri) + client = self.proxy_client_for(proxy_endpoint, endpoint) else - client = @internet.client_for(endpoint) + client = self.client_for(endpoint) end - if body = env[:body] + if body = env.body body = Body::Buffered.wrap(body) end - if headers = env[:request_headers] + if headers = env.request_headers headers = ::Protocol::HTTP::Headers[headers] end - method = env[:method].to_s.upcase + method = env.method.upcase request = ::Protocol::HTTP::Request.new(endpoint.scheme, endpoint.authority, method, endpoint.path, nil, headers, body) diff --git a/spec/async/http/faraday/adapter/proxy_spec.rb b/spec/async/http/faraday/adapter/proxy_spec.rb index 7bd427f..1d51025 100644 --- a/spec/async/http/faraday/adapter/proxy_spec.rb +++ b/spec/async/http/faraday/adapter/proxy_spec.rb @@ -30,16 +30,16 @@ include_context Async::RSpec::Reactor def get_response(url = endpoint.url, path = '/index', adapter_options: {}) - connection = Faraday.new(url: url) do |faraday| + connection = Faraday.new(url, proxy: ENV['PROXY_URL']) do |faraday| faraday.response :logger faraday.adapter :async_http, **adapter_options end - connection.get(path, proxy: ENV['PROXY_URL']) + connection.get(path) end it "can get remote resource via proxy" do - Async do + Sync do response = get_response('http://www.google.com', '/search?q=cats') expect(response).to be_success diff --git a/spec/async/http/faraday/adapter_spec.rb b/spec/async/http/faraday/adapter_spec.rb index 736c363..0fb4fca 100644 --- a/spec/async/http/faraday/adapter_spec.rb +++ b/spec/async/http/faraday/adapter_spec.rb @@ -34,7 +34,7 @@ } def run_server(response = Protocol::HTTP::Response[204], response_delay: nil) - Async do |task| + Sync do |task| begin server_task = task.async do app = Proc.new do @@ -52,11 +52,11 @@ def run_server(response = Protocol::HTTP::Response[204], response_delay: nil) ensure server_task.stop end - end.wait + end end def get_response(url = endpoint.url, path = '/index', adapter_options: {}) - connection = Faraday.new(url: url) do |faraday| + connection = Faraday.new(url) do |faraday| faraday.response :logger faraday.adapter :async_http, **adapter_options end @@ -80,7 +80,7 @@ def get_response(url = endpoint.url, path = '/index', adapter_options: {}) end it "can get remote resource" do - Async do + Sync do response = get_response('http://www.google.com', '/search?q=cats') expect(response).to be_success @@ -101,14 +101,6 @@ def get_response(url = endpoint.url, path = '/index', adapter_options: {}) end end - it 'closes connection automatically if persistent option is set to false' do - run_server do - expect do - get_response(adapter_options: { persistent: false }) - end.not_to raise_error - end - end - it 'raises an exception if request times out' do delay = 0.1 From 0cd170fc09d43cb67ed305812fe16f467ddb381f Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 17 Jun 2021 20:33:27 +1200 Subject: [PATCH 3/6] Explicit squid ports. --- .github/workflows/development.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 6e9fc66..e8dc6a2 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -35,6 +35,8 @@ jobs: services: squid: image: sameersbn/squid + ports: + - 3128:3128 steps: - uses: actions/checkout@v2 From 065559d0be9fdb213a10d187a6052dafcc79ba65 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 17 Jun 2021 20:38:35 +1200 Subject: [PATCH 4/6] Use squid directly. --- .github/workflows/development.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index e8dc6a2..52c4cbe 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -11,7 +11,7 @@ jobs: matrix: os: - ubuntu - # - macos + - macos ruby: - "2.6" @@ -31,12 +31,9 @@ jobs: - os: ubuntu ruby: head experimental: true - - services: - squid: - image: sameersbn/squid - ports: - - 3128:3128 + - os: ubuntu + ruby: "2.7" + proxy: true steps: - uses: actions/checkout@v2 @@ -45,8 +42,14 @@ jobs: ruby-version: ${{matrix.ruby}} bundler-cache: true - - name: Run tests - timeout-minutes: 5 + - name: Prepare squid + if: matrix.proxy env: PROXY_URL: "http://localhost:3128" + run: | + sudo apt-get install squid + sudo systemctl start squid + + - name: Run tests + timeout-minutes: 5 run: ${{matrix.env}} bundle exec rspec From 8f612d893f7fb4cbf2456c0a31dee1cd7fb13586 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Thu, 17 Jun 2021 20:52:40 +1200 Subject: [PATCH 5/6] Separate test for proxy. --- .github/workflows/development.yml | 36 ++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 52c4cbe..b07e98a 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -33,7 +33,7 @@ jobs: experimental: true - os: ubuntu ruby: "2.7" - proxy: true + proxy: http://localhost:3128 steps: - uses: actions/checkout@v2 @@ -42,14 +42,44 @@ jobs: ruby-version: ${{matrix.ruby}} bundler-cache: true + - name: Run tests + timeout-minutes: 5 + run: ${{matrix.env}} bundle exec rspec + + test-with-proxy: + runs-on: ${{matrix.os}}-latest + continue-on-error: ${{matrix.experimental}} + + strategy: + matrix: + os: + - ubuntu + ruby: + - "2.7" + + experimental: [false] + env: [""] + + proxy: + - http://localhost:3128 + + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{matrix.ruby}} + + - name: Install dependencies + run: ${{matrix.env}} bundle install + - name: Prepare squid if: matrix.proxy - env: - PROXY_URL: "http://localhost:3128" run: | sudo apt-get install squid sudo systemctl start squid - name: Run tests timeout-minutes: 5 + env: + PROXY_URL: ${{matrix.proxy}} run: ${{matrix.env}} bundle exec rspec From 63da0089df041aa6de60255b8e9440d43582e242 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sun, 11 Dec 2022 21:50:30 +1300 Subject: [PATCH 6/6] Prefer https. --- spec/async/http/faraday/adapter/proxy_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/async/http/faraday/adapter/proxy_spec.rb b/spec/async/http/faraday/adapter/proxy_spec.rb index 1d51025..db4254e 100644 --- a/spec/async/http/faraday/adapter/proxy_spec.rb +++ b/spec/async/http/faraday/adapter/proxy_spec.rb @@ -40,7 +40,7 @@ def get_response(url = endpoint.url, path = '/index', adapter_options: {}) it "can get remote resource via proxy" do Sync do - response = get_response('http://www.google.com', '/search?q=cats') + response = get_response('https://www.google.com', '/search?q=cats') expect(response).to be_success end